$ git diff --patch-with-stat --summary 899acbd766b157ef2446990460790c768e705dbe..8b71359564ed514682b6057824394ec385a64627
.abf.yml | 6 +-
0001-Fix-for-compilation-with-clang.patch | 42 -
0001-cpu-5.2-merge-graysky-s-patchset.patch | 6 +-
...ld-LLVMLinux-Set-compiler-flags-for-clang.patch | 59 -
...inux-Remove-warning-from-COMPATIBLE_IOCTL.patch | 37 -
0002-lib-Add-zstd-modules.patch | 13301 -------------------
0003-btrfs-Add-zstd-support.patch | 740 --
...Linux-Add-support-for-generating-LLVM-bit.patch | 94 -
...Linux-Make-asm-offset-generation-work-wit.patch | 121 -
0004-squashfs-Add-zstd-support.patch | 306 -
0005-crypto-Add-zstd-support.patch | 393 -
...VMLinux-Remove-nested-function-from-bcach.patch | 65 -
0006-apparmor-LLVMLinux-Remove-VLAIS.patch | 108 -
...inux-Remove-VLAIS-from-exofs-FIXME-Check-.patch | 91 -
...Linux-Use-Oz-instead-of-Os-when-using-cla.patch | 35 -
...x86-boot-LLVMLinux-Work-around-clang-PR39.patch | 71 -
...REAM-xen-LLVMLinux-Remove-VLAIS-from-xen-.patch | 76 -
...REAM-arm-LLVMLinux-Provide-__aeabi_-symbo.patch | 72 -
...REAM-arm-firmware-LLVMLinux-replace-naked.patch | 47 -
...ux-Remove-unreachable-from-naked-function.patch | 28 -
...nux-Fix-a-cast-to-type-not-present-in-uni.patch | 36 -
...nux-Fix-an-inline-asm-input-output-type-m.patch | 41 -
...nux-Silence-variable-self-assignment-warn.patch | 123 -
...nux-Silence-unicode-warnings-when-preproc.patch | 44 -
...e-attributes-error-and-warning-with-clang.patch | 30 -
...ed-references-to-acpi_idle_driver-on-aarc.patch | 51 -
...ware-LLVMLinux-fix-EFI-libstub-with-clang.patch | 46 -
...pto-LLVMLinux-Fix-inline-assembly-for-cla.patch | 46 -
...MLinux-Make-spin_lock_prefetch-asm-code-c.patch | 34 -
...on-t-use-attribute-externally_visible-whe.patch | 33 -
...LLVMLinux-Fix-building-x86_64-AES-extensi.patch | 32 -
...ux-Qualify-mul-as-mulq-to-make-clang-happ.patch | 33 -
...Linux-Add-Werror-to-cc-option-in-order-to.patch | 55 -
...LLVMLinux-Check-for-compiler-support-of-f.patch | 37 -
...-6602f080cb28745259e2fab1a4cf55eeb5894f93.patch | 166 +
0107-intel_idle-tweak-cpuidle-cstates.patch | 78 +-
...Don-t-declare-__force_order-in-kaslr_64.c.patch | 37 +
7a8b64d17e35810dc3176fe61208b45c15d25402.patch | 77 +
9d55bebd9816903b821a403a69a94190442ac043.patch | 68 +
...ble-building-ashmem-and-binder-as-modules.patch | 114 +-
arm64-common.config | 6 +
btrfs-zstd-backport-to-4.12-api.patch | 17 -
common-desktop.config | 2 +
common-server.config | 7 +-
common.config | 32 +-
export-symbols-needed-by-android-drivers.patch | 75 +-
extra-wifi-drivers-compile.patch | 50 +
extra-wifi-drivers-port-to-5.6.patch | 412 +
kernel-5.6-kvm-gcc10.patch | 12 +
...l-5.7-fewer-conditions-for-ARM64_PTR_AUTH.patch | 27 +
...l-5.8-nouveau-write-combining-only-on-x86.patch | 72 +
kernel-release.spec | 412 +-
linux-4.7-intel-dvi-duallink.patch | 18 +-
linux-4.8.1-buildfix.patch | 33 -
linux-5.5-corsair-strafe-quirks.patch | 22 +
linux-5.5.tar.sign | 19 -
linux-5.6-fix-disassembler-4args-detection.patch | 13 +
smpboot-no-stack-protector-for-gcc10.patch | 13 +
socket.h-include-bitsperlong.h.patch | 11 +
...ernel-and-ramfs-comp-and-decomp-with-zstd.patch | 407 +-
uksm-5.4.patch => uksm-5.6.patch | 118 +-
uksm-5.7.patch | 6936 ++++++++++
...tualBox-guest-shared-folder-vboxsf-support.diff | 3370 -----
...86-Add-support-for-ZSTD-compressed-kernel.patch | 102 -
vbox-6.1-fix-build-on-znver1-hosts.patch | 6 +-
x86_64-common.config | 3 +-
66 files changed, 8565 insertions(+), 20509 deletions(-)
delete mode 100644 0001-Fix-for-compilation-with-clang.patch
delete mode 100644 0001-kbuild-LLVMLinux-Set-compiler-flags-for-clang.patch
delete mode 100644 0002-fs-LLVMLinux-Remove-warning-from-COMPATIBLE_IOCTL.patch
delete mode 100644 0002-lib-Add-zstd-modules.patch
delete mode 100644 0003-btrfs-Add-zstd-support.patch
delete mode 100644 0003-kbuild-LLVMLinux-Add-support-for-generating-LLVM-bit.patch
delete mode 100644 0004-kbuild-LLVMLinux-Make-asm-offset-generation-work-wit.patch
delete mode 100644 0004-squashfs-Add-zstd-support.patch
delete mode 100644 0005-crypto-Add-zstd-support.patch
delete mode 100644 0005-md-sysfs-LLVMLinux-Remove-nested-function-from-bcach.patch
delete mode 100644 0006-apparmor-LLVMLinux-Remove-VLAIS.patch
delete mode 100644 0007-exofs-LLVMLinux-Remove-VLAIS-from-exofs-FIXME-Check-.patch
delete mode 100644 0011-Kbuild-LLVMLinux-Use-Oz-instead-of-Os-when-using-cla.patch
delete mode 100644 0012-WORKAROUND-x86-boot-LLVMLinux-Work-around-clang-PR39.patch
delete mode 100644 0013-DO-NOT-UPSTREAM-xen-LLVMLinux-Remove-VLAIS-from-xen-.patch
delete mode 100644 0014-DO-NOT-UPSTREAM-arm-LLVMLinux-Provide-__aeabi_-symbo.patch
delete mode 100644 0015-DO-NOT-UPSTREAM-arm-firmware-LLVMLinux-replace-naked.patch
delete mode 100644 0016-arm-LLVMLinux-Remove-unreachable-from-naked-function.patch
delete mode 100644 0017-MIPS-LLVMLinux-Fix-a-cast-to-type-not-present-in-uni.patch
delete mode 100644 0018-MIPS-LLVMLinux-Fix-an-inline-asm-input-output-type-m.patch
delete mode 100644 0019-MIPS-LLVMLinux-Silence-variable-self-assignment-warn.patch
delete mode 100644 0020-MIPS-LLVMLinux-Silence-unicode-warnings-when-preproc.patch
delete mode 100644 0021-Don-t-use-attributes-error-and-warning-with-clang.patch
delete mode 100644 0022-Fix-undefined-references-to-acpi_idle_driver-on-aarc.patch
delete mode 100644 0023-HACK-firmware-LLVMLinux-fix-EFI-libstub-with-clang.patch
delete mode 100644 0024-aarch64-crypto-LLVMLinux-Fix-inline-assembly-for-cla.patch
delete mode 100644 0025-aarch64-LLVMLinux-Make-spin_lock_prefetch-asm-code-c.patch
delete mode 100644 0026-LLVMLinux-Don-t-use-attribute-externally_visible-whe.patch
delete mode 100644 0027-x86-crypto-LLVMLinux-Fix-building-x86_64-AES-extensi.patch
delete mode 100644 0028-x86-LLVMLinux-Qualify-mul-as-mulq-to-make-clang-happ.patch
delete mode 100644 0029-kbuild-LLVMLinux-Add-Werror-to-cc-option-in-order-to.patch
delete mode 100644 0030-x86-kbuild-LLVMLinux-Check-for-compiler-support-of-f.patch
create mode 100644 01-revert-6602f080cb28745259e2fab1a4cf55eeb5894f93.patch
create mode 100644 2-0001-x86-Don-t-declare-__force_order-in-kaslr_64.c.patch
create mode 100644 7a8b64d17e35810dc3176fe61208b45c15d25402.patch
create mode 100644 9d55bebd9816903b821a403a69a94190442ac043.patch
delete mode 100644 btrfs-zstd-backport-to-4.12-api.patch
create mode 100644 extra-wifi-drivers-compile.patch
create mode 100644 extra-wifi-drivers-port-to-5.6.patch
create mode 100644 kernel-5.6-kvm-gcc10.patch
create mode 100644 kernel-5.7-fewer-conditions-for-ARM64_PTR_AUTH.patch
create mode 100644 kernel-5.8-nouveau-write-combining-only-on-x86.patch
delete mode 100644 linux-4.8.1-buildfix.patch
create mode 100644 linux-5.5-corsair-strafe-quirks.patch
delete mode 100644 linux-5.5.tar.sign
create mode 100644 linux-5.6-fix-disassembler-4args-detection.patch
create mode 100644 smpboot-no-stack-protector-for-gcc10.patch
create mode 100644 socket.h-include-bitsperlong.h.patch
rename v2-1-2-lib-Add-support-for-ZSTD-compressed-kernel.patch => support-kernel-and-ramfs-comp-and-decomp-with-zstd.patch (59%)
rename uksm-5.4.patch => uksm-5.6.patch (98%)
create mode 100644 uksm-5.7.patch
delete mode 100644 v19-fs-Add-VirtualBox-guest-shared-folder-vboxsf-support.diff
delete mode 100644 v2-2-2-x86-Add-support-for-ZSTD-compressed-kernel.patch
diff --git a/.abf.yml b/.abf.yml
index 8d4126e..cef6999 100644
--- a/.abf.yml
+++ b/.abf.yml
@@ -1,4 +1,6 @@
sources:
- linux-5.5.tar.xz: 527d2b01ecb9223ee041ba4b5d703cfe739783b2
+ extra-wifi-drivers-20200301.tar.zst: 3390c738c7d91250714ce0f88d26371e93bc40b8
saa716x-driver.tar.xz: f9b6ef1cd6f1f71f53d9a8aadfba2cf6b5c3d7b6
- patch-5.5.2.xz: b547b768b4e11b254cfd53f87d6ce7417b9c3356
+ linux-5.7.tar.xz: 07e40057b78f1c9dd2b042056325d99fcf9f8a08
+ linux-5.7.tar.sign: 9bdcc9e48ee0eca0baa8118b9a10ba2eda64363e
+ patch-5.7.11.xz: 22e11d00f94d641f0167db08f61a8feb2f231a25
diff --git a/0001-Fix-for-compilation-with-clang.patch b/0001-Fix-for-compilation-with-clang.patch
deleted file mode 100644
index b5da460..0000000
--- a/0001-Fix-for-compilation-with-clang.patch
+++ /dev/null
@@ -1,42 +0,0 @@
->From 14e1f86ad297cc3c2b575135699b8c08849d1c6a Mon Sep 17 00:00:00 2001
-From: Hans Petter Selasky <hps@selasky.org>
-Date: Tue, 25 Apr 2017 09:30:47 +0200
-Subject: [PATCH] Fix for compilation with clang
-
-The clang compiler treats the memory address of a structure field as
-tautologically true when compared to non-NULL. This breaks the macros
-in llist.h when compiled using clang. Fix this by not comparing the
-address of a structure field to NULL when determining the end of the
-llist. Refer to -Wtautological-pointer-compare in clang for more
-information.
-
-Signed-off-by: Hans Petter Selasky <hps@selasky.org>
----
- include/linux/llist.h | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/include/linux/llist.h b/include/linux/llist.h
-index 171baa90f6f6..76a6d618de75 100644
---- a/include/linux/llist.h
-+++ b/include/linux/llist.h
-@@ -126,7 +126,7 @@ static inline void init_llist_head(struct llist_head *list)
- */
- #define llist_for_each_entry(pos, node, member) \
- for ((pos) = llist_entry((node), typeof(*(pos)), member); \
-- &(pos)->member != NULL; \
-+ (pos) != llist_entry(NULL, typeof(*(pos)), member); \
- (pos) = llist_entry((pos)->member.next, typeof(*(pos)), member))
-
- /**
-@@ -148,7 +148,7 @@ static inline void init_llist_head(struct llist_head *list)
- */
- #define llist_for_each_entry_safe(pos, n, node, member) \
- for (pos = llist_entry((node), typeof(*pos), member); \
-- &pos->member != NULL && \
-+ (pos) != llist_entry(NULL, typeof(*(pos)), member) && \
- (n = llist_entry(pos->member.next, typeof(*n), member), true); \
- pos = n)
-
---
-2.11.1
-
diff --git a/0001-cpu-5.2-merge-graysky-s-patchset.patch b/0001-cpu-5.2-merge-graysky-s-patchset.patch
index 38091dc..ad3c196 100644
--- a/0001-cpu-5.2-merge-graysky-s-patchset.patch
+++ b/0001-cpu-5.2-merge-graysky-s-patchset.patch
@@ -1,6 +1,6 @@
-diff -up linux-5.5-rc1/arch/x86/include/asm/module.h.17~ linux-5.5-rc1/arch/x86/include/asm/module.h
---- linux-5.5-rc1/arch/x86/include/asm/module.h.17~ 2019-12-08 23:57:55.000000000 +0100
-+++ linux-5.5-rc1/arch/x86/include/asm/module.h 2019-12-10 17:00:36.851313621 +0100
+diff -up linux-5.5-rc1/arch/x86/include/asm/vermagic.h.17~ linux-5.5-rc1/arch/x86/include/asm/vermagic.h
+--- linux-5.5-rc1/arch/x86/include/asm/vermagic.h.17~ 2019-12-08 23:57:55.000000000 +0100
++++ linux-5.5-rc1/arch/x86/include/asm/vermagic.h 2019-12-10 17:00:36.851313621 +0100
@@ -27,6 +27,30 @@ struct mod_arch_specific {
#define MODULE_PROC_FAMILY "586MMX "
#elif defined CONFIG_MCORE2
diff --git a/0001-kbuild-LLVMLinux-Set-compiler-flags-for-clang.patch b/0001-kbuild-LLVMLinux-Set-compiler-flags-for-clang.patch
deleted file mode 100644
index e906bf9..0000000
--- a/0001-kbuild-LLVMLinux-Set-compiler-flags-for-clang.patch
+++ /dev/null
@@ -1,59 +0,0 @@
---- linux-4.10/Makefile.1000~ 2017-04-25 20:31:43.347811443 +0200
-+++ linux-4.10/Makefile 2017-04-25 20:32:20.593089243 +0200
-@@ -375,6 +375,19 @@ LDFLAGS_vmlinux =
- CFLAGS_GCOV := -fprofile-arcs -ftest-coverage -fno-tree-loop-im $(call cc-disable-warning,maybe-uninitialized,)
- CFLAGS_KCOV := $(call cc-option,-fsanitize-coverage=trace-pc,)
-
-+ifeq ($(cc-name),clang)
-+ifneq ($(CROSS_COMPILE),)
-+CLANG_TARGET := -target $(notdir $(CROSS_COMPILE:%-=%))
-+GCC_TOOLCHAIN := $(dir $(CROSS_COMPILE))
-+endif
-+ifneq ($(GCC_TOOLCHAIN),)
-+CLANG_GCC_TC := -gcc-toolchain $(GCC_TOOLCHAIN)
-+endif
-+ifneq ($(IA),1)
-+CLANG_IA_FLAG := -fno-integrated-as
-+endif
-+CLANG_FLAGS := $(CLANG_TARGET) $(CLANG_GCC_TC) $(CLANG_IA_FLAG) -meabi gnu
-+endif
-
- # Use USERINCLUDE when you must reference the UAPI directories only.
- USERINCLUDE := \
-@@ -400,13 +413,13 @@ KBUILD_CPPFLAGS := -D__KERNEL__
- KBUILD_CFLAGS := -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \
- -fno-strict-aliasing -fno-common \
- -Werror-implicit-function-declaration \
-- -Wno-format-security \
-+ -Wno-format-security $(CLANG_FLAGS) \
- -std=gnu89 $(call cc-option,-fno-PIE)
-
-
- KBUILD_AFLAGS_KERNEL :=
- KBUILD_CFLAGS_KERNEL :=
--KBUILD_AFLAGS := -D__ASSEMBLY__ $(call cc-option,-fno-PIE)
-+KBUILD_AFLAGS := -D__ASSEMBLY__ $(call cc-option,-fno-PIE) $(CLANG_FLAGS)
- KBUILD_AFLAGS_MODULE := -DMODULE
- KBUILD_CFLAGS_MODULE := -DMODULE
- KBUILD_LDFLAGS_MODULE := -T $(srctree)/scripts/module-common.lds
-@@ -700,9 +713,12 @@ KBUILD_CFLAGS += $(stackp-flag)
- ifeq ($(cc-name),clang)
- KBUILD_CPPFLAGS += $(call cc-option,-Qunused-arguments,)
- KBUILD_CPPFLAGS += $(call cc-option,-Wno-unknown-warning-option,)
-+KBUILD_CPPFLAGS += $(call cc-option,-fno-integrated-as,)
- KBUILD_CFLAGS += $(call cc-disable-warning, unused-variable)
- KBUILD_CFLAGS += $(call cc-disable-warning, format-invalid-specifier)
- KBUILD_CFLAGS += $(call cc-disable-warning, gnu)
-+KBUILD_CFLAGS += $(call cc-disable-warning, asm-operand-widths)
-+KBUILD_CFLAGS += $(call cc-disable-warning, initializer-overrides)
- # Quiet clang warning: comparison of unsigned expression < 0 is always false
- KBUILD_CFLAGS += $(call cc-disable-warning, tautological-compare)
- # CLANG uses a _MergedGlobals as optimization, but this breaks modpost, as the
-@@ -710,6 +726,7 @@ KBUILD_CFLAGS += $(call cc-disable-warni
- # See modpost pattern 2
- KBUILD_CFLAGS += $(call cc-option, -mno-global-merge,)
- KBUILD_CFLAGS += $(call cc-option, -fcatch-undefined-behavior)
-+KBUILD_CFLAGS += $(call cc-option, -fno-builtin)
- else
-
- # These warnings generated too much noise in a regular build.
diff --git a/0002-fs-LLVMLinux-Remove-warning-from-COMPATIBLE_IOCTL.patch b/0002-fs-LLVMLinux-Remove-warning-from-COMPATIBLE_IOCTL.patch
deleted file mode 100644
index dc11701..0000000
--- a/0002-fs-LLVMLinux-Remove-warning-from-COMPATIBLE_IOCTL.patch
+++ /dev/null
@@ -1,37 +0,0 @@
-From 73207dae13dbf5219a244376e6d3d6bc59b0a56b Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:06:22 +0100
-Subject: [PATCH 02/30] fs, LLVMLinux: Remove warning from COMPATIBLE_IOCTL
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-cmd in COMPATIBLE_IOCTL is always a u32, so cast it so there isn't a warning
-about an overflow in XFORM.
-
-Author: Mark Charlebois <charlebm@gmail.com>
-Signed-off-by: Mark Charlebois <charlebm@gmail.com>
-Signed-off-by: Behan Webster <behanw@converseincode.com>
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
-Acked-by: Arnd Bergmann <arnd@arndb.de>
----
- fs/compat_ioctl.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/fs/compat_ioctl.c b/fs/compat_ioctl.c
-index 11d087b2b28e..6116d5275a3e 100644
---- a/fs/compat_ioctl.c
-+++ b/fs/compat_ioctl.c
-@@ -833,7 +833,7 @@ static int compat_ioctl_preallocate(struct file *file,
- */
- #define XFORM(i) (((i) ^ ((i) << 27) ^ ((i) << 17)) & 0xffffffff)
-
--#define COMPATIBLE_IOCTL(cmd) XFORM(cmd),
-+#define COMPATIBLE_IOCTL(cmd) XFORM((u32)cmd),
- /* ioctl should not be warned about even if it's not implemented.
- Valid reasons to use this:
- - It is implemented with ->compat_ioctl on some device, but programs
---
-2.11.0
-
diff --git a/0002-lib-Add-zstd-modules.patch b/0002-lib-Add-zstd-modules.patch
deleted file mode 100644
index eb8b8b2..0000000
--- a/0002-lib-Add-zstd-modules.patch
+++ /dev/null
@@ -1,13301 +0,0 @@
-From b7f044163968d724be55bf4841fd80babe036dc2 Mon Sep 17 00:00:00 2001
-From: Nick Terrell <terrelln@fb.com>
-Date: Mon, 17 Jul 2017 17:08:19 -0700
-Subject: [PATCH v5 2/5] lib: Add zstd modules
-
-Add zstd compression and decompression kernel modules.
-zstd offers a wide varity of compression speed and quality trade-offs.
-It can compress at speeds approaching lz4, and quality approaching lzma.
-zstd decompressions at speeds more than twice as fast as zlib, and
-decompression speed remains roughly the same across all compression levels.
-
-The code was ported from the upstream zstd source repository. The
-`linux/zstd.h` header was modified to match linux kernel style.
-The cross-platform and allocation code was stripped out. Instead zstd
-requires the caller to pass a preallocated workspace. The source files
-were clang-formatted [1] to match the Linux Kernel style as much as
-possible. Otherwise, the code was unmodified. We would like to avoid
-as much further manual modification to the source code as possible, so it
-will be easier to keep the kernel zstd up to date.
-
-I benchmarked zstd compression as a special character device. I ran zstd
-and zlib compression at several levels, as well as performing no
-compression, which measure the time spent copying the data to kernel space.
-Data is passed to the compresser 4096 B at a time. The benchmark file is
-located in the upstream zstd source repository under
-`contrib/linux-kernel/zstd_compress_test.c` [2].
-
-I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
-The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
-16 GB of RAM, and a SSD. I benchmarked using `silesia.tar` [3], which is
-211,988,480 B large. Run the following commands for the benchmark:
-
- sudo modprobe zstd_compress_test
- sudo mknod zstd_compress_test c 245 0
- sudo cp silesia.tar zstd_compress_test
-
-The time is reported by the time of the userland `cp`.
-The MB/s is computed with
-
- 1,536,217,008 B / time(buffer size, hash)
-
-which includes the time to copy from userland.
-The Adjusted MB/s is computed with
-
- 1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)).
-
-The memory reported is the amount of memory the compressor requests.
-
-| Method | Size (B) | Time (s) | Ratio | MB/s | Adj MB/s | Mem (MB) |
-|----------|----------|----------|-------|---------|----------|----------|
-| none | 11988480 | 0.100 | 1 | 2119.88 | - | - |
-| zstd -1 | 73645762 | 1.044 | 2.878 | 203.05 | 224.56 | 1.23 |
-| zstd -3 | 66988878 | 1.761 | 3.165 | 120.38 | 127.63 | 2.47 |
-| zstd -5 | 65001259 | 2.563 | 3.261 | 82.71 | 86.07 | 2.86 |
-| zstd -10 | 60165346 | 13.242 | 3.523 | 16.01 | 16.13 | 13.22 |
-| zstd -15 | 58009756 | 47.601 | 3.654 | 4.45 | 4.46 | 21.61 |
-| zstd -19 | 54014593 | 102.835 | 3.925 | 2.06 | 2.06 | 60.15 |
-| zlib -1 | 77260026 | 2.895 | 2.744 | 73.23 | 75.85 | 0.27 |
-| zlib -3 | 72972206 | 4.116 | 2.905 | 51.50 | 52.79 | 0.27 |
-| zlib -6 | 68190360 | 9.633 | 3.109 | 22.01 | 22.24 | 0.27 |
-| zlib -9 | 67613382 | 22.554 | 3.135 | 9.40 | 9.44 | 0.27 |
-
-I benchmarked zstd decompression using the same method on the same machine.
-The benchmark file is located in the upstream zstd repo under
-`contrib/linux-kernel/zstd_decompress_test.c` [4]. The memory reported is
-the amount of memory required to decompress data compressed with the given
-compression level. If you know the maximum size of your input, you can
-reduce the memory usage of decompression irrespective of the compression
-level.
-
-| Method | Time (s) | MB/s | Adjusted MB/s | Memory (MB) |
-|----------|----------|---------|---------------|-------------|
-| none | 0.025 | 8479.54 | - | - |
-| zstd -1 | 0.358 | 592.15 | 636.60 | 0.84 |
-| zstd -3 | 0.396 | 535.32 | 571.40 | 1.46 |
-| zstd -5 | 0.396 | 535.32 | 571.40 | 1.46 |
-| zstd -10 | 0.374 | 566.81 | 607.42 | 2.51 |
-| zstd -15 | 0.379 | 559.34 | 598.84 | 4.61 |
-| zstd -19 | 0.412 | 514.54 | 547.77 | 8.80 |
-| zlib -1 | 0.940 | 225.52 | 231.68 | 0.04 |
-| zlib -3 | 0.883 | 240.08 | 247.07 | 0.04 |
-| zlib -6 | 0.844 | 251.17 | 258.84 | 0.04 |
-| zlib -9 | 0.837 | 253.27 | 287.64 | 0.04 |
-
-Tested in userland using the test-suite in the zstd repo under
-`contrib/linux-kernel/test/UserlandTest.cpp` [5] by mocking the kernel
-functions. Fuzz tested using libfuzzer [6] with the fuzz harnesses under
-`contrib/linux-kernel/test/{RoundTripCrash.c,DecompressCrash.c}` [7] [8]
-with ASAN, UBSAN, and MSAN. Additionaly, it was tested while testing the
-BtrFS and SquashFS patches coming next.
-
-[1] https://clang.llvm.org/docs/ClangFormat.html
-[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/zstd_compress_test.c
-[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
-[4] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/zstd_decompress_test.c
-[5] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/UserlandTest.cpp
-[6] http://llvm.org/docs/LibFuzzer.html
-[7] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/RoundTripCrash.c
-[8] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/DecompressCrash.c
-
-zstd source repository: https://github.com/facebook/zstd
-
-Signed-off-by: Nick Terrell <terrelln@fb.com>
----
-v1 -> v2:
-- Use div_u64() for division of u64s
-- Reduce stack usage of ZSTD_compressSequences(), ZSTD_buildSeqTable(),
- ZSTD_decompressSequencesLong(), FSE_buildDTable(), FSE_decompress_wksp(),
- HUF_writeCTable(), HUF_readStats(), HUF_readCTable(),
- HUF_compressWeights(), HUF_readDTableX2(), and HUF_readDTableX4()
-- No function uses more than 400 B of stack space
-
-v2 -> v3:
-- Work around gcc-7 bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388
-- Fix bug in dictionary compression from upstream commit cc1522351f
-
-v3 -> v4:
-- Fix minor compiler warnings
-
-v4 -> v5:
-- Fix rare compression bug from upstream commit 308047eb5d
-- Fix bug introduced in v3 when working around the gcc-7 bug
-
- include/linux/zstd.h | 1157 +++++++++++++++
- lib/Kconfig | 8 +
- lib/Makefile | 2 +
- lib/zstd/Makefile | 18 +
- lib/zstd/bitstream.h | 374 +++++
- lib/zstd/compress.c | 3484 +++++++++++++++++++++++++++++++++++++++++++++
- lib/zstd/decompress.c | 2528 ++++++++++++++++++++++++++++++++
- lib/zstd/entropy_common.c | 243 ++++
- lib/zstd/error_private.h | 53 +
- lib/zstd/fse.h | 575 ++++++++
- lib/zstd/fse_compress.c | 795 +++++++++++
- lib/zstd/fse_decompress.c | 332 +++++
- lib/zstd/huf.h | 212 +++
- lib/zstd/huf_compress.c | 770 ++++++++++
- lib/zstd/huf_decompress.c | 960 +++++++++++++
- lib/zstd/mem.h | 151 ++
- lib/zstd/zstd_common.c | 75 +
- lib/zstd/zstd_internal.h | 263 ++++
- lib/zstd/zstd_opt.h | 1014 +++++++++++++
- 19 files changed, 13014 insertions(+)
- create mode 100644 include/linux/zstd.h
- create mode 100644 lib/zstd/Makefile
- create mode 100644 lib/zstd/bitstream.h
- create mode 100644 lib/zstd/compress.c
- create mode 100644 lib/zstd/decompress.c
- create mode 100644 lib/zstd/entropy_common.c
- create mode 100644 lib/zstd/error_private.h
- create mode 100644 lib/zstd/fse.h
- create mode 100644 lib/zstd/fse_compress.c
- create mode 100644 lib/zstd/fse_decompress.c
- create mode 100644 lib/zstd/huf.h
- create mode 100644 lib/zstd/huf_compress.c
- create mode 100644 lib/zstd/huf_decompress.c
- create mode 100644 lib/zstd/mem.h
- create mode 100644 lib/zstd/zstd_common.c
- create mode 100644 lib/zstd/zstd_internal.h
- create mode 100644 lib/zstd/zstd_opt.h
-
-diff --git a/include/linux/zstd.h b/include/linux/zstd.h
-new file mode 100644
-index 0000000..249575e
---- /dev/null
-+++ b/include/linux/zstd.h
-@@ -0,0 +1,1157 @@
-+/*
-+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This source code is licensed under the BSD-style license found in the
-+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
-+ * An additional grant of patent rights can be found in the PATENTS file in the
-+ * same directory.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ */
-+
-+#ifndef ZSTD_H
-+#define ZSTD_H
-+
-+/* ====== Dependency ======*/
-+#include <linux/types.h> /* size_t */
-+
-+
-+/*-*****************************************************************************
-+ * Introduction
-+ *
-+ * zstd, short for Zstandard, is a fast lossless compression algorithm,
-+ * targeting real-time compression scenarios at zlib-level and better
-+ * compression ratios. The zstd compression library provides in-memory
-+ * compression and decompression functions. The library supports compression
-+ * levels from 1 up to ZSTD_maxCLevel() which is 22. Levels >= 20, labeled
-+ * ultra, should be used with caution, as they require more memory.
-+ * Compression can be done in:
-+ * - a single step, reusing a context (described as Explicit memory management)
-+ * - unbounded multiple steps (described as Streaming compression)
-+ * The compression ratio achievable on small data can be highly improved using
-+ * compression with a dictionary in:
-+ * - a single step (described as Simple dictionary API)
-+ * - a single step, reusing a dictionary (described as Fast dictionary API)
-+ ******************************************************************************/
-+
-+/*====== Helper functions ======*/
-+
-+/**
-+ * enum ZSTD_ErrorCode - zstd error codes
-+ *
-+ * Functions that return size_t can be checked for errors using ZSTD_isError()
-+ * and the ZSTD_ErrorCode can be extracted using ZSTD_getErrorCode().
-+ */
-+typedef enum {
-+ ZSTD_error_no_error,
-+ ZSTD_error_GENERIC,
-+ ZSTD_error_prefix_unknown,
-+ ZSTD_error_version_unsupported,
-+ ZSTD_error_parameter_unknown,
-+ ZSTD_error_frameParameter_unsupported,
-+ ZSTD_error_frameParameter_unsupportedBy32bits,
-+ ZSTD_error_frameParameter_windowTooLarge,
-+ ZSTD_error_compressionParameter_unsupported,
-+ ZSTD_error_init_missing,
-+ ZSTD_error_memory_allocation,
-+ ZSTD_error_stage_wrong,
-+ ZSTD_error_dstSize_tooSmall,
-+ ZSTD_error_srcSize_wrong,
-+ ZSTD_error_corruption_detected,
-+ ZSTD_error_checksum_wrong,
-+ ZSTD_error_tableLog_tooLarge,
-+ ZSTD_error_maxSymbolValue_tooLarge,
-+ ZSTD_error_maxSymbolValue_tooSmall,
-+ ZSTD_error_dictionary_corrupted,
-+ ZSTD_error_dictionary_wrong,
-+ ZSTD_error_dictionaryCreation_failed,
-+ ZSTD_error_maxCode
-+} ZSTD_ErrorCode;
-+
-+/**
-+ * ZSTD_maxCLevel() - maximum compression level available
-+ *
-+ * Return: Maximum compression level available.
-+ */
-+int ZSTD_maxCLevel(void);
-+/**
-+ * ZSTD_compressBound() - maximum compressed size in worst case scenario
-+ * @srcSize: The size of the data to compress.
-+ *
-+ * Return: The maximum compressed size in the worst case scenario.
-+ */
-+size_t ZSTD_compressBound(size_t srcSize);
-+/**
-+ * ZSTD_isError() - tells if a size_t function result is an error code
-+ * @code: The function result to check for error.
-+ *
-+ * Return: Non-zero iff the code is an error.
-+ */
-+static __attribute__((unused)) unsigned int ZSTD_isError(size_t code)
-+{
-+ return code > (size_t)-ZSTD_error_maxCode;
-+}
-+/**
-+ * ZSTD_getErrorCode() - translates an error function result to a ZSTD_ErrorCode
-+ * @functionResult: The result of a function for which ZSTD_isError() is true.
-+ *
-+ * Return: The ZSTD_ErrorCode corresponding to the functionResult or 0
-+ * if the functionResult isn't an error.
-+ */
-+static __attribute__((unused)) ZSTD_ErrorCode ZSTD_getErrorCode(
-+ size_t functionResult)
-+{
-+ if (!ZSTD_isError(functionResult))
-+ return (ZSTD_ErrorCode)0;
-+ return (ZSTD_ErrorCode)(0 - functionResult);
-+}
-+
-+/**
-+ * enum ZSTD_strategy - zstd compression search strategy
-+ *
-+ * From faster to stronger.
-+ */
-+typedef enum {
-+ ZSTD_fast,
-+ ZSTD_dfast,
-+ ZSTD_greedy,
-+ ZSTD_lazy,
-+ ZSTD_lazy2,
-+ ZSTD_btlazy2,
-+ ZSTD_btopt,
-+ ZSTD_btopt2
-+} ZSTD_strategy;
-+
-+/**
-+ * struct ZSTD_compressionParameters - zstd compression parameters
-+ * @windowLog: Log of the largest match distance. Larger means more
-+ * compression, and more memory needed during decompression.
-+ * @chainLog: Fully searched segment. Larger means more compression, slower,
-+ * and more memory (useless for fast).
-+ * @hashLog: Dispatch table. Larger means more compression,
-+ * slower, and more memory.
-+ * @searchLog: Number of searches. Larger means more compression and slower.
-+ * @searchLength: Match length searched. Larger means faster decompression,
-+ * sometimes less compression.
-+ * @targetLength: Acceptable match size for optimal parser (only). Larger means
-+ * more compression, and slower.
-+ * @strategy: The zstd compression strategy.
-+ */
-+typedef struct {
-+ unsigned int windowLog;
-+ unsigned int chainLog;
-+ unsigned int hashLog;
-+ unsigned int searchLog;
-+ unsigned int searchLength;
-+ unsigned int targetLength;
-+ ZSTD_strategy strategy;
-+} ZSTD_compressionParameters;
-+
-+/**
-+ * struct ZSTD_frameParameters - zstd frame parameters
-+ * @contentSizeFlag: Controls whether content size will be present in the frame
-+ * header (when known).
-+ * @checksumFlag: Controls whether a 32-bit checksum is generated at the end
-+ * of the frame for error detection.
-+ * @noDictIDFlag: Controls whether dictID will be saved into the frame header
-+ * when using dictionary compression.
-+ *
-+ * The default value is all fields set to 0.
-+ */
-+typedef struct {
-+ unsigned int contentSizeFlag;
-+ unsigned int checksumFlag;
-+ unsigned int noDictIDFlag;
-+} ZSTD_frameParameters;
-+
-+/**
-+ * struct ZSTD_parameters - zstd parameters
-+ * @cParams: The compression parameters.
-+ * @fParams: The frame parameters.
-+ */
-+typedef struct {
-+ ZSTD_compressionParameters cParams;
-+ ZSTD_frameParameters fParams;
-+} ZSTD_parameters;
-+
-+/**
-+ * ZSTD_getCParams() - returns ZSTD_compressionParameters for selected level
-+ * @compressionLevel: The compression level from 1 to ZSTD_maxCLevel().
-+ * @estimatedSrcSize: The estimated source size to compress or 0 if unknown.
-+ * @dictSize: The dictionary size or 0 if a dictionary isn't being used.
-+ *
-+ * Return: The selected ZSTD_compressionParameters.
-+ */
-+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel,
-+ unsigned long long estimatedSrcSize, size_t dictSize);
-+
-+/**
-+ * ZSTD_getParams() - returns ZSTD_parameters for selected level
-+ * @compressionLevel: The compression level from 1 to ZSTD_maxCLevel().
-+ * @estimatedSrcSize: The estimated source size to compress or 0 if unknown.
-+ * @dictSize: The dictionary size or 0 if a dictionary isn't being used.
-+ *
-+ * The same as ZSTD_getCParams() except also selects the default frame
-+ * parameters (all zero).
-+ *
-+ * Return: The selected ZSTD_parameters.
-+ */
-+ZSTD_parameters ZSTD_getParams(int compressionLevel,
-+ unsigned long long estimatedSrcSize, size_t dictSize);
-+
-+/*-*************************************
-+ * Explicit memory management
-+ **************************************/
-+
-+/**
-+ * ZSTD_CCtxWorkspaceBound() - amount of memory needed to initialize a ZSTD_CCtx
-+ * @cParams: The compression parameters to be used for compression.
-+ *
-+ * If multiple compression parameters might be used, the caller must call
-+ * ZSTD_CCtxWorkspaceBound() for each set of parameters and use the maximum
-+ * size.
-+ *
-+ * Return: A lower bound on the size of the workspace that is passed to
-+ * ZSTD_initCCtx().
-+ */
-+size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams);
-+
-+/**
-+ * struct ZSTD_CCtx - the zstd compression context
-+ *
-+ * When compressing many times it is recommended to allocate a context just once
-+ * and reuse it for each successive compression operation.
-+ */
-+typedef struct ZSTD_CCtx_s ZSTD_CCtx;
-+/**
-+ * ZSTD_initCCtx() - initialize a zstd compression context
-+ * @workspace: The workspace to emplace the context into. It must outlive
-+ * the returned context.
-+ * @workspaceSize: The size of workspace. Use ZSTD_CCtxWorkspaceBound() to
-+ * determine how large the workspace must be.
-+ *
-+ * Return: A compression context emplaced into workspace.
-+ */
-+ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize);
-+
-+/**
-+ * ZSTD_compressCCtx() - compress src into dst
-+ * @ctx: The context. Must have been initialized with a workspace at
-+ * least as large as ZSTD_CCtxWorkspaceBound(params.cParams).
-+ * @dst: The buffer to compress src into.
-+ * @dstCapacity: The size of the destination buffer. May be any size, but
-+ * ZSTD_compressBound(srcSize) is guaranteed to be large enough.
-+ * @src: The data to compress.
-+ * @srcSize: The size of the data to compress.
-+ * @params: The parameters to use for compression. See ZSTD_getParams().
-+ *
-+ * Return: The compressed size or an error, which can be checked using
-+ * ZSTD_isError().
-+ */
-+size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize, ZSTD_parameters params);
-+
-+/**
-+ * ZSTD_DCtxWorkspaceBound() - amount of memory needed to initialize a ZSTD_DCtx
-+ *
-+ * Return: A lower bound on the size of the workspace that is passed to
-+ * ZSTD_initDCtx().
-+ */
-+size_t ZSTD_DCtxWorkspaceBound(void);
-+
-+/**
-+ * struct ZSTD_DCtx - the zstd decompression context
-+ *
-+ * When decompressing many times it is recommended to allocate a context just
-+ * once and reuse it for each successive decompression operation.
-+ */
-+typedef struct ZSTD_DCtx_s ZSTD_DCtx;
-+/**
-+ * ZSTD_initDCtx() - initialize a zstd decompression context
-+ * @workspace: The workspace to emplace the context into. It must outlive
-+ * the returned context.
-+ * @workspaceSize: The size of workspace. Use ZSTD_DCtxWorkspaceBound() to
-+ * determine how large the workspace must be.
-+ *
-+ * Return: A decompression context emplaced into workspace.
-+ */
-+ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize);
-+
-+/**
-+ * ZSTD_decompressDCtx() - decompress zstd compressed src into dst
-+ * @ctx: The decompression context.
-+ * @dst: The buffer to decompress src into.
-+ * @dstCapacity: The size of the destination buffer. Must be at least as large
-+ * as the decompressed size. If the caller cannot upper bound the
-+ * decompressed size, then it's better to use the streaming API.
-+ * @src: The zstd compressed data to decompress. Multiple concatenated
-+ * frames and skippable frames are allowed.
-+ * @srcSize: The exact size of the data to decompress.
-+ *
-+ * Return: The decompressed size or an error, which can be checked using
-+ * ZSTD_isError().
-+ */
-+size_t ZSTD_decompressDCtx(ZSTD_DCtx *ctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize);
-+
-+/*-************************
-+ * Simple dictionary API
-+ **************************/
-+
-+/**
-+ * ZSTD_compress_usingDict() - compress src into dst using a dictionary
-+ * @ctx: The context. Must have been initialized with a workspace at
-+ * least as large as ZSTD_CCtxWorkspaceBound(params.cParams).
-+ * @dst: The buffer to compress src into.
-+ * @dstCapacity: The size of the destination buffer. May be any size, but
-+ * ZSTD_compressBound(srcSize) is guaranteed to be large enough.
-+ * @src: The data to compress.
-+ * @srcSize: The size of the data to compress.
-+ * @dict: The dictionary to use for compression.
-+ * @dictSize: The size of the dictionary.
-+ * @params: The parameters to use for compression. See ZSTD_getParams().
-+ *
-+ * Compression using a predefined dictionary. The same dictionary must be used
-+ * during decompression.
-+ *
-+ * Return: The compressed size or an error, which can be checked using
-+ * ZSTD_isError().
-+ */
-+size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize, const void *dict, size_t dictSize,
-+ ZSTD_parameters params);
-+
-+/**
-+ * ZSTD_decompress_usingDict() - decompress src into dst using a dictionary
-+ * @ctx: The decompression context.
-+ * @dst: The buffer to decompress src into.
-+ * @dstCapacity: The size of the destination buffer. Must be at least as large
-+ * as the decompressed size. If the caller cannot upper bound the
-+ * decompressed size, then it's better to use the streaming API.
-+ * @src: The zstd compressed data to decompress. Multiple concatenated
-+ * frames and skippable frames are allowed.
-+ * @srcSize: The exact size of the data to decompress.
-+ * @dict: The dictionary to use for decompression. The same dictionary
-+ * must've been used to compress the data.
-+ * @dictSize: The size of the dictionary.
-+ *
-+ * Return: The decompressed size or an error, which can be checked using
-+ * ZSTD_isError().
-+ */
-+size_t ZSTD_decompress_usingDict(ZSTD_DCtx *ctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize, const void *dict, size_t dictSize);
-+
-+/*-**************************
-+ * Fast dictionary API
-+ ***************************/
-+
-+/**
-+ * ZSTD_CDictWorkspaceBound() - memory needed to initialize a ZSTD_CDict
-+ * @cParams: The compression parameters to be used for compression.
-+ *
-+ * Return: A lower bound on the size of the workspace that is passed to
-+ * ZSTD_initCDict().
-+ */
-+size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams);
-+
-+/**
-+ * struct ZSTD_CDict - a digested dictionary to be used for compression
-+ */
-+typedef struct ZSTD_CDict_s ZSTD_CDict;
-+
-+/**
-+ * ZSTD_initCDict() - initialize a digested dictionary for compression
-+ * @dictBuffer: The dictionary to digest. The buffer is referenced by the
-+ * ZSTD_CDict so it must outlive the returned ZSTD_CDict.
-+ * @dictSize: The size of the dictionary.
-+ * @params: The parameters to use for compression. See ZSTD_getParams().
-+ * @workspace: The workspace. It must outlive the returned ZSTD_CDict.
-+ * @workspaceSize: The workspace size. Must be at least
-+ * ZSTD_CDictWorkspaceBound(params.cParams).
-+ *
-+ * When compressing multiple messages / blocks with the same dictionary it is
-+ * recommended to load it just once. The ZSTD_CDict merely references the
-+ * dictBuffer, so it must outlive the returned ZSTD_CDict.
-+ *
-+ * Return: The digested dictionary emplaced into workspace.
-+ */
-+ZSTD_CDict *ZSTD_initCDict(const void *dictBuffer, size_t dictSize,
-+ ZSTD_parameters params, void *workspace, size_t workspaceSize);
-+
-+/**
-+ * ZSTD_compress_usingCDict() - compress src into dst using a ZSTD_CDict
-+ * @ctx: The context. Must have been initialized with a workspace at
-+ * least as large as ZSTD_CCtxWorkspaceBound(cParams) where
-+ * cParams are the compression parameters used to initialize the
-+ * cdict.
-+ * @dst: The buffer to compress src into.
-+ * @dstCapacity: The size of the destination buffer. May be any size, but
-+ * ZSTD_compressBound(srcSize) is guaranteed to be large enough.
-+ * @src: The data to compress.
-+ * @srcSize: The size of the data to compress.
-+ * @cdict: The digested dictionary to use for compression.
-+ * @params: The parameters to use for compression. See ZSTD_getParams().
-+ *
-+ * Compression using a digested dictionary. The same dictionary must be used
-+ * during decompression.
-+ *
-+ * Return: The compressed size or an error, which can be checked using
-+ * ZSTD_isError().
-+ */
-+size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize, const ZSTD_CDict *cdict);
-+
-+
-+/**
-+ * ZSTD_DDictWorkspaceBound() - memory needed to initialize a ZSTD_DDict
-+ *
-+ * Return: A lower bound on the size of the workspace that is passed to
-+ * ZSTD_initDDict().
-+ */
-+size_t ZSTD_DDictWorkspaceBound(void);
-+
-+/**
-+ * struct ZSTD_DDict - a digested dictionary to be used for decompression
-+ */
-+typedef struct ZSTD_DDict_s ZSTD_DDict;
-+
-+/**
-+ * ZSTD_initDDict() - initialize a digested dictionary for decompression
-+ * @dictBuffer: The dictionary to digest. The buffer is referenced by the
-+ * ZSTD_DDict so it must outlive the returned ZSTD_DDict.
-+ * @dictSize: The size of the dictionary.
-+ * @workspace: The workspace. It must outlive the returned ZSTD_DDict.
-+ * @workspaceSize: The workspace size. Must be at least
-+ * ZSTD_DDictWorkspaceBound().
-+ *
-+ * When decompressing multiple messages / blocks with the same dictionary it is
-+ * recommended to load it just once. The ZSTD_DDict merely references the
-+ * dictBuffer, so it must outlive the returned ZSTD_DDict.
-+ *
-+ * Return: The digested dictionary emplaced into workspace.
-+ */
-+ZSTD_DDict *ZSTD_initDDict(const void *dictBuffer, size_t dictSize,
-+ void *workspace, size_t workspaceSize);
-+
-+/**
-+ * ZSTD_decompress_usingDDict() - decompress src into dst using a ZSTD_DDict
-+ * @ctx: The decompression context.
-+ * @dst: The buffer to decompress src into.
-+ * @dstCapacity: The size of the destination buffer. Must be at least as large
-+ * as the decompressed size. If the caller cannot upper bound the
-+ * decompressed size, then it's better to use the streaming API.
-+ * @src: The zstd compressed data to decompress. Multiple concatenated
-+ * frames and skippable frames are allowed.
-+ * @srcSize: The exact size of the data to decompress.
-+ * @ddict: The digested dictionary to use for decompression. The same
-+ * dictionary must've been used to compress the data.
-+ *
-+ * Return: The decompressed size or an error, which can be checked using
-+ * ZSTD_isError().
-+ */
-+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst,
-+ size_t dstCapacity, const void *src, size_t srcSize,
-+ const ZSTD_DDict *ddict);
-+
-+
-+/*-**************************
-+ * Streaming
-+ ***************************/
-+
-+/**
-+ * struct ZSTD_inBuffer - input buffer for streaming
-+ * @src: Start of the input buffer.
-+ * @size: Size of the input buffer.
-+ * @pos: Position where reading stopped. Will be updated.
-+ * Necessarily 0 <= pos <= size.
-+ */
-+typedef struct ZSTD_inBuffer_s {
-+ const void *src;
-+ size_t size;
-+ size_t pos;
-+} ZSTD_inBuffer;
-+
-+/**
-+ * struct ZSTD_outBuffer - output buffer for streaming
-+ * @dst: Start of the output buffer.
-+ * @size: Size of the output buffer.
-+ * @pos: Position where writing stopped. Will be updated.
-+ * Necessarily 0 <= pos <= size.
-+ */
-+typedef struct ZSTD_outBuffer_s {
-+ void *dst;
-+ size_t size;
-+ size_t pos;
-+} ZSTD_outBuffer;
-+
-+
-+
-+/*-*****************************************************************************
-+ * Streaming compression - HowTo
-+ *
-+ * A ZSTD_CStream object is required to track streaming operation.
-+ * Use ZSTD_initCStream() to initialize a ZSTD_CStream object.
-+ * ZSTD_CStream objects can be reused multiple times on consecutive compression
-+ * operations. It is recommended to re-use ZSTD_CStream in situations where many
-+ * streaming operations will be achieved consecutively. Use one separate
-+ * ZSTD_CStream per thread for parallel execution.
-+ *
-+ * Use ZSTD_compressStream() repetitively to consume input stream.
-+ * The function will automatically update both `pos` fields.
-+ * Note that it may not consume the entire input, in which case `pos < size`,
-+ * and it's up to the caller to present again remaining data.
-+ * It returns a hint for the preferred number of bytes to use as an input for
-+ * the next function call.
-+ *
-+ * At any moment, it's possible to flush whatever data remains within internal
-+ * buffer, using ZSTD_flushStream(). `output->pos` will be updated. There might
-+ * still be some content left within the internal buffer if `output->size` is
-+ * too small. It returns the number of bytes left in the internal buffer and
-+ * must be called until it returns 0.
-+ *
-+ * ZSTD_endStream() instructs to finish a frame. It will perform a flush and
-+ * write frame epilogue. The epilogue is required for decoders to consider a
-+ * frame completed. Similar to ZSTD_flushStream(), it may not be able to flush
-+ * the full content if `output->size` is too small. In which case, call again
-+ * ZSTD_endStream() to complete the flush. It returns the number of bytes left
-+ * in the internal buffer and must be called until it returns 0.
-+ ******************************************************************************/
-+
-+/**
-+ * ZSTD_CStreamWorkspaceBound() - memory needed to initialize a ZSTD_CStream
-+ * @cParams: The compression parameters to be used for compression.
-+ *
-+ * Return: A lower bound on the size of the workspace that is passed to
-+ * ZSTD_initCStream() and ZSTD_initCStream_usingCDict().
-+ */
-+size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams);
-+
-+/**
-+ * struct ZSTD_CStream - the zstd streaming compression context
-+ */
-+typedef struct ZSTD_CStream_s ZSTD_CStream;
-+
-+/*===== ZSTD_CStream management functions =====*/
-+/**
-+ * ZSTD_initCStream() - initialize a zstd streaming compression context
-+ * @params: The zstd compression parameters.
-+ * @pledgedSrcSize: If params.fParams.contentSizeFlag == 1 then the caller must
-+ * pass the source size (zero means empty source). Otherwise,
-+ * the caller may optionally pass the source size, or zero if
-+ * unknown.
-+ * @workspace: The workspace to emplace the context into. It must outlive
-+ * the returned context.
-+ * @workspaceSize: The size of workspace.
-+ * Use ZSTD_CStreamWorkspaceBound(params.cParams) to determine
-+ * how large the workspace must be.
-+ *
-+ * Return: The zstd streaming compression context.
-+ */
-+ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params,
-+ unsigned long long pledgedSrcSize, void *workspace,
-+ size_t workspaceSize);
-+
-+/**
-+ * ZSTD_initCStream_usingCDict() - initialize a streaming compression context
-+ * @cdict: The digested dictionary to use for compression.
-+ * @pledgedSrcSize: Optionally the source size, or zero if unknown.
-+ * @workspace: The workspace to emplace the context into. It must outlive
-+ * the returned context.
-+ * @workspaceSize: The size of workspace. Call ZSTD_CStreamWorkspaceBound()
-+ * with the cParams used to initialize the cdict to determine
-+ * how large the workspace must be.
-+ *
-+ * Return: The zstd streaming compression context.
-+ */
-+ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict,
-+ unsigned long long pledgedSrcSize, void *workspace,
-+ size_t workspaceSize);
-+
-+/*===== Streaming compression functions =====*/
-+/**
-+ * ZSTD_resetCStream() - reset the context using parameters from creation
-+ * @zcs: The zstd streaming compression context to reset.
-+ * @pledgedSrcSize: Optionally the source size, or zero if unknown.
-+ *
-+ * Resets the context using the parameters from creation. Skips dictionary
-+ * loading, since it can be reused. If `pledgedSrcSize` is non-zero the frame
-+ * content size is always written into the frame header.
-+ *
-+ * Return: Zero or an error, which can be checked using ZSTD_isError().
-+ */
-+size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize);
-+/**
-+ * ZSTD_compressStream() - streaming compress some of input into output
-+ * @zcs: The zstd streaming compression context.
-+ * @output: Destination buffer. `output->pos` is updated to indicate how much
-+ * compressed data was written.
-+ * @input: Source buffer. `input->pos` is updated to indicate how much data was
-+ * read. Note that it may not consume the entire input, in which case
-+ * `input->pos < input->size`, and it's up to the caller to present
-+ * remaining data again.
-+ *
-+ * The `input` and `output` buffers may be any size. Guaranteed to make some
-+ * forward progress if `input` and `output` are not empty.
-+ *
-+ * Return: A hint for the number of bytes to use as the input for the next
-+ * function call or an error, which can be checked using
-+ * ZSTD_isError().
-+ */
-+size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output,
-+ ZSTD_inBuffer *input);
-+/**
-+ * ZSTD_flushStream() - flush internal buffers into output
-+ * @zcs: The zstd streaming compression context.
-+ * @output: Destination buffer. `output->pos` is updated to indicate how much
-+ * compressed data was written.
-+ *
-+ * ZSTD_flushStream() must be called until it returns 0, meaning all the data
-+ * has been flushed. Since ZSTD_flushStream() causes a block to be ended,
-+ * calling it too often will degrade the compression ratio.
-+ *
-+ * Return: The number of bytes still present within internal buffers or an
-+ * error, which can be checked using ZSTD_isError().
-+ */
-+size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output);
-+/**
-+ * ZSTD_endStream() - flush internal buffers into output and end the frame
-+ * @zcs: The zstd streaming compression context.
-+ * @output: Destination buffer. `output->pos` is updated to indicate how much
-+ * compressed data was written.
-+ *
-+ * ZSTD_endStream() must be called until it returns 0, meaning all the data has
-+ * been flushed and the frame epilogue has been written.
-+ *
-+ * Return: The number of bytes still present within internal buffers or an
-+ * error, which can be checked using ZSTD_isError().
-+ */
-+size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output);
-+
-+/**
-+ * ZSTD_CStreamInSize() - recommended size for the input buffer
-+ *
-+ * Return: The recommended size for the input buffer.
-+ */
-+size_t ZSTD_CStreamInSize(void);
-+/**
-+ * ZSTD_CStreamOutSize() - recommended size for the output buffer
-+ *
-+ * When the output buffer is at least this large, it is guaranteed to be large
-+ * enough to flush at least one complete compressed block.
-+ *
-+ * Return: The recommended size for the output buffer.
-+ */
-+size_t ZSTD_CStreamOutSize(void);
-+
-+
-+
-+/*-*****************************************************************************
-+ * Streaming decompression - HowTo
-+ *
-+ * A ZSTD_DStream object is required to track streaming operations.
-+ * Use ZSTD_initDStream() to initialize a ZSTD_DStream object.
-+ * ZSTD_DStream objects can be re-used multiple times.
-+ *
-+ * Use ZSTD_decompressStream() repetitively to consume your input.
-+ * The function will update both `pos` fields.
-+ * If `input->pos < input->size`, some input has not been consumed.
-+ * It's up to the caller to present again remaining data.
-+ * If `output->pos < output->size`, decoder has flushed everything it could.
-+ * Returns 0 iff a frame is completely decoded and fully flushed.
-+ * Otherwise it returns a suggested next input size that will never load more
-+ * than the current frame.
-+ ******************************************************************************/
-+
-+/**
-+ * ZSTD_DStreamWorkspaceBound() - memory needed to initialize a ZSTD_DStream
-+ * @maxWindowSize: The maximum window size allowed for compressed frames.
-+ *
-+ * Return: A lower bound on the size of the workspace that is passed to
-+ * ZSTD_initDStream() and ZSTD_initDStream_usingDDict().
-+ */
-+size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize);
-+
-+/**
-+ * struct ZSTD_DStream - the zstd streaming decompression context
-+ */
-+typedef struct ZSTD_DStream_s ZSTD_DStream;
-+/*===== ZSTD_DStream management functions =====*/
-+/**
-+ * ZSTD_initDStream() - initialize a zstd streaming decompression context
-+ * @maxWindowSize: The maximum window size allowed for compressed frames.
-+ * @workspace: The workspace to emplace the context into. It must outlive
-+ * the returned context.
-+ * @workspaceSize: The size of workspace.
-+ * Use ZSTD_DStreamWorkspaceBound(maxWindowSize) to determine
-+ * how large the workspace must be.
-+ *
-+ * Return: The zstd streaming decompression context.
-+ */
-+ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace,
-+ size_t workspaceSize);
-+/**
-+ * ZSTD_initDStream_usingDDict() - initialize streaming decompression context
-+ * @maxWindowSize: The maximum window size allowed for compressed frames.
-+ * @ddict: The digested dictionary to use for decompression.
-+ * @workspace: The workspace to emplace the context into. It must outlive
-+ * the returned context.
-+ * @workspaceSize: The size of workspace.
-+ * Use ZSTD_DStreamWorkspaceBound(maxWindowSize) to determine
-+ * how large the workspace must be.
-+ *
-+ * Return: The zstd streaming decompression context.
-+ */
-+ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize,
-+ const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize);
-+
-+/*===== Streaming decompression functions =====*/
-+/**
-+ * ZSTD_resetDStream() - reset the context using parameters from creation
-+ * @zds: The zstd streaming decompression context to reset.
-+ *
-+ * Resets the context using the parameters from creation. Skips dictionary
-+ * loading, since it can be reused.
-+ *
-+ * Return: Zero or an error, which can be checked using ZSTD_isError().
-+ */
-+size_t ZSTD_resetDStream(ZSTD_DStream *zds);
-+/**
-+ * ZSTD_decompressStream() - streaming decompress some of input into output
-+ * @zds: The zstd streaming decompression context.
-+ * @output: Destination buffer. `output.pos` is updated to indicate how much
-+ * decompressed data was written.
-+ * @input: Source buffer. `input.pos` is updated to indicate how much data was
-+ * read. Note that it may not consume the entire input, in which case
-+ * `input.pos < input.size`, and it's up to the caller to present
-+ * remaining data again.
-+ *
-+ * The `input` and `output` buffers may be any size. Guaranteed to make some
-+ * forward progress if `input` and `output` are not empty.
-+ * ZSTD_decompressStream() will not consume the last byte of the frame until
-+ * the entire frame is flushed.
-+ *
-+ * Return: Returns 0 iff a frame is completely decoded and fully flushed.
-+ * Otherwise returns a hint for the number of bytes to use as the input
-+ * for the next function call or an error, which can be checked using
-+ * ZSTD_isError(). The size hint will never load more than the frame.
-+ */
-+size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output,
-+ ZSTD_inBuffer *input);
-+
-+/**
-+ * ZSTD_DStreamInSize() - recommended size for the input buffer
-+ *
-+ * Return: The recommended size for the input buffer.
-+ */
-+size_t ZSTD_DStreamInSize(void);
-+/**
-+ * ZSTD_DStreamOutSize() - recommended size for the output buffer
-+ *
-+ * When the output buffer is at least this large, it is guaranteed to be large
-+ * enough to flush at least one complete decompressed block.
-+ *
-+ * Return: The recommended size for the output buffer.
-+ */
-+size_t ZSTD_DStreamOutSize(void);
-+
-+
-+/* --- Constants ---*/
-+#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
-+#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
-+
-+#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
-+#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
-+
-+#define ZSTD_WINDOWLOG_MAX_32 27
-+#define ZSTD_WINDOWLOG_MAX_64 27
-+#define ZSTD_WINDOWLOG_MAX \
-+ ((unsigned int)(sizeof(size_t) == 4 \
-+ ? ZSTD_WINDOWLOG_MAX_32 \
-+ : ZSTD_WINDOWLOG_MAX_64))
-+#define ZSTD_WINDOWLOG_MIN 10
-+#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
-+#define ZSTD_HASHLOG_MIN 6
-+#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
-+#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
-+#define ZSTD_HASHLOG3_MAX 17
-+#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
-+#define ZSTD_SEARCHLOG_MIN 1
-+/* only for ZSTD_fast, other strategies are limited to 6 */
-+#define ZSTD_SEARCHLENGTH_MAX 7
-+/* only for ZSTD_btopt, other strategies are limited to 4 */
-+#define ZSTD_SEARCHLENGTH_MIN 3
-+#define ZSTD_TARGETLENGTH_MIN 4
-+#define ZSTD_TARGETLENGTH_MAX 999
-+
-+/* for static allocation */
-+#define ZSTD_FRAMEHEADERSIZE_MAX 18
-+#define ZSTD_FRAMEHEADERSIZE_MIN 6
-+static const size_t ZSTD_frameHeaderSize_prefix = 5;
-+static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
-+static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
-+/* magic number + skippable frame length */
-+static const size_t ZSTD_skippableHeaderSize = 8;
-+
-+
-+/*-*************************************
-+ * Compressed size functions
-+ **************************************/
-+
-+/**
-+ * ZSTD_findFrameCompressedSize() - returns the size of a compressed frame
-+ * @src: Source buffer. It should point to the start of a zstd encoded frame
-+ * or a skippable frame.
-+ * @srcSize: The size of the source buffer. It must be at least as large as the
-+ * size of the frame.
-+ *
-+ * Return: The compressed size of the frame pointed to by `src` or an error,
-+ * which can be check with ZSTD_isError().
-+ * Suitable to pass to ZSTD_decompress() or similar functions.
-+ */
-+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize);
-+
-+/*-*************************************
-+ * Decompressed size functions
-+ **************************************/
-+/**
-+ * ZSTD_getFrameContentSize() - returns the content size in a zstd frame header
-+ * @src: It should point to the start of a zstd encoded frame.
-+ * @srcSize: The size of the source buffer. It must be at least as large as the
-+ * frame header. `ZSTD_frameHeaderSize_max` is always large enough.
-+ *
-+ * Return: The frame content size stored in the frame header if known.
-+ * `ZSTD_CONTENTSIZE_UNKNOWN` if the content size isn't stored in the
-+ * frame header. `ZSTD_CONTENTSIZE_ERROR` on invalid input.
-+ */
-+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
-+
-+/**
-+ * ZSTD_findDecompressedSize() - returns decompressed size of a series of frames
-+ * @src: It should point to the start of a series of zstd encoded and/or
-+ * skippable frames.
-+ * @srcSize: The exact size of the series of frames.
-+ *
-+ * If any zstd encoded frame in the series doesn't have the frame content size
-+ * set, `ZSTD_CONTENTSIZE_UNKNOWN` is returned. But frame content size is always
-+ * set when using ZSTD_compress(). The decompressed size can be very large.
-+ * If the source is untrusted, the decompressed size could be wrong or
-+ * intentionally modified. Always ensure the result fits within the
-+ * application's authorized limits. ZSTD_findDecompressedSize() handles multiple
-+ * frames, and so it must traverse the input to read each frame header. This is
-+ * efficient as most of the data is skipped, however it does mean that all frame
-+ * data must be present and valid.
-+ *
-+ * Return: Decompressed size of all the data contained in the frames if known.
-+ * `ZSTD_CONTENTSIZE_UNKNOWN` if the decompressed size is unknown.
-+ * `ZSTD_CONTENTSIZE_ERROR` if an error occurred.
-+ */
-+unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize);
-+
-+/*-*************************************
-+ * Advanced compression functions
-+ **************************************/
-+/**
-+ * ZSTD_checkCParams() - ensure parameter values remain within authorized range
-+ * @cParams: The zstd compression parameters.
-+ *
-+ * Return: Zero or an error, which can be checked using ZSTD_isError().
-+ */
-+size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams);
-+
-+/**
-+ * ZSTD_adjustCParams() - optimize parameters for a given srcSize and dictSize
-+ * @srcSize: Optionally the estimated source size, or zero if unknown.
-+ * @dictSize: Optionally the estimated dictionary size, or zero if unknown.
-+ *
-+ * Return: The optimized parameters.
-+ */
-+ZSTD_compressionParameters ZSTD_adjustCParams(
-+ ZSTD_compressionParameters cParams, unsigned long long srcSize,
-+ size_t dictSize);
-+
-+/*--- Advanced decompression functions ---*/
-+
-+/**
-+ * ZSTD_isFrame() - returns true iff the buffer starts with a valid frame
-+ * @buffer: The source buffer to check.
-+ * @size: The size of the source buffer, must be at least 4 bytes.
-+ *
-+ * Return: True iff the buffer starts with a zstd or skippable frame identifier.
-+ */
-+unsigned int ZSTD_isFrame(const void *buffer, size_t size);
-+
-+/**
-+ * ZSTD_getDictID_fromDict() - returns the dictionary id stored in a dictionary
-+ * @dict: The dictionary buffer.
-+ * @dictSize: The size of the dictionary buffer.
-+ *
-+ * Return: The dictionary id stored within the dictionary or 0 if the
-+ * dictionary is not a zstd dictionary. If it returns 0 the
-+ * dictionary can still be loaded as a content-only dictionary.
-+ */
-+unsigned int ZSTD_getDictID_fromDict(const void *dict, size_t dictSize);
-+
-+/**
-+ * ZSTD_getDictID_fromDDict() - returns the dictionary id stored in a ZSTD_DDict
-+ * @ddict: The ddict to find the id of.
-+ *
-+ * Return: The dictionary id stored within `ddict` or 0 if the dictionary is not
-+ * a zstd dictionary. If it returns 0 `ddict` will be loaded as a
-+ * content-only dictionary.
-+ */
-+unsigned int ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict);
-+
-+/**
-+ * ZSTD_getDictID_fromFrame() - returns the dictionary id stored in a zstd frame
-+ * @src: Source buffer. It must be a zstd encoded frame.
-+ * @srcSize: The size of the source buffer. It must be at least as large as the
-+ * frame header. `ZSTD_frameHeaderSize_max` is always large enough.
-+ *
-+ * Return: The dictionary id required to decompress the frame stored within
-+ * `src` or 0 if the dictionary id could not be decoded. It can return
-+ * 0 if the frame does not require a dictionary, the dictionary id
-+ * wasn't stored in the frame, `src` is not a zstd frame, or `srcSize`
-+ * is too small.
-+ */
-+unsigned int ZSTD_getDictID_fromFrame(const void *src, size_t srcSize);
-+
-+/**
-+ * struct ZSTD_frameParams - zstd frame parameters stored in the frame header
-+ * @frameContentSize: The frame content size, or 0 if not present.
-+ * @windowSize: The window size, or 0 if the frame is a skippable frame.
-+ * @dictID: The dictionary id, or 0 if not present.
-+ * @checksumFlag: Whether a checksum was used.
-+ */
-+typedef struct {
-+ unsigned long long frameContentSize;
-+ unsigned int windowSize;
-+ unsigned int dictID;
-+ unsigned int checksumFlag;
-+} ZSTD_frameParams;
-+
-+/**
-+ * ZSTD_getFrameParams() - extracts parameters from a zstd or skippable frame
-+ * @fparamsPtr: On success the frame parameters are written here.
-+ * @src: The source buffer. It must point to a zstd or skippable frame.
-+ * @srcSize: The size of the source buffer. `ZSTD_frameHeaderSize_max` is
-+ * always large enough to succeed.
-+ *
-+ * Return: 0 on success. If more data is required it returns how many bytes
-+ * must be provided to make forward progress. Otherwise it returns
-+ * an error, which can be checked using ZSTD_isError().
-+ */
-+size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src,
-+ size_t srcSize);
-+
-+/*-*****************************************************************************
-+ * Buffer-less and synchronous inner streaming functions
-+ *
-+ * This is an advanced API, giving full control over buffer management, for
-+ * users which need direct control over memory.
-+ * But it's also a complex one, with many restrictions (documented below).
-+ * Prefer using normal streaming API for an easier experience
-+ ******************************************************************************/
-+
-+/*-*****************************************************************************
-+ * Buffer-less streaming compression (synchronous mode)
-+ *
-+ * A ZSTD_CCtx object is required to track streaming operations.
-+ * Use ZSTD_initCCtx() to initialize a context.
-+ * ZSTD_CCtx object can be re-used multiple times within successive compression
-+ * operations.
-+ *
-+ * Start by initializing a context.
-+ * Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary
-+ * compression,
-+ * or ZSTD_compressBegin_advanced(), for finer parameter control.
-+ * It's also possible to duplicate a reference context which has already been
-+ * initialized, using ZSTD_copyCCtx()
-+ *
-+ * Then, consume your input using ZSTD_compressContinue().
-+ * There are some important considerations to keep in mind when using this
-+ * advanced function :
-+ * - ZSTD_compressContinue() has no internal buffer. It uses externally provided
-+ * buffer only.
-+ * - Interface is synchronous : input is consumed entirely and produce 1+
-+ * (or more) compressed blocks.
-+ * - Caller must ensure there is enough space in `dst` to store compressed data
-+ * under worst case scenario. Worst case evaluation is provided by
-+ * ZSTD_compressBound().
-+ * ZSTD_compressContinue() doesn't guarantee recover after a failed
-+ * compression.
-+ * - ZSTD_compressContinue() presumes prior input ***is still accessible and
-+ * unmodified*** (up to maximum distance size, see WindowLog).
-+ * It remembers all previous contiguous blocks, plus one separated memory
-+ * segment (which can itself consists of multiple contiguous blocks)
-+ * - ZSTD_compressContinue() detects that prior input has been overwritten when
-+ * `src` buffer overlaps. In which case, it will "discard" the relevant memory
-+ * section from its history.
-+ *
-+ * Finish a frame with ZSTD_compressEnd(), which will write the last block(s)
-+ * and optional checksum. It's possible to use srcSize==0, in which case, it
-+ * will write a final empty block to end the frame. Without last block mark,
-+ * frames will be considered unfinished (corrupted) by decoders.
-+ *
-+ * `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new
-+ * frame.
-+ ******************************************************************************/
-+
-+/*===== Buffer-less streaming compression functions =====*/
-+size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel);
-+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict,
-+ size_t dictSize, int compressionLevel);
-+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict,
-+ size_t dictSize, ZSTD_parameters params,
-+ unsigned long long pledgedSrcSize);
-+size_t ZSTD_copyCCtx(ZSTD_CCtx *cctx, const ZSTD_CCtx *preparedCCtx,
-+ unsigned long long pledgedSrcSize);
-+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict,
-+ unsigned long long pledgedSrcSize);
-+size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize);
-+size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize);
-+
-+
-+
-+/*-*****************************************************************************
-+ * Buffer-less streaming decompression (synchronous mode)
-+ *
-+ * A ZSTD_DCtx object is required to track streaming operations.
-+ * Use ZSTD_initDCtx() to initialize a context.
-+ * A ZSTD_DCtx object can be re-used multiple times.
-+ *
-+ * First typical operation is to retrieve frame parameters, using
-+ * ZSTD_getFrameParams(). It fills a ZSTD_frameParams structure which provide
-+ * important information to correctly decode the frame, such as the minimum
-+ * rolling buffer size to allocate to decompress data (`windowSize`), and the
-+ * dictionary ID used.
-+ * Note: content size is optional, it may not be present. 0 means unknown.
-+ * Note that these values could be wrong, either because of data malformation,
-+ * or because an attacker is spoofing deliberate false information. As a
-+ * consequence, check that values remain within valid application range,
-+ * especially `windowSize`, before allocation. Each application can set its own
-+ * limit, depending on local restrictions. For extended interoperability, it is
-+ * recommended to support at least 8 MB.
-+ * Frame parameters are extracted from the beginning of the compressed frame.
-+ * Data fragment must be large enough to ensure successful decoding, typically
-+ * `ZSTD_frameHeaderSize_max` bytes.
-+ * Result: 0: successful decoding, the `ZSTD_frameParams` structure is filled.
-+ * >0: `srcSize` is too small, provide at least this many bytes.
-+ * errorCode, which can be tested using ZSTD_isError().
-+ *
-+ * Start decompression, with ZSTD_decompressBegin() or
-+ * ZSTD_decompressBegin_usingDict(). Alternatively, you can copy a prepared
-+ * context, using ZSTD_copyDCtx().
-+ *
-+ * Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue()
-+ * alternatively.
-+ * ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize'
-+ * to ZSTD_decompressContinue().
-+ * ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will
-+ * fail.
-+ *
-+ * The result of ZSTD_decompressContinue() is the number of bytes regenerated
-+ * within 'dst' (necessarily <= dstCapacity). It can be zero, which is not an
-+ * error; it just means ZSTD_decompressContinue() has decoded some metadata
-+ * item. It can also be an error code, which can be tested with ZSTD_isError().
-+ *
-+ * ZSTD_decompressContinue() needs previous data blocks during decompression, up
-+ * to `windowSize`. They should preferably be located contiguously, prior to
-+ * current block. Alternatively, a round buffer of sufficient size is also
-+ * possible. Sufficient size is determined by frame parameters.
-+ * ZSTD_decompressContinue() is very sensitive to contiguity, if 2 blocks don't
-+ * follow each other, make sure that either the compressor breaks contiguity at
-+ * the same place, or that previous contiguous segment is large enough to
-+ * properly handle maximum back-reference.
-+ *
-+ * A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
-+ * Context can then be reset to start a new decompression.
-+ *
-+ * Note: it's possible to know if next input to present is a header or a block,
-+ * using ZSTD_nextInputType(). This information is not required to properly
-+ * decode a frame.
-+ *
-+ * == Special case: skippable frames ==
-+ *
-+ * Skippable frames allow integration of user-defined data into a flow of
-+ * concatenated frames. Skippable frames will be ignored (skipped) by a
-+ * decompressor. The format of skippable frames is as follows:
-+ * a) Skippable frame ID - 4 Bytes, Little endian format, any value from
-+ * 0x184D2A50 to 0x184D2A5F
-+ * b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
-+ * c) Frame Content - any content (User Data) of length equal to Frame Size
-+ * For skippable frames ZSTD_decompressContinue() always returns 0.
-+ * For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0
-+ * what means that a frame is skippable.
-+ * Note: If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might
-+ * actually be a zstd encoded frame with no content. For purposes of
-+ * decompression, it is valid in both cases to skip the frame using
-+ * ZSTD_findFrameCompressedSize() to find its size in bytes.
-+ * It also returns frame size as fparamsPtr->frameContentSize.
-+ ******************************************************************************/
-+
-+/*===== Buffer-less streaming decompression functions =====*/
-+size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx);
-+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict,
-+ size_t dictSize);
-+void ZSTD_copyDCtx(ZSTD_DCtx *dctx, const ZSTD_DCtx *preparedDCtx);
-+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx);
-+size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize);
-+typedef enum {
-+ ZSTDnit_frameHeader,
-+ ZSTDnit_blockHeader,
-+ ZSTDnit_block,
-+ ZSTDnit_lastBlock,
-+ ZSTDnit_checksum,
-+ ZSTDnit_skippableFrame
-+} ZSTD_nextInputType_e;
-+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx);
-+
-+/*-*****************************************************************************
-+ * Block functions
-+ *
-+ * Block functions produce and decode raw zstd blocks, without frame metadata.
-+ * Frame metadata cost is typically ~18 bytes, which can be non-negligible for
-+ * very small blocks (< 100 bytes). User will have to take in charge required
-+ * information to regenerate data, such as compressed and content sizes.
-+ *
-+ * A few rules to respect:
-+ * - Compressing and decompressing require a context structure
-+ * + Use ZSTD_initCCtx() and ZSTD_initDCtx()
-+ * - It is necessary to init context before starting
-+ * + compression : ZSTD_compressBegin()
-+ * + decompression : ZSTD_decompressBegin()
-+ * + variants _usingDict() are also allowed
-+ * + copyCCtx() and copyDCtx() work too
-+ * - Block size is limited, it must be <= ZSTD_getBlockSizeMax()
-+ * + If you need to compress more, cut data into multiple blocks
-+ * + Consider using the regular ZSTD_compress() instead, as frame metadata
-+ * costs become negligible when source size is large.
-+ * - When a block is considered not compressible enough, ZSTD_compressBlock()
-+ * result will be zero. In which case, nothing is produced into `dst`.
-+ * + User must test for such outcome and deal directly with uncompressed data
-+ * + ZSTD_decompressBlock() doesn't accept uncompressed data as input!!!
-+ * + In case of multiple successive blocks, decoder must be informed of
-+ * uncompressed block existence to follow proper history. Use
-+ * ZSTD_insertBlock() in such a case.
-+ ******************************************************************************/
-+
-+/* Define for static allocation */
-+#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024)
-+/*===== Raw zstd block functions =====*/
-+size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx);
-+size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize);
-+size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity,
-+ const void *src, size_t srcSize);
-+size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart,
-+ size_t blockSize);
-+
-+#endif /* ZSTD_H */
-diff --git a/lib/Kconfig b/lib/Kconfig
-index 5e7541f..0d49ed0 100644
---- a/lib/Kconfig
-+++ b/lib/Kconfig
-@@ -249,6 +249,14 @@ config LZ4HC_COMPRESS
- config LZ4_DECOMPRESS
- tristate
-
-+config ZSTD_COMPRESS
-+ select XXHASH
-+ tristate
-+
-+config ZSTD_DECOMPRESS
-+ select XXHASH
-+ tristate
-+
- source "lib/xz/Kconfig"
-
- #
-diff --git a/lib/Makefile b/lib/Makefile
-index d06b68a..d5c8a4f 100644
---- a/lib/Makefile
-+++ b/lib/Makefile
-@@ -116,6 +116,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
- obj-$(CONFIG_LZ4_COMPRESS) += lz4/
- obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/
- obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
-+obj-$(CONFIG_ZSTD_COMPRESS) += zstd/
-+obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/
- obj-$(CONFIG_XZ_DEC) += xz/
- obj-$(CONFIG_RAID6_PQ) += raid6/
-
-diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile
-new file mode 100644
-index 0000000..dd0a359
---- /dev/null
-+++ b/lib/zstd/Makefile
-@@ -0,0 +1,18 @@
-+obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o
-+obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o
-+
-+ccflags-y += -O3
-+
-+# Object files unique to zstd_compress and zstd_decompress
-+zstd_compress-y := fse_compress.o huf_compress.o compress.o
-+zstd_decompress-y := huf_decompress.o decompress.o
-+
-+# These object files are shared between the modules.
-+# Always add them to zstd_compress.
-+# Unless both zstd_compress and zstd_decompress are built in
-+# then also add them to zstd_decompress.
-+zstd_compress-y += entropy_common.o fse_decompress.o zstd_common.o
-+
-+ifneq ($(CONFIG_ZSTD_COMPRESS)$(CONFIG_ZSTD_DECOMPRESS),yy)
-+ zstd_decompress-y += entropy_common.o fse_decompress.o zstd_common.o
-+endif
-diff --git a/lib/zstd/bitstream.h b/lib/zstd/bitstream.h
-new file mode 100644
-index 0000000..a826b99
---- /dev/null
-+++ b/lib/zstd/bitstream.h
-@@ -0,0 +1,374 @@
-+/*
-+ * bitstream
-+ * Part of FSE library
-+ * header file (to include)
-+ * Copyright (C) 2013-2016, Yann Collet.
-+ *
-+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are
-+ * met:
-+ *
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above
-+ * copyright notice, this list of conditions and the following disclaimer
-+ * in the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ *
-+ * You can contact the author at :
-+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-+ */
-+#ifndef BITSTREAM_H_MODULE
-+#define BITSTREAM_H_MODULE
-+
-+/*
-+* This API consists of small unitary functions, which must be inlined for best performance.
-+* Since link-time-optimization is not available for all compilers,
-+* these functions are defined into a .h to be included.
-+*/
-+
-+/*-****************************************
-+* Dependencies
-+******************************************/
-+#include "error_private.h" /* error codes and messages */
-+#include "mem.h" /* unaligned access routines */
-+
-+/*=========================================
-+* Target specific
-+=========================================*/
-+#define STREAM_ACCUMULATOR_MIN_32 25
-+#define STREAM_ACCUMULATOR_MIN_64 57
-+#define STREAM_ACCUMULATOR_MIN ((U32)(ZSTD_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
-+
-+/*-******************************************
-+* bitStream encoding API (write forward)
-+********************************************/
-+/* bitStream can mix input from multiple sources.
-+* A critical property of these streams is that they encode and decode in **reverse** direction.
-+* So the first bit sequence you add will be the last to be read, like a LIFO stack.
-+*/
-+typedef struct {
-+ size_t bitContainer;
-+ int bitPos;
-+ char *startPtr;
-+ char *ptr;
-+ char *endPtr;
-+} BIT_CStream_t;
-+
-+ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *dstBuffer, size_t dstCapacity);
-+ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits);
-+ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC);
-+ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC);
-+
-+/* Start with initCStream, providing the size of buffer to write into.
-+* bitStream will never write outside of this buffer.
-+* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
-+*
-+* bits are first added to a local register.
-+* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
-+* Writing data into memory is an explicit operation, performed by the flushBits function.
-+* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
-+* After a flushBits, a maximum of 7 bits might still be stored into local register.
-+*
-+* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
-+*
-+* Last operation is to close the bitStream.
-+* The function returns the final size of CStream in bytes.
-+* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
-+*/
-+
-+/*-********************************************
-+* bitStream decoding API (read backward)
-+**********************************************/
-+typedef struct {
-+ size_t bitContainer;
-+ unsigned bitsConsumed;
-+ const char *ptr;
-+ const char *start;
-+} BIT_DStream_t;
-+
-+typedef enum {
-+ BIT_DStream_unfinished = 0,
-+ BIT_DStream_endOfBuffer = 1,
-+ BIT_DStream_completed = 2,
-+ BIT_DStream_overflow = 3
-+} BIT_DStream_status; /* result of BIT_reloadDStream() */
-+/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
-+
-+ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize);
-+ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, unsigned nbBits);
-+ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD);
-+ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *bitD);
-+
-+/* Start by invoking BIT_initDStream().
-+* A chunk of the bitStream is then stored into a local register.
-+* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-+* You can then retrieve bitFields stored into the local register, **in reverse order**.
-+* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
-+* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
-+* Otherwise, it can be less than that, so proceed accordingly.
-+* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
-+*/
-+
-+/*-****************************************
-+* unsafe API
-+******************************************/
-+ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits);
-+/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
-+
-+ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC);
-+/* unsafe version; does not check buffer overflow */
-+
-+ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, unsigned nbBits);
-+/* faster, but works only if nbBits >= 1 */
-+
-+/*-**************************************************************
-+* Internal functions
-+****************************************************************/
-+ZSTD_STATIC unsigned BIT_highbit32(register U32 val) { return 31 - __builtin_clz(val); }
-+
-+/*===== Local Constants =====*/
-+static const unsigned BIT_mask[] = {0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF,
-+ 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
-+ 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF}; /* up to 26 bits */
-+
-+/*-**************************************************************
-+* bitStream encoding
-+****************************************************************/
-+/*! BIT_initCStream() :
-+ * `dstCapacity` must be > sizeof(void*)
-+ * @return : 0 if success,
-+ otherwise an error code (can be tested using ERR_isError() ) */
-+ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *startPtr, size_t dstCapacity)
-+{
-+ bitC->bitContainer = 0;
-+ bitC->bitPos = 0;
-+ bitC->startPtr = (char *)startPtr;
-+ bitC->ptr = bitC->startPtr;
-+ bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
-+ if (dstCapacity <= sizeof(bitC->ptr))
-+ return ERROR(dstSize_tooSmall);
-+ return 0;
-+}
-+
-+/*! BIT_addBits() :
-+ can add up to 26 bits into `bitC`.
-+ Does not check for register overflow ! */
-+ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits)
-+{
-+ bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
-+ bitC->bitPos += nbBits;
-+}
-+
-+/*! BIT_addBitsFast() :
-+ * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
-+ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits)
-+{
-+ bitC->bitContainer |= value << bitC->bitPos;
-+ bitC->bitPos += nbBits;
-+}
-+
-+/*! BIT_flushBitsFast() :
-+ * unsafe version; does not check buffer overflow */
-+ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC)
-+{
-+ size_t const nbBytes = bitC->bitPos >> 3;
-+ ZSTD_writeLEST(bitC->ptr, bitC->bitContainer);
-+ bitC->ptr += nbBytes;
-+ bitC->bitPos &= 7;
-+ bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
-+}
-+
-+/*! BIT_flushBits() :
-+ * safe version; check for buffer overflow, and prevents it.
-+ * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
-+ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC)
-+{
-+ size_t const nbBytes = bitC->bitPos >> 3;
-+ ZSTD_writeLEST(bitC->ptr, bitC->bitContainer);
-+ bitC->ptr += nbBytes;
-+ if (bitC->ptr > bitC->endPtr)
-+ bitC->ptr = bitC->endPtr;
-+ bitC->bitPos &= 7;
-+ bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
-+}
-+
-+/*! BIT_closeCStream() :
-+ * @return : size of CStream, in bytes,
-+ or 0 if it could not fit into dstBuffer */
-+ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC)
-+{
-+ BIT_addBitsFast(bitC, 1, 1); /* endMark */
-+ BIT_flushBits(bitC);
-+
-+ if (bitC->ptr >= bitC->endPtr)
-+ return 0; /* doesn't fit within authorized budget : cancel */
-+
-+ return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
-+}
-+
-+/*-********************************************************
-+* bitStream decoding
-+**********************************************************/
-+/*! BIT_initDStream() :
-+* Initialize a BIT_DStream_t.
-+* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
-+* `srcSize` must be the *exact* size of the bitStream, in bytes.
-+* @return : size of stream (== srcSize) or an errorCode if a problem is detected
-+*/
-+ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize)
-+{
-+ if (srcSize < 1) {
-+ memset(bitD, 0, sizeof(*bitD));
-+ return ERROR(srcSize_wrong);
-+ }
-+
-+ if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
-+ bitD->start = (const char *)srcBuffer;
-+ bitD->ptr = (const char *)srcBuffer + srcSize - sizeof(bitD->bitContainer);
-+ bitD->bitContainer = ZSTD_readLEST(bitD->ptr);
-+ {
-+ BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
-+ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
-+ if (lastByte == 0)
-+ return ERROR(GENERIC); /* endMark not present */
-+ }
-+ } else {
-+ bitD->start = (const char *)srcBuffer;
-+ bitD->ptr = bitD->start;
-+ bitD->bitContainer = *(const BYTE *)(bitD->start);
-+ switch (srcSize) {
-+ case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16);
-+ case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24);
-+ case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32);
-+ case 4: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[3]) << 24;
-+ case 3: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[2]) << 16;
-+ case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8;
-+ default:;
-+ }
-+ {
-+ BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
-+ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
-+ if (lastByte == 0)
-+ return ERROR(GENERIC); /* endMark not present */
-+ }
-+ bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8;
-+ }
-+
-+ return srcSize;
-+}
-+
-+ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { return bitContainer >> start; }
-+
-+ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { return (bitContainer >> start) & BIT_mask[nbBits]; }
-+
-+ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { return bitContainer & BIT_mask[nbBits]; }
-+
-+/*! BIT_lookBits() :
-+ * Provides next n bits from local register.
-+ * local register is not modified.
-+ * On 32-bits, maxNbBits==24.
-+ * On 64-bits, maxNbBits==56.
-+ * @return : value extracted
-+ */
-+ZSTD_STATIC size_t BIT_lookBits(const BIT_DStream_t *bitD, U32 nbBits)
-+{
-+ U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1;
-+ return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask - nbBits) & bitMask);
-+}
-+
-+/*! BIT_lookBitsFast() :
-+* unsafe version; only works only if nbBits >= 1 */
-+ZSTD_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t *bitD, U32 nbBits)
-+{
-+ U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1;
-+ return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask + 1) - nbBits) & bitMask);
-+}
-+
-+ZSTD_STATIC void BIT_skipBits(BIT_DStream_t *bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; }
-+
-+/*! BIT_readBits() :
-+ * Read (consume) next n bits from local register and update.
-+ * Pay attention to not read more than nbBits contained into local register.
-+ * @return : extracted value.
-+ */
-+ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, U32 nbBits)
-+{
-+ size_t const value = BIT_lookBits(bitD, nbBits);
-+ BIT_skipBits(bitD, nbBits);
-+ return value;
-+}
-+
-+/*! BIT_readBitsFast() :
-+* unsafe version; only works only if nbBits >= 1 */
-+ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, U32 nbBits)
-+{
-+ size_t const value = BIT_lookBitsFast(bitD, nbBits);
-+ BIT_skipBits(bitD, nbBits);
-+ return value;
-+}
-+
-+/*! BIT_reloadDStream() :
-+* Refill `bitD` from buffer previously set in BIT_initDStream() .
-+* This function is safe, it guarantees it will not read beyond src buffer.
-+* @return : status of `BIT_DStream_t` internal register.
-+ if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
-+ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD)
-+{
-+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer) * 8)) /* should not happen => corruption detected */
-+ return BIT_DStream_overflow;
-+
-+ if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
-+ bitD->ptr -= bitD->bitsConsumed >> 3;
-+ bitD->bitsConsumed &= 7;
-+ bitD->bitContainer = ZSTD_readLEST(bitD->ptr);
-+ return BIT_DStream_unfinished;
-+ }
-+ if (bitD->ptr == bitD->start) {
-+ if (bitD->bitsConsumed < sizeof(bitD->bitContainer) * 8)
-+ return BIT_DStream_endOfBuffer;
-+ return BIT_DStream_completed;
-+ }
-+ {
-+ U32 nbBytes = bitD->bitsConsumed >> 3;
-+ BIT_DStream_status result = BIT_DStream_unfinished;
-+ if (bitD->ptr - nbBytes < bitD->start) {
-+ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
-+ result = BIT_DStream_endOfBuffer;
-+ }
-+ bitD->ptr -= nbBytes;
-+ bitD->bitsConsumed -= nbBytes * 8;
-+ bitD->bitContainer = ZSTD_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
-+ return result;
-+ }
-+}
-+
-+/*! BIT_endOfDStream() :
-+* @return Tells if DStream has exactly reached its end (all bits consumed).
-+*/
-+ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *DStream)
-+{
-+ return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer) * 8));
-+}
-+
-+#endif /* BITSTREAM_H_MODULE */
-diff --git a/lib/zstd/compress.c b/lib/zstd/compress.c
-new file mode 100644
-index 0000000..f9166cf
---- /dev/null
-+++ b/lib/zstd/compress.c
-@@ -0,0 +1,3484 @@
-+/**
-+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This source code is licensed under the BSD-style license found in the
-+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
-+ * An additional grant of patent rights can be found in the PATENTS file in the
-+ * same directory.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ */
-+
-+/*-*************************************
-+* Dependencies
-+***************************************/
-+#include "fse.h"
-+#include "huf.h"
-+#include "mem.h"
-+#include "zstd_internal.h" /* includes zstd.h */
-+#include <linux/kernel.h>
-+#include <linux/module.h>
-+#include <linux/string.h> /* memset */
-+
-+/*-*************************************
-+* Constants
-+***************************************/
-+static const U32 g_searchStrength = 8; /* control skip over incompressible data */
-+#define HASH_READ_SIZE 8
-+typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
-+
-+/*-*************************************
-+* Helper functions
-+***************************************/
-+size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
-+
-+/*-*************************************
-+* Sequence storage
-+***************************************/
-+static void ZSTD_resetSeqStore(seqStore_t *ssPtr)
-+{
-+ ssPtr->lit = ssPtr->litStart;
-+ ssPtr->sequences = ssPtr->sequencesStart;
-+ ssPtr->longLengthID = 0;
-+}
-+
-+/*-*************************************
-+* Context memory management
-+***************************************/
-+struct ZSTD_CCtx_s {
-+ const BYTE *nextSrc; /* next block here to continue on curr prefix */
-+ const BYTE *base; /* All regular indexes relative to this position */
-+ const BYTE *dictBase; /* extDict indexes relative to this position */
-+ U32 dictLimit; /* below that point, need extDict */
-+ U32 lowLimit; /* below that point, no more data */
-+ U32 nextToUpdate; /* index from which to continue dictionary update */
-+ U32 nextToUpdate3; /* index from which to continue dictionary update */
-+ U32 hashLog3; /* dispatch table : larger == faster, more memory */
-+ U32 loadedDictEnd; /* index of end of dictionary */
-+ U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
-+ U32 forceRawDict; /* Force loading dictionary in "content-only" mode (no header analysis) */
-+ ZSTD_compressionStage_e stage;
-+ U32 rep[ZSTD_REP_NUM];
-+ U32 repToConfirm[ZSTD_REP_NUM];
-+ U32 dictID;
-+ ZSTD_parameters params;
-+ void *workSpace;
-+ size_t workSpaceSize;
-+ size_t blockSize;
-+ U64 frameContentSize;
-+ struct xxh64_state xxhState;
-+ ZSTD_customMem customMem;
-+
-+ seqStore_t seqStore; /* sequences storage ptrs */
-+ U32 *hashTable;
-+ U32 *hashTable3;
-+ U32 *chainTable;
-+ HUF_CElt *hufTable;
-+ U32 flagStaticTables;
-+ HUF_repeat flagStaticHufTable;
-+ FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
-+ FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
-+ FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
-+ unsigned tmpCounters[HUF_COMPRESS_WORKSPACE_SIZE_U32];
-+};
-+
-+size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams)
-+{
-+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
-+ U32 const divider = (cParams.searchLength == 3) ? 3 : 4;
-+ size_t const maxNbSeq = blockSize / divider;
-+ size_t const tokenSpace = blockSize + 11 * maxNbSeq;
-+ size_t const chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog);
-+ size_t const hSize = ((size_t)1) << cParams.hashLog;
-+ U32 const hashLog3 = (cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog);
-+ size_t const h3Size = ((size_t)1) << hashLog3;
-+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
-+ size_t const optSpace =
-+ ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
-+ size_t const workspaceSize = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
-+ (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
-+
-+ return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize);
-+}
-+
-+static ZSTD_CCtx *ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
-+{
-+ ZSTD_CCtx *cctx;
-+ if (!customMem.customAlloc || !customMem.customFree)
-+ return NULL;
-+ cctx = (ZSTD_CCtx *)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
-+ if (!cctx)
-+ return NULL;
-+ memset(cctx, 0, sizeof(ZSTD_CCtx));
-+ cctx->customMem = customMem;
-+ return cctx;
-+}
-+
-+ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize)
-+{
-+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
-+ ZSTD_CCtx *cctx = ZSTD_createCCtx_advanced(stackMem);
-+ if (cctx) {
-+ cctx->workSpace = ZSTD_stackAllocAll(cctx->customMem.opaque, &cctx->workSpaceSize);
-+ }
-+ return cctx;
-+}
-+
-+size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx)
-+{
-+ if (cctx == NULL)
-+ return 0; /* support free on NULL */
-+ ZSTD_free(cctx->workSpace, cctx->customMem);
-+ ZSTD_free(cctx, cctx->customMem);
-+ return 0; /* reserved as a potential error code in the future */
-+}
-+
-+const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx) /* hidden interface */ { return &(ctx->seqStore); }
-+
-+static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx *cctx) { return cctx->params; }
-+
-+/** ZSTD_checkParams() :
-+ ensure param values remain within authorized range.
-+ @return : 0, or an error code if one value is beyond authorized range */
-+size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
-+{
-+#define CLAMPCHECK(val, min, max) \
-+ { \
-+ if ((val < min) | (val > max)) \
-+ return ERROR(compressionParameter_unsupported); \
-+ }
-+ CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
-+ CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
-+ CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
-+ CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
-+ CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
-+ CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
-+ if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2)
-+ return ERROR(compressionParameter_unsupported);
-+ return 0;
-+}
-+
-+/** ZSTD_cycleLog() :
-+ * condition for correct operation : hashLog > 1 */
-+static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
-+{
-+ U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
-+ return hashLog - btScale;
-+}
-+
-+/** ZSTD_adjustCParams() :
-+ optimize `cPar` for a given input (`srcSize` and `dictSize`).
-+ mostly downsizing to reduce memory consumption and initialization.
-+ Both `srcSize` and `dictSize` are optional (use 0 if unknown),
-+ but if both are 0, no optimization can be done.
-+ Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
-+ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
-+{
-+ if (srcSize + dictSize == 0)
-+ return cPar; /* no size information available : no adjustment */
-+
-+ /* resize params, to use less memory when necessary */
-+ {
-+ U32 const minSrcSize = (srcSize == 0) ? 500 : 0;
-+ U64 const rSize = srcSize + dictSize + minSrcSize;
-+ if (rSize < ((U64)1 << ZSTD_WINDOWLOG_MAX)) {
-+ U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1);
-+ if (cPar.windowLog > srcLog)
-+ cPar.windowLog = srcLog;
-+ }
-+ }
-+ if (cPar.hashLog > cPar.windowLog)
-+ cPar.hashLog = cPar.windowLog;
-+ {
-+ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
-+ if (cycleLog > cPar.windowLog)
-+ cPar.chainLog -= (cycleLog - cPar.windowLog);
-+ }
-+
-+ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
-+ cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
-+
-+ return cPar;
-+}
-+
-+static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2)
-+{
-+ return (param1.cParams.hashLog == param2.cParams.hashLog) & (param1.cParams.chainLog == param2.cParams.chainLog) &
-+ (param1.cParams.strategy == param2.cParams.strategy) & ((param1.cParams.searchLength == 3) == (param2.cParams.searchLength == 3));
-+}
-+
-+/*! ZSTD_continueCCtx() :
-+ reuse CCtx without reset (note : requires no dictionary) */
-+static size_t ZSTD_continueCCtx(ZSTD_CCtx *cctx, ZSTD_parameters params, U64 frameContentSize)
-+{
-+ U32 const end = (U32)(cctx->nextSrc - cctx->base);
-+ cctx->params = params;
-+ cctx->frameContentSize = frameContentSize;
-+ cctx->lowLimit = end;
-+ cctx->dictLimit = end;
-+ cctx->nextToUpdate = end + 1;
-+ cctx->stage = ZSTDcs_init;
-+ cctx->dictID = 0;
-+ cctx->loadedDictEnd = 0;
-+ {
-+ int i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ cctx->rep[i] = repStartValue[i];
-+ }
-+ cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */
-+ xxh64_reset(&cctx->xxhState, 0);
-+ return 0;
-+}
-+
-+typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
-+
-+/*! ZSTD_resetCCtx_advanced() :
-+ note : `params` must be validated */
-+static size_t ZSTD_resetCCtx_advanced(ZSTD_CCtx *zc, ZSTD_parameters params, U64 frameContentSize, ZSTD_compResetPolicy_e const crp)
-+{
-+ if (crp == ZSTDcrp_continue)
-+ if (ZSTD_equivalentParams(params, zc->params)) {
-+ zc->flagStaticTables = 0;
-+ zc->flagStaticHufTable = HUF_repeat_none;
-+ return ZSTD_continueCCtx(zc, params, frameContentSize);
-+ }
-+
-+ {
-+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
-+ U32 const divider = (params.cParams.searchLength == 3) ? 3 : 4;
-+ size_t const maxNbSeq = blockSize / divider;
-+ size_t const tokenSpace = blockSize + 11 * maxNbSeq;
-+ size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
-+ size_t const hSize = ((size_t)1) << params.cParams.hashLog;
-+ U32 const hashLog3 = (params.cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
-+ size_t const h3Size = ((size_t)1) << hashLog3;
-+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
-+ void *ptr;
-+
-+ /* Check if workSpace is large enough, alloc a new one if needed */
-+ {
-+ size_t const optSpace = ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) +
-+ (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
-+ size_t const neededSpace = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
-+ (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
-+ if (zc->workSpaceSize < neededSpace) {
-+ ZSTD_free(zc->workSpace, zc->customMem);
-+ zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
-+ if (zc->workSpace == NULL)
-+ return ERROR(memory_allocation);
-+ zc->workSpaceSize = neededSpace;
-+ }
-+ }
-+
-+ if (crp != ZSTDcrp_noMemset)
-+ memset(zc->workSpace, 0, tableSpace); /* reset tables only */
-+ xxh64_reset(&zc->xxhState, 0);
-+ zc->hashLog3 = hashLog3;
-+ zc->hashTable = (U32 *)(zc->workSpace);
-+ zc->chainTable = zc->hashTable + hSize;
-+ zc->hashTable3 = zc->chainTable + chainSize;
-+ ptr = zc->hashTable3 + h3Size;
-+ zc->hufTable = (HUF_CElt *)ptr;
-+ zc->flagStaticTables = 0;
-+ zc->flagStaticHufTable = HUF_repeat_none;
-+ ptr = ((U32 *)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
-+
-+ zc->nextToUpdate = 1;
-+ zc->nextSrc = NULL;
-+ zc->base = NULL;
-+ zc->dictBase = NULL;
-+ zc->dictLimit = 0;
-+ zc->lowLimit = 0;
-+ zc->params = params;
-+ zc->blockSize = blockSize;
-+ zc->frameContentSize = frameContentSize;
-+ {
-+ int i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ zc->rep[i] = repStartValue[i];
-+ }
-+
-+ if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
-+ zc->seqStore.litFreq = (U32 *)ptr;
-+ zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1 << Litbits);
-+ zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL + 1);
-+ zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML + 1);
-+ ptr = zc->seqStore.offCodeFreq + (MaxOff + 1);
-+ zc->seqStore.matchTable = (ZSTD_match_t *)ptr;
-+ ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM + 1;
-+ zc->seqStore.priceTable = (ZSTD_optimal_t *)ptr;
-+ ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM + 1;
-+ zc->seqStore.litLengthSum = 0;
-+ }
-+ zc->seqStore.sequencesStart = (seqDef *)ptr;
-+ ptr = zc->seqStore.sequencesStart + maxNbSeq;
-+ zc->seqStore.llCode = (BYTE *)ptr;
-+ zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq;
-+ zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq;
-+ zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq;
-+
-+ zc->stage = ZSTDcs_init;
-+ zc->dictID = 0;
-+ zc->loadedDictEnd = 0;
-+
-+ return 0;
-+ }
-+}
-+
-+/* ZSTD_invalidateRepCodes() :
-+ * ensures next compression will not use repcodes from previous block.
-+ * Note : only works with regular variant;
-+ * do not use with extDict variant ! */
-+void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx)
-+{
-+ int i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ cctx->rep[i] = 0;
-+}
-+
-+/*! ZSTD_copyCCtx() :
-+* Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
-+* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
-+* @return : 0, or an error code */
-+size_t ZSTD_copyCCtx(ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize)
-+{
-+ if (srcCCtx->stage != ZSTDcs_init)
-+ return ERROR(stage_wrong);
-+
-+ memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
-+ {
-+ ZSTD_parameters params = srcCCtx->params;
-+ params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
-+ ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
-+ }
-+
-+ /* copy tables */
-+ {
-+ size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
-+ size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
-+ size_t const h3Size = (size_t)1 << srcCCtx->hashLog3;
-+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
-+ memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
-+ }
-+
-+ /* copy dictionary offsets */
-+ dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
-+ dstCCtx->nextToUpdate3 = srcCCtx->nextToUpdate3;
-+ dstCCtx->nextSrc = srcCCtx->nextSrc;
-+ dstCCtx->base = srcCCtx->base;
-+ dstCCtx->dictBase = srcCCtx->dictBase;
-+ dstCCtx->dictLimit = srcCCtx->dictLimit;
-+ dstCCtx->lowLimit = srcCCtx->lowLimit;
-+ dstCCtx->loadedDictEnd = srcCCtx->loadedDictEnd;
-+ dstCCtx->dictID = srcCCtx->dictID;
-+
-+ /* copy entropy tables */
-+ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
-+ dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable;
-+ if (srcCCtx->flagStaticTables) {
-+ memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable));
-+ memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable));
-+ memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
-+ }
-+ if (srcCCtx->flagStaticHufTable) {
-+ memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256 * 4);
-+ }
-+
-+ return 0;
-+}
-+
-+/*! ZSTD_reduceTable() :
-+* reduce table indexes by `reducerValue` */
-+static void ZSTD_reduceTable(U32 *const table, U32 const size, U32 const reducerValue)
-+{
-+ U32 u;
-+ for (u = 0; u < size; u++) {
-+ if (table[u] < reducerValue)
-+ table[u] = 0;
-+ else
-+ table[u] -= reducerValue;
-+ }
-+}
-+
-+/*! ZSTD_reduceIndex() :
-+* rescale all indexes to avoid future overflow (indexes are U32) */
-+static void ZSTD_reduceIndex(ZSTD_CCtx *zc, const U32 reducerValue)
-+{
-+ {
-+ U32 const hSize = 1 << zc->params.cParams.hashLog;
-+ ZSTD_reduceTable(zc->hashTable, hSize, reducerValue);
-+ }
-+
-+ {
-+ U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
-+ ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue);
-+ }
-+
-+ {
-+ U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
-+ ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue);
-+ }
-+}
-+
-+/*-*******************************************************
-+* Block entropic compression
-+*********************************************************/
-+
-+/* See doc/zstd_compression_format.md for detailed format description */
-+
-+size_t ZSTD_noCompressBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ if (srcSize + ZSTD_blockHeaderSize > dstCapacity)
-+ return ERROR(dstSize_tooSmall);
-+ memcpy((BYTE *)dst + ZSTD_blockHeaderSize, src, srcSize);
-+ ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw);
-+ return ZSTD_blockHeaderSize + srcSize;
-+}
-+
-+static size_t ZSTD_noCompressLiterals(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ BYTE *const ostart = (BYTE * const)dst;
-+ U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095);
-+
-+ if (srcSize + flSize > dstCapacity)
-+ return ERROR(dstSize_tooSmall);
-+
-+ switch (flSize) {
-+ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_basic + (srcSize << 3)); break;
-+ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1 << 2) + (srcSize << 4))); break;
-+ default: /*note : should not be necessary : flSize is within {1,2,3} */
-+ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3 << 2) + (srcSize << 4))); break;
-+ }
-+
-+ memcpy(ostart + flSize, src, srcSize);
-+ return srcSize + flSize;
-+}
-+
-+static size_t ZSTD_compressRleLiteralsBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ BYTE *const ostart = (BYTE * const)dst;
-+ U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095);
-+
-+ (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
-+
-+ switch (flSize) {
-+ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_rle + (srcSize << 3)); break;
-+ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1 << 2) + (srcSize << 4))); break;
-+ default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */
-+ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3 << 2) + (srcSize << 4))); break;
-+ }
-+
-+ ostart[flSize] = *(const BYTE *)src;
-+ return flSize + 1;
-+}
-+
-+static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
-+
-+static size_t ZSTD_compressLiterals(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ size_t const minGain = ZSTD_minGain(srcSize);
-+ size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
-+ BYTE *const ostart = (BYTE *)dst;
-+ U32 singleStream = srcSize < 256;
-+ symbolEncodingType_e hType = set_compressed;
-+ size_t cLitSize;
-+
-+/* small ? don't even attempt compression (speed opt) */
-+#define LITERAL_NOENTROPY 63
-+ {
-+ size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
-+ if (srcSize <= minLitSize)
-+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-+ }
-+
-+ if (dstCapacity < lhSize + 1)
-+ return ERROR(dstSize_tooSmall); /* not enough space for compression */
-+ {
-+ HUF_repeat repeat = zc->flagStaticHufTable;
-+ int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
-+ if (repeat == HUF_repeat_valid && lhSize == 3)
-+ singleStream = 1;
-+ cLitSize = singleStream ? HUF_compress1X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
-+ sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
-+ : HUF_compress4X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
-+ sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
-+ if (repeat != HUF_repeat_none) {
-+ hType = set_repeat;
-+ } /* reused the existing table */
-+ else {
-+ zc->flagStaticHufTable = HUF_repeat_check;
-+ } /* now have a table to reuse */
-+ }
-+
-+ if ((cLitSize == 0) | (cLitSize >= srcSize - minGain)) {
-+ zc->flagStaticHufTable = HUF_repeat_none;
-+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-+ }
-+ if (cLitSize == 1) {
-+ zc->flagStaticHufTable = HUF_repeat_none;
-+ return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
-+ }
-+
-+ /* Build header */
-+ switch (lhSize) {
-+ case 3: /* 2 - 2 - 10 - 10 */
-+ {
-+ U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 14);
-+ ZSTD_writeLE24(ostart, lhc);
-+ break;
-+ }
-+ case 4: /* 2 - 2 - 14 - 14 */
-+ {
-+ U32 const lhc = hType + (2 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 18);
-+ ZSTD_writeLE32(ostart, lhc);
-+ break;
-+ }
-+ default: /* should not be necessary, lhSize is only {3,4,5} */
-+ case 5: /* 2 - 2 - 18 - 18 */
-+ {
-+ U32 const lhc = hType + (3 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 22);
-+ ZSTD_writeLE32(ostart, lhc);
-+ ostart[4] = (BYTE)(cLitSize >> 10);
-+ break;
-+ }
-+ }
-+ return lhSize + cLitSize;
-+}
-+
-+static const BYTE LL_Code[64] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18,
-+ 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23,
-+ 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24};
-+
-+static const BYTE ML_Code[128] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
-+ 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38,
-+ 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
-+ 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 42,
-+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42};
-+
-+void ZSTD_seqToCodes(const seqStore_t *seqStorePtr)
-+{
-+ BYTE const LL_deltaCode = 19;
-+ BYTE const ML_deltaCode = 36;
-+ const seqDef *const sequences = seqStorePtr->sequencesStart;
-+ BYTE *const llCodeTable = seqStorePtr->llCode;
-+ BYTE *const ofCodeTable = seqStorePtr->ofCode;
-+ BYTE *const mlCodeTable = seqStorePtr->mlCode;
-+ U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
-+ U32 u;
-+ for (u = 0; u < nbSeq; u++) {
-+ U32 const llv = sequences[u].litLength;
-+ U32 const mlv = sequences[u].matchLength;
-+ llCodeTable[u] = (llv > 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv];
-+ ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset);
-+ mlCodeTable[u] = (mlv > 127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv];
-+ }
-+ if (seqStorePtr->longLengthID == 1)
-+ llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
-+ if (seqStorePtr->longLengthID == 2)
-+ mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
-+}
-+
-+ZSTD_STATIC size_t ZSTD_compressSequences_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity)
-+{
-+ const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
-+ const seqStore_t *seqStorePtr = &(zc->seqStore);
-+ FSE_CTable *CTable_LitLength = zc->litlengthCTable;
-+ FSE_CTable *CTable_OffsetBits = zc->offcodeCTable;
-+ FSE_CTable *CTable_MatchLength = zc->matchlengthCTable;
-+ U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
-+ const seqDef *const sequences = seqStorePtr->sequencesStart;
-+ const BYTE *const ofCodeTable = seqStorePtr->ofCode;
-+ const BYTE *const llCodeTable = seqStorePtr->llCode;
-+ const BYTE *const mlCodeTable = seqStorePtr->mlCode;
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *const oend = ostart + dstCapacity;
-+ BYTE *op = ostart;
-+ size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
-+ BYTE *seqHead;
-+
-+ U32 *count;
-+ S16 *norm;
-+ U32 *workspace;
-+ size_t workspaceSize = sizeof(zc->tmpCounters);
-+ {
-+ size_t spaceUsed32 = 0;
-+ count = (U32 *)zc->tmpCounters + spaceUsed32;
-+ spaceUsed32 += MaxSeq + 1;
-+ norm = (S16 *)((U32 *)zc->tmpCounters + spaceUsed32);
-+ spaceUsed32 += ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2;
-+
-+ workspace = (U32 *)zc->tmpCounters + spaceUsed32;
-+ workspaceSize -= (spaceUsed32 << 2);
-+ }
-+
-+ /* Compress literals */
-+ {
-+ const BYTE *const literals = seqStorePtr->litStart;
-+ size_t const litSize = seqStorePtr->lit - literals;
-+ size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize);
-+ if (ZSTD_isError(cSize))
-+ return cSize;
-+ op += cSize;
-+ }
-+
-+ /* Sequences Header */
-+ if ((oend - op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */)
-+ return ERROR(dstSize_tooSmall);
-+ if (nbSeq < 0x7F)
-+ *op++ = (BYTE)nbSeq;
-+ else if (nbSeq < LONGNBSEQ)
-+ op[0] = (BYTE)((nbSeq >> 8) + 0x80), op[1] = (BYTE)nbSeq, op += 2;
-+ else
-+ op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3;
-+ if (nbSeq == 0)
-+ return op - ostart;
-+
-+ /* seqHead : flags for FSE encoding type */
-+ seqHead = op++;
-+
-+#define MIN_SEQ_FOR_DYNAMIC_FSE 64
-+#define MAX_SEQ_FOR_STATIC_FSE 1000
-+
-+ /* convert length/distances into codes */
-+ ZSTD_seqToCodes(seqStorePtr);
-+
-+ /* CTable for Literal Lengths */
-+ {
-+ U32 max = MaxLL;
-+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace);
-+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
-+ *op++ = llCodeTable[0];
-+ FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
-+ LLtype = set_rle;
-+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
-+ LLtype = set_repeat;
-+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog - 1)))) {
-+ FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, workspace, workspaceSize);
-+ LLtype = set_basic;
-+ } else {
-+ size_t nbSeq_1 = nbSeq;
-+ const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
-+ if (count[llCodeTable[nbSeq - 1]] > 1) {
-+ count[llCodeTable[nbSeq - 1]]--;
-+ nbSeq_1--;
-+ }
-+ FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
-+ {
-+ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
-+ if (FSE_isError(NCountSize))
-+ return NCountSize;
-+ op += NCountSize;
-+ }
-+ FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, workspace, workspaceSize);
-+ LLtype = set_compressed;
-+ }
-+ }
-+
-+ /* CTable for Offsets */
-+ {
-+ U32 max = MaxOff;
-+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace);
-+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
-+ *op++ = ofCodeTable[0];
-+ FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
-+ Offtype = set_rle;
-+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
-+ Offtype = set_repeat;
-+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog - 1)))) {
-+ FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, workspace, workspaceSize);
-+ Offtype = set_basic;
-+ } else {
-+ size_t nbSeq_1 = nbSeq;
-+ const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
-+ if (count[ofCodeTable[nbSeq - 1]] > 1) {
-+ count[ofCodeTable[nbSeq - 1]]--;
-+ nbSeq_1--;
-+ }
-+ FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
-+ {
-+ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
-+ if (FSE_isError(NCountSize))
-+ return NCountSize;
-+ op += NCountSize;
-+ }
-+ FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, workspace, workspaceSize);
-+ Offtype = set_compressed;
-+ }
-+ }
-+
-+ /* CTable for MatchLengths */
-+ {
-+ U32 max = MaxML;
-+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace);
-+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
-+ *op++ = *mlCodeTable;
-+ FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
-+ MLtype = set_rle;
-+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
-+ MLtype = set_repeat;
-+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog - 1)))) {
-+ FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, workspace, workspaceSize);
-+ MLtype = set_basic;
-+ } else {
-+ size_t nbSeq_1 = nbSeq;
-+ const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
-+ if (count[mlCodeTable[nbSeq - 1]] > 1) {
-+ count[mlCodeTable[nbSeq - 1]]--;
-+ nbSeq_1--;
-+ }
-+ FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
-+ {
-+ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
-+ if (FSE_isError(NCountSize))
-+ return NCountSize;
-+ op += NCountSize;
-+ }
-+ FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, workspace, workspaceSize);
-+ MLtype = set_compressed;
-+ }
-+ }
-+
-+ *seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2));
-+ zc->flagStaticTables = 0;
-+
-+ /* Encoding Sequences */
-+ {
-+ BIT_CStream_t blockStream;
-+ FSE_CState_t stateMatchLength;
-+ FSE_CState_t stateOffsetBits;
-+ FSE_CState_t stateLitLength;
-+
-+ CHECK_E(BIT_initCStream(&blockStream, op, oend - op), dstSize_tooSmall); /* not enough space remaining */
-+
-+ /* first symbols */
-+ FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq - 1]);
-+ FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]);
-+ FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]);
-+ BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]);
-+ if (ZSTD_32bits())
-+ BIT_flushBits(&blockStream);
-+ BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]);
-+ if (ZSTD_32bits())
-+ BIT_flushBits(&blockStream);
-+ if (longOffsets) {
-+ U32 const ofBits = ofCodeTable[nbSeq - 1];
-+ int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
-+ if (extraBits) {
-+ BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits);
-+ BIT_flushBits(&blockStream);
-+ }
-+ BIT_addBits(&blockStream, sequences[nbSeq - 1].offset >> extraBits, ofBits - extraBits);
-+ } else {
-+ BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]);
-+ }
-+ BIT_flushBits(&blockStream);
-+
-+ {
-+ size_t n;
-+ for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */
-+ BYTE const llCode = llCodeTable[n];
-+ BYTE const ofCode = ofCodeTable[n];
-+ BYTE const mlCode = mlCodeTable[n];
-+ U32 const llBits = LL_bits[llCode];
-+ U32 const ofBits = ofCode; /* 32b*/ /* 64b*/
-+ U32 const mlBits = ML_bits[mlCode];
-+ /* (7)*/ /* (7)*/
-+ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
-+ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
-+ if (ZSTD_32bits())
-+ BIT_flushBits(&blockStream); /* (7)*/
-+ FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */
-+ if (ZSTD_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
-+ BIT_flushBits(&blockStream); /* (7)*/
-+ BIT_addBits(&blockStream, sequences[n].litLength, llBits);
-+ if (ZSTD_32bits() && ((llBits + mlBits) > 24))
-+ BIT_flushBits(&blockStream);
-+ BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
-+ if (ZSTD_32bits())
-+ BIT_flushBits(&blockStream); /* (7)*/
-+ if (longOffsets) {
-+ int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
-+ if (extraBits) {
-+ BIT_addBits(&blockStream, sequences[n].offset, extraBits);
-+ BIT_flushBits(&blockStream); /* (7)*/
-+ }
-+ BIT_addBits(&blockStream, sequences[n].offset >> extraBits, ofBits - extraBits); /* 31 */
-+ } else {
-+ BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */
-+ }
-+ BIT_flushBits(&blockStream); /* (7)*/
-+ }
-+ }
-+
-+ FSE_flushCState(&blockStream, &stateMatchLength);
-+ FSE_flushCState(&blockStream, &stateOffsetBits);
-+ FSE_flushCState(&blockStream, &stateLitLength);
-+
-+ {
-+ size_t const streamSize = BIT_closeCStream(&blockStream);
-+ if (streamSize == 0)
-+ return ERROR(dstSize_tooSmall); /* not enough space */
-+ op += streamSize;
-+ }
-+ }
-+ return op - ostart;
-+}
-+
-+ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize)
-+{
-+ size_t const cSize = ZSTD_compressSequences_internal(zc, dst, dstCapacity);
-+ size_t const minGain = ZSTD_minGain(srcSize);
-+ size_t const maxCSize = srcSize - minGain;
-+ /* If the srcSize <= dstCapacity, then there is enough space to write a
-+ * raw uncompressed block. Since we ran out of space, the block must not
-+ * be compressible, so fall back to a raw uncompressed block.
-+ */
-+ int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity;
-+ int i;
-+
-+ if (ZSTD_isError(cSize) && !uncompressibleError)
-+ return cSize;
-+ if (cSize >= maxCSize || uncompressibleError) {
-+ zc->flagStaticHufTable = HUF_repeat_none;
-+ return 0;
-+ }
-+ /* confirm repcodes */
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ zc->rep[i] = zc->repToConfirm[i];
-+ return cSize;
-+}
-+
-+/*! ZSTD_storeSeq() :
-+ Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
-+ `offsetCode` : distance to match, or 0 == repCode.
-+ `matchCode` : matchLength - MINMATCH
-+*/
-+ZSTD_STATIC void ZSTD_storeSeq(seqStore_t *seqStorePtr, size_t litLength, const void *literals, U32 offsetCode, size_t matchCode)
-+{
-+ /* copy Literals */
-+ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
-+ seqStorePtr->lit += litLength;
-+
-+ /* literal Length */
-+ if (litLength > 0xFFFF) {
-+ seqStorePtr->longLengthID = 1;
-+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
-+ }
-+ seqStorePtr->sequences[0].litLength = (U16)litLength;
-+
-+ /* match offset */
-+ seqStorePtr->sequences[0].offset = offsetCode + 1;
-+
-+ /* match Length */
-+ if (matchCode > 0xFFFF) {
-+ seqStorePtr->longLengthID = 2;
-+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
-+ }
-+ seqStorePtr->sequences[0].matchLength = (U16)matchCode;
-+
-+ seqStorePtr->sequences++;
-+}
-+
-+/*-*************************************
-+* Match length counter
-+***************************************/
-+static unsigned ZSTD_NbCommonBytes(register size_t val)
-+{
-+ if (ZSTD_isLittleEndian()) {
-+ if (ZSTD_64bits()) {
-+ return (__builtin_ctzll((U64)val) >> 3);
-+ } else { /* 32 bits */
-+ return (__builtin_ctz((U32)val) >> 3);
-+ }
-+ } else { /* Big Endian CPU */
-+ if (ZSTD_64bits()) {
-+ return (__builtin_clzll(val) >> 3);
-+ } else { /* 32 bits */
-+ return (__builtin_clz((U32)val) >> 3);
-+ }
-+ }
-+}
-+
-+static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit)
-+{
-+ const BYTE *const pStart = pIn;
-+ const BYTE *const pInLoopLimit = pInLimit - (sizeof(size_t) - 1);
-+
-+ while (pIn < pInLoopLimit) {
-+ size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn);
-+ if (!diff) {
-+ pIn += sizeof(size_t);
-+ pMatch += sizeof(size_t);
-+ continue;
-+ }
-+ pIn += ZSTD_NbCommonBytes(diff);
-+ return (size_t)(pIn - pStart);
-+ }
-+ if (ZSTD_64bits())
-+ if ((pIn < (pInLimit - 3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) {
-+ pIn += 4;
-+ pMatch += 4;
-+ }
-+ if ((pIn < (pInLimit - 1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) {
-+ pIn += 2;
-+ pMatch += 2;
-+ }
-+ if ((pIn < pInLimit) && (*pMatch == *pIn))
-+ pIn++;
-+ return (size_t)(pIn - pStart);
-+}
-+
-+/** ZSTD_count_2segments() :
-+* can count match length with `ip` & `match` in 2 different segments.
-+* convention : on reaching mEnd, match count continue starting from iStart
-+*/
-+static size_t ZSTD_count_2segments(const BYTE *ip, const BYTE *match, const BYTE *iEnd, const BYTE *mEnd, const BYTE *iStart)
-+{
-+ const BYTE *const vEnd = MIN(ip + (mEnd - match), iEnd);
-+ size_t const matchLength = ZSTD_count(ip, match, vEnd);
-+ if (match + matchLength != mEnd)
-+ return matchLength;
-+ return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd);
-+}
-+
-+/*-*************************************
-+* Hashes
-+***************************************/
-+static const U32 prime3bytes = 506832829U;
-+static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32 - 24)) * prime3bytes) >> (32 - h); }
-+ZSTD_STATIC size_t ZSTD_hash3Ptr(const void *ptr, U32 h) { return ZSTD_hash3(ZSTD_readLE32(ptr), h); } /* only in zstd_opt.h */
-+
-+static const U32 prime4bytes = 2654435761U;
-+static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32 - h); }
-+static size_t ZSTD_hash4Ptr(const void *ptr, U32 h) { return ZSTD_hash4(ZSTD_read32(ptr), h); }
-+
-+static const U64 prime5bytes = 889523592379ULL;
-+static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64 - 40)) * prime5bytes) >> (64 - h)); }
-+static size_t ZSTD_hash5Ptr(const void *p, U32 h) { return ZSTD_hash5(ZSTD_readLE64(p), h); }
-+
-+static const U64 prime6bytes = 227718039650203ULL;
-+static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64 - 48)) * prime6bytes) >> (64 - h)); }
-+static size_t ZSTD_hash6Ptr(const void *p, U32 h) { return ZSTD_hash6(ZSTD_readLE64(p), h); }
-+
-+static const U64 prime7bytes = 58295818150454627ULL;
-+static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64 - 56)) * prime7bytes) >> (64 - h)); }
-+static size_t ZSTD_hash7Ptr(const void *p, U32 h) { return ZSTD_hash7(ZSTD_readLE64(p), h); }
-+
-+static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
-+static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u)*prime8bytes) >> (64 - h)); }
-+static size_t ZSTD_hash8Ptr(const void *p, U32 h) { return ZSTD_hash8(ZSTD_readLE64(p), h); }
-+
-+static size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls)
-+{
-+ switch (mls) {
-+ // case 3: return ZSTD_hash3Ptr(p, hBits);
-+ default:
-+ case 4: return ZSTD_hash4Ptr(p, hBits);
-+ case 5: return ZSTD_hash5Ptr(p, hBits);
-+ case 6: return ZSTD_hash6Ptr(p, hBits);
-+ case 7: return ZSTD_hash7Ptr(p, hBits);
-+ case 8: return ZSTD_hash8Ptr(p, hBits);
-+ }
-+}
-+
-+/*-*************************************
-+* Fast Scan
-+***************************************/
-+static void ZSTD_fillHashTable(ZSTD_CCtx *zc, const void *end, const U32 mls)
-+{
-+ U32 *const hashTable = zc->hashTable;
-+ U32 const hBits = zc->params.cParams.hashLog;
-+ const BYTE *const base = zc->base;
-+ const BYTE *ip = base + zc->nextToUpdate;
-+ const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE;
-+ const size_t fastHashFillStep = 3;
-+
-+ while (ip <= iend) {
-+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
-+ ip += fastHashFillStep;
-+ }
-+}
-+
-+FORCE_INLINE
-+void ZSTD_compressBlock_fast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
-+{
-+ U32 *const hashTable = cctx->hashTable;
-+ U32 const hBits = cctx->params.cParams.hashLog;
-+ seqStore_t *seqStorePtr = &(cctx->seqStore);
-+ const BYTE *const base = cctx->base;
-+ const BYTE *const istart = (const BYTE *)src;
-+ const BYTE *ip = istart;
-+ const BYTE *anchor = istart;
-+ const U32 lowestIndex = cctx->dictLimit;
-+ const BYTE *const lowest = base + lowestIndex;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *const ilimit = iend - HASH_READ_SIZE;
-+ U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1];
-+ U32 offsetSaved = 0;
-+
-+ /* init */
-+ ip += (ip == lowest);
-+ {
-+ U32 const maxRep = (U32)(ip - lowest);
-+ if (offset_2 > maxRep)
-+ offsetSaved = offset_2, offset_2 = 0;
-+ if (offset_1 > maxRep)
-+ offsetSaved = offset_1, offset_1 = 0;
-+ }
-+
-+ /* Main Search Loop */
-+ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
-+ size_t mLength;
-+ size_t const h = ZSTD_hashPtr(ip, hBits, mls);
-+ U32 const curr = (U32)(ip - base);
-+ U32 const matchIndex = hashTable[h];
-+ const BYTE *match = base + matchIndex;
-+ hashTable[h] = curr; /* update hash table */
-+
-+ if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) {
-+ mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4;
-+ ip++;
-+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
-+ } else {
-+ U32 offset;
-+ if ((matchIndex <= lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) {
-+ ip += ((ip - anchor) >> g_searchStrength) + 1;
-+ continue;
-+ }
-+ mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
-+ offset = (U32)(ip - match);
-+ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
-+ ip--;
-+ match--;
-+ mLength++;
-+ } /* catch up */
-+ offset_2 = offset_1;
-+ offset_1 = offset;
-+
-+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
-+ }
-+
-+ /* match found */
-+ ip += mLength;
-+ anchor = ip;
-+
-+ if (ip <= ilimit) {
-+ /* Fill Table */
-+ hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; /* here because curr+2 could be > iend-8 */
-+ hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base);
-+ /* check immediate repcode */
-+ while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
-+ /* store sequence */
-+ size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
-+ {
-+ U32 const tmpOff = offset_2;
-+ offset_2 = offset_1;
-+ offset_1 = tmpOff;
-+ } /* swap offset_2 <=> offset_1 */
-+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
-+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
-+ ip += rLength;
-+ anchor = ip;
-+ continue; /* faster when present ... (?) */
-+ }
-+ }
-+ }
-+
-+ /* save reps for next block */
-+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
-+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
-+
-+ /* Last Literals */
-+ {
-+ size_t const lastLLSize = iend - anchor;
-+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
-+ seqStorePtr->lit += lastLLSize;
-+ }
-+}
-+
-+static void ZSTD_compressBlock_fast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+ const U32 mls = ctx->params.cParams.searchLength;
-+ switch (mls) {
-+ default: /* includes case 3 */
-+ case 4: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return;
-+ case 5: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return;
-+ case 6: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return;
-+ case 7: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return;
-+ }
-+}
-+
-+static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
-+{
-+ U32 *hashTable = ctx->hashTable;
-+ const U32 hBits = ctx->params.cParams.hashLog;
-+ seqStore_t *seqStorePtr = &(ctx->seqStore);
-+ const BYTE *const base = ctx->base;
-+ const BYTE *const dictBase = ctx->dictBase;
-+ const BYTE *const istart = (const BYTE *)src;
-+ const BYTE *ip = istart;
-+ const BYTE *anchor = istart;
-+ const U32 lowestIndex = ctx->lowLimit;
-+ const BYTE *const dictStart = dictBase + lowestIndex;
-+ const U32 dictLimit = ctx->dictLimit;
-+ const BYTE *const lowPrefixPtr = base + dictLimit;
-+ const BYTE *const dictEnd = dictBase + dictLimit;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *const ilimit = iend - 8;
-+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
-+
-+ /* Search Loop */
-+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
-+ const size_t h = ZSTD_hashPtr(ip, hBits, mls);
-+ const U32 matchIndex = hashTable[h];
-+ const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base;
-+ const BYTE *match = matchBase + matchIndex;
-+ const U32 curr = (U32)(ip - base);
-+ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
-+ const BYTE *repBase = repIndex < dictLimit ? dictBase : base;
-+ const BYTE *repMatch = repBase + repIndex;
-+ size_t mLength;
-+ hashTable[h] = curr; /* update hash table */
-+
-+ if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
-+ (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
-+ const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
-+ mLength = ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
-+ ip++;
-+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
-+ } else {
-+ if ((matchIndex < lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) {
-+ ip += ((ip - anchor) >> g_searchStrength) + 1;
-+ continue;
-+ }
-+ {
-+ const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
-+ const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
-+ U32 offset;
-+ mLength = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
-+ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
-+ ip--;
-+ match--;
-+ mLength++;
-+ } /* catch up */
-+ offset = curr - matchIndex;
-+ offset_2 = offset_1;
-+ offset_1 = offset;
-+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
-+ }
-+ }
-+
-+ /* found a match : store it */
-+ ip += mLength;
-+ anchor = ip;
-+
-+ if (ip <= ilimit) {
-+ /* Fill Table */
-+ hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2;
-+ hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base);
-+ /* check immediate repcode */
-+ while (ip <= ilimit) {
-+ U32 const curr2 = (U32)(ip - base);
-+ U32 const repIndex2 = curr2 - offset_2;
-+ const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
-+ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
-+ && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
-+ const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
-+ size_t repLength2 =
-+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
-+ U32 tmpOffset = offset_2;
-+ offset_2 = offset_1;
-+ offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
-+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
-+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2;
-+ ip += repLength2;
-+ anchor = ip;
-+ continue;
-+ }
-+ break;
-+ }
-+ }
-+ }
-+
-+ /* save reps for next block */
-+ ctx->repToConfirm[0] = offset_1;
-+ ctx->repToConfirm[1] = offset_2;
-+
-+ /* Last Literals */
-+ {
-+ size_t const lastLLSize = iend - anchor;
-+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
-+ seqStorePtr->lit += lastLLSize;
-+ }
-+}
-+
-+static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+ U32 const mls = ctx->params.cParams.searchLength;
-+ switch (mls) {
-+ default: /* includes case 3 */
-+ case 4: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return;
-+ case 5: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return;
-+ case 6: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return;
-+ case 7: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return;
-+ }
-+}
-+
-+/*-*************************************
-+* Double Fast
-+***************************************/
-+static void ZSTD_fillDoubleHashTable(ZSTD_CCtx *cctx, const void *end, const U32 mls)
-+{
-+ U32 *const hashLarge = cctx->hashTable;
-+ U32 const hBitsL = cctx->params.cParams.hashLog;
-+ U32 *const hashSmall = cctx->chainTable;
-+ U32 const hBitsS = cctx->params.cParams.chainLog;
-+ const BYTE *const base = cctx->base;
-+ const BYTE *ip = base + cctx->nextToUpdate;
-+ const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE;
-+ const size_t fastHashFillStep = 3;
-+
-+ while (ip <= iend) {
-+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base);
-+ hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base);
-+ ip += fastHashFillStep;
-+ }
-+}
-+
-+FORCE_INLINE
-+void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
-+{
-+ U32 *const hashLong = cctx->hashTable;
-+ const U32 hBitsL = cctx->params.cParams.hashLog;
-+ U32 *const hashSmall = cctx->chainTable;
-+ const U32 hBitsS = cctx->params.cParams.chainLog;
-+ seqStore_t *seqStorePtr = &(cctx->seqStore);
-+ const BYTE *const base = cctx->base;
-+ const BYTE *const istart = (const BYTE *)src;
-+ const BYTE *ip = istart;
-+ const BYTE *anchor = istart;
-+ const U32 lowestIndex = cctx->dictLimit;
-+ const BYTE *const lowest = base + lowestIndex;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *const ilimit = iend - HASH_READ_SIZE;
-+ U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1];
-+ U32 offsetSaved = 0;
-+
-+ /* init */
-+ ip += (ip == lowest);
-+ {
-+ U32 const maxRep = (U32)(ip - lowest);
-+ if (offset_2 > maxRep)
-+ offsetSaved = offset_2, offset_2 = 0;
-+ if (offset_1 > maxRep)
-+ offsetSaved = offset_1, offset_1 = 0;
-+ }
-+
-+ /* Main Search Loop */
-+ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
-+ size_t mLength;
-+ size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
-+ size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
-+ U32 const curr = (U32)(ip - base);
-+ U32 const matchIndexL = hashLong[h2];
-+ U32 const matchIndexS = hashSmall[h];
-+ const BYTE *matchLong = base + matchIndexL;
-+ const BYTE *match = base + matchIndexS;
-+ hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
-+
-+ if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { /* note : by construction, offset_1 <= curr */
-+ mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4;
-+ ip++;
-+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
-+ } else {
-+ U32 offset;
-+ if ((matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) {
-+ mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8;
-+ offset = (U32)(ip - matchLong);
-+ while (((ip > anchor) & (matchLong > lowest)) && (ip[-1] == matchLong[-1])) {
-+ ip--;
-+ matchLong--;
-+ mLength++;
-+ } /* catch up */
-+ } else if ((matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) {
-+ size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8);
-+ U32 const matchIndex3 = hashLong[h3];
-+ const BYTE *match3 = base + matchIndex3;
-+ hashLong[h3] = curr + 1;
-+ if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) {
-+ mLength = ZSTD_count(ip + 9, match3 + 8, iend) + 8;
-+ ip++;
-+ offset = (U32)(ip - match3);
-+ while (((ip > anchor) & (match3 > lowest)) && (ip[-1] == match3[-1])) {
-+ ip--;
-+ match3--;
-+ mLength++;
-+ } /* catch up */
-+ } else {
-+ mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
-+ offset = (U32)(ip - match);
-+ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
-+ ip--;
-+ match--;
-+ mLength++;
-+ } /* catch up */
-+ }
-+ } else {
-+ ip += ((ip - anchor) >> g_searchStrength) + 1;
-+ continue;
-+ }
-+
-+ offset_2 = offset_1;
-+ offset_1 = offset;
-+
-+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
-+ }
-+
-+ /* match found */
-+ ip += mLength;
-+ anchor = ip;
-+
-+ if (ip <= ilimit) {
-+ /* Fill Table */
-+ hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] =
-+ curr + 2; /* here because curr+2 could be > iend-8 */
-+ hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base);
-+
-+ /* check immediate repcode */
-+ while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
-+ /* store sequence */
-+ size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
-+ {
-+ U32 const tmpOff = offset_2;
-+ offset_2 = offset_1;
-+ offset_1 = tmpOff;
-+ } /* swap offset_2 <=> offset_1 */
-+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base);
-+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base);
-+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
-+ ip += rLength;
-+ anchor = ip;
-+ continue; /* faster when present ... (?) */
-+ }
-+ }
-+ }
-+
-+ /* save reps for next block */
-+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
-+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
-+
-+ /* Last Literals */
-+ {
-+ size_t const lastLLSize = iend - anchor;
-+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
-+ seqStorePtr->lit += lastLLSize;
-+ }
-+}
-+
-+static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+ const U32 mls = ctx->params.cParams.searchLength;
-+ switch (mls) {
-+ default: /* includes case 3 */
-+ case 4: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return;
-+ case 5: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return;
-+ case 6: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return;
-+ case 7: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return;
-+ }
-+}
-+
-+static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
-+{
-+ U32 *const hashLong = ctx->hashTable;
-+ U32 const hBitsL = ctx->params.cParams.hashLog;
-+ U32 *const hashSmall = ctx->chainTable;
-+ U32 const hBitsS = ctx->params.cParams.chainLog;
-+ seqStore_t *seqStorePtr = &(ctx->seqStore);
-+ const BYTE *const base = ctx->base;
-+ const BYTE *const dictBase = ctx->dictBase;
-+ const BYTE *const istart = (const BYTE *)src;
-+ const BYTE *ip = istart;
-+ const BYTE *anchor = istart;
-+ const U32 lowestIndex = ctx->lowLimit;
-+ const BYTE *const dictStart = dictBase + lowestIndex;
-+ const U32 dictLimit = ctx->dictLimit;
-+ const BYTE *const lowPrefixPtr = base + dictLimit;
-+ const BYTE *const dictEnd = dictBase + dictLimit;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *const ilimit = iend - 8;
-+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
-+
-+ /* Search Loop */
-+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
-+ const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
-+ const U32 matchIndex = hashSmall[hSmall];
-+ const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base;
-+ const BYTE *match = matchBase + matchIndex;
-+
-+ const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8);
-+ const U32 matchLongIndex = hashLong[hLong];
-+ const BYTE *matchLongBase = matchLongIndex < dictLimit ? dictBase : base;
-+ const BYTE *matchLong = matchLongBase + matchLongIndex;
-+
-+ const U32 curr = (U32)(ip - base);
-+ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
-+ const BYTE *repBase = repIndex < dictLimit ? dictBase : base;
-+ const BYTE *repMatch = repBase + repIndex;
-+ size_t mLength;
-+ hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
-+
-+ if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
-+ (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
-+ const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
-+ mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, lowPrefixPtr) + 4;
-+ ip++;
-+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
-+ } else {
-+ if ((matchLongIndex > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) {
-+ const BYTE *matchEnd = matchLongIndex < dictLimit ? dictEnd : iend;
-+ const BYTE *lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr;
-+ U32 offset;
-+ mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, lowPrefixPtr) + 8;
-+ offset = curr - matchLongIndex;
-+ while (((ip > anchor) & (matchLong > lowMatchPtr)) && (ip[-1] == matchLong[-1])) {
-+ ip--;
-+ matchLong--;
-+ mLength++;
-+ } /* catch up */
-+ offset_2 = offset_1;
-+ offset_1 = offset;
-+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
-+
-+ } else if ((matchIndex > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) {
-+ size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8);
-+ U32 const matchIndex3 = hashLong[h3];
-+ const BYTE *const match3Base = matchIndex3 < dictLimit ? dictBase : base;
-+ const BYTE *match3 = match3Base + matchIndex3;
-+ U32 offset;
-+ hashLong[h3] = curr + 1;
-+ if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) {
-+ const BYTE *matchEnd = matchIndex3 < dictLimit ? dictEnd : iend;
-+ const BYTE *lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr;
-+ mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, lowPrefixPtr) + 8;
-+ ip++;
-+ offset = curr + 1 - matchIndex3;
-+ while (((ip > anchor) & (match3 > lowMatchPtr)) && (ip[-1] == match3[-1])) {
-+ ip--;
-+ match3--;
-+ mLength++;
-+ } /* catch up */
-+ } else {
-+ const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
-+ const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
-+ mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, lowPrefixPtr) + 4;
-+ offset = curr - matchIndex;
-+ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
-+ ip--;
-+ match--;
-+ mLength++;
-+ } /* catch up */
-+ }
-+ offset_2 = offset_1;
-+ offset_1 = offset;
-+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
-+
-+ } else {
-+ ip += ((ip - anchor) >> g_searchStrength) + 1;
-+ continue;
-+ }
-+ }
-+
-+ /* found a match : store it */
-+ ip += mLength;
-+ anchor = ip;
-+
-+ if (ip <= ilimit) {
-+ /* Fill Table */
-+ hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = curr + 2;
-+ hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = curr + 2;
-+ hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base);
-+ hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = (U32)(ip - 2 - base);
-+ /* check immediate repcode */
-+ while (ip <= ilimit) {
-+ U32 const curr2 = (U32)(ip - base);
-+ U32 const repIndex2 = curr2 - offset_2;
-+ const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
-+ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
-+ && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
-+ const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
-+ size_t const repLength2 =
-+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
-+ U32 tmpOffset = offset_2;
-+ offset_2 = offset_1;
-+ offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
-+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
-+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2;
-+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2;
-+ ip += repLength2;
-+ anchor = ip;
-+ continue;
-+ }
-+ break;
-+ }
-+ }
-+ }
-+
-+ /* save reps for next block */
-+ ctx->repToConfirm[0] = offset_1;
-+ ctx->repToConfirm[1] = offset_2;
-+
-+ /* Last Literals */
-+ {
-+ size_t const lastLLSize = iend - anchor;
-+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
-+ seqStorePtr->lit += lastLLSize;
-+ }
-+}
-+
-+static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+ U32 const mls = ctx->params.cParams.searchLength;
-+ switch (mls) {
-+ default: /* includes case 3 */
-+ case 4: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return;
-+ case 5: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return;
-+ case 6: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return;
-+ case 7: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return;
-+ }
-+}
-+
-+/*-*************************************
-+* Binary Tree search
-+***************************************/
-+/** ZSTD_insertBt1() : add one or multiple positions to tree.
-+* ip : assumed <= iend-8 .
-+* @return : nb of positions added */
-+static U32 ZSTD_insertBt1(ZSTD_CCtx *zc, const BYTE *const ip, const U32 mls, const BYTE *const iend, U32 nbCompares, U32 extDict)
-+{
-+ U32 *const hashTable = zc->hashTable;
-+ U32 const hashLog = zc->params.cParams.hashLog;
-+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
-+ U32 *const bt = zc->chainTable;
-+ U32 const btLog = zc->params.cParams.chainLog - 1;
-+ U32 const btMask = (1 << btLog) - 1;
-+ U32 matchIndex = hashTable[h];
-+ size_t commonLengthSmaller = 0, commonLengthLarger = 0;
-+ const BYTE *const base = zc->base;
-+ const BYTE *const dictBase = zc->dictBase;
-+ const U32 dictLimit = zc->dictLimit;
-+ const BYTE *const dictEnd = dictBase + dictLimit;
-+ const BYTE *const prefixStart = base + dictLimit;
-+ const BYTE *match;
-+ const U32 curr = (U32)(ip - base);
-+ const U32 btLow = btMask >= curr ? 0 : curr - btMask;
-+ U32 *smallerPtr = bt + 2 * (curr & btMask);
-+ U32 *largerPtr = smallerPtr + 1;
-+ U32 dummy32; /* to be nullified at the end */
-+ U32 const windowLow = zc->lowLimit;
-+ U32 matchEndIdx = curr + 8;
-+ size_t bestLength = 8;
-+
-+ hashTable[h] = curr; /* Update Hash Table */
-+
-+ while (nbCompares-- && (matchIndex > windowLow)) {
-+ U32 *const nextPtr = bt + 2 * (matchIndex & btMask);
-+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
-+
-+ if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
-+ match = base + matchIndex;
-+ if (match[matchLength] == ip[matchLength])
-+ matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1;
-+ } else {
-+ match = dictBase + matchIndex;
-+ matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart);
-+ if (matchIndex + matchLength >= dictLimit)
-+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
-+ }
-+
-+ if (matchLength > bestLength) {
-+ bestLength = matchLength;
-+ if (matchLength > matchEndIdx - matchIndex)
-+ matchEndIdx = matchIndex + (U32)matchLength;
-+ }
-+
-+ if (ip + matchLength == iend) /* equal : no way to know if inf or sup */
-+ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
-+
-+ if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */
-+ /* match is smaller than curr */
-+ *smallerPtr = matchIndex; /* update smaller idx */
-+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
-+ if (matchIndex <= btLow) {
-+ smallerPtr = &dummy32;
-+ break;
-+ } /* beyond tree size, stop the search */
-+ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
-+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
-+ } else {
-+ /* match is larger than curr */
-+ *largerPtr = matchIndex;
-+ commonLengthLarger = matchLength;
-+ if (matchIndex <= btLow) {
-+ largerPtr = &dummy32;
-+ break;
-+ } /* beyond tree size, stop the search */
-+ largerPtr = nextPtr;
-+ matchIndex = nextPtr[0];
-+ }
-+ }
-+
-+ *smallerPtr = *largerPtr = 0;
-+ if (bestLength > 384)
-+ return MIN(192, (U32)(bestLength - 384)); /* speed optimization */
-+ if (matchEndIdx > curr + 8)
-+ return matchEndIdx - curr - 8;
-+ return 1;
-+}
-+
-+static size_t ZSTD_insertBtAndFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, size_t *offsetPtr, U32 nbCompares, const U32 mls,
-+ U32 extDict)
-+{
-+ U32 *const hashTable = zc->hashTable;
-+ U32 const hashLog = zc->params.cParams.hashLog;
-+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
-+ U32 *const bt = zc->chainTable;
-+ U32 const btLog = zc->params.cParams.chainLog - 1;
-+ U32 const btMask = (1 << btLog) - 1;
-+ U32 matchIndex = hashTable[h];
-+ size_t commonLengthSmaller = 0, commonLengthLarger = 0;
-+ const BYTE *const base = zc->base;
-+ const BYTE *const dictBase = zc->dictBase;
-+ const U32 dictLimit = zc->dictLimit;
-+ const BYTE *const dictEnd = dictBase + dictLimit;
-+ const BYTE *const prefixStart = base + dictLimit;
-+ const U32 curr = (U32)(ip - base);
-+ const U32 btLow = btMask >= curr ? 0 : curr - btMask;
-+ const U32 windowLow = zc->lowLimit;
-+ U32 *smallerPtr = bt + 2 * (curr & btMask);
-+ U32 *largerPtr = bt + 2 * (curr & btMask) + 1;
-+ U32 matchEndIdx = curr + 8;
-+ U32 dummy32; /* to be nullified at the end */
-+ size_t bestLength = 0;
-+
-+ hashTable[h] = curr; /* Update Hash Table */
-+
-+ while (nbCompares-- && (matchIndex > windowLow)) {
-+ U32 *const nextPtr = bt + 2 * (matchIndex & btMask);
-+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
-+ const BYTE *match;
-+
-+ if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
-+ match = base + matchIndex;
-+ if (match[matchLength] == ip[matchLength])
-+ matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1;
-+ } else {
-+ match = dictBase + matchIndex;
-+ matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart);
-+ if (matchIndex + matchLength >= dictLimit)
-+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
-+ }
-+
-+ if (matchLength > bestLength) {
-+ if (matchLength > matchEndIdx - matchIndex)
-+ matchEndIdx = matchIndex + (U32)matchLength;
-+ if ((4 * (int)(matchLength - bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)offsetPtr[0] + 1)))
-+ bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex;
-+ if (ip + matchLength == iend) /* equal : no way to know if inf or sup */
-+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
-+ }
-+
-+ if (match[matchLength] < ip[matchLength]) {
-+ /* match is smaller than curr */
-+ *smallerPtr = matchIndex; /* update smaller idx */
-+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
-+ if (matchIndex <= btLow) {
-+ smallerPtr = &dummy32;
-+ break;
-+ } /* beyond tree size, stop the search */
-+ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
-+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
-+ } else {
-+ /* match is larger than curr */
-+ *largerPtr = matchIndex;
-+ commonLengthLarger = matchLength;
-+ if (matchIndex <= btLow) {
-+ largerPtr = &dummy32;
-+ break;
-+ } /* beyond tree size, stop the search */
-+ largerPtr = nextPtr;
-+ matchIndex = nextPtr[0];
-+ }
-+ }
-+
-+ *smallerPtr = *largerPtr = 0;
-+
-+ zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1;
-+ return bestLength;
-+}
-+
-+static void ZSTD_updateTree(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
-+{
-+ const BYTE *const base = zc->base;
-+ const U32 target = (U32)(ip - base);
-+ U32 idx = zc->nextToUpdate;
-+
-+ while (idx < target)
-+ idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 0);
-+}
-+
-+/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
-+static size_t ZSTD_BtFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls)
-+{
-+ if (ip < zc->base + zc->nextToUpdate)
-+ return 0; /* skipped area */
-+ ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
-+ return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0);
-+}
-+
-+static size_t ZSTD_BtFindBestMatch_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
-+ const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 matchLengthSearch)
-+{
-+ switch (matchLengthSearch) {
-+ default: /* includes case 3 */
-+ case 4: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
-+ case 5: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
-+ case 7:
-+ case 6: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
-+ }
-+}
-+
-+static void ZSTD_updateTree_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
-+{
-+ const BYTE *const base = zc->base;
-+ const U32 target = (U32)(ip - base);
-+ U32 idx = zc->nextToUpdate;
-+
-+ while (idx < target)
-+ idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 1);
-+}
-+
-+/** Tree updater, providing best match */
-+static size_t ZSTD_BtFindBestMatch_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
-+ const U32 mls)
-+{
-+ if (ip < zc->base + zc->nextToUpdate)
-+ return 0; /* skipped area */
-+ ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
-+ return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1);
-+}
-+
-+static size_t ZSTD_BtFindBestMatch_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
-+ const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
-+ const U32 matchLengthSearch)
-+{
-+ switch (matchLengthSearch) {
-+ default: /* includes case 3 */
-+ case 4: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
-+ case 5: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
-+ case 7:
-+ case 6: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
-+ }
-+}
-+
-+/* *********************************
-+* Hash Chain
-+***********************************/
-+#define NEXT_IN_CHAIN(d, mask) chainTable[(d)&mask]
-+
-+/* Update chains up to ip (excluded)
-+ Assumption : always within prefix (i.e. not within extDict) */
-+FORCE_INLINE
-+U32 ZSTD_insertAndFindFirstIndex(ZSTD_CCtx *zc, const BYTE *ip, U32 mls)
-+{
-+ U32 *const hashTable = zc->hashTable;
-+ const U32 hashLog = zc->params.cParams.hashLog;
-+ U32 *const chainTable = zc->chainTable;
-+ const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1;
-+ const BYTE *const base = zc->base;
-+ const U32 target = (U32)(ip - base);
-+ U32 idx = zc->nextToUpdate;
-+
-+ while (idx < target) { /* catch up */
-+ size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls);
-+ NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
-+ hashTable[h] = idx;
-+ idx++;
-+ }
-+
-+ zc->nextToUpdate = target;
-+ return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
-+}
-+
-+/* inlining is important to hardwire a hot branch (template emulation) */
-+FORCE_INLINE
-+size_t ZSTD_HcFindBestMatch_generic(ZSTD_CCtx *zc, /* Index table will be updated */
-+ const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls,
-+ const U32 extDict)
-+{
-+ U32 *const chainTable = zc->chainTable;
-+ const U32 chainSize = (1 << zc->params.cParams.chainLog);
-+ const U32 chainMask = chainSize - 1;
-+ const BYTE *const base = zc->base;
-+ const BYTE *const dictBase = zc->dictBase;
-+ const U32 dictLimit = zc->dictLimit;
-+ const BYTE *const prefixStart = base + dictLimit;
-+ const BYTE *const dictEnd = dictBase + dictLimit;
-+ const U32 lowLimit = zc->lowLimit;
-+ const U32 curr = (U32)(ip - base);
-+ const U32 minChain = curr > chainSize ? curr - chainSize : 0;
-+ int nbAttempts = maxNbAttempts;
-+ size_t ml = EQUAL_READ32 - 1;
-+
-+ /* HC4 match finder */
-+ U32 matchIndex = ZSTD_insertAndFindFirstIndex(zc, ip, mls);
-+
-+ for (; (matchIndex > lowLimit) & (nbAttempts > 0); nbAttempts--) {
-+ const BYTE *match;
-+ size_t currMl = 0;
-+ if ((!extDict) || matchIndex >= dictLimit) {
-+ match = base + matchIndex;
-+ if (match[ml] == ip[ml]) /* potentially better */
-+ currMl = ZSTD_count(ip, match, iLimit);
-+ } else {
-+ match = dictBase + matchIndex;
-+ if (ZSTD_read32(match) == ZSTD_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
-+ currMl = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32;
-+ }
-+
-+ /* save best solution */
-+ if (currMl > ml) {
-+ ml = currMl;
-+ *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE;
-+ if (ip + currMl == iLimit)
-+ break; /* best possible, and avoid read overflow*/
-+ }
-+
-+ if (matchIndex <= minChain)
-+ break;
-+ matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
-+ }
-+
-+ return ml;
-+}
-+
-+FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
-+ const U32 matchLengthSearch)
-+{
-+ switch (matchLengthSearch) {
-+ default: /* includes case 3 */
-+ case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0);
-+ case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0);
-+ case 7:
-+ case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0);
-+ }
-+}
-+
-+FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
-+ const U32 matchLengthSearch)
-+{
-+ switch (matchLengthSearch) {
-+ default: /* includes case 3 */
-+ case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1);
-+ case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1);
-+ case 7:
-+ case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1);
-+ }
-+}
-+
-+/* *******************************
-+* Common parser - lazy strategy
-+*********************************/
-+FORCE_INLINE
-+void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
-+{
-+ seqStore_t *seqStorePtr = &(ctx->seqStore);
-+ const BYTE *const istart = (const BYTE *)src;
-+ const BYTE *ip = istart;
-+ const BYTE *anchor = istart;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *const ilimit = iend - 8;
-+ const BYTE *const base = ctx->base + ctx->dictLimit;
-+
-+ U32 const maxSearches = 1 << ctx->params.cParams.searchLog;
-+ U32 const mls = ctx->params.cParams.searchLength;
-+
-+ typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
-+ searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
-+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset = 0;
-+
-+ /* init */
-+ ip += (ip == base);
-+ ctx->nextToUpdate3 = ctx->nextToUpdate;
-+ {
-+ U32 const maxRep = (U32)(ip - base);
-+ if (offset_2 > maxRep)
-+ savedOffset = offset_2, offset_2 = 0;
-+ if (offset_1 > maxRep)
-+ savedOffset = offset_1, offset_1 = 0;
-+ }
-+
-+ /* Match Loop */
-+ while (ip < ilimit) {
-+ size_t matchLength = 0;
-+ size_t offset = 0;
-+ const BYTE *start = ip + 1;
-+
-+ /* check repCode */
-+ if ((offset_1 > 0) & (ZSTD_read32(ip + 1) == ZSTD_read32(ip + 1 - offset_1))) {
-+ /* repcode : we take it */
-+ matchLength = ZSTD_count(ip + 1 + EQUAL_READ32, ip + 1 + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
-+ if (depth == 0)
-+ goto _storeSequence;
-+ }
-+
-+ /* first search (depth 0) */
-+ {
-+ size_t offsetFound = 99999999;
-+ size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
-+ if (ml2 > matchLength)
-+ matchLength = ml2, start = ip, offset = offsetFound;
-+ }
-+
-+ if (matchLength < EQUAL_READ32) {
-+ ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
-+ continue;
-+ }
-+
-+ /* let's try to find a better solution */
-+ if (depth >= 1)
-+ while (ip < ilimit) {
-+ ip++;
-+ if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) {
-+ size_t const mlRep = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
-+ int const gain2 = (int)(mlRep * 3);
-+ int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1);
-+ if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
-+ matchLength = mlRep, offset = 0, start = ip;
-+ }
-+ {
-+ size_t offset2 = 99999999;
-+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
-+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
-+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
-+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
-+ matchLength = ml2, offset = offset2, start = ip;
-+ continue; /* search a better one */
-+ }
-+ }
-+
-+ /* let's find an even better one */
-+ if ((depth == 2) && (ip < ilimit)) {
-+ ip++;
-+ if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) {
-+ size_t const ml2 = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
-+ int const gain2 = (int)(ml2 * 4);
-+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1);
-+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
-+ matchLength = ml2, offset = 0, start = ip;
-+ }
-+ {
-+ size_t offset2 = 99999999;
-+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
-+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
-+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
-+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
-+ matchLength = ml2, offset = offset2, start = ip;
-+ continue;
-+ }
-+ }
-+ }
-+ break; /* nothing found : store previous solution */
-+ }
-+
-+ /* NOTE:
-+ * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior.
-+ * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which
-+ * overflows the pointer, which is undefined behavior.
-+ */
-+ /* catch up */
-+ if (offset) {
-+ while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) &&
-+ (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1])) /* only search for offset within prefix */
-+ {
-+ start--;
-+ matchLength++;
-+ }
-+ offset_2 = offset_1;
-+ offset_1 = (U32)(offset - ZSTD_REP_MOVE);
-+ }
-+
-+ /* store sequence */
-+_storeSequence:
-+ {
-+ size_t const litLength = start - anchor;
-+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
-+ anchor = ip = start + matchLength;
-+ }
-+
-+ /* check immediate repcode */
-+ while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
-+ /* store sequence */
-+ matchLength = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_2, iend) + EQUAL_READ32;
-+ offset = offset_2;
-+ offset_2 = offset_1;
-+ offset_1 = (U32)offset; /* swap repcodes */
-+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
-+ ip += matchLength;
-+ anchor = ip;
-+ continue; /* faster when present ... (?) */
-+ }
-+ }
-+
-+ /* Save reps for next block */
-+ ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
-+ ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
-+
-+ /* Last Literals */
-+ {
-+ size_t const lastLLSize = iend - anchor;
-+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
-+ seqStorePtr->lit += lastLLSize;
-+ }
-+}
-+
-+static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); }
-+
-+static void ZSTD_compressBlock_lazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); }
-+
-+static void ZSTD_compressBlock_lazy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); }
-+
-+static void ZSTD_compressBlock_greedy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); }
-+
-+FORCE_INLINE
-+void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
-+{
-+ seqStore_t *seqStorePtr = &(ctx->seqStore);
-+ const BYTE *const istart = (const BYTE *)src;
-+ const BYTE *ip = istart;
-+ const BYTE *anchor = istart;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *const ilimit = iend - 8;
-+ const BYTE *const base = ctx->base;
-+ const U32 dictLimit = ctx->dictLimit;
-+ const U32 lowestIndex = ctx->lowLimit;
-+ const BYTE *const prefixStart = base + dictLimit;
-+ const BYTE *const dictBase = ctx->dictBase;
-+ const BYTE *const dictEnd = dictBase + dictLimit;
-+ const BYTE *const dictStart = dictBase + ctx->lowLimit;
-+
-+ const U32 maxSearches = 1 << ctx->params.cParams.searchLog;
-+ const U32 mls = ctx->params.cParams.searchLength;
-+
-+ typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
-+ searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS;
-+
-+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
-+
-+ /* init */
-+ ctx->nextToUpdate3 = ctx->nextToUpdate;
-+ ip += (ip == prefixStart);
-+
-+ /* Match Loop */
-+ while (ip < ilimit) {
-+ size_t matchLength = 0;
-+ size_t offset = 0;
-+ const BYTE *start = ip + 1;
-+ U32 curr = (U32)(ip - base);
-+
-+ /* check repCode */
-+ {
-+ const U32 repIndex = (U32)(curr + 1 - offset_1);
-+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
-+ const BYTE *const repMatch = repBase + repIndex;
-+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
-+ if (ZSTD_read32(ip + 1) == ZSTD_read32(repMatch)) {
-+ /* repcode detected we should take it */
-+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
-+ matchLength =
-+ ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
-+ if (depth == 0)
-+ goto _storeSequence;
-+ }
-+ }
-+
-+ /* first search (depth 0) */
-+ {
-+ size_t offsetFound = 99999999;
-+ size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
-+ if (ml2 > matchLength)
-+ matchLength = ml2, start = ip, offset = offsetFound;
-+ }
-+
-+ if (matchLength < EQUAL_READ32) {
-+ ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
-+ continue;
-+ }
-+
-+ /* let's try to find a better solution */
-+ if (depth >= 1)
-+ while (ip < ilimit) {
-+ ip++;
-+ curr++;
-+ /* check repCode */
-+ if (offset) {
-+ const U32 repIndex = (U32)(curr - offset_1);
-+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
-+ const BYTE *const repMatch = repBase + repIndex;
-+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
-+ if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
-+ /* repcode detected */
-+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
-+ size_t const repLength =
-+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) +
-+ EQUAL_READ32;
-+ int const gain2 = (int)(repLength * 3);
-+ int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1);
-+ if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
-+ matchLength = repLength, offset = 0, start = ip;
-+ }
-+ }
-+
-+ /* search match, depth 1 */
-+ {
-+ size_t offset2 = 99999999;
-+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
-+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
-+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
-+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
-+ matchLength = ml2, offset = offset2, start = ip;
-+ continue; /* search a better one */
-+ }
-+ }
-+
-+ /* let's find an even better one */
-+ if ((depth == 2) && (ip < ilimit)) {
-+ ip++;
-+ curr++;
-+ /* check repCode */
-+ if (offset) {
-+ const U32 repIndex = (U32)(curr - offset_1);
-+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
-+ const BYTE *const repMatch = repBase + repIndex;
-+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
-+ if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
-+ /* repcode detected */
-+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
-+ size_t repLength = ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend,
-+ repEnd, prefixStart) +
-+ EQUAL_READ32;
-+ int gain2 = (int)(repLength * 4);
-+ int gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1);
-+ if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
-+ matchLength = repLength, offset = 0, start = ip;
-+ }
-+ }
-+
-+ /* search match, depth 2 */
-+ {
-+ size_t offset2 = 99999999;
-+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
-+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
-+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
-+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
-+ matchLength = ml2, offset = offset2, start = ip;
-+ continue;
-+ }
-+ }
-+ }
-+ break; /* nothing found : store previous solution */
-+ }
-+
-+ /* catch up */
-+ if (offset) {
-+ U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE));
-+ const BYTE *match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
-+ const BYTE *const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
-+ while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) {
-+ start--;
-+ match--;
-+ matchLength++;
-+ } /* catch up */
-+ offset_2 = offset_1;
-+ offset_1 = (U32)(offset - ZSTD_REP_MOVE);
-+ }
-+
-+ /* store sequence */
-+ _storeSequence : {
-+ size_t const litLength = start - anchor;
-+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
-+ anchor = ip = start + matchLength;
-+ }
-+
-+ /* check immediate repcode */
-+ while (ip <= ilimit) {
-+ const U32 repIndex = (U32)((ip - base) - offset_2);
-+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
-+ const BYTE *const repMatch = repBase + repIndex;
-+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
-+ if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
-+ /* repcode detected we should take it */
-+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
-+ matchLength =
-+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
-+ offset = offset_2;
-+ offset_2 = offset_1;
-+ offset_1 = (U32)offset; /* swap offset history */
-+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
-+ ip += matchLength;
-+ anchor = ip;
-+ continue; /* faster when present ... (?) */
-+ }
-+ break;
-+ }
-+ }
-+
-+ /* Save reps for next block */
-+ ctx->repToConfirm[0] = offset_1;
-+ ctx->repToConfirm[1] = offset_2;
-+
-+ /* Last Literals */
-+ {
-+ size_t const lastLLSize = iend - anchor;
-+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
-+ seqStorePtr->lit += lastLLSize;
-+ }
-+}
-+
-+void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); }
-+
-+static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+ ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1);
-+}
-+
-+static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+ ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2);
-+}
-+
-+static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+ ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2);
-+}
-+
-+/* The optimal parser */
-+#include "zstd_opt.h"
-+
-+static void ZSTD_compressBlock_btopt(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+#ifdef ZSTD_OPT_H_91842398743
-+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
-+#else
-+ (void)ctx;
-+ (void)src;
-+ (void)srcSize;
-+ return;
-+#endif
-+}
-+
-+static void ZSTD_compressBlock_btopt2(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+#ifdef ZSTD_OPT_H_91842398743
-+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
-+#else
-+ (void)ctx;
-+ (void)src;
-+ (void)srcSize;
-+ return;
-+#endif
-+}
-+
-+static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+#ifdef ZSTD_OPT_H_91842398743
-+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
-+#else
-+ (void)ctx;
-+ (void)src;
-+ (void)srcSize;
-+ return;
-+#endif
-+}
-+
-+static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
-+{
-+#ifdef ZSTD_OPT_H_91842398743
-+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
-+#else
-+ (void)ctx;
-+ (void)src;
-+ (void)srcSize;
-+ return;
-+#endif
-+}
-+
-+typedef void (*ZSTD_blockCompressor)(ZSTD_CCtx *ctx, const void *src, size_t srcSize);
-+
-+static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
-+{
-+ static const ZSTD_blockCompressor blockCompressor[2][8] = {
-+ {ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2,
-+ ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2},
-+ {ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,
-+ ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict}};
-+
-+ return blockCompressor[extDict][(U32)strat];
-+}
-+
-+static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
-+ const BYTE *const base = zc->base;
-+ const BYTE *const istart = (const BYTE *)src;
-+ const U32 curr = (U32)(istart - base);
-+ if (srcSize < MIN_CBLOCK_SIZE + ZSTD_blockHeaderSize + 1)
-+ return 0; /* don't even attempt compression below a certain srcSize */
-+ ZSTD_resetSeqStore(&(zc->seqStore));
-+ if (curr > zc->nextToUpdate + 384)
-+ zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */
-+ blockCompressor(zc, src, srcSize);
-+ return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
-+}
-+
-+/*! ZSTD_compress_generic() :
-+* Compress a chunk of data into one or multiple blocks.
-+* All blocks will be terminated, all input will be consumed.
-+* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
-+* Frame is supposed already started (header already produced)
-+* @return : compressed size, or an error code
-+*/
-+static size_t ZSTD_compress_generic(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastFrameChunk)
-+{
-+ size_t blockSize = cctx->blockSize;
-+ size_t remaining = srcSize;
-+ const BYTE *ip = (const BYTE *)src;
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *op = ostart;
-+ U32 const maxDist = 1 << cctx->params.cParams.windowLog;
-+
-+ if (cctx->params.fParams.checksumFlag && srcSize)
-+ xxh64_update(&cctx->xxhState, src, srcSize);
-+
-+ while (remaining) {
-+ U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
-+ size_t cSize;
-+
-+ if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
-+ return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
-+ if (remaining < blockSize)
-+ blockSize = remaining;
-+
-+ /* preemptive overflow correction */
-+ if (cctx->lowLimit > (3U << 29)) {
-+ U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
-+ U32 const curr = (U32)(ip - cctx->base);
-+ U32 const newCurr = (curr & cycleMask) + (1 << cctx->params.cParams.windowLog);
-+ U32 const correction = curr - newCurr;
-+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30);
-+ ZSTD_reduceIndex(cctx, correction);
-+ cctx->base += correction;
-+ cctx->dictBase += correction;
-+ cctx->lowLimit -= correction;
-+ cctx->dictLimit -= correction;
-+ if (cctx->nextToUpdate < correction)
-+ cctx->nextToUpdate = 0;
-+ else
-+ cctx->nextToUpdate -= correction;
-+ }
-+
-+ if ((U32)(ip + blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
-+ /* enforce maxDist */
-+ U32 const newLowLimit = (U32)(ip + blockSize - cctx->base) - maxDist;
-+ if (cctx->lowLimit < newLowLimit)
-+ cctx->lowLimit = newLowLimit;
-+ if (cctx->dictLimit < cctx->lowLimit)
-+ cctx->dictLimit = cctx->lowLimit;
-+ }
-+
-+ cSize = ZSTD_compressBlock_internal(cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize);
-+ if (ZSTD_isError(cSize))
-+ return cSize;
-+
-+ if (cSize == 0) { /* block is not compressible */
-+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(blockSize << 3);
-+ if (blockSize + ZSTD_blockHeaderSize > dstCapacity)
-+ return ERROR(dstSize_tooSmall);
-+ ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */
-+ memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
-+ cSize = ZSTD_blockHeaderSize + blockSize;
-+ } else {
-+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed) << 1) + (U32)(cSize << 3);
-+ ZSTD_writeLE24(op, cBlockHeader24);
-+ cSize += ZSTD_blockHeaderSize;
-+ }
-+
-+ remaining -= blockSize;
-+ dstCapacity -= cSize;
-+ ip += blockSize;
-+ op += cSize;
-+ }
-+
-+ if (lastFrameChunk && (op > ostart))
-+ cctx->stage = ZSTDcs_ending;
-+ return op - ostart;
-+}
-+
-+static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
-+{
-+ BYTE *const op = (BYTE *)dst;
-+ U32 const dictIDSizeCode = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */
-+ U32 const checksumFlag = params.fParams.checksumFlag > 0;
-+ U32 const windowSize = 1U << params.cParams.windowLog;
-+ U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
-+ BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
-+ U32 const fcsCode =
-+ params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
-+ BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
-+ size_t pos;
-+
-+ if (dstCapacity < ZSTD_frameHeaderSize_max)
-+ return ERROR(dstSize_tooSmall);
-+
-+ ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
-+ op[4] = frameHeaderDecriptionByte;
-+ pos = 5;
-+ if (!singleSegment)
-+ op[pos++] = windowLogByte;
-+ switch (dictIDSizeCode) {
-+ default: /* impossible */
-+ case 0: break;
-+ case 1:
-+ op[pos] = (BYTE)(dictID);
-+ pos++;
-+ break;
-+ case 2:
-+ ZSTD_writeLE16(op + pos, (U16)dictID);
-+ pos += 2;
-+ break;
-+ case 3:
-+ ZSTD_writeLE32(op + pos, dictID);
-+ pos += 4;
-+ break;
-+ }
-+ switch (fcsCode) {
-+ default: /* impossible */
-+ case 0:
-+ if (singleSegment)
-+ op[pos++] = (BYTE)(pledgedSrcSize);
-+ break;
-+ case 1:
-+ ZSTD_writeLE16(op + pos, (U16)(pledgedSrcSize - 256));
-+ pos += 2;
-+ break;
-+ case 2:
-+ ZSTD_writeLE32(op + pos, (U32)(pledgedSrcSize));
-+ pos += 4;
-+ break;
-+ case 3:
-+ ZSTD_writeLE64(op + pos, (U64)(pledgedSrcSize));
-+ pos += 8;
-+ break;
-+ }
-+ return pos;
-+}
-+
-+static size_t ZSTD_compressContinue_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame, U32 lastFrameChunk)
-+{
-+ const BYTE *const ip = (const BYTE *)src;
-+ size_t fhSize = 0;
-+
-+ if (cctx->stage == ZSTDcs_created)
-+ return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
-+
-+ if (frame && (cctx->stage == ZSTDcs_init)) {
-+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID);
-+ if (ZSTD_isError(fhSize))
-+ return fhSize;
-+ dstCapacity -= fhSize;
-+ dst = (char *)dst + fhSize;
-+ cctx->stage = ZSTDcs_ongoing;
-+ }
-+
-+ /* Check if blocks follow each other */
-+ if (src != cctx->nextSrc) {
-+ /* not contiguous */
-+ ptrdiff_t const delta = cctx->nextSrc - ip;
-+ cctx->lowLimit = cctx->dictLimit;
-+ cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base);
-+ cctx->dictBase = cctx->base;
-+ cctx->base -= delta;
-+ cctx->nextToUpdate = cctx->dictLimit;
-+ if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE)
-+ cctx->lowLimit = cctx->dictLimit; /* too small extDict */
-+ }
-+
-+ /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
-+ if ((ip + srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
-+ ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase;
-+ U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx;
-+ cctx->lowLimit = lowLimitMax;
-+ }
-+
-+ cctx->nextSrc = ip + srcSize;
-+
-+ if (srcSize) {
-+ size_t const cSize = frame ? ZSTD_compress_generic(cctx, dst, dstCapacity, src, srcSize, lastFrameChunk)
-+ : ZSTD_compressBlock_internal(cctx, dst, dstCapacity, src, srcSize);
-+ if (ZSTD_isError(cSize))
-+ return cSize;
-+ return cSize + fhSize;
-+ } else
-+ return fhSize;
-+}
-+
-+size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0);
-+}
-+
-+size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx) { return MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); }
-+
-+size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
-+ if (srcSize > blockSizeMax)
-+ return ERROR(srcSize_wrong);
-+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0);
-+}
-+
-+/*! ZSTD_loadDictionaryContent() :
-+ * @return : 0, or an error code
-+ */
-+static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx *zc, const void *src, size_t srcSize)
-+{
-+ const BYTE *const ip = (const BYTE *)src;
-+ const BYTE *const iend = ip + srcSize;
-+
-+ /* input becomes curr prefix */
-+ zc->lowLimit = zc->dictLimit;
-+ zc->dictLimit = (U32)(zc->nextSrc - zc->base);
-+ zc->dictBase = zc->base;
-+ zc->base += ip - zc->nextSrc;
-+ zc->nextToUpdate = zc->dictLimit;
-+ zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base);
-+
-+ zc->nextSrc = iend;
-+ if (srcSize <= HASH_READ_SIZE)
-+ return 0;
-+
-+ switch (zc->params.cParams.strategy) {
-+ case ZSTD_fast: ZSTD_fillHashTable(zc, iend, zc->params.cParams.searchLength); break;
-+
-+ case ZSTD_dfast: ZSTD_fillDoubleHashTable(zc, iend, zc->params.cParams.searchLength); break;
-+
-+ case ZSTD_greedy:
-+ case ZSTD_lazy:
-+ case ZSTD_lazy2:
-+ if (srcSize >= HASH_READ_SIZE)
-+ ZSTD_insertAndFindFirstIndex(zc, iend - HASH_READ_SIZE, zc->params.cParams.searchLength);
-+ break;
-+
-+ case ZSTD_btlazy2:
-+ case ZSTD_btopt:
-+ case ZSTD_btopt2:
-+ if (srcSize >= HASH_READ_SIZE)
-+ ZSTD_updateTree(zc, iend - HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength);
-+ break;
-+
-+ default:
-+ return ERROR(GENERIC); /* strategy doesn't exist; impossible */
-+ }
-+
-+ zc->nextToUpdate = (U32)(iend - zc->base);
-+ return 0;
-+}
-+
-+/* Dictionaries that assign zero probability to symbols that show up causes problems
-+ when FSE encoding. Refuse dictionaries that assign zero probability to symbols
-+ that we may encounter during compression.
-+ NOTE: This behavior is not standard and could be improved in the future. */
-+static size_t ZSTD_checkDictNCount(short *normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue)
-+{
-+ U32 s;
-+ if (dictMaxSymbolValue < maxSymbolValue)
-+ return ERROR(dictionary_corrupted);
-+ for (s = 0; s <= maxSymbolValue; ++s) {
-+ if (normalizedCounter[s] == 0)
-+ return ERROR(dictionary_corrupted);
-+ }
-+ return 0;
-+}
-+
-+/* Dictionary format :
-+ * See :
-+ * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format
-+ */
-+/*! ZSTD_loadZstdDictionary() :
-+ * @return : 0, or an error code
-+ * assumptions : magic number supposed already checked
-+ * dictSize supposed > 8
-+ */
-+static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
-+{
-+ const BYTE *dictPtr = (const BYTE *)dict;
-+ const BYTE *const dictEnd = dictPtr + dictSize;
-+ short offcodeNCount[MaxOff + 1];
-+ unsigned offcodeMaxValue = MaxOff;
-+
-+ dictPtr += 4; /* skip magic number */
-+ cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr);
-+ dictPtr += 4;
-+
-+ {
-+ size_t const hufHeaderSize = HUF_readCTable_wksp(cctx->hufTable, 255, dictPtr, dictEnd - dictPtr, cctx->tmpCounters, sizeof(cctx->tmpCounters));
-+ if (HUF_isError(hufHeaderSize))
-+ return ERROR(dictionary_corrupted);
-+ dictPtr += hufHeaderSize;
-+ }
-+
-+ {
-+ unsigned offcodeLog;
-+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
-+ if (FSE_isError(offcodeHeaderSize))
-+ return ERROR(dictionary_corrupted);
-+ if (offcodeLog > OffFSELog)
-+ return ERROR(dictionary_corrupted);
-+ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
-+ CHECK_E(FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)),
-+ dictionary_corrupted);
-+ dictPtr += offcodeHeaderSize;
-+ }
-+
-+ {
-+ short matchlengthNCount[MaxML + 1];
-+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
-+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
-+ if (FSE_isError(matchlengthHeaderSize))
-+ return ERROR(dictionary_corrupted);
-+ if (matchlengthLog > MLFSELog)
-+ return ERROR(dictionary_corrupted);
-+ /* Every match length code must have non-zero probability */
-+ CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
-+ CHECK_E(
-+ FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)),
-+ dictionary_corrupted);
-+ dictPtr += matchlengthHeaderSize;
-+ }
-+
-+ {
-+ short litlengthNCount[MaxLL + 1];
-+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
-+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
-+ if (FSE_isError(litlengthHeaderSize))
-+ return ERROR(dictionary_corrupted);
-+ if (litlengthLog > LLFSELog)
-+ return ERROR(dictionary_corrupted);
-+ /* Every literal length code must have non-zero probability */
-+ CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
-+ CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)),
-+ dictionary_corrupted);
-+ dictPtr += litlengthHeaderSize;
-+ }
-+
-+ if (dictPtr + 12 > dictEnd)
-+ return ERROR(dictionary_corrupted);
-+ cctx->rep[0] = ZSTD_readLE32(dictPtr + 0);
-+ cctx->rep[1] = ZSTD_readLE32(dictPtr + 4);
-+ cctx->rep[2] = ZSTD_readLE32(dictPtr + 8);
-+ dictPtr += 12;
-+
-+ {
-+ size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
-+ U32 offcodeMax = MaxOff;
-+ if (dictContentSize <= ((U32)-1) - 128 KB) {
-+ U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */
-+ offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */
-+ }
-+ /* All offset values <= dictContentSize + 128 KB must be representable */
-+ CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
-+ /* All repCodes must be <= dictContentSize and != 0*/
-+ {
-+ U32 u;
-+ for (u = 0; u < 3; u++) {
-+ if (cctx->rep[u] == 0)
-+ return ERROR(dictionary_corrupted);
-+ if (cctx->rep[u] > dictContentSize)
-+ return ERROR(dictionary_corrupted);
-+ }
-+ }
-+
-+ cctx->flagStaticTables = 1;
-+ cctx->flagStaticHufTable = HUF_repeat_valid;
-+ return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize);
-+ }
-+}
-+
-+/** ZSTD_compress_insertDictionary() :
-+* @return : 0, or an error code */
-+static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
-+{
-+ if ((dict == NULL) || (dictSize <= 8))
-+ return 0;
-+
-+ /* dict as pure content */
-+ if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict))
-+ return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
-+
-+ /* dict as zstd dictionary */
-+ return ZSTD_loadZstdDictionary(cctx, dict, dictSize);
-+}
-+
-+/*! ZSTD_compressBegin_internal() :
-+* @return : 0, or an error code */
-+static size_t ZSTD_compressBegin_internal(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize)
-+{
-+ ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
-+ CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp));
-+ return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
-+}
-+
-+/*! ZSTD_compressBegin_advanced() :
-+* @return : 0, or an error code */
-+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
-+{
-+ /* compression parameters verification and optimization */
-+ CHECK_F(ZSTD_checkCParams(params.cParams));
-+ return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
-+}
-+
-+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel)
-+{
-+ ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
-+ return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
-+}
-+
-+size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel) { return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); }
-+
-+/*! ZSTD_writeEpilogue() :
-+* Ends a frame.
-+* @return : nb of bytes written into dst (or an error code) */
-+static size_t ZSTD_writeEpilogue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity)
-+{
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *op = ostart;
-+ size_t fhSize = 0;
-+
-+ if (cctx->stage == ZSTDcs_created)
-+ return ERROR(stage_wrong); /* init missing */
-+
-+ /* special case : empty frame */
-+ if (cctx->stage == ZSTDcs_init) {
-+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
-+ if (ZSTD_isError(fhSize))
-+ return fhSize;
-+ dstCapacity -= fhSize;
-+ op += fhSize;
-+ cctx->stage = ZSTDcs_ongoing;
-+ }
-+
-+ if (cctx->stage != ZSTDcs_ending) {
-+ /* write one last empty block, make it the "last" block */
-+ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0;
-+ if (dstCapacity < 4)
-+ return ERROR(dstSize_tooSmall);
-+ ZSTD_writeLE32(op, cBlockHeader24);
-+ op += ZSTD_blockHeaderSize;
-+ dstCapacity -= ZSTD_blockHeaderSize;
-+ }
-+
-+ if (cctx->params.fParams.checksumFlag) {
-+ U32 const checksum = (U32)xxh64_digest(&cctx->xxhState);
-+ if (dstCapacity < 4)
-+ return ERROR(dstSize_tooSmall);
-+ ZSTD_writeLE32(op, checksum);
-+ op += 4;
-+ }
-+
-+ cctx->stage = ZSTDcs_created; /* return to "created but no init" status */
-+ return op - ostart;
-+}
-+
-+size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ size_t endResult;
-+ size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1);
-+ if (ZSTD_isError(cSize))
-+ return cSize;
-+ endResult = ZSTD_writeEpilogue(cctx, (char *)dst + cSize, dstCapacity - cSize);
-+ if (ZSTD_isError(endResult))
-+ return endResult;
-+ return cSize + endResult;
-+}
-+
-+static size_t ZSTD_compress_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
-+ ZSTD_parameters params)
-+{
-+ CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize));
-+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
-+}
-+
-+size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
-+ ZSTD_parameters params)
-+{
-+ return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
-+}
-+
-+size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, ZSTD_parameters params)
-+{
-+ return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params);
-+}
-+
-+/* ===== Dictionary API ===== */
-+
-+struct ZSTD_CDict_s {
-+ void *dictBuffer;
-+ const void *dictContent;
-+ size_t dictContentSize;
-+ ZSTD_CCtx *refContext;
-+}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
-+
-+size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) { return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict)); }
-+
-+static ZSTD_CDict *ZSTD_createCDict_advanced(const void *dictBuffer, size_t dictSize, unsigned byReference, ZSTD_parameters params, ZSTD_customMem customMem)
-+{
-+ if (!customMem.customAlloc || !customMem.customFree)
-+ return NULL;
-+
-+ {
-+ ZSTD_CDict *const cdict = (ZSTD_CDict *)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
-+ ZSTD_CCtx *const cctx = ZSTD_createCCtx_advanced(customMem);
-+
-+ if (!cdict || !cctx) {
-+ ZSTD_free(cdict, customMem);
-+ ZSTD_freeCCtx(cctx);
-+ return NULL;
-+ }
-+
-+ if ((byReference) || (!dictBuffer) || (!dictSize)) {
-+ cdict->dictBuffer = NULL;
-+ cdict->dictContent = dictBuffer;
-+ } else {
-+ void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
-+ if (!internalBuffer) {
-+ ZSTD_free(cctx, customMem);
-+ ZSTD_free(cdict, customMem);
-+ return NULL;
-+ }
-+ memcpy(internalBuffer, dictBuffer, dictSize);
-+ cdict->dictBuffer = internalBuffer;
-+ cdict->dictContent = internalBuffer;
-+ }
-+
-+ {
-+ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
-+ if (ZSTD_isError(errorCode)) {
-+ ZSTD_free(cdict->dictBuffer, customMem);
-+ ZSTD_free(cdict, customMem);
-+ ZSTD_freeCCtx(cctx);
-+ return NULL;
-+ }
-+ }
-+
-+ cdict->refContext = cctx;
-+ cdict->dictContentSize = dictSize;
-+ return cdict;
-+ }
-+}
-+
-+ZSTD_CDict *ZSTD_initCDict(const void *dict, size_t dictSize, ZSTD_parameters params, void *workspace, size_t workspaceSize)
-+{
-+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
-+ return ZSTD_createCDict_advanced(dict, dictSize, 1, params, stackMem);
-+}
-+
-+size_t ZSTD_freeCDict(ZSTD_CDict *cdict)
-+{
-+ if (cdict == NULL)
-+ return 0; /* support free on NULL */
-+ {
-+ ZSTD_customMem const cMem = cdict->refContext->customMem;
-+ ZSTD_freeCCtx(cdict->refContext);
-+ ZSTD_free(cdict->dictBuffer, cMem);
-+ ZSTD_free(cdict, cMem);
-+ return 0;
-+ }
-+}
-+
-+static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict *cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); }
-+
-+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize)
-+{
-+ if (cdict->dictContentSize)
-+ CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
-+ else {
-+ ZSTD_parameters params = cdict->refContext->params;
-+ params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
-+ CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize));
-+ }
-+ return 0;
-+}
-+
-+/*! ZSTD_compress_usingCDict() :
-+* Compression using a digested Dictionary.
-+* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
-+* Note that compression level is decided during dictionary creation */
-+size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict)
-+{
-+ CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));
-+
-+ if (cdict->refContext->params.fParams.contentSizeFlag == 1) {
-+ cctx->params.fParams.contentSizeFlag = 1;
-+ cctx->frameContentSize = srcSize;
-+ } else {
-+ cctx->params.fParams.contentSizeFlag = 0;
-+ }
-+
-+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
-+}
-+
-+/* ******************************************************************
-+* Streaming
-+********************************************************************/
-+
-+typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
-+
-+struct ZSTD_CStream_s {
-+ ZSTD_CCtx *cctx;
-+ ZSTD_CDict *cdictLocal;
-+ const ZSTD_CDict *cdict;
-+ char *inBuff;
-+ size_t inBuffSize;
-+ size_t inToCompress;
-+ size_t inBuffPos;
-+ size_t inBuffTarget;
-+ size_t blockSize;
-+ char *outBuff;
-+ size_t outBuffSize;
-+ size_t outBuffContentSize;
-+ size_t outBuffFlushedSize;
-+ ZSTD_cStreamStage stage;
-+ U32 checksum;
-+ U32 frameEnded;
-+ U64 pledgedSrcSize;
-+ U64 inputProcessed;
-+ ZSTD_parameters params;
-+ ZSTD_customMem customMem;
-+}; /* typedef'd to ZSTD_CStream within "zstd.h" */
-+
-+size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams)
-+{
-+ size_t const inBuffSize = (size_t)1 << cParams.windowLog;
-+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, inBuffSize);
-+ size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
-+
-+ return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize);
-+}
-+
-+ZSTD_CStream *ZSTD_createCStream_advanced(ZSTD_customMem customMem)
-+{
-+ ZSTD_CStream *zcs;
-+
-+ if (!customMem.customAlloc || !customMem.customFree)
-+ return NULL;
-+
-+ zcs = (ZSTD_CStream *)ZSTD_malloc(sizeof(ZSTD_CStream), customMem);
-+ if (zcs == NULL)
-+ return NULL;
-+ memset(zcs, 0, sizeof(ZSTD_CStream));
-+ memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
-+ zcs->cctx = ZSTD_createCCtx_advanced(customMem);
-+ if (zcs->cctx == NULL) {
-+ ZSTD_freeCStream(zcs);
-+ return NULL;
-+ }
-+ return zcs;
-+}
-+
-+size_t ZSTD_freeCStream(ZSTD_CStream *zcs)
-+{
-+ if (zcs == NULL)
-+ return 0; /* support free on NULL */
-+ {
-+ ZSTD_customMem const cMem = zcs->customMem;
-+ ZSTD_freeCCtx(zcs->cctx);
-+ zcs->cctx = NULL;
-+ ZSTD_freeCDict(zcs->cdictLocal);
-+ zcs->cdictLocal = NULL;
-+ ZSTD_free(zcs->inBuff, cMem);
-+ zcs->inBuff = NULL;
-+ ZSTD_free(zcs->outBuff, cMem);
-+ zcs->outBuff = NULL;
-+ ZSTD_free(zcs, cMem);
-+ return 0;
-+ }
-+}
-+
-+/*====== Initialization ======*/
-+
-+size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
-+size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */; }
-+
-+static size_t ZSTD_resetCStream_internal(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize)
-+{
-+ if (zcs->inBuffSize == 0)
-+ return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
-+
-+ if (zcs->cdict)
-+ CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
-+ else
-+ CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
-+
-+ zcs->inToCompress = 0;
-+ zcs->inBuffPos = 0;
-+ zcs->inBuffTarget = zcs->blockSize;
-+ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
-+ zcs->stage = zcss_load;
-+ zcs->frameEnded = 0;
-+ zcs->pledgedSrcSize = pledgedSrcSize;
-+ zcs->inputProcessed = 0;
-+ return 0; /* ready to go */
-+}
-+
-+size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize)
-+{
-+
-+ zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
-+
-+ return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
-+}
-+
-+static size_t ZSTD_initCStream_advanced(ZSTD_CStream *zcs, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
-+{
-+ /* allocate buffers */
-+ {
-+ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
-+ if (zcs->inBuffSize < neededInBuffSize) {
-+ zcs->inBuffSize = neededInBuffSize;
-+ ZSTD_free(zcs->inBuff, zcs->customMem);
-+ zcs->inBuff = (char *)ZSTD_malloc(neededInBuffSize, zcs->customMem);
-+ if (zcs->inBuff == NULL)
-+ return ERROR(memory_allocation);
-+ }
-+ zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
-+ }
-+ if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize) + 1) {
-+ zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize) + 1;
-+ ZSTD_free(zcs->outBuff, zcs->customMem);
-+ zcs->outBuff = (char *)ZSTD_malloc(zcs->outBuffSize, zcs->customMem);
-+ if (zcs->outBuff == NULL)
-+ return ERROR(memory_allocation);
-+ }
-+
-+ if (dict && dictSize >= 8) {
-+ ZSTD_freeCDict(zcs->cdictLocal);
-+ zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
-+ if (zcs->cdictLocal == NULL)
-+ return ERROR(memory_allocation);
-+ zcs->cdict = zcs->cdictLocal;
-+ } else
-+ zcs->cdict = NULL;
-+
-+ zcs->checksum = params.fParams.checksumFlag > 0;
-+ zcs->params = params;
-+
-+ return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
-+}
-+
-+ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize)
-+{
-+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
-+ ZSTD_CStream *const zcs = ZSTD_createCStream_advanced(stackMem);
-+ if (zcs) {
-+ size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
-+ if (ZSTD_isError(code)) {
-+ return NULL;
-+ }
-+ }
-+ return zcs;
-+}
-+
-+ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize)
-+{
-+ ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
-+ ZSTD_CStream *const zcs = ZSTD_initCStream(params, pledgedSrcSize, workspace, workspaceSize);
-+ if (zcs) {
-+ zcs->cdict = cdict;
-+ if (ZSTD_isError(ZSTD_resetCStream_internal(zcs, pledgedSrcSize))) {
-+ return NULL;
-+ }
-+ }
-+ return zcs;
-+}
-+
-+/*====== Compression ======*/
-+
-+typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e;
-+
-+ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ size_t const length = MIN(dstCapacity, srcSize);
-+ memcpy(dst, src, length);
-+ return length;
-+}
-+
-+static size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, void *dst, size_t *dstCapacityPtr, const void *src, size_t *srcSizePtr, ZSTD_flush_e const flush)
-+{
-+ U32 someMoreWork = 1;
-+ const char *const istart = (const char *)src;
-+ const char *const iend = istart + *srcSizePtr;
-+ const char *ip = istart;
-+ char *const ostart = (char *)dst;
-+ char *const oend = ostart + *dstCapacityPtr;
-+ char *op = ostart;
-+
-+ while (someMoreWork) {
-+ switch (zcs->stage) {
-+ case zcss_init:
-+ return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
-+
-+ case zcss_load:
-+ /* complete inBuffer */
-+ {
-+ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
-+ size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip);
-+ zcs->inBuffPos += loaded;
-+ ip += loaded;
-+ if ((zcs->inBuffPos == zcs->inToCompress) || (!flush && (toLoad != loaded))) {
-+ someMoreWork = 0;
-+ break; /* not enough input to get a full block : stop there, wait for more */
-+ }
-+ }
-+ /* compress curr block (note : this stage cannot be stopped in the middle) */
-+ {
-+ void *cDst;
-+ size_t cSize;
-+ size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
-+ size_t oSize = oend - op;
-+ if (oSize >= ZSTD_compressBound(iSize))
-+ cDst = op; /* compress directly into output buffer (avoid flush stage) */
-+ else
-+ cDst = zcs->outBuff, oSize = zcs->outBuffSize;
-+ cSize = (flush == zsf_end) ? ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize)
-+ : ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
-+ if (ZSTD_isError(cSize))
-+ return cSize;
-+ if (flush == zsf_end)
-+ zcs->frameEnded = 1;
-+ /* prepare next block */
-+ zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
-+ if (zcs->inBuffTarget > zcs->inBuffSize)
-+ zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */
-+ zcs->inToCompress = zcs->inBuffPos;
-+ if (cDst == op) {
-+ op += cSize;
-+ break;
-+ } /* no need to flush */
-+ zcs->outBuffContentSize = cSize;
-+ zcs->outBuffFlushedSize = 0;
-+ zcs->stage = zcss_flush; /* pass-through to flush stage */
-+ }
-+
-+ case zcss_flush: {
-+ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
-+ size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
-+ op += flushed;
-+ zcs->outBuffFlushedSize += flushed;
-+ if (toFlush != flushed) {
-+ someMoreWork = 0;
-+ break;
-+ } /* dst too small to store flushed data : stop there */
-+ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
-+ zcs->stage = zcss_load;
-+ break;
-+ }
-+
-+ case zcss_final:
-+ someMoreWork = 0; /* do nothing */
-+ break;
-+
-+ default:
-+ return ERROR(GENERIC); /* impossible */
-+ }
-+ }
-+
-+ *srcSizePtr = ip - istart;
-+ *dstCapacityPtr = op - ostart;
-+ zcs->inputProcessed += *srcSizePtr;
-+ if (zcs->frameEnded)
-+ return 0;
-+ {
-+ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
-+ if (hintInSize == 0)
-+ hintInSize = zcs->blockSize;
-+ return hintInSize;
-+ }
-+}
-+
-+size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
-+{
-+ size_t sizeRead = input->size - input->pos;
-+ size_t sizeWritten = output->size - output->pos;
-+ size_t const result =
-+ ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, (const char *)(input->src) + input->pos, &sizeRead, zsf_gather);
-+ input->pos += sizeRead;
-+ output->pos += sizeWritten;
-+ return result;
-+}
-+
-+/*====== Finalize ======*/
-+
-+/*! ZSTD_flushStream() :
-+* @return : amount of data remaining to flush */
-+size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
-+{
-+ size_t srcSize = 0;
-+ size_t sizeWritten = output->size - output->pos;
-+ size_t const result = ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, &srcSize,
-+ &srcSize, /* use a valid src address instead of NULL */
-+ zsf_flush);
-+ output->pos += sizeWritten;
-+ if (ZSTD_isError(result))
-+ return result;
-+ return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
-+}
-+
-+size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
-+{
-+ BYTE *const ostart = (BYTE *)(output->dst) + output->pos;
-+ BYTE *const oend = (BYTE *)(output->dst) + output->size;
-+ BYTE *op = ostart;
-+
-+ if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
-+ return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */
-+
-+ if (zcs->stage != zcss_final) {
-+ /* flush whatever remains */
-+ size_t srcSize = 0;
-+ size_t sizeWritten = output->size - output->pos;
-+ size_t const notEnded =
-+ ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */
-+ size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
-+ op += sizeWritten;
-+ if (remainingToFlush) {
-+ output->pos += sizeWritten;
-+ return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4);
-+ }
-+ /* create epilogue */
-+ zcs->stage = zcss_final;
-+ zcs->outBuffContentSize = !notEnded ? 0 : ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL,
-+ 0); /* write epilogue, including final empty block, into outBuff */
-+ }
-+
-+ /* flush epilogue */
-+ {
-+ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
-+ size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
-+ op += flushed;
-+ zcs->outBuffFlushedSize += flushed;
-+ output->pos += op - ostart;
-+ if (toFlush == flushed)
-+ zcs->stage = zcss_init; /* end reached */
-+ return toFlush - flushed;
-+ }
-+}
-+
-+/*-===== Pre-defined compression levels =====-*/
-+
-+#define ZSTD_DEFAULT_CLEVEL 1
-+#define ZSTD_MAX_CLEVEL 22
-+int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
-+
-+static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = {
-+ {
-+ /* "default" */
-+ /* W, C, H, S, L, TL, strat */
-+ {18, 12, 12, 1, 7, 16, ZSTD_fast}, /* level 0 - never used */
-+ {19, 13, 14, 1, 7, 16, ZSTD_fast}, /* level 1 */
-+ {19, 15, 16, 1, 6, 16, ZSTD_fast}, /* level 2 */
-+ {20, 16, 17, 1, 5, 16, ZSTD_dfast}, /* level 3.*/
-+ {20, 18, 18, 1, 5, 16, ZSTD_dfast}, /* level 4.*/
-+ {20, 15, 18, 3, 5, 16, ZSTD_greedy}, /* level 5 */
-+ {21, 16, 19, 2, 5, 16, ZSTD_lazy}, /* level 6 */
-+ {21, 17, 20, 3, 5, 16, ZSTD_lazy}, /* level 7 */
-+ {21, 18, 20, 3, 5, 16, ZSTD_lazy2}, /* level 8 */
-+ {21, 20, 20, 3, 5, 16, ZSTD_lazy2}, /* level 9 */
-+ {21, 19, 21, 4, 5, 16, ZSTD_lazy2}, /* level 10 */
-+ {22, 20, 22, 4, 5, 16, ZSTD_lazy2}, /* level 11 */
-+ {22, 20, 22, 5, 5, 16, ZSTD_lazy2}, /* level 12 */
-+ {22, 21, 22, 5, 5, 16, ZSTD_lazy2}, /* level 13 */
-+ {22, 21, 22, 6, 5, 16, ZSTD_lazy2}, /* level 14 */
-+ {22, 21, 21, 5, 5, 16, ZSTD_btlazy2}, /* level 15 */
-+ {23, 22, 22, 5, 5, 16, ZSTD_btlazy2}, /* level 16 */
-+ {23, 21, 22, 4, 5, 24, ZSTD_btopt}, /* level 17 */
-+ {23, 23, 22, 6, 5, 32, ZSTD_btopt}, /* level 18 */
-+ {23, 23, 22, 6, 3, 48, ZSTD_btopt}, /* level 19 */
-+ {25, 25, 23, 7, 3, 64, ZSTD_btopt2}, /* level 20 */
-+ {26, 26, 23, 7, 3, 256, ZSTD_btopt2}, /* level 21 */
-+ {27, 27, 25, 9, 3, 512, ZSTD_btopt2}, /* level 22 */
-+ },
-+ {
-+ /* for srcSize <= 256 KB */
-+ /* W, C, H, S, L, T, strat */
-+ {0, 0, 0, 0, 0, 0, ZSTD_fast}, /* level 0 - not used */
-+ {18, 13, 14, 1, 6, 8, ZSTD_fast}, /* level 1 */
-+ {18, 14, 13, 1, 5, 8, ZSTD_dfast}, /* level 2 */
-+ {18, 16, 15, 1, 5, 8, ZSTD_dfast}, /* level 3 */
-+ {18, 15, 17, 1, 5, 8, ZSTD_greedy}, /* level 4.*/
-+ {18, 16, 17, 4, 5, 8, ZSTD_greedy}, /* level 5.*/
-+ {18, 16, 17, 3, 5, 8, ZSTD_lazy}, /* level 6.*/
-+ {18, 17, 17, 4, 4, 8, ZSTD_lazy}, /* level 7 */
-+ {18, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */
-+ {18, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */
-+ {18, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */
-+ {18, 18, 17, 6, 4, 8, ZSTD_lazy2}, /* level 11.*/
-+ {18, 18, 17, 7, 4, 8, ZSTD_lazy2}, /* level 12.*/
-+ {18, 19, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13 */
-+ {18, 18, 18, 4, 4, 16, ZSTD_btopt}, /* level 14.*/
-+ {18, 18, 18, 4, 3, 16, ZSTD_btopt}, /* level 15.*/
-+ {18, 19, 18, 6, 3, 32, ZSTD_btopt}, /* level 16.*/
-+ {18, 19, 18, 8, 3, 64, ZSTD_btopt}, /* level 17.*/
-+ {18, 19, 18, 9, 3, 128, ZSTD_btopt}, /* level 18.*/
-+ {18, 19, 18, 10, 3, 256, ZSTD_btopt}, /* level 19.*/
-+ {18, 19, 18, 11, 3, 512, ZSTD_btopt2}, /* level 20.*/
-+ {18, 19, 18, 12, 3, 512, ZSTD_btopt2}, /* level 21.*/
-+ {18, 19, 18, 13, 3, 512, ZSTD_btopt2}, /* level 22.*/
-+ },
-+ {
-+ /* for srcSize <= 128 KB */
-+ /* W, C, H, S, L, T, strat */
-+ {17, 12, 12, 1, 7, 8, ZSTD_fast}, /* level 0 - not used */
-+ {17, 12, 13, 1, 6, 8, ZSTD_fast}, /* level 1 */
-+ {17, 13, 16, 1, 5, 8, ZSTD_fast}, /* level 2 */
-+ {17, 16, 16, 2, 5, 8, ZSTD_dfast}, /* level 3 */
-+ {17, 13, 15, 3, 4, 8, ZSTD_greedy}, /* level 4 */
-+ {17, 15, 17, 4, 4, 8, ZSTD_greedy}, /* level 5 */
-+ {17, 16, 17, 3, 4, 8, ZSTD_lazy}, /* level 6 */
-+ {17, 15, 17, 4, 4, 8, ZSTD_lazy2}, /* level 7 */
-+ {17, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */
-+ {17, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */
-+ {17, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */
-+ {17, 17, 17, 7, 4, 8, ZSTD_lazy2}, /* level 11 */
-+ {17, 17, 17, 8, 4, 8, ZSTD_lazy2}, /* level 12 */
-+ {17, 18, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13.*/
-+ {17, 17, 17, 7, 3, 8, ZSTD_btopt}, /* level 14.*/
-+ {17, 17, 17, 7, 3, 16, ZSTD_btopt}, /* level 15.*/
-+ {17, 18, 17, 7, 3, 32, ZSTD_btopt}, /* level 16.*/
-+ {17, 18, 17, 7, 3, 64, ZSTD_btopt}, /* level 17.*/
-+ {17, 18, 17, 7, 3, 256, ZSTD_btopt}, /* level 18.*/
-+ {17, 18, 17, 8, 3, 256, ZSTD_btopt}, /* level 19.*/
-+ {17, 18, 17, 9, 3, 256, ZSTD_btopt2}, /* level 20.*/
-+ {17, 18, 17, 10, 3, 256, ZSTD_btopt2}, /* level 21.*/
-+ {17, 18, 17, 11, 3, 512, ZSTD_btopt2}, /* level 22.*/
-+ },
-+ {
-+ /* for srcSize <= 16 KB */
-+ /* W, C, H, S, L, T, strat */
-+ {14, 12, 12, 1, 7, 6, ZSTD_fast}, /* level 0 - not used */
-+ {14, 14, 14, 1, 6, 6, ZSTD_fast}, /* level 1 */
-+ {14, 14, 14, 1, 4, 6, ZSTD_fast}, /* level 2 */
-+ {14, 14, 14, 1, 4, 6, ZSTD_dfast}, /* level 3.*/
-+ {14, 14, 14, 4, 4, 6, ZSTD_greedy}, /* level 4.*/
-+ {14, 14, 14, 3, 4, 6, ZSTD_lazy}, /* level 5.*/
-+ {14, 14, 14, 4, 4, 6, ZSTD_lazy2}, /* level 6 */
-+ {14, 14, 14, 5, 4, 6, ZSTD_lazy2}, /* level 7 */
-+ {14, 14, 14, 6, 4, 6, ZSTD_lazy2}, /* level 8.*/
-+ {14, 15, 14, 6, 4, 6, ZSTD_btlazy2}, /* level 9.*/
-+ {14, 15, 14, 3, 3, 6, ZSTD_btopt}, /* level 10.*/
-+ {14, 15, 14, 6, 3, 8, ZSTD_btopt}, /* level 11.*/
-+ {14, 15, 14, 6, 3, 16, ZSTD_btopt}, /* level 12.*/
-+ {14, 15, 14, 6, 3, 24, ZSTD_btopt}, /* level 13.*/
-+ {14, 15, 15, 6, 3, 48, ZSTD_btopt}, /* level 14.*/
-+ {14, 15, 15, 6, 3, 64, ZSTD_btopt}, /* level 15.*/
-+ {14, 15, 15, 6, 3, 96, ZSTD_btopt}, /* level 16.*/
-+ {14, 15, 15, 6, 3, 128, ZSTD_btopt}, /* level 17.*/
-+ {14, 15, 15, 6, 3, 256, ZSTD_btopt}, /* level 18.*/
-+ {14, 15, 15, 7, 3, 256, ZSTD_btopt}, /* level 19.*/
-+ {14, 15, 15, 8, 3, 256, ZSTD_btopt2}, /* level 20.*/
-+ {14, 15, 15, 9, 3, 256, ZSTD_btopt2}, /* level 21.*/
-+ {14, 15, 15, 10, 3, 256, ZSTD_btopt2}, /* level 22.*/
-+ },
-+};
-+
-+/*! ZSTD_getCParams() :
-+* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
-+* Size values are optional, provide 0 if not known or unused */
-+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
-+{
-+ ZSTD_compressionParameters cp;
-+ size_t const addedSize = srcSize ? 0 : 500;
-+ U64 const rSize = srcSize + dictSize ? srcSize + dictSize + addedSize : (U64)-1;
-+ U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
-+ if (compressionLevel <= 0)
-+ compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */
-+ if (compressionLevel > ZSTD_MAX_CLEVEL)
-+ compressionLevel = ZSTD_MAX_CLEVEL;
-+ cp = ZSTD_defaultCParameters[tableID][compressionLevel];
-+ if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */
-+ if (cp.windowLog > ZSTD_WINDOWLOG_MAX)
-+ cp.windowLog = ZSTD_WINDOWLOG_MAX;
-+ if (cp.chainLog > ZSTD_CHAINLOG_MAX)
-+ cp.chainLog = ZSTD_CHAINLOG_MAX;
-+ if (cp.hashLog > ZSTD_HASHLOG_MAX)
-+ cp.hashLog = ZSTD_HASHLOG_MAX;
-+ }
-+ cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
-+ return cp;
-+}
-+
-+/*! ZSTD_getParams() :
-+* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`).
-+* All fields of `ZSTD_frameParameters` are set to default (0) */
-+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
-+{
-+ ZSTD_parameters params;
-+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
-+ memset(¶ms, 0, sizeof(params));
-+ params.cParams = cParams;
-+ return params;
-+}
-+
-+EXPORT_SYMBOL(ZSTD_maxCLevel);
-+EXPORT_SYMBOL(ZSTD_compressBound);
-+
-+EXPORT_SYMBOL(ZSTD_CCtxWorkspaceBound);
-+EXPORT_SYMBOL(ZSTD_initCCtx);
-+EXPORT_SYMBOL(ZSTD_compressCCtx);
-+EXPORT_SYMBOL(ZSTD_compress_usingDict);
-+
-+EXPORT_SYMBOL(ZSTD_CDictWorkspaceBound);
-+EXPORT_SYMBOL(ZSTD_initCDict);
-+EXPORT_SYMBOL(ZSTD_compress_usingCDict);
-+
-+EXPORT_SYMBOL(ZSTD_CStreamWorkspaceBound);
-+EXPORT_SYMBOL(ZSTD_initCStream);
-+EXPORT_SYMBOL(ZSTD_initCStream_usingCDict);
-+EXPORT_SYMBOL(ZSTD_resetCStream);
-+EXPORT_SYMBOL(ZSTD_compressStream);
-+EXPORT_SYMBOL(ZSTD_flushStream);
-+EXPORT_SYMBOL(ZSTD_endStream);
-+EXPORT_SYMBOL(ZSTD_CStreamInSize);
-+EXPORT_SYMBOL(ZSTD_CStreamOutSize);
-+
-+EXPORT_SYMBOL(ZSTD_getCParams);
-+EXPORT_SYMBOL(ZSTD_getParams);
-+EXPORT_SYMBOL(ZSTD_checkCParams);
-+EXPORT_SYMBOL(ZSTD_adjustCParams);
-+
-+EXPORT_SYMBOL(ZSTD_compressBegin);
-+EXPORT_SYMBOL(ZSTD_compressBegin_usingDict);
-+EXPORT_SYMBOL(ZSTD_compressBegin_advanced);
-+EXPORT_SYMBOL(ZSTD_copyCCtx);
-+EXPORT_SYMBOL(ZSTD_compressBegin_usingCDict);
-+EXPORT_SYMBOL(ZSTD_compressContinue);
-+EXPORT_SYMBOL(ZSTD_compressEnd);
-+
-+EXPORT_SYMBOL(ZSTD_getBlockSizeMax);
-+EXPORT_SYMBOL(ZSTD_compressBlock);
-+
-+MODULE_LICENSE("Dual BSD/GPL");
-+MODULE_DESCRIPTION("Zstd Compressor");
-diff --git a/lib/zstd/decompress.c b/lib/zstd/decompress.c
-new file mode 100644
-index 0000000..b178467
---- /dev/null
-+++ b/lib/zstd/decompress.c
-@@ -0,0 +1,2528 @@
-+/**
-+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This source code is licensed under the BSD-style license found in the
-+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
-+ * An additional grant of patent rights can be found in the PATENTS file in the
-+ * same directory.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ */
-+
-+/* ***************************************************************
-+* Tuning parameters
-+*****************************************************************/
-+/*!
-+* MAXWINDOWSIZE_DEFAULT :
-+* maximum window size accepted by DStream, by default.
-+* Frames requiring more memory will be rejected.
-+*/
-+#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
-+#define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
-+#endif
-+
-+/*-*******************************************************
-+* Dependencies
-+*********************************************************/
-+#include "fse.h"
-+#include "huf.h"
-+#include "mem.h" /* low level memory routines */
-+#include "zstd_internal.h"
-+#include <linux/kernel.h>
-+#include <linux/module.h>
-+#include <linux/string.h> /* memcpy, memmove, memset */
-+
-+#define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
-+
-+/*-*************************************
-+* Macros
-+***************************************/
-+#define ZSTD_isError ERR_isError /* for inlining */
-+#define FSE_isError ERR_isError
-+#define HUF_isError ERR_isError
-+
-+/*_*******************************************************
-+* Memory operations
-+**********************************************************/
-+static void ZSTD_copy4(void *dst, const void *src) { memcpy(dst, src, 4); }
-+
-+/*-*************************************************************
-+* Context management
-+***************************************************************/
-+typedef enum {
-+ ZSTDds_getFrameHeaderSize,
-+ ZSTDds_decodeFrameHeader,
-+ ZSTDds_decodeBlockHeader,
-+ ZSTDds_decompressBlock,
-+ ZSTDds_decompressLastBlock,
-+ ZSTDds_checkChecksum,
-+ ZSTDds_decodeSkippableHeader,
-+ ZSTDds_skipFrame
-+} ZSTD_dStage;
-+
-+typedef struct {
-+ FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
-+ FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
-+ FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
-+ HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
-+ U64 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32 / 2];
-+ U32 rep[ZSTD_REP_NUM];
-+} ZSTD_entropyTables_t;
-+
-+struct ZSTD_DCtx_s {
-+ const FSE_DTable *LLTptr;
-+ const FSE_DTable *MLTptr;
-+ const FSE_DTable *OFTptr;
-+ const HUF_DTable *HUFptr;
-+ ZSTD_entropyTables_t entropy;
-+ const void *previousDstEnd; /* detect continuity */
-+ const void *base; /* start of curr segment */
-+ const void *vBase; /* virtual start of previous segment if it was just before curr one */
-+ const void *dictEnd; /* end of previous segment */
-+ size_t expected;
-+ ZSTD_frameParams fParams;
-+ blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
-+ ZSTD_dStage stage;
-+ U32 litEntropy;
-+ U32 fseEntropy;
-+ struct xxh64_state xxhState;
-+ size_t headerSize;
-+ U32 dictID;
-+ const BYTE *litPtr;
-+ ZSTD_customMem customMem;
-+ size_t litSize;
-+ size_t rleSize;
-+ BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
-+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
-+}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
-+
-+size_t ZSTD_DCtxWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx)); }
-+
-+size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx)
-+{
-+ dctx->expected = ZSTD_frameHeaderSize_prefix;
-+ dctx->stage = ZSTDds_getFrameHeaderSize;
-+ dctx->previousDstEnd = NULL;
-+ dctx->base = NULL;
-+ dctx->vBase = NULL;
-+ dctx->dictEnd = NULL;
-+ dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
-+ dctx->litEntropy = dctx->fseEntropy = 0;
-+ dctx->dictID = 0;
-+ ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
-+ memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
-+ dctx->LLTptr = dctx->entropy.LLTable;
-+ dctx->MLTptr = dctx->entropy.MLTable;
-+ dctx->OFTptr = dctx->entropy.OFTable;
-+ dctx->HUFptr = dctx->entropy.hufTable;
-+ return 0;
-+}
-+
-+ZSTD_DCtx *ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
-+{
-+ ZSTD_DCtx *dctx;
-+
-+ if (!customMem.customAlloc || !customMem.customFree)
-+ return NULL;
-+
-+ dctx = (ZSTD_DCtx *)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
-+ if (!dctx)
-+ return NULL;
-+ memcpy(&dctx->customMem, &customMem, sizeof(customMem));
-+ ZSTD_decompressBegin(dctx);
-+ return dctx;
-+}
-+
-+ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize)
-+{
-+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
-+ return ZSTD_createDCtx_advanced(stackMem);
-+}
-+
-+size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx)
-+{
-+ if (dctx == NULL)
-+ return 0; /* support free on NULL */
-+ ZSTD_free(dctx, dctx->customMem);
-+ return 0; /* reserved as a potential error code in the future */
-+}
-+
-+void ZSTD_copyDCtx(ZSTD_DCtx *dstDCtx, const ZSTD_DCtx *srcDCtx)
-+{
-+ size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
-+ memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
-+}
-+
-+static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict);
-+
-+/*-*************************************************************
-+* Decompression section
-+***************************************************************/
-+
-+/*! ZSTD_isFrame() :
-+ * Tells if the content of `buffer` starts with a valid Frame Identifier.
-+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
-+ * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
-+ * Note 3 : Skippable Frame Identifiers are considered valid. */
-+unsigned ZSTD_isFrame(const void *buffer, size_t size)
-+{
-+ if (size < 4)
-+ return 0;
-+ {
-+ U32 const magic = ZSTD_readLE32(buffer);
-+ if (magic == ZSTD_MAGICNUMBER)
-+ return 1;
-+ if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START)
-+ return 1;
-+ }
-+ return 0;
-+}
-+
-+/** ZSTD_frameHeaderSize() :
-+* srcSize must be >= ZSTD_frameHeaderSize_prefix.
-+* @return : size of the Frame Header */
-+static size_t ZSTD_frameHeaderSize(const void *src, size_t srcSize)
-+{
-+ if (srcSize < ZSTD_frameHeaderSize_prefix)
-+ return ERROR(srcSize_wrong);
-+ {
-+ BYTE const fhd = ((const BYTE *)src)[4];
-+ U32 const dictID = fhd & 3;
-+ U32 const singleSegment = (fhd >> 5) & 1;
-+ U32 const fcsId = fhd >> 6;
-+ return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + (singleSegment && !fcsId);
-+ }
-+}
-+
-+/** ZSTD_getFrameParams() :
-+* decode Frame Header, or require larger `srcSize`.
-+* @return : 0, `fparamsPtr` is correctly filled,
-+* >0, `srcSize` is too small, result is expected `srcSize`,
-+* or an error code, which can be tested using ZSTD_isError() */
-+size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, size_t srcSize)
-+{
-+ const BYTE *ip = (const BYTE *)src;
-+
-+ if (srcSize < ZSTD_frameHeaderSize_prefix)
-+ return ZSTD_frameHeaderSize_prefix;
-+ if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) {
-+ if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
-+ if (srcSize < ZSTD_skippableHeaderSize)
-+ return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
-+ memset(fparamsPtr, 0, sizeof(*fparamsPtr));
-+ fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4);
-+ fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
-+ return 0;
-+ }
-+ return ERROR(prefix_unknown);
-+ }
-+
-+ /* ensure there is enough `srcSize` to fully read/decode frame header */
-+ {
-+ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
-+ if (srcSize < fhsize)
-+ return fhsize;
-+ }
-+
-+ {
-+ BYTE const fhdByte = ip[4];
-+ size_t pos = 5;
-+ U32 const dictIDSizeCode = fhdByte & 3;
-+ U32 const checksumFlag = (fhdByte >> 2) & 1;
-+ U32 const singleSegment = (fhdByte >> 5) & 1;
-+ U32 const fcsID = fhdByte >> 6;
-+ U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
-+ U32 windowSize = 0;
-+ U32 dictID = 0;
-+ U64 frameContentSize = 0;
-+ if ((fhdByte & 0x08) != 0)
-+ return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
-+ if (!singleSegment) {
-+ BYTE const wlByte = ip[pos++];
-+ U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
-+ if (windowLog > ZSTD_WINDOWLOG_MAX)
-+ return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
-+ windowSize = (1U << windowLog);
-+ windowSize += (windowSize >> 3) * (wlByte & 7);
-+ }
-+
-+ switch (dictIDSizeCode) {
-+ default: /* impossible */
-+ case 0: break;
-+ case 1:
-+ dictID = ip[pos];
-+ pos++;
-+ break;
-+ case 2:
-+ dictID = ZSTD_readLE16(ip + pos);
-+ pos += 2;
-+ break;
-+ case 3:
-+ dictID = ZSTD_readLE32(ip + pos);
-+ pos += 4;
-+ break;
-+ }
-+ switch (fcsID) {
-+ default: /* impossible */
-+ case 0:
-+ if (singleSegment)
-+ frameContentSize = ip[pos];
-+ break;
-+ case 1: frameContentSize = ZSTD_readLE16(ip + pos) + 256; break;
-+ case 2: frameContentSize = ZSTD_readLE32(ip + pos); break;
-+ case 3: frameContentSize = ZSTD_readLE64(ip + pos); break;
-+ }
-+ if (!windowSize)
-+ windowSize = (U32)frameContentSize;
-+ if (windowSize > windowSizeMax)
-+ return ERROR(frameParameter_windowTooLarge);
-+ fparamsPtr->frameContentSize = frameContentSize;
-+ fparamsPtr->windowSize = windowSize;
-+ fparamsPtr->dictID = dictID;
-+ fparamsPtr->checksumFlag = checksumFlag;
-+ }
-+ return 0;
-+}
-+
-+/** ZSTD_getFrameContentSize() :
-+* compatible with legacy mode
-+* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
-+* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
-+* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
-+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
-+{
-+ {
-+ ZSTD_frameParams fParams;
-+ if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0)
-+ return ZSTD_CONTENTSIZE_ERROR;
-+ if (fParams.windowSize == 0) {
-+ /* Either skippable or empty frame, size == 0 either way */
-+ return 0;
-+ } else if (fParams.frameContentSize != 0) {
-+ return fParams.frameContentSize;
-+ } else {
-+ return ZSTD_CONTENTSIZE_UNKNOWN;
-+ }
-+ }
-+}
-+
-+/** ZSTD_findDecompressedSize() :
-+ * compatible with legacy mode
-+ * `srcSize` must be the exact length of some number of ZSTD compressed and/or
-+ * skippable frames
-+ * @return : decompressed size of the frames contained */
-+unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize)
-+{
-+ {
-+ unsigned long long totalDstSize = 0;
-+ while (srcSize >= ZSTD_frameHeaderSize_prefix) {
-+ const U32 magicNumber = ZSTD_readLE32(src);
-+
-+ if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
-+ size_t skippableSize;
-+ if (srcSize < ZSTD_skippableHeaderSize)
-+ return ERROR(srcSize_wrong);
-+ skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
-+ if (srcSize < skippableSize) {
-+ return ZSTD_CONTENTSIZE_ERROR;
-+ }
-+
-+ src = (const BYTE *)src + skippableSize;
-+ srcSize -= skippableSize;
-+ continue;
-+ }
-+
-+ {
-+ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
-+ if (ret >= ZSTD_CONTENTSIZE_ERROR)
-+ return ret;
-+
-+ /* check for overflow */
-+ if (totalDstSize + ret < totalDstSize)
-+ return ZSTD_CONTENTSIZE_ERROR;
-+ totalDstSize += ret;
-+ }
-+ {
-+ size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
-+ if (ZSTD_isError(frameSrcSize)) {
-+ return ZSTD_CONTENTSIZE_ERROR;
-+ }
-+
-+ src = (const BYTE *)src + frameSrcSize;
-+ srcSize -= frameSrcSize;
-+ }
-+ }
-+
-+ if (srcSize) {
-+ return ZSTD_CONTENTSIZE_ERROR;
-+ }
-+
-+ return totalDstSize;
-+ }
-+}
-+
-+/** ZSTD_decodeFrameHeader() :
-+* `headerSize` must be the size provided by ZSTD_frameHeaderSize().
-+* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
-+static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx *dctx, const void *src, size_t headerSize)
-+{
-+ size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize);
-+ if (ZSTD_isError(result))
-+ return result; /* invalid header */
-+ if (result > 0)
-+ return ERROR(srcSize_wrong); /* headerSize too small */
-+ if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
-+ return ERROR(dictionary_wrong);
-+ if (dctx->fParams.checksumFlag)
-+ xxh64_reset(&dctx->xxhState, 0);
-+ return 0;
-+}
-+
-+typedef struct {
-+ blockType_e blockType;
-+ U32 lastBlock;
-+ U32 origSize;
-+} blockProperties_t;
-+
-+/*! ZSTD_getcBlockSize() :
-+* Provides the size of compressed block from block header `src` */
-+size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t *bpPtr)
-+{
-+ if (srcSize < ZSTD_blockHeaderSize)
-+ return ERROR(srcSize_wrong);
-+ {
-+ U32 const cBlockHeader = ZSTD_readLE24(src);
-+ U32 const cSize = cBlockHeader >> 3;
-+ bpPtr->lastBlock = cBlockHeader & 1;
-+ bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
-+ bpPtr->origSize = cSize; /* only useful for RLE */
-+ if (bpPtr->blockType == bt_rle)
-+ return 1;
-+ if (bpPtr->blockType == bt_reserved)
-+ return ERROR(corruption_detected);
-+ return cSize;
-+ }
-+}
-+
-+static size_t ZSTD_copyRawBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ if (srcSize > dstCapacity)
-+ return ERROR(dstSize_tooSmall);
-+ memcpy(dst, src, srcSize);
-+ return srcSize;
-+}
-+
-+static size_t ZSTD_setRleBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize, size_t regenSize)
-+{
-+ if (srcSize != 1)
-+ return ERROR(srcSize_wrong);
-+ if (regenSize > dstCapacity)
-+ return ERROR(dstSize_tooSmall);
-+ memset(dst, *(const BYTE *)src, regenSize);
-+ return regenSize;
-+}
-+
-+/*! ZSTD_decodeLiteralsBlock() :
-+ @return : nb of bytes read from src (< srcSize ) */
-+size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
-+{
-+ if (srcSize < MIN_CBLOCK_SIZE)
-+ return ERROR(corruption_detected);
-+
-+ {
-+ const BYTE *const istart = (const BYTE *)src;
-+ symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
-+
-+ switch (litEncType) {
-+ case set_repeat:
-+ if (dctx->litEntropy == 0)
-+ return ERROR(dictionary_corrupted);
-+ /* fall-through */
-+ case set_compressed:
-+ if (srcSize < 5)
-+ return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
-+ {
-+ size_t lhSize, litSize, litCSize;
-+ U32 singleStream = 0;
-+ U32 const lhlCode = (istart[0] >> 2) & 3;
-+ U32 const lhc = ZSTD_readLE32(istart);
-+ switch (lhlCode) {
-+ case 0:
-+ case 1:
-+ default: /* note : default is impossible, since lhlCode into [0..3] */
-+ /* 2 - 2 - 10 - 10 */
-+ singleStream = !lhlCode;
-+ lhSize = 3;
-+ litSize = (lhc >> 4) & 0x3FF;
-+ litCSize = (lhc >> 14) & 0x3FF;
-+ break;
-+ case 2:
-+ /* 2 - 2 - 14 - 14 */
-+ lhSize = 4;
-+ litSize = (lhc >> 4) & 0x3FFF;
-+ litCSize = lhc >> 18;
-+ break;
-+ case 3:
-+ /* 2 - 2 - 18 - 18 */
-+ lhSize = 5;
-+ litSize = (lhc >> 4) & 0x3FFFF;
-+ litCSize = (lhc >> 22) + (istart[4] << 10);
-+ break;
-+ }
-+ if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
-+ return ERROR(corruption_detected);
-+ if (litCSize + lhSize > srcSize)
-+ return ERROR(corruption_detected);
-+
-+ if (HUF_isError(
-+ (litEncType == set_repeat)
-+ ? (singleStream ? HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr)
-+ : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr))
-+ : (singleStream
-+ ? HUF_decompress1X2_DCtx_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize,
-+ dctx->entropy.workspace, sizeof(dctx->entropy.workspace))
-+ : HUF_decompress4X_hufOnly_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize,
-+ dctx->entropy.workspace, sizeof(dctx->entropy.workspace)))))
-+ return ERROR(corruption_detected);
-+
-+ dctx->litPtr = dctx->litBuffer;
-+ dctx->litSize = litSize;
-+ dctx->litEntropy = 1;
-+ if (litEncType == set_compressed)
-+ dctx->HUFptr = dctx->entropy.hufTable;
-+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
-+ return litCSize + lhSize;
-+ }
-+
-+ case set_basic: {
-+ size_t litSize, lhSize;
-+ U32 const lhlCode = ((istart[0]) >> 2) & 3;
-+ switch (lhlCode) {
-+ case 0:
-+ case 2:
-+ default: /* note : default is impossible, since lhlCode into [0..3] */
-+ lhSize = 1;
-+ litSize = istart[0] >> 3;
-+ break;
-+ case 1:
-+ lhSize = 2;
-+ litSize = ZSTD_readLE16(istart) >> 4;
-+ break;
-+ case 3:
-+ lhSize = 3;
-+ litSize = ZSTD_readLE24(istart) >> 4;
-+ break;
-+ }
-+
-+ if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
-+ if (litSize + lhSize > srcSize)
-+ return ERROR(corruption_detected);
-+ memcpy(dctx->litBuffer, istart + lhSize, litSize);
-+ dctx->litPtr = dctx->litBuffer;
-+ dctx->litSize = litSize;
-+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
-+ return lhSize + litSize;
-+ }
-+ /* direct reference into compressed stream */
-+ dctx->litPtr = istart + lhSize;
-+ dctx->litSize = litSize;
-+ return lhSize + litSize;
-+ }
-+
-+ case set_rle: {
-+ U32 const lhlCode = ((istart[0]) >> 2) & 3;
-+ size_t litSize, lhSize;
-+ switch (lhlCode) {
-+ case 0:
-+ case 2:
-+ default: /* note : default is impossible, since lhlCode into [0..3] */
-+ lhSize = 1;
-+ litSize = istart[0] >> 3;
-+ break;
-+ case 1:
-+ lhSize = 2;
-+ litSize = ZSTD_readLE16(istart) >> 4;
-+ break;
-+ case 3:
-+ lhSize = 3;
-+ litSize = ZSTD_readLE24(istart) >> 4;
-+ if (srcSize < 4)
-+ return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
-+ break;
-+ }
-+ if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
-+ return ERROR(corruption_detected);
-+ memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
-+ dctx->litPtr = dctx->litBuffer;
-+ dctx->litSize = litSize;
-+ return lhSize + 1;
-+ }
-+ default:
-+ return ERROR(corruption_detected); /* impossible */
-+ }
-+ }
-+}
-+
-+typedef union {
-+ FSE_decode_t realData;
-+ U32 alignedBy4;
-+} FSE_decode_t4;
-+
-+static const FSE_decode_t4 LL_defaultDTable[(1 << LL_DEFAULTNORMLOG) + 1] = {
-+ {{LL_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
-+ {{0, 0, 4}}, /* 0 : base, symbol, bits */
-+ {{16, 0, 4}},
-+ {{32, 1, 5}},
-+ {{0, 3, 5}},
-+ {{0, 4, 5}},
-+ {{0, 6, 5}},
-+ {{0, 7, 5}},
-+ {{0, 9, 5}},
-+ {{0, 10, 5}},
-+ {{0, 12, 5}},
-+ {{0, 14, 6}},
-+ {{0, 16, 5}},
-+ {{0, 18, 5}},
-+ {{0, 19, 5}},
-+ {{0, 21, 5}},
-+ {{0, 22, 5}},
-+ {{0, 24, 5}},
-+ {{32, 25, 5}},
-+ {{0, 26, 5}},
-+ {{0, 27, 6}},
-+ {{0, 29, 6}},
-+ {{0, 31, 6}},
-+ {{32, 0, 4}},
-+ {{0, 1, 4}},
-+ {{0, 2, 5}},
-+ {{32, 4, 5}},
-+ {{0, 5, 5}},
-+ {{32, 7, 5}},
-+ {{0, 8, 5}},
-+ {{32, 10, 5}},
-+ {{0, 11, 5}},
-+ {{0, 13, 6}},
-+ {{32, 16, 5}},
-+ {{0, 17, 5}},
-+ {{32, 19, 5}},
-+ {{0, 20, 5}},
-+ {{32, 22, 5}},
-+ {{0, 23, 5}},
-+ {{0, 25, 4}},
-+ {{16, 25, 4}},
-+ {{32, 26, 5}},
-+ {{0, 28, 6}},
-+ {{0, 30, 6}},
-+ {{48, 0, 4}},
-+ {{16, 1, 4}},
-+ {{32, 2, 5}},
-+ {{32, 3, 5}},
-+ {{32, 5, 5}},
-+ {{32, 6, 5}},
-+ {{32, 8, 5}},
-+ {{32, 9, 5}},
-+ {{32, 11, 5}},
-+ {{32, 12, 5}},
-+ {{0, 15, 6}},
-+ {{32, 17, 5}},
-+ {{32, 18, 5}},
-+ {{32, 20, 5}},
-+ {{32, 21, 5}},
-+ {{32, 23, 5}},
-+ {{32, 24, 5}},
-+ {{0, 35, 6}},
-+ {{0, 34, 6}},
-+ {{0, 33, 6}},
-+ {{0, 32, 6}},
-+}; /* LL_defaultDTable */
-+
-+static const FSE_decode_t4 ML_defaultDTable[(1 << ML_DEFAULTNORMLOG) + 1] = {
-+ {{ML_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
-+ {{0, 0, 6}}, /* 0 : base, symbol, bits */
-+ {{0, 1, 4}},
-+ {{32, 2, 5}},
-+ {{0, 3, 5}},
-+ {{0, 5, 5}},
-+ {{0, 6, 5}},
-+ {{0, 8, 5}},
-+ {{0, 10, 6}},
-+ {{0, 13, 6}},
-+ {{0, 16, 6}},
-+ {{0, 19, 6}},
-+ {{0, 22, 6}},
-+ {{0, 25, 6}},
-+ {{0, 28, 6}},
-+ {{0, 31, 6}},
-+ {{0, 33, 6}},
-+ {{0, 35, 6}},
-+ {{0, 37, 6}},
-+ {{0, 39, 6}},
-+ {{0, 41, 6}},
-+ {{0, 43, 6}},
-+ {{0, 45, 6}},
-+ {{16, 1, 4}},
-+ {{0, 2, 4}},
-+ {{32, 3, 5}},
-+ {{0, 4, 5}},
-+ {{32, 6, 5}},
-+ {{0, 7, 5}},
-+ {{0, 9, 6}},
-+ {{0, 12, 6}},
-+ {{0, 15, 6}},
-+ {{0, 18, 6}},
-+ {{0, 21, 6}},
-+ {{0, 24, 6}},
-+ {{0, 27, 6}},
-+ {{0, 30, 6}},
-+ {{0, 32, 6}},
-+ {{0, 34, 6}},
-+ {{0, 36, 6}},
-+ {{0, 38, 6}},
-+ {{0, 40, 6}},
-+ {{0, 42, 6}},
-+ {{0, 44, 6}},
-+ {{32, 1, 4}},
-+ {{48, 1, 4}},
-+ {{16, 2, 4}},
-+ {{32, 4, 5}},
-+ {{32, 5, 5}},
-+ {{32, 7, 5}},
-+ {{32, 8, 5}},
-+ {{0, 11, 6}},
-+ {{0, 14, 6}},
-+ {{0, 17, 6}},
-+ {{0, 20, 6}},
-+ {{0, 23, 6}},
-+ {{0, 26, 6}},
-+ {{0, 29, 6}},
-+ {{0, 52, 6}},
-+ {{0, 51, 6}},
-+ {{0, 50, 6}},
-+ {{0, 49, 6}},
-+ {{0, 48, 6}},
-+ {{0, 47, 6}},
-+ {{0, 46, 6}},
-+}; /* ML_defaultDTable */
-+
-+static const FSE_decode_t4 OF_defaultDTable[(1 << OF_DEFAULTNORMLOG) + 1] = {
-+ {{OF_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
-+ {{0, 0, 5}}, /* 0 : base, symbol, bits */
-+ {{0, 6, 4}},
-+ {{0, 9, 5}},
-+ {{0, 15, 5}},
-+ {{0, 21, 5}},
-+ {{0, 3, 5}},
-+ {{0, 7, 4}},
-+ {{0, 12, 5}},
-+ {{0, 18, 5}},
-+ {{0, 23, 5}},
-+ {{0, 5, 5}},
-+ {{0, 8, 4}},
-+ {{0, 14, 5}},
-+ {{0, 20, 5}},
-+ {{0, 2, 5}},
-+ {{16, 7, 4}},
-+ {{0, 11, 5}},
-+ {{0, 17, 5}},
-+ {{0, 22, 5}},
-+ {{0, 4, 5}},
-+ {{16, 8, 4}},
-+ {{0, 13, 5}},
-+ {{0, 19, 5}},
-+ {{0, 1, 5}},
-+ {{16, 6, 4}},
-+ {{0, 10, 5}},
-+ {{0, 16, 5}},
-+ {{0, 28, 5}},
-+ {{0, 27, 5}},
-+ {{0, 26, 5}},
-+ {{0, 25, 5}},
-+ {{0, 24, 5}},
-+}; /* OF_defaultDTable */
-+
-+/*! ZSTD_buildSeqTable() :
-+ @return : nb bytes read from src,
-+ or an error code if it fails, testable with ZSTD_isError()
-+*/
-+static size_t ZSTD_buildSeqTable(FSE_DTable *DTableSpace, const FSE_DTable **DTablePtr, symbolEncodingType_e type, U32 max, U32 maxLog, const void *src,
-+ size_t srcSize, const FSE_decode_t4 *defaultTable, U32 flagRepeatTable, void *workspace, size_t workspaceSize)
-+{
-+ const void *const tmpPtr = defaultTable; /* bypass strict aliasing */
-+ switch (type) {
-+ case set_rle:
-+ if (!srcSize)
-+ return ERROR(srcSize_wrong);
-+ if ((*(const BYTE *)src) > max)
-+ return ERROR(corruption_detected);
-+ FSE_buildDTable_rle(DTableSpace, *(const BYTE *)src);
-+ *DTablePtr = DTableSpace;
-+ return 1;
-+ case set_basic: *DTablePtr = (const FSE_DTable *)tmpPtr; return 0;
-+ case set_repeat:
-+ if (!flagRepeatTable)
-+ return ERROR(corruption_detected);
-+ return 0;
-+ default: /* impossible */
-+ case set_compressed: {
-+ U32 tableLog;
-+ S16 *norm = (S16 *)workspace;
-+ size_t const spaceUsed32 = ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2;
-+
-+ if ((spaceUsed32 << 2) > workspaceSize)
-+ return ERROR(GENERIC);
-+ workspace = (U32 *)workspace + spaceUsed32;
-+ workspaceSize -= (spaceUsed32 << 2);
-+ {
-+ size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
-+ if (FSE_isError(headerSize))
-+ return ERROR(corruption_detected);
-+ if (tableLog > maxLog)
-+ return ERROR(corruption_detected);
-+ FSE_buildDTable_wksp(DTableSpace, norm, max, tableLog, workspace, workspaceSize);
-+ *DTablePtr = DTableSpace;
-+ return headerSize;
-+ }
-+ }
-+ }
-+}
-+
-+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx *dctx, int *nbSeqPtr, const void *src, size_t srcSize)
-+{
-+ const BYTE *const istart = (const BYTE *const)src;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *ip = istart;
-+
-+ /* check */
-+ if (srcSize < MIN_SEQUENCES_SIZE)
-+ return ERROR(srcSize_wrong);
-+
-+ /* SeqHead */
-+ {
-+ int nbSeq = *ip++;
-+ if (!nbSeq) {
-+ *nbSeqPtr = 0;
-+ return 1;
-+ }
-+ if (nbSeq > 0x7F) {
-+ if (nbSeq == 0xFF) {
-+ if (ip + 2 > iend)
-+ return ERROR(srcSize_wrong);
-+ nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip += 2;
-+ } else {
-+ if (ip >= iend)
-+ return ERROR(srcSize_wrong);
-+ nbSeq = ((nbSeq - 0x80) << 8) + *ip++;
-+ }
-+ }
-+ *nbSeqPtr = nbSeq;
-+ }
-+
-+ /* FSE table descriptors */
-+ if (ip + 4 > iend)
-+ return ERROR(srcSize_wrong); /* minimum possible size */
-+ {
-+ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
-+ symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
-+ symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
-+ ip++;
-+
-+ /* Build DTables */
-+ {
-+ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, LLtype, MaxLL, LLFSELog, ip, iend - ip,
-+ LL_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace));
-+ if (ZSTD_isError(llhSize))
-+ return ERROR(corruption_detected);
-+ ip += llhSize;
-+ }
-+ {
-+ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, OFtype, MaxOff, OffFSELog, ip, iend - ip,
-+ OF_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace));
-+ if (ZSTD_isError(ofhSize))
-+ return ERROR(corruption_detected);
-+ ip += ofhSize;
-+ }
-+ {
-+ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, MLtype, MaxML, MLFSELog, ip, iend - ip,
-+ ML_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace));
-+ if (ZSTD_isError(mlhSize))
-+ return ERROR(corruption_detected);
-+ ip += mlhSize;
-+ }
-+ }
-+
-+ return ip - istart;
-+}
-+
-+typedef struct {
-+ size_t litLength;
-+ size_t matchLength;
-+ size_t offset;
-+ const BYTE *match;
-+} seq_t;
-+
-+typedef struct {
-+ BIT_DStream_t DStream;
-+ FSE_DState_t stateLL;
-+ FSE_DState_t stateOffb;
-+ FSE_DState_t stateML;
-+ size_t prevOffset[ZSTD_REP_NUM];
-+ const BYTE *base;
-+ size_t pos;
-+ uPtrDiff gotoDict;
-+} seqState_t;
-+
-+FORCE_NOINLINE
-+size_t ZSTD_execSequenceLast7(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
-+ const BYTE *const vBase, const BYTE *const dictEnd)
-+{
-+ BYTE *const oLitEnd = op + sequence.litLength;
-+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
-+ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
-+ BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
-+ const BYTE *const iLitEnd = *litPtr + sequence.litLength;
-+ const BYTE *match = oLitEnd - sequence.offset;
-+
-+ /* check */
-+ if (oMatchEnd > oend)
-+ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
-+ if (iLitEnd > litLimit)
-+ return ERROR(corruption_detected); /* over-read beyond lit buffer */
-+ if (oLitEnd <= oend_w)
-+ return ERROR(GENERIC); /* Precondition */
-+
-+ /* copy literals */
-+ if (op < oend_w) {
-+ ZSTD_wildcopy(op, *litPtr, oend_w - op);
-+ *litPtr += oend_w - op;
-+ op = oend_w;
-+ }
-+ while (op < oLitEnd)
-+ *op++ = *(*litPtr)++;
-+
-+ /* copy Match */
-+ if (sequence.offset > (size_t)(oLitEnd - base)) {
-+ /* offset beyond prefix */
-+ if (sequence.offset > (size_t)(oLitEnd - vBase))
-+ return ERROR(corruption_detected);
-+ match = dictEnd - (base - match);
-+ if (match + sequence.matchLength <= dictEnd) {
-+ memmove(oLitEnd, match, sequence.matchLength);
-+ return sequenceLength;
-+ }
-+ /* span extDict & currPrefixSegment */
-+ {
-+ size_t const length1 = dictEnd - match;
-+ memmove(oLitEnd, match, length1);
-+ op = oLitEnd + length1;
-+ sequence.matchLength -= length1;
-+ match = base;
-+ }
-+ }
-+ while (op < oMatchEnd)
-+ *op++ = *match++;
-+ return sequenceLength;
-+}
-+
-+static seq_t ZSTD_decodeSequence(seqState_t *seqState)
-+{
-+ seq_t seq;
-+
-+ U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
-+ U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
-+ U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */
-+
-+ U32 const llBits = LL_bits[llCode];
-+ U32 const mlBits = ML_bits[mlCode];
-+ U32 const ofBits = ofCode;
-+ U32 const totalBits = llBits + mlBits + ofBits;
-+
-+ static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
-+ 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
-+
-+ static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
-+ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
-+ 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
-+
-+ static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
-+ 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
-+ 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
-+
-+ /* sequence */
-+ {
-+ size_t offset;
-+ if (!ofCode)
-+ offset = 0;
-+ else {
-+ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
-+ if (ZSTD_32bits())
-+ BIT_reloadDStream(&seqState->DStream);
-+ }
-+
-+ if (ofCode <= 1) {
-+ offset += (llCode == 0);
-+ if (offset) {
-+ size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
-+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
-+ if (offset != 1)
-+ seqState->prevOffset[2] = seqState->prevOffset[1];
-+ seqState->prevOffset[1] = seqState->prevOffset[0];
-+ seqState->prevOffset[0] = offset = temp;
-+ } else {
-+ offset = seqState->prevOffset[0];
-+ }
-+ } else {
-+ seqState->prevOffset[2] = seqState->prevOffset[1];
-+ seqState->prevOffset[1] = seqState->prevOffset[0];
-+ seqState->prevOffset[0] = offset;
-+ }
-+ seq.offset = offset;
-+ }
-+
-+ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
-+ if (ZSTD_32bits() && (mlBits + llBits > 24))
-+ BIT_reloadDStream(&seqState->DStream);
-+
-+ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
-+ if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
-+ BIT_reloadDStream(&seqState->DStream);
-+
-+ /* ANS state update */
-+ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
-+ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
-+ if (ZSTD_32bits())
-+ BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
-+ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
-+
-+ seq.match = NULL;
-+
-+ return seq;
-+}
-+
-+FORCE_INLINE
-+size_t ZSTD_execSequence(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
-+ const BYTE *const vBase, const BYTE *const dictEnd)
-+{
-+ BYTE *const oLitEnd = op + sequence.litLength;
-+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
-+ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
-+ BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
-+ const BYTE *const iLitEnd = *litPtr + sequence.litLength;
-+ const BYTE *match = oLitEnd - sequence.offset;
-+
-+ /* check */
-+ if (oMatchEnd > oend)
-+ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
-+ if (iLitEnd > litLimit)
-+ return ERROR(corruption_detected); /* over-read beyond lit buffer */
-+ if (oLitEnd > oend_w)
-+ return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
-+
-+ /* copy Literals */
-+ ZSTD_copy8(op, *litPtr);
-+ if (sequence.litLength > 8)
-+ ZSTD_wildcopy(op + 8, (*litPtr) + 8,
-+ sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
-+ op = oLitEnd;
-+ *litPtr = iLitEnd; /* update for next sequence */
-+
-+ /* copy Match */
-+ if (sequence.offset > (size_t)(oLitEnd - base)) {
-+ /* offset beyond prefix */
-+ if (sequence.offset > (size_t)(oLitEnd - vBase))
-+ return ERROR(corruption_detected);
-+ match = dictEnd + (match - base);
-+ if (match + sequence.matchLength <= dictEnd) {
-+ memmove(oLitEnd, match, sequence.matchLength);
-+ return sequenceLength;
-+ }
-+ /* span extDict & currPrefixSegment */
-+ {
-+ size_t const length1 = dictEnd - match;
-+ memmove(oLitEnd, match, length1);
-+ op = oLitEnd + length1;
-+ sequence.matchLength -= length1;
-+ match = base;
-+ if (op > oend_w || sequence.matchLength < MINMATCH) {
-+ U32 i;
-+ for (i = 0; i < sequence.matchLength; ++i)
-+ op[i] = match[i];
-+ return sequenceLength;
-+ }
-+ }
-+ }
-+ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
-+
-+ /* match within prefix */
-+ if (sequence.offset < 8) {
-+ /* close range match, overlap */
-+ static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
-+ static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */
-+ int const sub2 = dec64table[sequence.offset];
-+ op[0] = match[0];
-+ op[1] = match[1];
-+ op[2] = match[2];
-+ op[3] = match[3];
-+ match += dec32table[sequence.offset];
-+ ZSTD_copy4(op + 4, match);
-+ match -= sub2;
-+ } else {
-+ ZSTD_copy8(op, match);
-+ }
-+ op += 8;
-+ match += 8;
-+
-+ if (oMatchEnd > oend - (16 - MINMATCH)) {
-+ if (op < oend_w) {
-+ ZSTD_wildcopy(op, match, oend_w - op);
-+ match += oend_w - op;
-+ op = oend_w;
-+ }
-+ while (op < oMatchEnd)
-+ *op++ = *match++;
-+ } else {
-+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
-+ }
-+ return sequenceLength;
-+}
-+
-+static size_t ZSTD_decompressSequences(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
-+{
-+ const BYTE *ip = (const BYTE *)seqStart;
-+ const BYTE *const iend = ip + seqSize;
-+ BYTE *const ostart = (BYTE * const)dst;
-+ BYTE *const oend = ostart + maxDstSize;
-+ BYTE *op = ostart;
-+ const BYTE *litPtr = dctx->litPtr;
-+ const BYTE *const litEnd = litPtr + dctx->litSize;
-+ const BYTE *const base = (const BYTE *)(dctx->base);
-+ const BYTE *const vBase = (const BYTE *)(dctx->vBase);
-+ const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd);
-+ int nbSeq;
-+
-+ /* Build Decoding Tables */
-+ {
-+ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
-+ if (ZSTD_isError(seqHSize))
-+ return seqHSize;
-+ ip += seqHSize;
-+ }
-+
-+ /* Regen sequences */
-+ if (nbSeq) {
-+ seqState_t seqState;
-+ dctx->fseEntropy = 1;
-+ {
-+ U32 i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ seqState.prevOffset[i] = dctx->entropy.rep[i];
-+ }
-+ CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected);
-+ FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
-+ FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
-+ FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
-+
-+ for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) {
-+ nbSeq--;
-+ {
-+ seq_t const sequence = ZSTD_decodeSequence(&seqState);
-+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
-+ if (ZSTD_isError(oneSeqSize))
-+ return oneSeqSize;
-+ op += oneSeqSize;
-+ }
-+ }
-+
-+ /* check if reached exact end */
-+ if (nbSeq)
-+ return ERROR(corruption_detected);
-+ /* save reps for next block */
-+ {
-+ U32 i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
-+ }
-+ }
-+
-+ /* last literal segment */
-+ {
-+ size_t const lastLLSize = litEnd - litPtr;
-+ if (lastLLSize > (size_t)(oend - op))
-+ return ERROR(dstSize_tooSmall);
-+ memcpy(op, litPtr, lastLLSize);
-+ op += lastLLSize;
-+ }
-+
-+ return op - ostart;
-+}
-+
-+FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t *seqState, int const longOffsets)
-+{
-+ seq_t seq;
-+
-+ U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
-+ U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
-+ U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */
-+
-+ U32 const llBits = LL_bits[llCode];
-+ U32 const mlBits = ML_bits[mlCode];
-+ U32 const ofBits = ofCode;
-+ U32 const totalBits = llBits + mlBits + ofBits;
-+
-+ static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
-+ 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
-+
-+ static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
-+ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
-+ 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
-+
-+ static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
-+ 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
-+ 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
-+
-+ /* sequence */
-+ {
-+ size_t offset;
-+ if (!ofCode)
-+ offset = 0;
-+ else {
-+ if (longOffsets) {
-+ int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
-+ offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
-+ if (ZSTD_32bits() || extraBits)
-+ BIT_reloadDStream(&seqState->DStream);
-+ if (extraBits)
-+ offset += BIT_readBitsFast(&seqState->DStream, extraBits);
-+ } else {
-+ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
-+ if (ZSTD_32bits())
-+ BIT_reloadDStream(&seqState->DStream);
-+ }
-+ }
-+
-+ if (ofCode <= 1) {
-+ offset += (llCode == 0);
-+ if (offset) {
-+ size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
-+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
-+ if (offset != 1)
-+ seqState->prevOffset[2] = seqState->prevOffset[1];
-+ seqState->prevOffset[1] = seqState->prevOffset[0];
-+ seqState->prevOffset[0] = offset = temp;
-+ } else {
-+ offset = seqState->prevOffset[0];
-+ }
-+ } else {
-+ seqState->prevOffset[2] = seqState->prevOffset[1];
-+ seqState->prevOffset[1] = seqState->prevOffset[0];
-+ seqState->prevOffset[0] = offset;
-+ }
-+ seq.offset = offset;
-+ }
-+
-+ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
-+ if (ZSTD_32bits() && (mlBits + llBits > 24))
-+ BIT_reloadDStream(&seqState->DStream);
-+
-+ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
-+ if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
-+ BIT_reloadDStream(&seqState->DStream);
-+
-+ {
-+ size_t const pos = seqState->pos + seq.litLength;
-+ seq.match = seqState->base + pos - seq.offset; /* single memory segment */
-+ if (seq.offset > pos)
-+ seq.match += seqState->gotoDict; /* separate memory segment */
-+ seqState->pos = pos + seq.matchLength;
-+ }
-+
-+ /* ANS state update */
-+ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
-+ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
-+ if (ZSTD_32bits())
-+ BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
-+ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
-+
-+ return seq;
-+}
-+
-+static seq_t ZSTD_decodeSequenceLong(seqState_t *seqState, unsigned const windowSize)
-+{
-+ if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) {
-+ return ZSTD_decodeSequenceLong_generic(seqState, 1);
-+ } else {
-+ return ZSTD_decodeSequenceLong_generic(seqState, 0);
-+ }
-+}
-+
-+FORCE_INLINE
-+size_t ZSTD_execSequenceLong(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
-+ const BYTE *const vBase, const BYTE *const dictEnd)
-+{
-+ BYTE *const oLitEnd = op + sequence.litLength;
-+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
-+ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
-+ BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
-+ const BYTE *const iLitEnd = *litPtr + sequence.litLength;
-+ const BYTE *match = sequence.match;
-+
-+ /* check */
-+ if (oMatchEnd > oend)
-+ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
-+ if (iLitEnd > litLimit)
-+ return ERROR(corruption_detected); /* over-read beyond lit buffer */
-+ if (oLitEnd > oend_w)
-+ return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
-+
-+ /* copy Literals */
-+ ZSTD_copy8(op, *litPtr);
-+ if (sequence.litLength > 8)
-+ ZSTD_wildcopy(op + 8, (*litPtr) + 8,
-+ sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
-+ op = oLitEnd;
-+ *litPtr = iLitEnd; /* update for next sequence */
-+
-+ /* copy Match */
-+ if (sequence.offset > (size_t)(oLitEnd - base)) {
-+ /* offset beyond prefix */
-+ if (sequence.offset > (size_t)(oLitEnd - vBase))
-+ return ERROR(corruption_detected);
-+ if (match + sequence.matchLength <= dictEnd) {
-+ memmove(oLitEnd, match, sequence.matchLength);
-+ return sequenceLength;
-+ }
-+ /* span extDict & currPrefixSegment */
-+ {
-+ size_t const length1 = dictEnd - match;
-+ memmove(oLitEnd, match, length1);
-+ op = oLitEnd + length1;
-+ sequence.matchLength -= length1;
-+ match = base;
-+ if (op > oend_w || sequence.matchLength < MINMATCH) {
-+ U32 i;
-+ for (i = 0; i < sequence.matchLength; ++i)
-+ op[i] = match[i];
-+ return sequenceLength;
-+ }
-+ }
-+ }
-+ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
-+
-+ /* match within prefix */
-+ if (sequence.offset < 8) {
-+ /* close range match, overlap */
-+ static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
-+ static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */
-+ int const sub2 = dec64table[sequence.offset];
-+ op[0] = match[0];
-+ op[1] = match[1];
-+ op[2] = match[2];
-+ op[3] = match[3];
-+ match += dec32table[sequence.offset];
-+ ZSTD_copy4(op + 4, match);
-+ match -= sub2;
-+ } else {
-+ ZSTD_copy8(op, match);
-+ }
-+ op += 8;
-+ match += 8;
-+
-+ if (oMatchEnd > oend - (16 - MINMATCH)) {
-+ if (op < oend_w) {
-+ ZSTD_wildcopy(op, match, oend_w - op);
-+ match += oend_w - op;
-+ op = oend_w;
-+ }
-+ while (op < oMatchEnd)
-+ *op++ = *match++;
-+ } else {
-+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
-+ }
-+ return sequenceLength;
-+}
-+
-+static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
-+{
-+ const BYTE *ip = (const BYTE *)seqStart;
-+ const BYTE *const iend = ip + seqSize;
-+ BYTE *const ostart = (BYTE * const)dst;
-+ BYTE *const oend = ostart + maxDstSize;
-+ BYTE *op = ostart;
-+ const BYTE *litPtr = dctx->litPtr;
-+ const BYTE *const litEnd = litPtr + dctx->litSize;
-+ const BYTE *const base = (const BYTE *)(dctx->base);
-+ const BYTE *const vBase = (const BYTE *)(dctx->vBase);
-+ const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd);
-+ unsigned const windowSize = dctx->fParams.windowSize;
-+ int nbSeq;
-+
-+ /* Build Decoding Tables */
-+ {
-+ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
-+ if (ZSTD_isError(seqHSize))
-+ return seqHSize;
-+ ip += seqHSize;
-+ }
-+
-+ /* Regen sequences */
-+ if (nbSeq) {
-+#define STORED_SEQS 4
-+#define STOSEQ_MASK (STORED_SEQS - 1)
-+#define ADVANCED_SEQS 4
-+ seq_t *sequences = (seq_t *)dctx->entropy.workspace;
-+ int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
-+ seqState_t seqState;
-+ int seqNb;
-+ ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.workspace) >= sizeof(seq_t) * STORED_SEQS);
-+ dctx->fseEntropy = 1;
-+ {
-+ U32 i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ seqState.prevOffset[i] = dctx->entropy.rep[i];
-+ }
-+ seqState.base = base;
-+ seqState.pos = (size_t)(op - base);
-+ seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
-+ CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected);
-+ FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
-+ FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
-+ FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
-+
-+ /* prepare in advance */
-+ for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb < seqAdvance; seqNb++) {
-+ sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize);
-+ }
-+ if (seqNb < seqAdvance)
-+ return ERROR(corruption_detected);
-+
-+ /* decode and decompress */
-+ for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb < nbSeq; seqNb++) {
-+ seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize);
-+ size_t const oneSeqSize =
-+ ZSTD_execSequenceLong(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
-+ if (ZSTD_isError(oneSeqSize))
-+ return oneSeqSize;
-+ ZSTD_PREFETCH(sequence.match);
-+ sequences[seqNb & STOSEQ_MASK] = sequence;
-+ op += oneSeqSize;
-+ }
-+ if (seqNb < nbSeq)
-+ return ERROR(corruption_detected);
-+
-+ /* finish queue */
-+ seqNb -= seqAdvance;
-+ for (; seqNb < nbSeq; seqNb++) {
-+ size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
-+ if (ZSTD_isError(oneSeqSize))
-+ return oneSeqSize;
-+ op += oneSeqSize;
-+ }
-+
-+ /* save reps for next block */
-+ {
-+ U32 i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
-+ }
-+ }
-+
-+ /* last literal segment */
-+ {
-+ size_t const lastLLSize = litEnd - litPtr;
-+ if (lastLLSize > (size_t)(oend - op))
-+ return ERROR(dstSize_tooSmall);
-+ memcpy(op, litPtr, lastLLSize);
-+ op += lastLLSize;
-+ }
-+
-+ return op - ostart;
-+}
-+
-+static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{ /* blockType == blockCompressed */
-+ const BYTE *ip = (const BYTE *)src;
-+
-+ if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX)
-+ return ERROR(srcSize_wrong);
-+
-+ /* Decode literals section */
-+ {
-+ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
-+ if (ZSTD_isError(litCSize))
-+ return litCSize;
-+ ip += litCSize;
-+ srcSize -= litCSize;
-+ }
-+ if (sizeof(size_t) > 4) /* do not enable prefetching on 32-bits x86, as it's performance detrimental */
-+ /* likely because of register pressure */
-+ /* if that's the correct cause, then 32-bits ARM should be affected differently */
-+ /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */
-+ if (dctx->fParams.windowSize > (1 << 23))
-+ return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
-+ return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
-+}
-+
-+static void ZSTD_checkContinuity(ZSTD_DCtx *dctx, const void *dst)
-+{
-+ if (dst != dctx->previousDstEnd) { /* not contiguous */
-+ dctx->dictEnd = dctx->previousDstEnd;
-+ dctx->vBase = (const char *)dst - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base));
-+ dctx->base = dst;
-+ dctx->previousDstEnd = dst;
-+ }
-+}
-+
-+size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ size_t dSize;
-+ ZSTD_checkContinuity(dctx, dst);
-+ dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
-+ dctx->previousDstEnd = (char *)dst + dSize;
-+ return dSize;
-+}
-+
-+/** ZSTD_insertBlock() :
-+ insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
-+size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, size_t blockSize)
-+{
-+ ZSTD_checkContinuity(dctx, blockStart);
-+ dctx->previousDstEnd = (const char *)blockStart + blockSize;
-+ return blockSize;
-+}
-+
-+size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t length)
-+{
-+ if (length > dstCapacity)
-+ return ERROR(dstSize_tooSmall);
-+ memset(dst, byte, length);
-+ return length;
-+}
-+
-+/** ZSTD_findFrameCompressedSize() :
-+ * compatible with legacy mode
-+ * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
-+ * `srcSize` must be at least as large as the frame contained
-+ * @return : the compressed size of the frame starting at `src` */
-+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
-+{
-+ if (srcSize >= ZSTD_skippableHeaderSize && (ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
-+ return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE *)src + 4);
-+ } else {
-+ const BYTE *ip = (const BYTE *)src;
-+ const BYTE *const ipstart = ip;
-+ size_t remainingSize = srcSize;
-+ ZSTD_frameParams fParams;
-+
-+ size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
-+ if (ZSTD_isError(headerSize))
-+ return headerSize;
-+
-+ /* Frame Header */
-+ {
-+ size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
-+ if (ZSTD_isError(ret))
-+ return ret;
-+ if (ret > 0)
-+ return ERROR(srcSize_wrong);
-+ }
-+
-+ ip += headerSize;
-+ remainingSize -= headerSize;
-+
-+ /* Loop on each block */
-+ while (1) {
-+ blockProperties_t blockProperties;
-+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
-+ if (ZSTD_isError(cBlockSize))
-+ return cBlockSize;
-+
-+ if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
-+ return ERROR(srcSize_wrong);
-+
-+ ip += ZSTD_blockHeaderSize + cBlockSize;
-+ remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
-+
-+ if (blockProperties.lastBlock)
-+ break;
-+ }
-+
-+ if (fParams.checksumFlag) { /* Frame content checksum */
-+ if (remainingSize < 4)
-+ return ERROR(srcSize_wrong);
-+ ip += 4;
-+ remainingSize -= 4;
-+ }
-+
-+ return ip - ipstart;
-+ }
-+}
-+
-+/*! ZSTD_decompressFrame() :
-+* @dctx must be properly initialized */
-+static size_t ZSTD_decompressFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void **srcPtr, size_t *srcSizePtr)
-+{
-+ const BYTE *ip = (const BYTE *)(*srcPtr);
-+ BYTE *const ostart = (BYTE * const)dst;
-+ BYTE *const oend = ostart + dstCapacity;
-+ BYTE *op = ostart;
-+ size_t remainingSize = *srcSizePtr;
-+
-+ /* check */
-+ if (remainingSize < ZSTD_frameHeaderSize_min + ZSTD_blockHeaderSize)
-+ return ERROR(srcSize_wrong);
-+
-+ /* Frame Header */
-+ {
-+ size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix);
-+ if (ZSTD_isError(frameHeaderSize))
-+ return frameHeaderSize;
-+ if (remainingSize < frameHeaderSize + ZSTD_blockHeaderSize)
-+ return ERROR(srcSize_wrong);
-+ CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize));
-+ ip += frameHeaderSize;
-+ remainingSize -= frameHeaderSize;
-+ }
-+
-+ /* Loop on each block */
-+ while (1) {
-+ size_t decodedSize;
-+ blockProperties_t blockProperties;
-+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
-+ if (ZSTD_isError(cBlockSize))
-+ return cBlockSize;
-+
-+ ip += ZSTD_blockHeaderSize;
-+ remainingSize -= ZSTD_blockHeaderSize;
-+ if (cBlockSize > remainingSize)
-+ return ERROR(srcSize_wrong);
-+
-+ switch (blockProperties.blockType) {
-+ case bt_compressed: decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend - op, ip, cBlockSize); break;
-+ case bt_raw: decodedSize = ZSTD_copyRawBlock(op, oend - op, ip, cBlockSize); break;
-+ case bt_rle: decodedSize = ZSTD_generateNxBytes(op, oend - op, *ip, blockProperties.origSize); break;
-+ case bt_reserved:
-+ default: return ERROR(corruption_detected);
-+ }
-+
-+ if (ZSTD_isError(decodedSize))
-+ return decodedSize;
-+ if (dctx->fParams.checksumFlag)
-+ xxh64_update(&dctx->xxhState, op, decodedSize);
-+ op += decodedSize;
-+ ip += cBlockSize;
-+ remainingSize -= cBlockSize;
-+ if (blockProperties.lastBlock)
-+ break;
-+ }
-+
-+ if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
-+ U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState);
-+ U32 checkRead;
-+ if (remainingSize < 4)
-+ return ERROR(checksum_wrong);
-+ checkRead = ZSTD_readLE32(ip);
-+ if (checkRead != checkCalc)
-+ return ERROR(checksum_wrong);
-+ ip += 4;
-+ remainingSize -= 4;
-+ }
-+
-+ /* Allow caller to get size read */
-+ *srcPtr = ip;
-+ *srcSizePtr = remainingSize;
-+ return op - ostart;
-+}
-+
-+static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict);
-+static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict);
-+
-+static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
-+ const ZSTD_DDict *ddict)
-+{
-+ void *const dststart = dst;
-+
-+ if (ddict) {
-+ if (dict) {
-+ /* programmer error, these two cases should be mutually exclusive */
-+ return ERROR(GENERIC);
-+ }
-+
-+ dict = ZSTD_DDictDictContent(ddict);
-+ dictSize = ZSTD_DDictDictSize(ddict);
-+ }
-+
-+ while (srcSize >= ZSTD_frameHeaderSize_prefix) {
-+ U32 magicNumber;
-+
-+ magicNumber = ZSTD_readLE32(src);
-+ if (magicNumber != ZSTD_MAGICNUMBER) {
-+ if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
-+ size_t skippableSize;
-+ if (srcSize < ZSTD_skippableHeaderSize)
-+ return ERROR(srcSize_wrong);
-+ skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
-+ if (srcSize < skippableSize) {
-+ return ERROR(srcSize_wrong);
-+ }
-+
-+ src = (const BYTE *)src + skippableSize;
-+ srcSize -= skippableSize;
-+ continue;
-+ } else {
-+ return ERROR(prefix_unknown);
-+ }
-+ }
-+
-+ if (ddict) {
-+ /* we were called from ZSTD_decompress_usingDDict */
-+ ZSTD_refDDict(dctx, ddict);
-+ } else {
-+ /* this will initialize correctly with no dict if dict == NULL, so
-+ * use this in all cases but ddict */
-+ CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
-+ }
-+ ZSTD_checkContinuity(dctx, dst);
-+
-+ {
-+ const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize);
-+ if (ZSTD_isError(res))
-+ return res;
-+ /* don't need to bounds check this, ZSTD_decompressFrame will have
-+ * already */
-+ dst = (BYTE *)dst + res;
-+ dstCapacity -= res;
-+ }
-+ }
-+
-+ if (srcSize)
-+ return ERROR(srcSize_wrong); /* input not entirely consumed */
-+
-+ return (BYTE *)dst - (BYTE *)dststart;
-+}
-+
-+size_t ZSTD_decompress_usingDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize)
-+{
-+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
-+}
-+
-+size_t ZSTD_decompressDCtx(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
-+}
-+
-+/*-**************************************
-+* Advanced Streaming Decompression API
-+* Bufferless and synchronous
-+****************************************/
-+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx) { return dctx->expected; }
-+
-+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx)
-+{
-+ switch (dctx->stage) {
-+ default: /* should not happen */
-+ case ZSTDds_getFrameHeaderSize:
-+ case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader;
-+ case ZSTDds_decodeBlockHeader: return ZSTDnit_blockHeader;
-+ case ZSTDds_decompressBlock: return ZSTDnit_block;
-+ case ZSTDds_decompressLastBlock: return ZSTDnit_lastBlock;
-+ case ZSTDds_checkChecksum: return ZSTDnit_checksum;
-+ case ZSTDds_decodeSkippableHeader:
-+ case ZSTDds_skipFrame: return ZSTDnit_skippableFrame;
-+ }
-+}
-+
-+int ZSTD_isSkipFrame(ZSTD_DCtx *dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */
-+
-+/** ZSTD_decompressContinue() :
-+* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
-+* or an error code, which can be tested using ZSTD_isError() */
-+size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ /* Sanity check */
-+ if (srcSize != dctx->expected)
-+ return ERROR(srcSize_wrong);
-+ if (dstCapacity)
-+ ZSTD_checkContinuity(dctx, dst);
-+
-+ switch (dctx->stage) {
-+ case ZSTDds_getFrameHeaderSize:
-+ if (srcSize != ZSTD_frameHeaderSize_prefix)
-+ return ERROR(srcSize_wrong); /* impossible */
-+ if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
-+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
-+ dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */
-+ dctx->stage = ZSTDds_decodeSkippableHeader;
-+ return 0;
-+ }
-+ dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
-+ if (ZSTD_isError(dctx->headerSize))
-+ return dctx->headerSize;
-+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
-+ if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) {
-+ dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix;
-+ dctx->stage = ZSTDds_decodeFrameHeader;
-+ return 0;
-+ }
-+ dctx->expected = 0; /* not necessary to copy more */
-+
-+ case ZSTDds_decodeFrameHeader:
-+ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
-+ CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
-+ dctx->expected = ZSTD_blockHeaderSize;
-+ dctx->stage = ZSTDds_decodeBlockHeader;
-+ return 0;
-+
-+ case ZSTDds_decodeBlockHeader: {
-+ blockProperties_t bp;
-+ size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
-+ if (ZSTD_isError(cBlockSize))
-+ return cBlockSize;
-+ dctx->expected = cBlockSize;
-+ dctx->bType = bp.blockType;
-+ dctx->rleSize = bp.origSize;
-+ if (cBlockSize) {
-+ dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
-+ return 0;
-+ }
-+ /* empty block */
-+ if (bp.lastBlock) {
-+ if (dctx->fParams.checksumFlag) {
-+ dctx->expected = 4;
-+ dctx->stage = ZSTDds_checkChecksum;
-+ } else {
-+ dctx->expected = 0; /* end of frame */
-+ dctx->stage = ZSTDds_getFrameHeaderSize;
-+ }
-+ } else {
-+ dctx->expected = 3; /* go directly to next header */
-+ dctx->stage = ZSTDds_decodeBlockHeader;
-+ }
-+ return 0;
-+ }
-+ case ZSTDds_decompressLastBlock:
-+ case ZSTDds_decompressBlock: {
-+ size_t rSize;
-+ switch (dctx->bType) {
-+ case bt_compressed: rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break;
-+ case bt_raw: rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break;
-+ case bt_rle: rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); break;
-+ case bt_reserved: /* should never happen */
-+ default: return ERROR(corruption_detected);
-+ }
-+ if (ZSTD_isError(rSize))
-+ return rSize;
-+ if (dctx->fParams.checksumFlag)
-+ xxh64_update(&dctx->xxhState, dst, rSize);
-+
-+ if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
-+ if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
-+ dctx->expected = 4;
-+ dctx->stage = ZSTDds_checkChecksum;
-+ } else {
-+ dctx->expected = 0; /* ends here */
-+ dctx->stage = ZSTDds_getFrameHeaderSize;
-+ }
-+ } else {
-+ dctx->stage = ZSTDds_decodeBlockHeader;
-+ dctx->expected = ZSTD_blockHeaderSize;
-+ dctx->previousDstEnd = (char *)dst + rSize;
-+ }
-+ return rSize;
-+ }
-+ case ZSTDds_checkChecksum: {
-+ U32 const h32 = (U32)xxh64_digest(&dctx->xxhState);
-+ U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */
-+ if (check32 != h32)
-+ return ERROR(checksum_wrong);
-+ dctx->expected = 0;
-+ dctx->stage = ZSTDds_getFrameHeaderSize;
-+ return 0;
-+ }
-+ case ZSTDds_decodeSkippableHeader: {
-+ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
-+ dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4);
-+ dctx->stage = ZSTDds_skipFrame;
-+ return 0;
-+ }
-+ case ZSTDds_skipFrame: {
-+ dctx->expected = 0;
-+ dctx->stage = ZSTDds_getFrameHeaderSize;
-+ return 0;
-+ }
-+ default:
-+ return ERROR(GENERIC); /* impossible */
-+ }
-+}
-+
-+static size_t ZSTD_refDictContent(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
-+{
-+ dctx->dictEnd = dctx->previousDstEnd;
-+ dctx->vBase = (const char *)dict - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base));
-+ dctx->base = dict;
-+ dctx->previousDstEnd = (const char *)dict + dictSize;
-+ return 0;
-+}
-+
-+/* ZSTD_loadEntropy() :
-+ * dict : must point at beginning of a valid zstd dictionary
-+ * @return : size of entropy tables read */
-+static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t *entropy, const void *const dict, size_t const dictSize)
-+{
-+ const BYTE *dictPtr = (const BYTE *)dict;
-+ const BYTE *const dictEnd = dictPtr + dictSize;
-+
-+ if (dictSize <= 8)
-+ return ERROR(dictionary_corrupted);
-+ dictPtr += 8; /* skip header = magic + dictID */
-+
-+ {
-+ size_t const hSize = HUF_readDTableX4_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, entropy->workspace, sizeof(entropy->workspace));
-+ if (HUF_isError(hSize))
-+ return ERROR(dictionary_corrupted);
-+ dictPtr += hSize;
-+ }
-+
-+ {
-+ short offcodeNCount[MaxOff + 1];
-+ U32 offcodeMaxValue = MaxOff, offcodeLog;
-+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
-+ if (FSE_isError(offcodeHeaderSize))
-+ return ERROR(dictionary_corrupted);
-+ if (offcodeLog > OffFSELog)
-+ return ERROR(dictionary_corrupted);
-+ CHECK_E(FSE_buildDTable_wksp(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted);
-+ dictPtr += offcodeHeaderSize;
-+ }
-+
-+ {
-+ short matchlengthNCount[MaxML + 1];
-+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
-+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
-+ if (FSE_isError(matchlengthHeaderSize))
-+ return ERROR(dictionary_corrupted);
-+ if (matchlengthLog > MLFSELog)
-+ return ERROR(dictionary_corrupted);
-+ CHECK_E(FSE_buildDTable_wksp(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted);
-+ dictPtr += matchlengthHeaderSize;
-+ }
-+
-+ {
-+ short litlengthNCount[MaxLL + 1];
-+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
-+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
-+ if (FSE_isError(litlengthHeaderSize))
-+ return ERROR(dictionary_corrupted);
-+ if (litlengthLog > LLFSELog)
-+ return ERROR(dictionary_corrupted);
-+ CHECK_E(FSE_buildDTable_wksp(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted);
-+ dictPtr += litlengthHeaderSize;
-+ }
-+
-+ if (dictPtr + 12 > dictEnd)
-+ return ERROR(dictionary_corrupted);
-+ {
-+ int i;
-+ size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12));
-+ for (i = 0; i < 3; i++) {
-+ U32 const rep = ZSTD_readLE32(dictPtr);
-+ dictPtr += 4;
-+ if (rep == 0 || rep >= dictContentSize)
-+ return ERROR(dictionary_corrupted);
-+ entropy->rep[i] = rep;
-+ }
-+ }
-+
-+ return dictPtr - (const BYTE *)dict;
-+}
-+
-+static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
-+{
-+ if (dictSize < 8)
-+ return ZSTD_refDictContent(dctx, dict, dictSize);
-+ {
-+ U32 const magic = ZSTD_readLE32(dict);
-+ if (magic != ZSTD_DICT_MAGIC) {
-+ return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
-+ }
-+ }
-+ dctx->dictID = ZSTD_readLE32((const char *)dict + 4);
-+
-+ /* load entropy tables */
-+ {
-+ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
-+ if (ZSTD_isError(eSize))
-+ return ERROR(dictionary_corrupted);
-+ dict = (const char *)dict + eSize;
-+ dictSize -= eSize;
-+ }
-+ dctx->litEntropy = dctx->fseEntropy = 1;
-+
-+ /* reference dictionary content */
-+ return ZSTD_refDictContent(dctx, dict, dictSize);
-+}
-+
-+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
-+{
-+ CHECK_F(ZSTD_decompressBegin(dctx));
-+ if (dict && dictSize)
-+ CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
-+ return 0;
-+}
-+
-+/* ====== ZSTD_DDict ====== */
-+
-+struct ZSTD_DDict_s {
-+ void *dictBuffer;
-+ const void *dictContent;
-+ size_t dictSize;
-+ ZSTD_entropyTables_t entropy;
-+ U32 dictID;
-+ U32 entropyPresent;
-+ ZSTD_customMem cMem;
-+}; /* typedef'd to ZSTD_DDict within "zstd.h" */
-+
-+size_t ZSTD_DDictWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict)); }
-+
-+static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict) { return ddict->dictContent; }
-+
-+static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict) { return ddict->dictSize; }
-+
-+static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict)
-+{
-+ ZSTD_decompressBegin(dstDCtx); /* init */
-+ if (ddict) { /* support refDDict on NULL */
-+ dstDCtx->dictID = ddict->dictID;
-+ dstDCtx->base = ddict->dictContent;
-+ dstDCtx->vBase = ddict->dictContent;
-+ dstDCtx->dictEnd = (const BYTE *)ddict->dictContent + ddict->dictSize;
-+ dstDCtx->previousDstEnd = dstDCtx->dictEnd;
-+ if (ddict->entropyPresent) {
-+ dstDCtx->litEntropy = 1;
-+ dstDCtx->fseEntropy = 1;
-+ dstDCtx->LLTptr = ddict->entropy.LLTable;
-+ dstDCtx->MLTptr = ddict->entropy.MLTable;
-+ dstDCtx->OFTptr = ddict->entropy.OFTable;
-+ dstDCtx->HUFptr = ddict->entropy.hufTable;
-+ dstDCtx->entropy.rep[0] = ddict->entropy.rep[0];
-+ dstDCtx->entropy.rep[1] = ddict->entropy.rep[1];
-+ dstDCtx->entropy.rep[2] = ddict->entropy.rep[2];
-+ } else {
-+ dstDCtx->litEntropy = 0;
-+ dstDCtx->fseEntropy = 0;
-+ }
-+ }
-+}
-+
-+static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict *ddict)
-+{
-+ ddict->dictID = 0;
-+ ddict->entropyPresent = 0;
-+ if (ddict->dictSize < 8)
-+ return 0;
-+ {
-+ U32 const magic = ZSTD_readLE32(ddict->dictContent);
-+ if (magic != ZSTD_DICT_MAGIC)
-+ return 0; /* pure content mode */
-+ }
-+ ddict->dictID = ZSTD_readLE32((const char *)ddict->dictContent + 4);
-+
-+ /* load entropy tables */
-+ CHECK_E(ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted);
-+ ddict->entropyPresent = 1;
-+ return 0;
-+}
-+
-+static ZSTD_DDict *ZSTD_createDDict_advanced(const void *dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
-+{
-+ if (!customMem.customAlloc || !customMem.customFree)
-+ return NULL;
-+
-+ {
-+ ZSTD_DDict *const ddict = (ZSTD_DDict *)ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
-+ if (!ddict)
-+ return NULL;
-+ ddict->cMem = customMem;
-+
-+ if ((byReference) || (!dict) || (!dictSize)) {
-+ ddict->dictBuffer = NULL;
-+ ddict->dictContent = dict;
-+ } else {
-+ void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
-+ if (!internalBuffer) {
-+ ZSTD_freeDDict(ddict);
-+ return NULL;
-+ }
-+ memcpy(internalBuffer, dict, dictSize);
-+ ddict->dictBuffer = internalBuffer;
-+ ddict->dictContent = internalBuffer;
-+ }
-+ ddict->dictSize = dictSize;
-+ ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
-+ /* parse dictionary content */
-+ {
-+ size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
-+ if (ZSTD_isError(errorCode)) {
-+ ZSTD_freeDDict(ddict);
-+ return NULL;
-+ }
-+ }
-+
-+ return ddict;
-+ }
-+}
-+
-+/*! ZSTD_initDDict() :
-+* Create a digested dictionary, to start decompression without startup delay.
-+* `dict` content is copied inside DDict.
-+* Consequently, `dict` can be released after `ZSTD_DDict` creation */
-+ZSTD_DDict *ZSTD_initDDict(const void *dict, size_t dictSize, void *workspace, size_t workspaceSize)
-+{
-+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
-+ return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem);
-+}
-+
-+size_t ZSTD_freeDDict(ZSTD_DDict *ddict)
-+{
-+ if (ddict == NULL)
-+ return 0; /* support free on NULL */
-+ {
-+ ZSTD_customMem const cMem = ddict->cMem;
-+ ZSTD_free(ddict->dictBuffer, cMem);
-+ ZSTD_free(ddict, cMem);
-+ return 0;
-+ }
-+}
-+
-+/*! ZSTD_getDictID_fromDict() :
-+ * Provides the dictID stored within dictionary.
-+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
-+ * It can still be loaded, but as a content-only dictionary. */
-+unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize)
-+{
-+ if (dictSize < 8)
-+ return 0;
-+ if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC)
-+ return 0;
-+ return ZSTD_readLE32((const char *)dict + 4);
-+}
-+
-+/*! ZSTD_getDictID_fromDDict() :
-+ * Provides the dictID of the dictionary loaded into `ddict`.
-+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
-+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
-+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict)
-+{
-+ if (ddict == NULL)
-+ return 0;
-+ return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
-+}
-+
-+/*! ZSTD_getDictID_fromFrame() :
-+ * Provides the dictID required to decompressed the frame stored within `src`.
-+ * If @return == 0, the dictID could not be decoded.
-+ * This could for one of the following reasons :
-+ * - The frame does not require a dictionary to be decoded (most common case).
-+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
-+ * Note : this use case also happens when using a non-conformant dictionary.
-+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
-+ * - This is not a Zstandard frame.
-+ * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
-+unsigned ZSTD_getDictID_fromFrame(const void *src, size_t srcSize)
-+{
-+ ZSTD_frameParams zfp = {0, 0, 0, 0};
-+ size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
-+ if (ZSTD_isError(hError))
-+ return 0;
-+ return zfp.dictID;
-+}
-+
-+/*! ZSTD_decompress_usingDDict() :
-+* Decompression using a pre-digested Dictionary
-+* Use dictionary without significant overhead. */
-+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_DDict *ddict)
-+{
-+ /* pass content and size in case legacy frames are encountered */
-+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, NULL, 0, ddict);
-+}
-+
-+/*=====================================
-+* Streaming decompression
-+*====================================*/
-+
-+typedef enum { zdss_init, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
-+
-+/* *** Resource management *** */
-+struct ZSTD_DStream_s {
-+ ZSTD_DCtx *dctx;
-+ ZSTD_DDict *ddictLocal;
-+ const ZSTD_DDict *ddict;
-+ ZSTD_frameParams fParams;
-+ ZSTD_dStreamStage stage;
-+ char *inBuff;
-+ size_t inBuffSize;
-+ size_t inPos;
-+ size_t maxWindowSize;
-+ char *outBuff;
-+ size_t outBuffSize;
-+ size_t outStart;
-+ size_t outEnd;
-+ size_t blockSize;
-+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */
-+ size_t lhSize;
-+ ZSTD_customMem customMem;
-+ void *legacyContext;
-+ U32 previousLegacyVersion;
-+ U32 legacyVersion;
-+ U32 hostageByte;
-+}; /* typedef'd to ZSTD_DStream within "zstd.h" */
-+
-+size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize)
-+{
-+ size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
-+ size_t const inBuffSize = blockSize;
-+ size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
-+ return ZSTD_DCtxWorkspaceBound() + ZSTD_ALIGN(sizeof(ZSTD_DStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize);
-+}
-+
-+static ZSTD_DStream *ZSTD_createDStream_advanced(ZSTD_customMem customMem)
-+{
-+ ZSTD_DStream *zds;
-+
-+ if (!customMem.customAlloc || !customMem.customFree)
-+ return NULL;
-+
-+ zds = (ZSTD_DStream *)ZSTD_malloc(sizeof(ZSTD_DStream), customMem);
-+ if (zds == NULL)
-+ return NULL;
-+ memset(zds, 0, sizeof(ZSTD_DStream));
-+ memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
-+ zds->dctx = ZSTD_createDCtx_advanced(customMem);
-+ if (zds->dctx == NULL) {
-+ ZSTD_freeDStream(zds);
-+ return NULL;
-+ }
-+ zds->stage = zdss_init;
-+ zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
-+ return zds;
-+}
-+
-+ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, size_t workspaceSize)
-+{
-+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
-+ ZSTD_DStream *zds = ZSTD_createDStream_advanced(stackMem);
-+ if (!zds) {
-+ return NULL;
-+ }
-+
-+ zds->maxWindowSize = maxWindowSize;
-+ zds->stage = zdss_loadHeader;
-+ zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
-+ ZSTD_freeDDict(zds->ddictLocal);
-+ zds->ddictLocal = NULL;
-+ zds->ddict = zds->ddictLocal;
-+ zds->legacyVersion = 0;
-+ zds->hostageByte = 0;
-+
-+ {
-+ size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
-+ size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
-+
-+ zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem);
-+ zds->inBuffSize = blockSize;
-+ zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem);
-+ zds->outBuffSize = neededOutSize;
-+ if (zds->inBuff == NULL || zds->outBuff == NULL) {
-+ ZSTD_freeDStream(zds);
-+ return NULL;
-+ }
-+ }
-+ return zds;
-+}
-+
-+ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize)
-+{
-+ ZSTD_DStream *zds = ZSTD_initDStream(maxWindowSize, workspace, workspaceSize);
-+ if (zds) {
-+ zds->ddict = ddict;
-+ }
-+ return zds;
-+}
-+
-+size_t ZSTD_freeDStream(ZSTD_DStream *zds)
-+{
-+ if (zds == NULL)
-+ return 0; /* support free on null */
-+ {
-+ ZSTD_customMem const cMem = zds->customMem;
-+ ZSTD_freeDCtx(zds->dctx);
-+ zds->dctx = NULL;
-+ ZSTD_freeDDict(zds->ddictLocal);
-+ zds->ddictLocal = NULL;
-+ ZSTD_free(zds->inBuff, cMem);
-+ zds->inBuff = NULL;
-+ ZSTD_free(zds->outBuff, cMem);
-+ zds->outBuff = NULL;
-+ ZSTD_free(zds, cMem);
-+ return 0;
-+ }
-+}
-+
-+/* *** Initialization *** */
-+
-+size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
-+size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
-+
-+size_t ZSTD_resetDStream(ZSTD_DStream *zds)
-+{
-+ zds->stage = zdss_loadHeader;
-+ zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
-+ zds->legacyVersion = 0;
-+ zds->hostageByte = 0;
-+ return ZSTD_frameHeaderSize_prefix;
-+}
-+
-+/* ***** Decompression ***** */
-+
-+ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
-+{
-+ size_t const length = MIN(dstCapacity, srcSize);
-+ memcpy(dst, src, length);
-+ return length;
-+}
-+
-+size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
-+{
-+ const char *const istart = (const char *)(input->src) + input->pos;
-+ const char *const iend = (const char *)(input->src) + input->size;
-+ const char *ip = istart;
-+ char *const ostart = (char *)(output->dst) + output->pos;
-+ char *const oend = (char *)(output->dst) + output->size;
-+ char *op = ostart;
-+ U32 someMoreWork = 1;
-+
-+ while (someMoreWork) {
-+ switch (zds->stage) {
-+ case zdss_init:
-+ ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
-+ /* fall-through */
-+
-+ case zdss_loadHeader: {
-+ size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
-+ if (ZSTD_isError(hSize))
-+ return hSize;
-+ if (hSize != 0) { /* need more input */
-+ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
-+ if (toLoad > (size_t)(iend - ip)) { /* not enough input to load full header */
-+ memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip);
-+ zds->lhSize += iend - ip;
-+ input->pos = input->size;
-+ return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) +
-+ ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
-+ }
-+ memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad);
-+ zds->lhSize = hSize;
-+ ip += toLoad;
-+ break;
-+ }
-+
-+ /* check for single-pass mode opportunity */
-+ if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
-+ && (U64)(size_t)(oend - op) >= zds->fParams.frameContentSize) {
-+ size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart);
-+ if (cSize <= (size_t)(iend - istart)) {
-+ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend - op, istart, cSize, zds->ddict);
-+ if (ZSTD_isError(decompressedSize))
-+ return decompressedSize;
-+ ip = istart + cSize;
-+ op += decompressedSize;
-+ zds->dctx->expected = 0;
-+ zds->stage = zdss_init;
-+ someMoreWork = 0;
-+ break;
-+ }
-+ }
-+
-+ /* Consume header */
-+ ZSTD_refDDict(zds->dctx, zds->ddict);
-+ {
-+ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
-+ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
-+ {
-+ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
-+ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer + h1Size, h2Size));
-+ }
-+ }
-+
-+ zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
-+ if (zds->fParams.windowSize > zds->maxWindowSize)
-+ return ERROR(frameParameter_windowTooLarge);
-+
-+ /* Buffers are preallocated, but double check */
-+ {
-+ size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
-+ size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
-+ if (zds->inBuffSize < blockSize) {
-+ return ERROR(GENERIC);
-+ }
-+ if (zds->outBuffSize < neededOutSize) {
-+ return ERROR(GENERIC);
-+ }
-+ zds->blockSize = blockSize;
-+ }
-+ zds->stage = zdss_read;
-+ }
-+ /* pass-through */
-+
-+ case zdss_read: {
-+ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
-+ if (neededInSize == 0) { /* end of frame */
-+ zds->stage = zdss_init;
-+ someMoreWork = 0;
-+ break;
-+ }
-+ if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */
-+ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
-+ size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart,
-+ (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize);
-+ if (ZSTD_isError(decodedSize))
-+ return decodedSize;
-+ ip += neededInSize;
-+ if (!decodedSize && !isSkipFrame)
-+ break; /* this was just a header */
-+ zds->outEnd = zds->outStart + decodedSize;
-+ zds->stage = zdss_flush;
-+ break;
-+ }
-+ if (ip == iend) {
-+ someMoreWork = 0;
-+ break;
-+ } /* no more input */
-+ zds->stage = zdss_load;
-+ /* pass-through */
-+ }
-+
-+ case zdss_load: {
-+ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
-+ size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
-+ size_t loadedSize;
-+ if (toLoad > zds->inBuffSize - zds->inPos)
-+ return ERROR(corruption_detected); /* should never happen */
-+ loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip);
-+ ip += loadedSize;
-+ zds->inPos += loadedSize;
-+ if (loadedSize < toLoad) {
-+ someMoreWork = 0;
-+ break;
-+ } /* not enough input, wait for more */
-+
-+ /* decode loaded input */
-+ {
-+ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
-+ size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
-+ zds->inBuff, neededInSize);
-+ if (ZSTD_isError(decodedSize))
-+ return decodedSize;
-+ zds->inPos = 0; /* input is consumed */
-+ if (!decodedSize && !isSkipFrame) {
-+ zds->stage = zdss_read;
-+ break;
-+ } /* this was just a header */
-+ zds->outEnd = zds->outStart + decodedSize;
-+ zds->stage = zdss_flush;
-+ /* pass-through */
-+ }
-+ }
-+
-+ case zdss_flush: {
-+ size_t const toFlushSize = zds->outEnd - zds->outStart;
-+ size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize);
-+ op += flushedSize;
-+ zds->outStart += flushedSize;
-+ if (flushedSize == toFlushSize) { /* flush completed */
-+ zds->stage = zdss_read;
-+ if (zds->outStart + zds->blockSize > zds->outBuffSize)
-+ zds->outStart = zds->outEnd = 0;
-+ break;
-+ }
-+ /* cannot complete flush */
-+ someMoreWork = 0;
-+ break;
-+ }
-+ default:
-+ return ERROR(GENERIC); /* impossible */
-+ }
-+ }
-+
-+ /* result */
-+ input->pos += (size_t)(ip - istart);
-+ output->pos += (size_t)(op - ostart);
-+ {
-+ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
-+ if (!nextSrcSizeHint) { /* frame fully decoded */
-+ if (zds->outEnd == zds->outStart) { /* output fully flushed */
-+ if (zds->hostageByte) {
-+ if (input->pos >= input->size) {
-+ zds->stage = zdss_read;
-+ return 1;
-+ } /* can't release hostage (not present) */
-+ input->pos++; /* release hostage */
-+ }
-+ return 0;
-+ }
-+ if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
-+ input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
-+ zds->hostageByte = 1;
-+ }
-+ return 1;
-+ }
-+ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */
-+ if (zds->inPos > nextSrcSizeHint)
-+ return ERROR(GENERIC); /* should never happen */
-+ nextSrcSizeHint -= zds->inPos; /* already loaded*/
-+ return nextSrcSizeHint;
-+ }
-+}
-+
-+EXPORT_SYMBOL(ZSTD_DCtxWorkspaceBound);
-+EXPORT_SYMBOL(ZSTD_initDCtx);
-+EXPORT_SYMBOL(ZSTD_decompressDCtx);
-+EXPORT_SYMBOL(ZSTD_decompress_usingDict);
-+
-+EXPORT_SYMBOL(ZSTD_DDictWorkspaceBound);
-+EXPORT_SYMBOL(ZSTD_initDDict);
-+EXPORT_SYMBOL(ZSTD_decompress_usingDDict);
-+
-+EXPORT_SYMBOL(ZSTD_DStreamWorkspaceBound);
-+EXPORT_SYMBOL(ZSTD_initDStream);
-+EXPORT_SYMBOL(ZSTD_initDStream_usingDDict);
-+EXPORT_SYMBOL(ZSTD_resetDStream);
-+EXPORT_SYMBOL(ZSTD_decompressStream);
-+EXPORT_SYMBOL(ZSTD_DStreamInSize);
-+EXPORT_SYMBOL(ZSTD_DStreamOutSize);
-+
-+EXPORT_SYMBOL(ZSTD_findFrameCompressedSize);
-+EXPORT_SYMBOL(ZSTD_getFrameContentSize);
-+EXPORT_SYMBOL(ZSTD_findDecompressedSize);
-+
-+EXPORT_SYMBOL(ZSTD_isFrame);
-+EXPORT_SYMBOL(ZSTD_getDictID_fromDict);
-+EXPORT_SYMBOL(ZSTD_getDictID_fromDDict);
-+EXPORT_SYMBOL(ZSTD_getDictID_fromFrame);
-+
-+EXPORT_SYMBOL(ZSTD_getFrameParams);
-+EXPORT_SYMBOL(ZSTD_decompressBegin);
-+EXPORT_SYMBOL(ZSTD_decompressBegin_usingDict);
-+EXPORT_SYMBOL(ZSTD_copyDCtx);
-+EXPORT_SYMBOL(ZSTD_nextSrcSizeToDecompress);
-+EXPORT_SYMBOL(ZSTD_decompressContinue);
-+EXPORT_SYMBOL(ZSTD_nextInputType);
-+
-+EXPORT_SYMBOL(ZSTD_decompressBlock);
-+EXPORT_SYMBOL(ZSTD_insertBlock);
-+
-+MODULE_LICENSE("Dual BSD/GPL");
-+MODULE_DESCRIPTION("Zstd Decompressor");
-diff --git a/lib/zstd/entropy_common.c b/lib/zstd/entropy_common.c
-new file mode 100644
-index 0000000..2b0a643
---- /dev/null
-+++ b/lib/zstd/entropy_common.c
-@@ -0,0 +1,243 @@
-+/*
-+ * Common functions of New Generation Entropy library
-+ * Copyright (C) 2016, Yann Collet.
-+ *
-+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are
-+ * met:
-+ *
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above
-+ * copyright notice, this list of conditions and the following disclaimer
-+ * in the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ *
-+ * You can contact the author at :
-+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-+ */
-+
-+/* *************************************
-+* Dependencies
-+***************************************/
-+#include "error_private.h" /* ERR_*, ERROR */
-+#include "fse.h"
-+#include "huf.h"
-+#include "mem.h"
-+
-+/*=== Version ===*/
-+unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
-+
-+/*=== Error Management ===*/
-+unsigned FSE_isError(size_t code) { return ERR_isError(code); }
-+
-+unsigned HUF_isError(size_t code) { return ERR_isError(code); }
-+
-+/*-**************************************************************
-+* FSE NCount encoding-decoding
-+****************************************************************/
-+size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSVPtr, unsigned *tableLogPtr, const void *headerBuffer, size_t hbSize)
-+{
-+ const BYTE *const istart = (const BYTE *)headerBuffer;
-+ const BYTE *const iend = istart + hbSize;
-+ const BYTE *ip = istart;
-+ int nbBits;
-+ int remaining;
-+ int threshold;
-+ U32 bitStream;
-+ int bitCount;
-+ unsigned charnum = 0;
-+ int previous0 = 0;
-+
-+ if (hbSize < 4)
-+ return ERROR(srcSize_wrong);
-+ bitStream = ZSTD_readLE32(ip);
-+ nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
-+ if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX)
-+ return ERROR(tableLog_tooLarge);
-+ bitStream >>= 4;
-+ bitCount = 4;
-+ *tableLogPtr = nbBits;
-+ remaining = (1 << nbBits) + 1;
-+ threshold = 1 << nbBits;
-+ nbBits++;
-+
-+ while ((remaining > 1) & (charnum <= *maxSVPtr)) {
-+ if (previous0) {
-+ unsigned n0 = charnum;
-+ while ((bitStream & 0xFFFF) == 0xFFFF) {
-+ n0 += 24;
-+ if (ip < iend - 5) {
-+ ip += 2;
-+ bitStream = ZSTD_readLE32(ip) >> bitCount;
-+ } else {
-+ bitStream >>= 16;
-+ bitCount += 16;
-+ }
-+ }
-+ while ((bitStream & 3) == 3) {
-+ n0 += 3;
-+ bitStream >>= 2;
-+ bitCount += 2;
-+ }
-+ n0 += bitStream & 3;
-+ bitCount += 2;
-+ if (n0 > *maxSVPtr)
-+ return ERROR(maxSymbolValue_tooSmall);
-+ while (charnum < n0)
-+ normalizedCounter[charnum++] = 0;
-+ if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) {
-+ ip += bitCount >> 3;
-+ bitCount &= 7;
-+ bitStream = ZSTD_readLE32(ip) >> bitCount;
-+ } else {
-+ bitStream >>= 2;
-+ }
-+ }
-+ {
-+ int const max = (2 * threshold - 1) - remaining;
-+ int count;
-+
-+ if ((bitStream & (threshold - 1)) < (U32)max) {
-+ count = bitStream & (threshold - 1);
-+ bitCount += nbBits - 1;
-+ } else {
-+ count = bitStream & (2 * threshold - 1);
-+ if (count >= threshold)
-+ count -= max;
-+ bitCount += nbBits;
-+ }
-+
-+ count--; /* extra accuracy */
-+ remaining -= count < 0 ? -count : count; /* -1 means +1 */
-+ normalizedCounter[charnum++] = (short)count;
-+ previous0 = !count;
-+ while (remaining < threshold) {
-+ nbBits--;
-+ threshold >>= 1;
-+ }
-+
-+ if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) {
-+ ip += bitCount >> 3;
-+ bitCount &= 7;
-+ } else {
-+ bitCount -= (int)(8 * (iend - 4 - ip));
-+ ip = iend - 4;
-+ }
-+ bitStream = ZSTD_readLE32(ip) >> (bitCount & 31);
-+ }
-+ } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
-+ if (remaining != 1)
-+ return ERROR(corruption_detected);
-+ if (bitCount > 32)
-+ return ERROR(corruption_detected);
-+ *maxSVPtr = charnum - 1;
-+
-+ ip += (bitCount + 7) >> 3;
-+ return ip - istart;
-+}
-+
-+/*! HUF_readStats() :
-+ Read compact Huffman tree, saved by HUF_writeCTable().
-+ `huffWeight` is destination buffer.
-+ `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
-+ @return : size read from `src` , or an error Code .
-+ Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
-+*/
-+size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
-+{
-+ U32 weightTotal;
-+ const BYTE *ip = (const BYTE *)src;
-+ size_t iSize;
-+ size_t oSize;
-+
-+ if (!srcSize)
-+ return ERROR(srcSize_wrong);
-+ iSize = ip[0];
-+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
-+
-+ if (iSize >= 128) { /* special header */
-+ oSize = iSize - 127;
-+ iSize = ((oSize + 1) / 2);
-+ if (iSize + 1 > srcSize)
-+ return ERROR(srcSize_wrong);
-+ if (oSize >= hwSize)
-+ return ERROR(corruption_detected);
-+ ip += 1;
-+ {
-+ U32 n;
-+ for (n = 0; n < oSize; n += 2) {
-+ huffWeight[n] = ip[n / 2] >> 4;
-+ huffWeight[n + 1] = ip[n / 2] & 15;
-+ }
-+ }
-+ } else { /* header compressed with FSE (normal case) */
-+ if (iSize + 1 > srcSize)
-+ return ERROR(srcSize_wrong);
-+ oSize = FSE_decompress_wksp(huffWeight, hwSize - 1, ip + 1, iSize, 6, workspace, workspaceSize); /* max (hwSize-1) values decoded, as last one is implied */
-+ if (FSE_isError(oSize))
-+ return oSize;
-+ }
-+
-+ /* collect weight stats */
-+ memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
-+ weightTotal = 0;
-+ {
-+ U32 n;
-+ for (n = 0; n < oSize; n++) {
-+ if (huffWeight[n] >= HUF_TABLELOG_MAX)
-+ return ERROR(corruption_detected);
-+ rankStats[huffWeight[n]]++;
-+ weightTotal += (1 << huffWeight[n]) >> 1;
-+ }
-+ }
-+ if (weightTotal == 0)
-+ return ERROR(corruption_detected);
-+
-+ /* get last non-null symbol weight (implied, total must be 2^n) */
-+ {
-+ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
-+ if (tableLog > HUF_TABLELOG_MAX)
-+ return ERROR(corruption_detected);
-+ *tableLogPtr = tableLog;
-+ /* determine last weight */
-+ {
-+ U32 const total = 1 << tableLog;
-+ U32 const rest = total - weightTotal;
-+ U32 const verif = 1 << BIT_highbit32(rest);
-+ U32 const lastWeight = BIT_highbit32(rest) + 1;
-+ if (verif != rest)
-+ return ERROR(corruption_detected); /* last value must be a clean power of 2 */
-+ huffWeight[oSize] = (BYTE)lastWeight;
-+ rankStats[lastWeight]++;
-+ }
-+ }
-+
-+ /* check tree construction validity */
-+ if ((rankStats[1] < 2) || (rankStats[1] & 1))
-+ return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
-+
-+ /* results */
-+ *nbSymbolsPtr = (U32)(oSize + 1);
-+ return iSize + 1;
-+}
-diff --git a/lib/zstd/error_private.h b/lib/zstd/error_private.h
-new file mode 100644
-index 0000000..1a60b31
---- /dev/null
-+++ b/lib/zstd/error_private.h
-@@ -0,0 +1,53 @@
-+/**
-+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This source code is licensed under the BSD-style license found in the
-+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
-+ * An additional grant of patent rights can be found in the PATENTS file in the
-+ * same directory.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ */
-+
-+/* Note : this module is expected to remain private, do not expose it */
-+
-+#ifndef ERROR_H_MODULE
-+#define ERROR_H_MODULE
-+
-+/* ****************************************
-+* Dependencies
-+******************************************/
-+#include <linux/types.h> /* size_t */
-+#include <linux/zstd.h> /* enum list */
-+
-+/* ****************************************
-+* Compiler-specific
-+******************************************/
-+#define ERR_STATIC static __attribute__((unused))
-+
-+/*-****************************************
-+* Customization (error_public.h)
-+******************************************/
-+typedef ZSTD_ErrorCode ERR_enum;
-+#define PREFIX(name) ZSTD_error_##name
-+
-+/*-****************************************
-+* Error codes handling
-+******************************************/
-+#define ERROR(name) ((size_t)-PREFIX(name))
-+
-+ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
-+
-+ERR_STATIC ERR_enum ERR_getErrorCode(size_t code)
-+{
-+ if (!ERR_isError(code))
-+ return (ERR_enum)0;
-+ return (ERR_enum)(0 - code);
-+}
-+
-+#endif /* ERROR_H_MODULE */
-diff --git a/lib/zstd/fse.h b/lib/zstd/fse.h
-new file mode 100644
-index 0000000..7460ab0
---- /dev/null
-+++ b/lib/zstd/fse.h
-@@ -0,0 +1,575 @@
-+/*
-+ * FSE : Finite State Entropy codec
-+ * Public Prototypes declaration
-+ * Copyright (C) 2013-2016, Yann Collet.
-+ *
-+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are
-+ * met:
-+ *
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above
-+ * copyright notice, this list of conditions and the following disclaimer
-+ * in the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ *
-+ * You can contact the author at :
-+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-+ */
-+#ifndef FSE_H
-+#define FSE_H
-+
-+/*-*****************************************
-+* Dependencies
-+******************************************/
-+#include <linux/types.h> /* size_t, ptrdiff_t */
-+
-+/*-*****************************************
-+* FSE_PUBLIC_API : control library symbols visibility
-+******************************************/
-+#define FSE_PUBLIC_API
-+
-+/*------ Version ------*/
-+#define FSE_VERSION_MAJOR 0
-+#define FSE_VERSION_MINOR 9
-+#define FSE_VERSION_RELEASE 0
-+
-+#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
-+#define FSE_QUOTE(str) #str
-+#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
-+#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
-+
-+#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR * 100 * 100 + FSE_VERSION_MINOR * 100 + FSE_VERSION_RELEASE)
-+FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
-+
-+/*-*****************************************
-+* Tool functions
-+******************************************/
-+FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
-+
-+/* Error Management */
-+FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
-+
-+/*-*****************************************
-+* FSE detailed API
-+******************************************/
-+/*!
-+FSE_compress() does the following:
-+1. count symbol occurrence from source[] into table count[]
-+2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
-+3. save normalized counters to memory buffer using writeNCount()
-+4. build encoding table 'CTable' from normalized counters
-+5. encode the data stream using encoding table 'CTable'
-+
-+FSE_decompress() does the following:
-+1. read normalized counters with readNCount()
-+2. build decoding table 'DTable' from normalized counters
-+3. decode the data stream using decoding table 'DTable'
-+
-+The following API allows targeting specific sub-functions for advanced tasks.
-+For example, it's possible to compress several blocks using the same 'CTable',
-+or to save and provide normalized distribution using external method.
-+*/
-+
-+/* *** COMPRESSION *** */
-+/*! FSE_optimalTableLog():
-+ dynamically downsize 'tableLog' when conditions are met.
-+ It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
-+ @return : recommended tableLog (necessarily <= 'maxTableLog') */
-+FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
-+
-+/*! FSE_normalizeCount():
-+ normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
-+ 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
-+ @return : tableLog,
-+ or an errorCode, which can be tested using FSE_isError() */
-+FSE_PUBLIC_API size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue);
-+
-+/*! FSE_NCountWriteBound():
-+ Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
-+ Typically useful for allocation purpose. */
-+FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
-+
-+/*! FSE_writeNCount():
-+ Compactly save 'normalizedCounter' into 'buffer'.
-+ @return : size of the compressed table,
-+ or an errorCode, which can be tested using FSE_isError(). */
-+FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
-+
-+/*! Constructor and Destructor of FSE_CTable.
-+ Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
-+typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
-+
-+/*! FSE_compress_usingCTable():
-+ Compress `src` using `ct` into `dst` which must be already allocated.
-+ @return : size of compressed data (<= `dstCapacity`),
-+ or 0 if compressed data could not fit into `dst`,
-+ or an errorCode, which can be tested using FSE_isError() */
-+FSE_PUBLIC_API size_t FSE_compress_usingCTable(void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct);
-+
-+/*!
-+Tutorial :
-+----------
-+The first step is to count all symbols. FSE_count() does this job very fast.
-+Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
-+'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
-+maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
-+FSE_count() will return the number of occurrence of the most frequent symbol.
-+This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
-+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
-+
-+The next step is to normalize the frequencies.
-+FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
-+It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
-+You can use 'tableLog'==0 to mean "use default tableLog value".
-+If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
-+which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
-+
-+The result of FSE_normalizeCount() will be saved into a table,
-+called 'normalizedCounter', which is a table of signed short.
-+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
-+The return value is tableLog if everything proceeded as expected.
-+It is 0 if there is a single symbol within distribution.
-+If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
-+
-+'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
-+'buffer' must be already allocated.
-+For guaranteed success, buffer size must be at least FSE_headerBound().
-+The result of the function is the number of bytes written into 'buffer'.
-+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
-+
-+'normalizedCounter' can then be used to create the compression table 'CTable'.
-+The space required by 'CTable' must be already allocated, using FSE_createCTable().
-+You can then use FSE_buildCTable() to fill 'CTable'.
-+If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
-+
-+'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
-+Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
-+The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
-+If it returns '0', compressed data could not fit into 'dst'.
-+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
-+*/
-+
-+/* *** DECOMPRESSION *** */
-+
-+/*! FSE_readNCount():
-+ Read compactly saved 'normalizedCounter' from 'rBuffer'.
-+ @return : size read from 'rBuffer',
-+ or an errorCode, which can be tested using FSE_isError().
-+ maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
-+FSE_PUBLIC_API size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize);
-+
-+/*! Constructor and Destructor of FSE_DTable.
-+ Note that its size depends on 'tableLog' */
-+typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
-+
-+/*! FSE_buildDTable():
-+ Builds 'dt', which must be already allocated, using FSE_createDTable().
-+ return : 0, or an errorCode, which can be tested using FSE_isError() */
-+FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize);
-+
-+/*! FSE_decompress_usingDTable():
-+ Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
-+ into `dst` which must be already allocated.
-+ @return : size of regenerated data (necessarily <= `dstCapacity`),
-+ or an errorCode, which can be tested using FSE_isError() */
-+FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt);
-+
-+/*!
-+Tutorial :
-+----------
-+(Note : these functions only decompress FSE-compressed blocks.
-+ If block is uncompressed, use memcpy() instead
-+ If block is a single repeated byte, use memset() instead )
-+
-+The first step is to obtain the normalized frequencies of symbols.
-+This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
-+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
-+In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
-+or size the table to handle worst case situations (typically 256).
-+FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
-+The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
-+Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
-+If there is an error, the function will return an error code, which can be tested using FSE_isError().
-+
-+The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
-+This is performed by the function FSE_buildDTable().
-+The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
-+If there is an error, the function will return an error code, which can be tested using FSE_isError().
-+
-+`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
-+`cSrcSize` must be strictly correct, otherwise decompression will fail.
-+FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
-+If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
-+*/
-+
-+/* *** Dependency *** */
-+#include "bitstream.h"
-+
-+/* *****************************************
-+* Static allocation
-+*******************************************/
-+/* FSE buffer bounds */
-+#define FSE_NCOUNTBOUND 512
-+#define FSE_BLOCKBOUND(size) (size + (size >> 7))
-+#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
-+
-+/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
-+#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2))
-+#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog))
-+
-+/* *****************************************
-+* FSE advanced API
-+*******************************************/
-+/* FSE_count_wksp() :
-+ * Same as FSE_count(), but using an externally provided scratch buffer.
-+ * `workSpace` size must be table of >= `1024` unsigned
-+ */
-+size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace);
-+
-+/* FSE_countFast_wksp() :
-+ * Same as FSE_countFast(), but using an externally provided scratch buffer.
-+ * `workSpace` must be a table of minimum `1024` unsigned
-+ */
-+size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize, unsigned *workSpace);
-+
-+/*! FSE_count_simple
-+ * Same as FSE_countFast(), but does not use any additional memory (not even on stack).
-+ * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
-+*/
-+size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize);
-+
-+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
-+/**< same as FSE_optimalTableLog(), which used `minus==2` */
-+
-+size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits);
-+/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
-+
-+size_t FSE_buildCTable_rle(FSE_CTable *ct, unsigned char symbolValue);
-+/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
-+
-+/* FSE_buildCTable_wksp() :
-+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
-+ * `wkspSize` must be >= `(1<<tableLog)`.
-+ */
-+size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, size_t wkspSize);
-+
-+size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits);
-+/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
-+
-+size_t FSE_buildDTable_rle(FSE_DTable *dt, unsigned char symbolValue);
-+/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
-+
-+size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize);
-+/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
-+
-+/* *****************************************
-+* FSE symbol compression API
-+*******************************************/
-+/*!
-+ This API consists of small unitary functions, which highly benefit from being inlined.
-+ Hence their body are included in next section.
-+*/
-+typedef struct {
-+ ptrdiff_t value;
-+ const void *stateTable;
-+ const void *symbolTT;
-+ unsigned stateLog;
-+} FSE_CState_t;
-+
-+static void FSE_initCState(FSE_CState_t *CStatePtr, const FSE_CTable *ct);
-+
-+static void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *CStatePtr, unsigned symbol);
-+
-+static void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *CStatePtr);
-+
-+/**<
-+These functions are inner components of FSE_compress_usingCTable().
-+They allow the creation of custom streams, mixing multiple tables and bit sources.
-+
-+A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
-+So the first symbol you will encode is the last you will decode, like a LIFO stack.
-+
-+You will need a few variables to track your CStream. They are :
-+
-+FSE_CTable ct; // Provided by FSE_buildCTable()
-+BIT_CStream_t bitStream; // bitStream tracking structure
-+FSE_CState_t state; // State tracking structure (can have several)
-+
-+
-+The first thing to do is to init bitStream and state.
-+ size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
-+ FSE_initCState(&state, ct);
-+
-+Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
-+You can then encode your input data, byte after byte.
-+FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
-+Remember decoding will be done in reverse direction.
-+ FSE_encodeByte(&bitStream, &state, symbol);
-+
-+At any time, you can also add any bit sequence.
-+Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
-+ BIT_addBits(&bitStream, bitField, nbBits);
-+
-+The above methods don't commit data to memory, they just store it into local register, for speed.
-+Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
-+Writing data to memory is a manual operation, performed by the flushBits function.
-+ BIT_flushBits(&bitStream);
-+
-+Your last FSE encoding operation shall be to flush your last state value(s).
-+ FSE_flushState(&bitStream, &state);
-+
-+Finally, you must close the bitStream.
-+The function returns the size of CStream in bytes.
-+If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
-+If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
-+ size_t size = BIT_closeCStream(&bitStream);
-+*/
-+
-+/* *****************************************
-+* FSE symbol decompression API
-+*******************************************/
-+typedef struct {
-+ size_t state;
-+ const void *table; /* precise table may vary, depending on U16 */
-+} FSE_DState_t;
-+
-+static void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt);
-+
-+static unsigned char FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
-+
-+static unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr);
-+
-+/**<
-+Let's now decompose FSE_decompress_usingDTable() into its unitary components.
-+You will decode FSE-encoded symbols from the bitStream,
-+and also any other bitFields you put in, **in reverse order**.
-+
-+You will need a few variables to track your bitStream. They are :
-+
-+BIT_DStream_t DStream; // Stream context
-+FSE_DState_t DState; // State context. Multiple ones are possible
-+FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
-+
-+The first thing to do is to init the bitStream.
-+ errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
-+
-+You should then retrieve your initial state(s)
-+(in reverse flushing order if you have several ones) :
-+ errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
-+
-+You can then decode your data, symbol after symbol.
-+For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
-+Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
-+ unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
-+
-+You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
-+Note : maximum allowed nbBits is 25, for 32-bits compatibility
-+ size_t bitField = BIT_readBits(&DStream, nbBits);
-+
-+All above operations only read from local register (which size depends on size_t).
-+Refueling the register from memory is manually performed by the reload method.
-+ endSignal = FSE_reloadDStream(&DStream);
-+
-+BIT_reloadDStream() result tells if there is still some more data to read from DStream.
-+BIT_DStream_unfinished : there is still some data left into the DStream.
-+BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
-+BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
-+BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
-+
-+When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
-+to properly detect the exact end of stream.
-+After each decoded symbol, check if DStream is fully consumed using this simple test :
-+ BIT_reloadDStream(&DStream) >= BIT_DStream_completed
-+
-+When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
-+Checking if DStream has reached its end is performed by :
-+ BIT_endOfDStream(&DStream);
-+Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
-+ FSE_endOfDState(&DState);
-+*/
-+
-+/* *****************************************
-+* FSE unsafe API
-+*******************************************/
-+static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
-+/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
-+
-+/* *****************************************
-+* Implementation of inlined functions
-+*******************************************/
-+typedef struct {
-+ int deltaFindState;
-+ U32 deltaNbBits;
-+} FSE_symbolCompressionTransform; /* total 8 bytes */
-+
-+ZSTD_STATIC void FSE_initCState(FSE_CState_t *statePtr, const FSE_CTable *ct)
-+{
-+ const void *ptr = ct;
-+ const U16 *u16ptr = (const U16 *)ptr;
-+ const U32 tableLog = ZSTD_read16(ptr);
-+ statePtr->value = (ptrdiff_t)1 << tableLog;
-+ statePtr->stateTable = u16ptr + 2;
-+ statePtr->symbolTT = ((const U32 *)ct + 1 + (tableLog ? (1 << (tableLog - 1)) : 1));
-+ statePtr->stateLog = tableLog;
-+}
-+
-+/*! FSE_initCState2() :
-+* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
-+* uses the smallest state value possible, saving the cost of this symbol */
-+ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol)
-+{
-+ FSE_initCState(statePtr, ct);
-+ {
-+ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol];
-+ const U16 *stateTable = (const U16 *)(statePtr->stateTable);
-+ U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16);
-+ statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
-+ statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
-+ }
-+}
-+
-+ZSTD_STATIC void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *statePtr, U32 symbol)
-+{
-+ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol];
-+ const U16 *const stateTable = (const U16 *)(statePtr->stateTable);
-+ U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
-+ BIT_addBits(bitC, statePtr->value, nbBitsOut);
-+ statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
-+}
-+
-+ZSTD_STATIC void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *statePtr)
-+{
-+ BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
-+ BIT_flushBits(bitC);
-+}
-+
-+/* ====== Decompression ====== */
-+
-+typedef struct {
-+ U16 tableLog;
-+ U16 fastMode;
-+} FSE_DTableHeader; /* sizeof U32 */
-+
-+typedef struct {
-+ unsigned short newState;
-+ unsigned char symbol;
-+ unsigned char nbBits;
-+} FSE_decode_t; /* size == U32 */
-+
-+ZSTD_STATIC void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt)
-+{
-+ const void *ptr = dt;
-+ const FSE_DTableHeader *const DTableH = (const FSE_DTableHeader *)ptr;
-+ DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
-+ BIT_reloadDStream(bitD);
-+ DStatePtr->table = dt + 1;
-+}
-+
-+ZSTD_STATIC BYTE FSE_peekSymbol(const FSE_DState_t *DStatePtr)
-+{
-+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
-+ return DInfo.symbol;
-+}
-+
-+ZSTD_STATIC void FSE_updateState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
-+{
-+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
-+ U32 const nbBits = DInfo.nbBits;
-+ size_t const lowBits = BIT_readBits(bitD, nbBits);
-+ DStatePtr->state = DInfo.newState + lowBits;
-+}
-+
-+ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
-+{
-+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
-+ U32 const nbBits = DInfo.nbBits;
-+ BYTE const symbol = DInfo.symbol;
-+ size_t const lowBits = BIT_readBits(bitD, nbBits);
-+
-+ DStatePtr->state = DInfo.newState + lowBits;
-+ return symbol;
-+}
-+
-+/*! FSE_decodeSymbolFast() :
-+ unsafe, only works if no symbol has a probability > 50% */
-+ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
-+{
-+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
-+ U32 const nbBits = DInfo.nbBits;
-+ BYTE const symbol = DInfo.symbol;
-+ size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
-+
-+ DStatePtr->state = DInfo.newState + lowBits;
-+ return symbol;
-+}
-+
-+ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; }
-+
-+/* **************************************************************
-+* Tuning parameters
-+****************************************************************/
-+/*!MEMORY_USAGE :
-+* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
-+* Increasing memory usage improves compression ratio
-+* Reduced memory usage can improve speed, due to cache effect
-+* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
-+#ifndef FSE_MAX_MEMORY_USAGE
-+#define FSE_MAX_MEMORY_USAGE 14
-+#endif
-+#ifndef FSE_DEFAULT_MEMORY_USAGE
-+#define FSE_DEFAULT_MEMORY_USAGE 13
-+#endif
-+
-+/*!FSE_MAX_SYMBOL_VALUE :
-+* Maximum symbol value authorized.
-+* Required for proper stack allocation */
-+#ifndef FSE_MAX_SYMBOL_VALUE
-+#define FSE_MAX_SYMBOL_VALUE 255
-+#endif
-+
-+/* **************************************************************
-+* template functions type & suffix
-+****************************************************************/
-+#define FSE_FUNCTION_TYPE BYTE
-+#define FSE_FUNCTION_EXTENSION
-+#define FSE_DECODE_TYPE FSE_decode_t
-+
-+/* ***************************************************************
-+* Constants
-+*****************************************************************/
-+#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE - 2)
-+#define FSE_MAX_TABLESIZE (1U << FSE_MAX_TABLELOG)
-+#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE - 1)
-+#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE - 2)
-+#define FSE_MIN_TABLELOG 5
-+
-+#define FSE_TABLELOG_ABSOLUTE_MAX 15
-+#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
-+#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
-+#endif
-+
-+#define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3)
-+
-+#endif /* FSE_H */
-diff --git a/lib/zstd/fse_compress.c b/lib/zstd/fse_compress.c
-new file mode 100644
-index 0000000..ef3d174
---- /dev/null
-+++ b/lib/zstd/fse_compress.c
-@@ -0,0 +1,795 @@
-+/*
-+ * FSE : Finite State Entropy encoder
-+ * Copyright (C) 2013-2015, Yann Collet.
-+ *
-+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are
-+ * met:
-+ *
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above
-+ * copyright notice, this list of conditions and the following disclaimer
-+ * in the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ *
-+ * You can contact the author at :
-+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-+ */
-+
-+/* **************************************************************
-+* Compiler specifics
-+****************************************************************/
-+#define FORCE_INLINE static __always_inline
-+
-+/* **************************************************************
-+* Includes
-+****************************************************************/
-+#include "bitstream.h"
-+#include "fse.h"
-+#include <linux/compiler.h>
-+#include <linux/kernel.h>
-+#include <linux/math64.h>
-+#include <linux/string.h> /* memcpy, memset */
-+
-+/* **************************************************************
-+* Error Management
-+****************************************************************/
-+#define FSE_STATIC_ASSERT(c) \
-+ { \
-+ enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
-+ } /* use only *after* variable declarations */
-+
-+/* **************************************************************
-+* Templates
-+****************************************************************/
-+/*
-+ designed to be included
-+ for type-specific functions (template emulation in C)
-+ Objective is to write these functions only once, for improved maintenance
-+*/
-+
-+/* safety checks */
-+#ifndef FSE_FUNCTION_EXTENSION
-+#error "FSE_FUNCTION_EXTENSION must be defined"
-+#endif
-+#ifndef FSE_FUNCTION_TYPE
-+#error "FSE_FUNCTION_TYPE must be defined"
-+#endif
-+
-+/* Function names */
-+#define FSE_CAT(X, Y) X##Y
-+#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
-+#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
-+
-+/* Function templates */
-+
-+/* FSE_buildCTable_wksp() :
-+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
-+ * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
-+ * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
-+ */
-+size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize)
-+{
-+ U32 const tableSize = 1 << tableLog;
-+ U32 const tableMask = tableSize - 1;
-+ void *const ptr = ct;
-+ U16 *const tableU16 = ((U16 *)ptr) + 2;
-+ void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableLog ? tableSize >> 1 : 1);
-+ FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT);
-+ U32 const step = FSE_TABLESTEP(tableSize);
-+ U32 highThreshold = tableSize - 1;
-+
-+ U32 *cumul;
-+ FSE_FUNCTION_TYPE *tableSymbol;
-+ size_t spaceUsed32 = 0;
-+
-+ cumul = (U32 *)workspace + spaceUsed32;
-+ spaceUsed32 += FSE_MAX_SYMBOL_VALUE + 2;
-+ tableSymbol = (FSE_FUNCTION_TYPE *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += ALIGN(sizeof(FSE_FUNCTION_TYPE) * ((size_t)1 << tableLog), sizeof(U32)) >> 2;
-+
-+ if ((spaceUsed32 << 2) > workspaceSize)
-+ return ERROR(tableLog_tooLarge);
-+ workspace = (U32 *)workspace + spaceUsed32;
-+ workspaceSize -= (spaceUsed32 << 2);
-+
-+ /* CTable header */
-+ tableU16[-2] = (U16)tableLog;
-+ tableU16[-1] = (U16)maxSymbolValue;
-+
-+ /* For explanations on how to distribute symbol values over the table :
-+ * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
-+
-+ /* symbol start positions */
-+ {
-+ U32 u;
-+ cumul[0] = 0;
-+ for (u = 1; u <= maxSymbolValue + 1; u++) {
-+ if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */
-+ cumul[u] = cumul[u - 1] + 1;
-+ tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1);
-+ } else {
-+ cumul[u] = cumul[u - 1] + normalizedCounter[u - 1];
-+ }
-+ }
-+ cumul[maxSymbolValue + 1] = tableSize + 1;
-+ }
-+
-+ /* Spread symbols */
-+ {
-+ U32 position = 0;
-+ U32 symbol;
-+ for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
-+ int nbOccurences;
-+ for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
-+ tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
-+ position = (position + step) & tableMask;
-+ while (position > highThreshold)
-+ position = (position + step) & tableMask; /* Low proba area */
-+ }
-+ }
-+
-+ if (position != 0)
-+ return ERROR(GENERIC); /* Must have gone through all positions */
-+ }
-+
-+ /* Build table */
-+ {
-+ U32 u;
-+ for (u = 0; u < tableSize; u++) {
-+ FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
-+ tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */
-+ }
-+ }
-+
-+ /* Build Symbol Transformation Table */
-+ {
-+ unsigned total = 0;
-+ unsigned s;
-+ for (s = 0; s <= maxSymbolValue; s++) {
-+ switch (normalizedCounter[s]) {
-+ case 0: break;
-+
-+ case -1:
-+ case 1:
-+ symbolTT[s].deltaNbBits = (tableLog << 16) - (1 << tableLog);
-+ symbolTT[s].deltaFindState = total - 1;
-+ total++;
-+ break;
-+ default: {
-+ U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1);
-+ U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
-+ symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
-+ symbolTT[s].deltaFindState = total - normalizedCounter[s];
-+ total += normalizedCounter[s];
-+ }
-+ }
-+ }
-+ }
-+
-+ return 0;
-+}
-+
-+/*-**************************************************************
-+* FSE NCount encoding-decoding
-+****************************************************************/
-+size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
-+{
-+ size_t const maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3;
-+ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
-+}
-+
-+static size_t FSE_writeNCount_generic(void *header, size_t headerBufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
-+ unsigned writeIsSafe)
-+{
-+ BYTE *const ostart = (BYTE *)header;
-+ BYTE *out = ostart;
-+ BYTE *const oend = ostart + headerBufferSize;
-+ int nbBits;
-+ const int tableSize = 1 << tableLog;
-+ int remaining;
-+ int threshold;
-+ U32 bitStream;
-+ int bitCount;
-+ unsigned charnum = 0;
-+ int previous0 = 0;
-+
-+ bitStream = 0;
-+ bitCount = 0;
-+ /* Table Size */
-+ bitStream += (tableLog - FSE_MIN_TABLELOG) << bitCount;
-+ bitCount += 4;
-+
-+ /* Init */
-+ remaining = tableSize + 1; /* +1 for extra accuracy */
-+ threshold = tableSize;
-+ nbBits = tableLog + 1;
-+
-+ while (remaining > 1) { /* stops at 1 */
-+ if (previous0) {
-+ unsigned start = charnum;
-+ while (!normalizedCounter[charnum])
-+ charnum++;
-+ while (charnum >= start + 24) {
-+ start += 24;
-+ bitStream += 0xFFFFU << bitCount;
-+ if ((!writeIsSafe) && (out > oend - 2))
-+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
-+ out[0] = (BYTE)bitStream;
-+ out[1] = (BYTE)(bitStream >> 8);
-+ out += 2;
-+ bitStream >>= 16;
-+ }
-+ while (charnum >= start + 3) {
-+ start += 3;
-+ bitStream += 3 << bitCount;
-+ bitCount += 2;
-+ }
-+ bitStream += (charnum - start) << bitCount;
-+ bitCount += 2;
-+ if (bitCount > 16) {
-+ if ((!writeIsSafe) && (out > oend - 2))
-+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
-+ out[0] = (BYTE)bitStream;
-+ out[1] = (BYTE)(bitStream >> 8);
-+ out += 2;
-+ bitStream >>= 16;
-+ bitCount -= 16;
-+ }
-+ }
-+ {
-+ int count = normalizedCounter[charnum++];
-+ int const max = (2 * threshold - 1) - remaining;
-+ remaining -= count < 0 ? -count : count;
-+ count++; /* +1 for extra accuracy */
-+ if (count >= threshold)
-+ count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
-+ bitStream += count << bitCount;
-+ bitCount += nbBits;
-+ bitCount -= (count < max);
-+ previous0 = (count == 1);
-+ if (remaining < 1)
-+ return ERROR(GENERIC);
-+ while (remaining < threshold)
-+ nbBits--, threshold >>= 1;
-+ }
-+ if (bitCount > 16) {
-+ if ((!writeIsSafe) && (out > oend - 2))
-+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
-+ out[0] = (BYTE)bitStream;
-+ out[1] = (BYTE)(bitStream >> 8);
-+ out += 2;
-+ bitStream >>= 16;
-+ bitCount -= 16;
-+ }
-+ }
-+
-+ /* flush remaining bitStream */
-+ if ((!writeIsSafe) && (out > oend - 2))
-+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
-+ out[0] = (BYTE)bitStream;
-+ out[1] = (BYTE)(bitStream >> 8);
-+ out += (bitCount + 7) / 8;
-+
-+ if (charnum > maxSymbolValue + 1)
-+ return ERROR(GENERIC);
-+
-+ return (out - ostart);
-+}
-+
-+size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
-+{
-+ if (tableLog > FSE_MAX_TABLELOG)
-+ return ERROR(tableLog_tooLarge); /* Unsupported */
-+ if (tableLog < FSE_MIN_TABLELOG)
-+ return ERROR(GENERIC); /* Unsupported */
-+
-+ if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
-+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
-+
-+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
-+}
-+
-+/*-**************************************************************
-+* Counting histogram
-+****************************************************************/
-+/*! FSE_count_simple
-+ This function counts byte values within `src`, and store the histogram into table `count`.
-+ It doesn't use any additional memory.
-+ But this function is unsafe : it doesn't check that all values within `src` can fit into `count`.
-+ For this reason, prefer using a table `count` with 256 elements.
-+ @return : count of most numerous element
-+*/
-+size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize)
-+{
-+ const BYTE *ip = (const BYTE *)src;
-+ const BYTE *const end = ip + srcSize;
-+ unsigned maxSymbolValue = *maxSymbolValuePtr;
-+ unsigned max = 0;
-+
-+ memset(count, 0, (maxSymbolValue + 1) * sizeof(*count));
-+ if (srcSize == 0) {
-+ *maxSymbolValuePtr = 0;
-+ return 0;
-+ }
-+
-+ while (ip < end)
-+ count[*ip++]++;
-+
-+ while (!count[maxSymbolValue])
-+ maxSymbolValue--;
-+ *maxSymbolValuePtr = maxSymbolValue;
-+
-+ {
-+ U32 s;
-+ for (s = 0; s <= maxSymbolValue; s++)
-+ if (count[s] > max)
-+ max = count[s];
-+ }
-+
-+ return (size_t)max;
-+}
-+
-+/* FSE_count_parallel_wksp() :
-+ * Same as FSE_count_parallel(), but using an externally provided scratch buffer.
-+ * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */
-+static size_t FSE_count_parallel_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned checkMax,
-+ unsigned *const workSpace)
-+{
-+ const BYTE *ip = (const BYTE *)source;
-+ const BYTE *const iend = ip + sourceSize;
-+ unsigned maxSymbolValue = *maxSymbolValuePtr;
-+ unsigned max = 0;
-+ U32 *const Counting1 = workSpace;
-+ U32 *const Counting2 = Counting1 + 256;
-+ U32 *const Counting3 = Counting2 + 256;
-+ U32 *const Counting4 = Counting3 + 256;
-+
-+ memset(Counting1, 0, 4 * 256 * sizeof(unsigned));
-+
-+ /* safety checks */
-+ if (!sourceSize) {
-+ memset(count, 0, maxSymbolValue + 1);
-+ *maxSymbolValuePtr = 0;
-+ return 0;
-+ }
-+ if (!maxSymbolValue)
-+ maxSymbolValue = 255; /* 0 == default */
-+
-+ /* by stripes of 16 bytes */
-+ {
-+ U32 cached = ZSTD_read32(ip);
-+ ip += 4;
-+ while (ip < iend - 15) {
-+ U32 c = cached;
-+ cached = ZSTD_read32(ip);
-+ ip += 4;
-+ Counting1[(BYTE)c]++;
-+ Counting2[(BYTE)(c >> 8)]++;
-+ Counting3[(BYTE)(c >> 16)]++;
-+ Counting4[c >> 24]++;
-+ c = cached;
-+ cached = ZSTD_read32(ip);
-+ ip += 4;
-+ Counting1[(BYTE)c]++;
-+ Counting2[(BYTE)(c >> 8)]++;
-+ Counting3[(BYTE)(c >> 16)]++;
-+ Counting4[c >> 24]++;
-+ c = cached;
-+ cached = ZSTD_read32(ip);
-+ ip += 4;
-+ Counting1[(BYTE)c]++;
-+ Counting2[(BYTE)(c >> 8)]++;
-+ Counting3[(BYTE)(c >> 16)]++;
-+ Counting4[c >> 24]++;
-+ c = cached;
-+ cached = ZSTD_read32(ip);
-+ ip += 4;
-+ Counting1[(BYTE)c]++;
-+ Counting2[(BYTE)(c >> 8)]++;
-+ Counting3[(BYTE)(c >> 16)]++;
-+ Counting4[c >> 24]++;
-+ }
-+ ip -= 4;
-+ }
-+
-+ /* finish last symbols */
-+ while (ip < iend)
-+ Counting1[*ip++]++;
-+
-+ if (checkMax) { /* verify stats will fit into destination table */
-+ U32 s;
-+ for (s = 255; s > maxSymbolValue; s--) {
-+ Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
-+ if (Counting1[s])
-+ return ERROR(maxSymbolValue_tooSmall);
-+ }
-+ }
-+
-+ {
-+ U32 s;
-+ for (s = 0; s <= maxSymbolValue; s++) {
-+ count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
-+ if (count[s] > max)
-+ max = count[s];
-+ }
-+ }
-+
-+ while (!count[maxSymbolValue])
-+ maxSymbolValue--;
-+ *maxSymbolValuePtr = maxSymbolValue;
-+ return (size_t)max;
-+}
-+
-+/* FSE_countFast_wksp() :
-+ * Same as FSE_countFast(), but using an externally provided scratch buffer.
-+ * `workSpace` size must be table of >= `1024` unsigned */
-+size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
-+{
-+ if (sourceSize < 1500)
-+ return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize);
-+ return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
-+}
-+
-+/* FSE_count_wksp() :
-+ * Same as FSE_count(), but using an externally provided scratch buffer.
-+ * `workSpace` size must be table of >= `1024` unsigned */
-+size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
-+{
-+ if (*maxSymbolValuePtr < 255)
-+ return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
-+ *maxSymbolValuePtr = 255;
-+ return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
-+}
-+
-+/*-**************************************************************
-+* FSE Compression Code
-+****************************************************************/
-+/*! FSE_sizeof_CTable() :
-+ FSE_CTable is a variable size structure which contains :
-+ `U16 tableLog;`
-+ `U16 maxSymbolValue;`
-+ `U16 nextStateNumber[1 << tableLog];` // This size is variable
-+ `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable
-+Allocation is manual (C standard does not support variable-size structures).
-+*/
-+size_t FSE_sizeof_CTable(unsigned maxSymbolValue, unsigned tableLog)
-+{
-+ if (tableLog > FSE_MAX_TABLELOG)
-+ return ERROR(tableLog_tooLarge);
-+ return FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32);
-+}
-+
-+/* provides the minimum logSize to safely represent a distribution */
-+static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
-+{
-+ U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1;
-+ U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
-+ U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
-+ return minBits;
-+}
-+
-+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
-+{
-+ U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
-+ U32 tableLog = maxTableLog;
-+ U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
-+ if (tableLog == 0)
-+ tableLog = FSE_DEFAULT_TABLELOG;
-+ if (maxBitsSrc < tableLog)
-+ tableLog = maxBitsSrc; /* Accuracy can be reduced */
-+ if (minBits > tableLog)
-+ tableLog = minBits; /* Need a minimum to safely represent all symbol values */
-+ if (tableLog < FSE_MIN_TABLELOG)
-+ tableLog = FSE_MIN_TABLELOG;
-+ if (tableLog > FSE_MAX_TABLELOG)
-+ tableLog = FSE_MAX_TABLELOG;
-+ return tableLog;
-+}
-+
-+unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
-+{
-+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
-+}
-+
-+/* Secondary normalization method.
-+ To be used when primary method fails. */
-+
-+static size_t FSE_normalizeM2(short *norm, U32 tableLog, const unsigned *count, size_t total, U32 maxSymbolValue)
-+{
-+ short const NOT_YET_ASSIGNED = -2;
-+ U32 s;
-+ U32 distributed = 0;
-+ U32 ToDistribute;
-+
-+ /* Init */
-+ U32 const lowThreshold = (U32)(total >> tableLog);
-+ U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
-+
-+ for (s = 0; s <= maxSymbolValue; s++) {
-+ if (count[s] == 0) {
-+ norm[s] = 0;
-+ continue;
-+ }
-+ if (count[s] <= lowThreshold) {
-+ norm[s] = -1;
-+ distributed++;
-+ total -= count[s];
-+ continue;
-+ }
-+ if (count[s] <= lowOne) {
-+ norm[s] = 1;
-+ distributed++;
-+ total -= count[s];
-+ continue;
-+ }
-+
-+ norm[s] = NOT_YET_ASSIGNED;
-+ }
-+ ToDistribute = (1 << tableLog) - distributed;
-+
-+ if ((total / ToDistribute) > lowOne) {
-+ /* risk of rounding to zero */
-+ lowOne = (U32)((total * 3) / (ToDistribute * 2));
-+ for (s = 0; s <= maxSymbolValue; s++) {
-+ if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
-+ norm[s] = 1;
-+ distributed++;
-+ total -= count[s];
-+ continue;
-+ }
-+ }
-+ ToDistribute = (1 << tableLog) - distributed;
-+ }
-+
-+ if (distributed == maxSymbolValue + 1) {
-+ /* all values are pretty poor;
-+ probably incompressible data (should have already been detected);
-+ find max, then give all remaining points to max */
-+ U32 maxV = 0, maxC = 0;
-+ for (s = 0; s <= maxSymbolValue; s++)
-+ if (count[s] > maxC)
-+ maxV = s, maxC = count[s];
-+ norm[maxV] += (short)ToDistribute;
-+ return 0;
-+ }
-+
-+ if (total == 0) {
-+ /* all of the symbols were low enough for the lowOne or lowThreshold */
-+ for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1))
-+ if (norm[s] > 0)
-+ ToDistribute--, norm[s]++;
-+ return 0;
-+ }
-+
-+ {
-+ U64 const vStepLog = 62 - tableLog;
-+ U64 const mid = (1ULL << (vStepLog - 1)) - 1;
-+ U64 const rStep = div_u64((((U64)1 << vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
-+ U64 tmpTotal = mid;
-+ for (s = 0; s <= maxSymbolValue; s++) {
-+ if (norm[s] == NOT_YET_ASSIGNED) {
-+ U64 const end = tmpTotal + (count[s] * rStep);
-+ U32 const sStart = (U32)(tmpTotal >> vStepLog);
-+ U32 const sEnd = (U32)(end >> vStepLog);
-+ U32 const weight = sEnd - sStart;
-+ if (weight < 1)
-+ return ERROR(GENERIC);
-+ norm[s] = (short)weight;
-+ tmpTotal = end;
-+ }
-+ }
-+ }
-+
-+ return 0;
-+}
-+
-+size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t total, unsigned maxSymbolValue)
-+{
-+ /* Sanity checks */
-+ if (tableLog == 0)
-+ tableLog = FSE_DEFAULT_TABLELOG;
-+ if (tableLog < FSE_MIN_TABLELOG)
-+ return ERROR(GENERIC); /* Unsupported size */
-+ if (tableLog > FSE_MAX_TABLELOG)
-+ return ERROR(tableLog_tooLarge); /* Unsupported size */
-+ if (tableLog < FSE_minTableLog(total, maxSymbolValue))
-+ return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
-+
-+ {
-+ U32 const rtbTable[] = {0, 473195, 504333, 520860, 550000, 700000, 750000, 830000};
-+ U64 const scale = 62 - tableLog;
-+ U64 const step = div_u64((U64)1 << 62, (U32)total); /* <== here, one division ! */
-+ U64 const vStep = 1ULL << (scale - 20);
-+ int stillToDistribute = 1 << tableLog;
-+ unsigned s;
-+ unsigned largest = 0;
-+ short largestP = 0;
-+ U32 lowThreshold = (U32)(total >> tableLog);
-+
-+ for (s = 0; s <= maxSymbolValue; s++) {
-+ if (count[s] == total)
-+ return 0; /* rle special case */
-+ if (count[s] == 0) {
-+ normalizedCounter[s] = 0;
-+ continue;
-+ }
-+ if (count[s] <= lowThreshold) {
-+ normalizedCounter[s] = -1;
-+ stillToDistribute--;
-+ } else {
-+ short proba = (short)((count[s] * step) >> scale);
-+ if (proba < 8) {
-+ U64 restToBeat = vStep * rtbTable[proba];
-+ proba += (count[s] * step) - ((U64)proba << scale) > restToBeat;
-+ }
-+ if (proba > largestP)
-+ largestP = proba, largest = s;
-+ normalizedCounter[s] = proba;
-+ stillToDistribute -= proba;
-+ }
-+ }
-+ if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
-+ /* corner case, need another normalization method */
-+ size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
-+ if (FSE_isError(errorCode))
-+ return errorCode;
-+ } else
-+ normalizedCounter[largest] += (short)stillToDistribute;
-+ }
-+
-+ return tableLog;
-+}
-+
-+/* fake FSE_CTable, for raw (uncompressed) input */
-+size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits)
-+{
-+ const unsigned tableSize = 1 << nbBits;
-+ const unsigned tableMask = tableSize - 1;
-+ const unsigned maxSymbolValue = tableMask;
-+ void *const ptr = ct;
-+ U16 *const tableU16 = ((U16 *)ptr) + 2;
-+ void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableSize >> 1); /* assumption : tableLog >= 1 */
-+ FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT);
-+ unsigned s;
-+
-+ /* Sanity checks */
-+ if (nbBits < 1)
-+ return ERROR(GENERIC); /* min size */
-+
-+ /* header */
-+ tableU16[-2] = (U16)nbBits;
-+ tableU16[-1] = (U16)maxSymbolValue;
-+
-+ /* Build table */
-+ for (s = 0; s < tableSize; s++)
-+ tableU16[s] = (U16)(tableSize + s);
-+
-+ /* Build Symbol Transformation Table */
-+ {
-+ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
-+ for (s = 0; s <= maxSymbolValue; s++) {
-+ symbolTT[s].deltaNbBits = deltaNbBits;
-+ symbolTT[s].deltaFindState = s - 1;
-+ }
-+ }
-+
-+ return 0;
-+}
-+
-+/* fake FSE_CTable, for rle input (always same symbol) */
-+size_t FSE_buildCTable_rle(FSE_CTable *ct, BYTE symbolValue)
-+{
-+ void *ptr = ct;
-+ U16 *tableU16 = ((U16 *)ptr) + 2;
-+ void *FSCTptr = (U32 *)ptr + 2;
-+ FSE_symbolCompressionTransform *symbolTT = (FSE_symbolCompressionTransform *)FSCTptr;
-+
-+ /* header */
-+ tableU16[-2] = (U16)0;
-+ tableU16[-1] = (U16)symbolValue;
-+
-+ /* Build table */
-+ tableU16[0] = 0;
-+ tableU16[1] = 0; /* just in case */
-+
-+ /* Build Symbol Transformation Table */
-+ symbolTT[symbolValue].deltaNbBits = 0;
-+ symbolTT[symbolValue].deltaFindState = 0;
-+
-+ return 0;
-+}
-+
-+static size_t FSE_compress_usingCTable_generic(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct, const unsigned fast)
-+{
-+ const BYTE *const istart = (const BYTE *)src;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *ip = iend;
-+
-+ BIT_CStream_t bitC;
-+ FSE_CState_t CState1, CState2;
-+
-+ /* init */
-+ if (srcSize <= 2)
-+ return 0;
-+ {
-+ size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
-+ if (FSE_isError(initError))
-+ return 0; /* not enough space available to write a bitstream */
-+ }
-+
-+#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
-+
-+ if (srcSize & 1) {
-+ FSE_initCState2(&CState1, ct, *--ip);
-+ FSE_initCState2(&CState2, ct, *--ip);
-+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
-+ FSE_FLUSHBITS(&bitC);
-+ } else {
-+ FSE_initCState2(&CState2, ct, *--ip);
-+ FSE_initCState2(&CState1, ct, *--ip);
-+ }
-+
-+ /* join to mod 4 */
-+ srcSize -= 2;
-+ if ((sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) && (srcSize & 2)) { /* test bit 2 */
-+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
-+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
-+ FSE_FLUSHBITS(&bitC);
-+ }
-+
-+ /* 2 or 4 encoding per loop */
-+ while (ip > istart) {
-+
-+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
-+
-+ if (sizeof(bitC.bitContainer) * 8 < FSE_MAX_TABLELOG * 2 + 7) /* this test must be static */
-+ FSE_FLUSHBITS(&bitC);
-+
-+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
-+
-+ if (sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) { /* this test must be static */
-+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
-+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
-+ }
-+
-+ FSE_FLUSHBITS(&bitC);
-+ }
-+
-+ FSE_flushCState(&bitC, &CState2);
-+ FSE_flushCState(&bitC, &CState1);
-+ return BIT_closeCStream(&bitC);
-+}
-+
-+size_t FSE_compress_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct)
-+{
-+ unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
-+
-+ if (fast)
-+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
-+ else
-+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
-+}
-+
-+size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
-diff --git a/lib/zstd/fse_decompress.c b/lib/zstd/fse_decompress.c
-new file mode 100644
-index 0000000..a84300e
---- /dev/null
-+++ b/lib/zstd/fse_decompress.c
-@@ -0,0 +1,332 @@
-+/*
-+ * FSE : Finite State Entropy decoder
-+ * Copyright (C) 2013-2015, Yann Collet.
-+ *
-+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are
-+ * met:
-+ *
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above
-+ * copyright notice, this list of conditions and the following disclaimer
-+ * in the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ *
-+ * You can contact the author at :
-+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-+ */
-+
-+/* **************************************************************
-+* Compiler specifics
-+****************************************************************/
-+#define FORCE_INLINE static __always_inline
-+
-+/* **************************************************************
-+* Includes
-+****************************************************************/
-+#include "bitstream.h"
-+#include "fse.h"
-+#include <linux/compiler.h>
-+#include <linux/kernel.h>
-+#include <linux/string.h> /* memcpy, memset */
-+
-+/* **************************************************************
-+* Error Management
-+****************************************************************/
-+#define FSE_isError ERR_isError
-+#define FSE_STATIC_ASSERT(c) \
-+ { \
-+ enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
-+ } /* use only *after* variable declarations */
-+
-+/* check and forward error code */
-+#define CHECK_F(f) \
-+ { \
-+ size_t const e = f; \
-+ if (FSE_isError(e)) \
-+ return e; \
-+ }
-+
-+/* **************************************************************
-+* Templates
-+****************************************************************/
-+/*
-+ designed to be included
-+ for type-specific functions (template emulation in C)
-+ Objective is to write these functions only once, for improved maintenance
-+*/
-+
-+/* safety checks */
-+#ifndef FSE_FUNCTION_EXTENSION
-+#error "FSE_FUNCTION_EXTENSION must be defined"
-+#endif
-+#ifndef FSE_FUNCTION_TYPE
-+#error "FSE_FUNCTION_TYPE must be defined"
-+#endif
-+
-+/* Function names */
-+#define FSE_CAT(X, Y) X##Y
-+#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
-+#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
-+
-+/* Function templates */
-+
-+size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize)
-+{
-+ void *const tdPtr = dt + 1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
-+ FSE_DECODE_TYPE *const tableDecode = (FSE_DECODE_TYPE *)(tdPtr);
-+ U16 *symbolNext = (U16 *)workspace;
-+
-+ U32 const maxSV1 = maxSymbolValue + 1;
-+ U32 const tableSize = 1 << tableLog;
-+ U32 highThreshold = tableSize - 1;
-+
-+ /* Sanity Checks */
-+ if (workspaceSize < sizeof(U16) * (FSE_MAX_SYMBOL_VALUE + 1))
-+ return ERROR(tableLog_tooLarge);
-+ if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE)
-+ return ERROR(maxSymbolValue_tooLarge);
-+ if (tableLog > FSE_MAX_TABLELOG)
-+ return ERROR(tableLog_tooLarge);
-+
-+ /* Init, lay down lowprob symbols */
-+ {
-+ FSE_DTableHeader DTableH;
-+ DTableH.tableLog = (U16)tableLog;
-+ DTableH.fastMode = 1;
-+ {
-+ S16 const largeLimit = (S16)(1 << (tableLog - 1));
-+ U32 s;
-+ for (s = 0; s < maxSV1; s++) {
-+ if (normalizedCounter[s] == -1) {
-+ tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
-+ symbolNext[s] = 1;
-+ } else {
-+ if (normalizedCounter[s] >= largeLimit)
-+ DTableH.fastMode = 0;
-+ symbolNext[s] = normalizedCounter[s];
-+ }
-+ }
-+ }
-+ memcpy(dt, &DTableH, sizeof(DTableH));
-+ }
-+
-+ /* Spread symbols */
-+ {
-+ U32 const tableMask = tableSize - 1;
-+ U32 const step = FSE_TABLESTEP(tableSize);
-+ U32 s, position = 0;
-+ for (s = 0; s < maxSV1; s++) {
-+ int i;
-+ for (i = 0; i < normalizedCounter[s]; i++) {
-+ tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
-+ position = (position + step) & tableMask;
-+ while (position > highThreshold)
-+ position = (position + step) & tableMask; /* lowprob area */
-+ }
-+ }
-+ if (position != 0)
-+ return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
-+ }
-+
-+ /* Build Decoding table */
-+ {
-+ U32 u;
-+ for (u = 0; u < tableSize; u++) {
-+ FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
-+ U16 nextState = symbolNext[symbol]++;
-+ tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState));
-+ tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize);
-+ }
-+ }
-+
-+ return 0;
-+}
-+
-+/*-*******************************************************
-+* Decompression (Byte symbols)
-+*********************************************************/
-+size_t FSE_buildDTable_rle(FSE_DTable *dt, BYTE symbolValue)
-+{
-+ void *ptr = dt;
-+ FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
-+ void *dPtr = dt + 1;
-+ FSE_decode_t *const cell = (FSE_decode_t *)dPtr;
-+
-+ DTableH->tableLog = 0;
-+ DTableH->fastMode = 0;
-+
-+ cell->newState = 0;
-+ cell->symbol = symbolValue;
-+ cell->nbBits = 0;
-+
-+ return 0;
-+}
-+
-+size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits)
-+{
-+ void *ptr = dt;
-+ FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
-+ void *dPtr = dt + 1;
-+ FSE_decode_t *const dinfo = (FSE_decode_t *)dPtr;
-+ const unsigned tableSize = 1 << nbBits;
-+ const unsigned tableMask = tableSize - 1;
-+ const unsigned maxSV1 = tableMask + 1;
-+ unsigned s;
-+
-+ /* Sanity checks */
-+ if (nbBits < 1)
-+ return ERROR(GENERIC); /* min size */
-+
-+ /* Build Decoding Table */
-+ DTableH->tableLog = (U16)nbBits;
-+ DTableH->fastMode = 1;
-+ for (s = 0; s < maxSV1; s++) {
-+ dinfo[s].newState = 0;
-+ dinfo[s].symbol = (BYTE)s;
-+ dinfo[s].nbBits = (BYTE)nbBits;
-+ }
-+
-+ return 0;
-+}
-+
-+FORCE_INLINE size_t FSE_decompress_usingDTable_generic(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt,
-+ const unsigned fast)
-+{
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *op = ostart;
-+ BYTE *const omax = op + maxDstSize;
-+ BYTE *const olimit = omax - 3;
-+
-+ BIT_DStream_t bitD;
-+ FSE_DState_t state1;
-+ FSE_DState_t state2;
-+
-+ /* Init */
-+ CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
-+
-+ FSE_initDState(&state1, &bitD, dt);
-+ FSE_initDState(&state2, &bitD, dt);
-+
-+#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
-+
-+ /* 4 symbols per loop */
-+ for (; (BIT_reloadDStream(&bitD) == BIT_DStream_unfinished) & (op < olimit); op += 4) {
-+ op[0] = FSE_GETSYMBOL(&state1);
-+
-+ if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
-+ BIT_reloadDStream(&bitD);
-+
-+ op[1] = FSE_GETSYMBOL(&state2);
-+
-+ if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
-+ {
-+ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) {
-+ op += 2;
-+ break;
-+ }
-+ }
-+
-+ op[2] = FSE_GETSYMBOL(&state1);
-+
-+ if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
-+ BIT_reloadDStream(&bitD);
-+
-+ op[3] = FSE_GETSYMBOL(&state2);
-+ }
-+
-+ /* tail */
-+ /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
-+ while (1) {
-+ if (op > (omax - 2))
-+ return ERROR(dstSize_tooSmall);
-+ *op++ = FSE_GETSYMBOL(&state1);
-+ if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
-+ *op++ = FSE_GETSYMBOL(&state2);
-+ break;
-+ }
-+
-+ if (op > (omax - 2))
-+ return ERROR(dstSize_tooSmall);
-+ *op++ = FSE_GETSYMBOL(&state2);
-+ if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
-+ *op++ = FSE_GETSYMBOL(&state1);
-+ break;
-+ }
-+ }
-+
-+ return op - ostart;
-+}
-+
-+size_t FSE_decompress_usingDTable(void *dst, size_t originalSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt)
-+{
-+ const void *ptr = dt;
-+ const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr;
-+ const U32 fastMode = DTableH->fastMode;
-+
-+ /* select fast mode (static) */
-+ if (fastMode)
-+ return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
-+ return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
-+}
-+
-+size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize)
-+{
-+ const BYTE *const istart = (const BYTE *)cSrc;
-+ const BYTE *ip = istart;
-+ unsigned tableLog;
-+ unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
-+ size_t NCountLength;
-+
-+ FSE_DTable *dt;
-+ short *counting;
-+ size_t spaceUsed32 = 0;
-+
-+ FSE_STATIC_ASSERT(sizeof(FSE_DTable) == sizeof(U32));
-+
-+ dt = (FSE_DTable *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += FSE_DTABLE_SIZE_U32(maxLog);
-+ counting = (short *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += ALIGN(sizeof(short) * (FSE_MAX_SYMBOL_VALUE + 1), sizeof(U32)) >> 2;
-+
-+ if ((spaceUsed32 << 2) > workspaceSize)
-+ return ERROR(tableLog_tooLarge);
-+ workspace = (U32 *)workspace + spaceUsed32;
-+ workspaceSize -= (spaceUsed32 << 2);
-+
-+ /* normal FSE decoding mode */
-+ NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
-+ if (FSE_isError(NCountLength))
-+ return NCountLength;
-+ // if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining
-+ // case : NCountLength==cSrcSize */
-+ if (tableLog > maxLog)
-+ return ERROR(tableLog_tooLarge);
-+ ip += NCountLength;
-+ cSrcSize -= NCountLength;
-+
-+ CHECK_F(FSE_buildDTable_wksp(dt, counting, maxSymbolValue, tableLog, workspace, workspaceSize));
-+
-+ return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, dt); /* always return, even if it is an error code */
-+}
-diff --git a/lib/zstd/huf.h b/lib/zstd/huf.h
-new file mode 100644
-index 0000000..2143da2
---- /dev/null
-+++ b/lib/zstd/huf.h
-@@ -0,0 +1,212 @@
-+/*
-+ * Huffman coder, part of New Generation Entropy library
-+ * header file
-+ * Copyright (C) 2013-2016, Yann Collet.
-+ *
-+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are
-+ * met:
-+ *
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above
-+ * copyright notice, this list of conditions and the following disclaimer
-+ * in the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ *
-+ * You can contact the author at :
-+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-+ */
-+#ifndef HUF_H_298734234
-+#define HUF_H_298734234
-+
-+/* *** Dependencies *** */
-+#include <linux/types.h> /* size_t */
-+
-+/* *** Tool functions *** */
-+#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
-+size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
-+
-+/* Error Management */
-+unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
-+
-+/* *** Advanced function *** */
-+
-+/** HUF_compress4X_wksp() :
-+* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
-+size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
-+ size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
-+
-+/* *** Dependencies *** */
-+#include "mem.h" /* U32 */
-+
-+/* *** Constants *** */
-+#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
-+#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
-+#define HUF_SYMBOLVALUE_MAX 255
-+
-+#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
-+#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
-+#error "HUF_TABLELOG_MAX is too large !"
-+#endif
-+
-+/* ****************************************
-+* Static allocation
-+******************************************/
-+/* HUF buffer bounds */
-+#define HUF_CTABLEBOUND 129
-+#define HUF_BLOCKBOUND(size) (size + (size >> 8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
-+#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
-+
-+/* static allocation of HUF's Compression Table */
-+#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
-+ U32 name##hb[maxSymbolValue + 1]; \
-+ void *name##hv = &(name##hb); \
-+ HUF_CElt *name = (HUF_CElt *)(name##hv) /* no final ; */
-+
-+/* static allocation of HUF's DTable */
-+typedef U32 HUF_DTable;
-+#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog)))
-+#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = {((U32)((maxTableLog)-1) * 0x01000001)}
-+#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = {((U32)(maxTableLog)*0x01000001)}
-+
-+/* The workspace must have alignment at least 4 and be at least this large */
-+#define HUF_COMPRESS_WORKSPACE_SIZE (6 << 10)
-+#define HUF_COMPRESS_WORKSPACE_SIZE_U32 (HUF_COMPRESS_WORKSPACE_SIZE / sizeof(U32))
-+
-+/* The workspace must have alignment at least 4 and be at least this large */
-+#define HUF_DECOMPRESS_WORKSPACE_SIZE (3 << 10)
-+#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
-+
-+/* ****************************************
-+* Advanced decompression functions
-+******************************************/
-+size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize); /**< decodes RLE and uncompressed */
-+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
-+ size_t workspaceSize); /**< considers RLE and uncompressed as errors */
-+size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
-+ size_t workspaceSize); /**< single-symbol decoder */
-+size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
-+ size_t workspaceSize); /**< double-symbols decoder */
-+
-+/* ****************************************
-+* HUF detailed API
-+******************************************/
-+/*!
-+HUF_compress() does the following:
-+1. count symbol occurrence from source[] into table count[] using FSE_count()
-+2. (optional) refine tableLog using HUF_optimalTableLog()
-+3. build Huffman table from count using HUF_buildCTable()
-+4. save Huffman table to memory buffer using HUF_writeCTable_wksp()
-+5. encode the data stream using HUF_compress4X_usingCTable()
-+
-+The following API allows targeting specific sub-functions for advanced tasks.
-+For example, it's possible to compress several blocks using the same 'CTable',
-+or to save and regenerate 'CTable' using external methods.
-+*/
-+/* FSE_count() : find it within "fse.h" */
-+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
-+typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
-+size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, unsigned maxSymbolValue, unsigned huffLog, void *workspace, size_t workspaceSize);
-+size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable);
-+
-+typedef enum {
-+ HUF_repeat_none, /**< Cannot use the previous table */
-+ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
-+ 4}X_repeat */
-+ HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
-+} HUF_repeat;
-+/** HUF_compress4X_repeat() :
-+* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
-+* If it uses hufTable it does not modify hufTable or repeat.
-+* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
-+* If preferRepeat then the old table will always be used if valid. */
-+size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
-+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
-+ int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
-+
-+/** HUF_buildCTable_wksp() :
-+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
-+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
-+ */
-+size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize);
-+
-+/*! HUF_readStats() :
-+ Read compact Huffman tree, saved by HUF_writeCTable().
-+ `huffWeight` is destination buffer.
-+ @return : size read from `src` , or an error Code .
-+ Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
-+size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize,
-+ void *workspace, size_t workspaceSize);
-+
-+/** HUF_readCTable() :
-+* Loading a CTable saved with HUF_writeCTable() */
-+size_t HUF_readCTable_wksp(HUF_CElt *CTable, unsigned maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize);
-+
-+/*
-+HUF_decompress() does the following:
-+1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
-+2. build Huffman table from save, using HUF_readDTableXn()
-+3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
-+*/
-+
-+/** HUF_selectDecoder() :
-+* Tells which decoder is likely to decode faster,
-+* based on a set of pre-determined metrics.
-+* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
-+* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
-+U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize);
-+
-+size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize);
-+size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize);
-+
-+size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
-+size_t HUF_decompress4X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
-+size_t HUF_decompress4X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
-+
-+/* single stream variants */
-+
-+size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
-+ size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
-+size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable);
-+/** HUF_compress1X_repeat() :
-+* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
-+* If it uses hufTable it does not modify hufTable or repeat.
-+* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
-+* If preferRepeat then the old table will always be used if valid. */
-+size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
-+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
-+ int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
-+
-+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize);
-+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
-+ size_t workspaceSize); /**< single-symbol decoder */
-+size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
-+ size_t workspaceSize); /**< double-symbols decoder */
-+
-+size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize,
-+ const HUF_DTable *DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
-+size_t HUF_decompress1X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
-+size_t HUF_decompress1X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
-+
-+#endif /* HUF_H_298734234 */
-diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c
-new file mode 100644
-index 0000000..40055a7
---- /dev/null
-+++ b/lib/zstd/huf_compress.c
-@@ -0,0 +1,770 @@
-+/*
-+ * Huffman encoder, part of New Generation Entropy library
-+ * Copyright (C) 2013-2016, Yann Collet.
-+ *
-+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are
-+ * met:
-+ *
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above
-+ * copyright notice, this list of conditions and the following disclaimer
-+ * in the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ *
-+ * You can contact the author at :
-+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-+ */
-+
-+/* **************************************************************
-+* Includes
-+****************************************************************/
-+#include "bitstream.h"
-+#include "fse.h" /* header compression */
-+#include "huf.h"
-+#include <linux/kernel.h>
-+#include <linux/string.h> /* memcpy, memset */
-+
-+/* **************************************************************
-+* Error Management
-+****************************************************************/
-+#define HUF_STATIC_ASSERT(c) \
-+ { \
-+ enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
-+ } /* use only *after* variable declarations */
-+#define CHECK_V_F(e, f) \
-+ size_t const e = f; \
-+ if (ERR_isError(e)) \
-+ return f
-+#define CHECK_F(f) \
-+ { \
-+ CHECK_V_F(_var_err__, f); \
-+ }
-+
-+/* **************************************************************
-+* Utils
-+****************************************************************/
-+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
-+{
-+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
-+}
-+
-+/* *******************************************************
-+* HUF : Huffman block compression
-+*********************************************************/
-+/* HUF_compressWeights() :
-+ * Same as FSE_compress(), but dedicated to huff0's weights compression.
-+ * The use case needs much less stack memory.
-+ * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
-+ */
-+#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
-+size_t HUF_compressWeights_wksp(void *dst, size_t dstSize, const void *weightTable, size_t wtSize, void *workspace, size_t workspaceSize)
-+{
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *op = ostart;
-+ BYTE *const oend = ostart + dstSize;
-+
-+ U32 maxSymbolValue = HUF_TABLELOG_MAX;
-+ U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
-+
-+ FSE_CTable *CTable;
-+ U32 *count;
-+ S16 *norm;
-+ size_t spaceUsed32 = 0;
-+
-+ HUF_STATIC_ASSERT(sizeof(FSE_CTable) == sizeof(U32));
-+
-+ CTable = (FSE_CTable *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX);
-+ count = (U32 *)workspace + spaceUsed32;
-+ spaceUsed32 += HUF_TABLELOG_MAX + 1;
-+ norm = (S16 *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += ALIGN(sizeof(S16) * (HUF_TABLELOG_MAX + 1), sizeof(U32)) >> 2;
-+
-+ if ((spaceUsed32 << 2) > workspaceSize)
-+ return ERROR(tableLog_tooLarge);
-+ workspace = (U32 *)workspace + spaceUsed32;
-+ workspaceSize -= (spaceUsed32 << 2);
-+
-+ /* init conditions */
-+ if (wtSize <= 1)
-+ return 0; /* Not compressible */
-+
-+ /* Scan input and build symbol stats */
-+ {
-+ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize));
-+ if (maxCount == wtSize)
-+ return 1; /* only a single symbol in src : rle */
-+ if (maxCount == 1)
-+ return 0; /* each symbol present maximum once => not compressible */
-+ }
-+
-+ tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
-+ CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue));
-+
-+ /* Write table description header */
-+ {
-+ CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog));
-+ op += hSize;
-+ }
-+
-+ /* Compress */
-+ CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, workspace, workspaceSize));
-+ {
-+ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable));
-+ if (cSize == 0)
-+ return 0; /* not enough space for compressed data */
-+ op += cSize;
-+ }
-+
-+ return op - ostart;
-+}
-+
-+struct HUF_CElt_s {
-+ U16 val;
-+ BYTE nbBits;
-+}; /* typedef'd to HUF_CElt within "huf.h" */
-+
-+/*! HUF_writeCTable_wksp() :
-+ `CTable` : Huffman tree to save, using huf representation.
-+ @return : size of saved CTable */
-+size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, U32 maxSymbolValue, U32 huffLog, void *workspace, size_t workspaceSize)
-+{
-+ BYTE *op = (BYTE *)dst;
-+ U32 n;
-+
-+ BYTE *bitsToWeight;
-+ BYTE *huffWeight;
-+ size_t spaceUsed32 = 0;
-+
-+ bitsToWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += ALIGN(HUF_TABLELOG_MAX + 1, sizeof(U32)) >> 2;
-+ huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX, sizeof(U32)) >> 2;
-+
-+ if ((spaceUsed32 << 2) > workspaceSize)
-+ return ERROR(tableLog_tooLarge);
-+ workspace = (U32 *)workspace + spaceUsed32;
-+ workspaceSize -= (spaceUsed32 << 2);
-+
-+ /* check conditions */
-+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
-+ return ERROR(maxSymbolValue_tooLarge);
-+
-+ /* convert to weight */
-+ bitsToWeight[0] = 0;
-+ for (n = 1; n < huffLog + 1; n++)
-+ bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
-+ for (n = 0; n < maxSymbolValue; n++)
-+ huffWeight[n] = bitsToWeight[CTable[n].nbBits];
-+
-+ /* attempt weights compression by FSE */
-+ {
-+ CHECK_V_F(hSize, HUF_compressWeights_wksp(op + 1, maxDstSize - 1, huffWeight, maxSymbolValue, workspace, workspaceSize));
-+ if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */
-+ op[0] = (BYTE)hSize;
-+ return hSize + 1;
-+ }
-+ }
-+
-+ /* write raw values as 4-bits (max : 15) */
-+ if (maxSymbolValue > (256 - 128))
-+ return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
-+ if (((maxSymbolValue + 1) / 2) + 1 > maxDstSize)
-+ return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
-+ op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1));
-+ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
-+ for (n = 0; n < maxSymbolValue; n += 2)
-+ op[(n / 2) + 1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n + 1]);
-+ return ((maxSymbolValue + 1) / 2) + 1;
-+}
-+
-+size_t HUF_readCTable_wksp(HUF_CElt *CTable, U32 maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
-+{
-+ U32 *rankVal;
-+ BYTE *huffWeight;
-+ U32 tableLog = 0;
-+ U32 nbSymbols = 0;
-+ size_t readSize;
-+ size_t spaceUsed32 = 0;
-+
-+ rankVal = (U32 *)workspace + spaceUsed32;
-+ spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1;
-+ huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
-+
-+ if ((spaceUsed32 << 2) > workspaceSize)
-+ return ERROR(tableLog_tooLarge);
-+ workspace = (U32 *)workspace + spaceUsed32;
-+ workspaceSize -= (spaceUsed32 << 2);
-+
-+ /* get symbol weights */
-+ readSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize);
-+ if (ERR_isError(readSize))
-+ return readSize;
-+
-+ /* check result */
-+ if (tableLog > HUF_TABLELOG_MAX)
-+ return ERROR(tableLog_tooLarge);
-+ if (nbSymbols > maxSymbolValue + 1)
-+ return ERROR(maxSymbolValue_tooSmall);
-+
-+ /* Prepare base value per rank */
-+ {
-+ U32 n, nextRankStart = 0;
-+ for (n = 1; n <= tableLog; n++) {
-+ U32 curr = nextRankStart;
-+ nextRankStart += (rankVal[n] << (n - 1));
-+ rankVal[n] = curr;
-+ }
-+ }
-+
-+ /* fill nbBits */
-+ {
-+ U32 n;
-+ for (n = 0; n < nbSymbols; n++) {
-+ const U32 w = huffWeight[n];
-+ CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
-+ }
-+ }
-+
-+ /* fill val */
-+ {
-+ U16 nbPerRank[HUF_TABLELOG_MAX + 2] = {0}; /* support w=0=>n=tableLog+1 */
-+ U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0};
-+ {
-+ U32 n;
-+ for (n = 0; n < nbSymbols; n++)
-+ nbPerRank[CTable[n].nbBits]++;
-+ }
-+ /* determine stating value per rank */
-+ valPerRank[tableLog + 1] = 0; /* for w==0 */
-+ {
-+ U16 min = 0;
-+ U32 n;
-+ for (n = tableLog; n > 0; n--) { /* start at n=tablelog <-> w=1 */
-+ valPerRank[n] = min; /* get starting value within each rank */
-+ min += nbPerRank[n];
-+ min >>= 1;
-+ }
-+ }
-+ /* assign value within rank, symbol order */
-+ {
-+ U32 n;
-+ for (n = 0; n <= maxSymbolValue; n++)
-+ CTable[n].val = valPerRank[CTable[n].nbBits]++;
-+ }
-+ }
-+
-+ return readSize;
-+}
-+
-+typedef struct nodeElt_s {
-+ U32 count;
-+ U16 parent;
-+ BYTE byte;
-+ BYTE nbBits;
-+} nodeElt;
-+
-+static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits)
-+{
-+ const U32 largestBits = huffNode[lastNonNull].nbBits;
-+ if (largestBits <= maxNbBits)
-+ return largestBits; /* early exit : no elt > maxNbBits */
-+
-+ /* there are several too large elements (at least >= 2) */
-+ {
-+ int totalCost = 0;
-+ const U32 baseCost = 1 << (largestBits - maxNbBits);
-+ U32 n = lastNonNull;
-+
-+ while (huffNode[n].nbBits > maxNbBits) {
-+ totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
-+ huffNode[n].nbBits = (BYTE)maxNbBits;
-+ n--;
-+ } /* n stops at huffNode[n].nbBits <= maxNbBits */
-+ while (huffNode[n].nbBits == maxNbBits)
-+ n--; /* n end at index of smallest symbol using < maxNbBits */
-+
-+ /* renorm totalCost */
-+ totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
-+
-+ /* repay normalized cost */
-+ {
-+ U32 const noSymbol = 0xF0F0F0F0;
-+ U32 rankLast[HUF_TABLELOG_MAX + 2];
-+ int pos;
-+
-+ /* Get pos of last (smallest) symbol per rank */
-+ memset(rankLast, 0xF0, sizeof(rankLast));
-+ {
-+ U32 currNbBits = maxNbBits;
-+ for (pos = n; pos >= 0; pos--) {
-+ if (huffNode[pos].nbBits >= currNbBits)
-+ continue;
-+ currNbBits = huffNode[pos].nbBits; /* < maxNbBits */
-+ rankLast[maxNbBits - currNbBits] = pos;
-+ }
-+ }
-+
-+ while (totalCost > 0) {
-+ U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
-+ for (; nBitsToDecrease > 1; nBitsToDecrease--) {
-+ U32 highPos = rankLast[nBitsToDecrease];
-+ U32 lowPos = rankLast[nBitsToDecrease - 1];
-+ if (highPos == noSymbol)
-+ continue;
-+ if (lowPos == noSymbol)
-+ break;
-+ {
-+ U32 const highTotal = huffNode[highPos].count;
-+ U32 const lowTotal = 2 * huffNode[lowPos].count;
-+ if (highTotal <= lowTotal)
-+ break;
-+ }
-+ }
-+ /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
-+ /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
-+ while ((nBitsToDecrease <= HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol))
-+ nBitsToDecrease++;
-+ totalCost -= 1 << (nBitsToDecrease - 1);
-+ if (rankLast[nBitsToDecrease - 1] == noSymbol)
-+ rankLast[nBitsToDecrease - 1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
-+ huffNode[rankLast[nBitsToDecrease]].nbBits++;
-+ if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
-+ rankLast[nBitsToDecrease] = noSymbol;
-+ else {
-+ rankLast[nBitsToDecrease]--;
-+ if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease)
-+ rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
-+ }
-+ } /* while (totalCost > 0) */
-+
-+ while (totalCost < 0) { /* Sometimes, cost correction overshoot */
-+ if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0
-+ (using maxNbBits) */
-+ while (huffNode[n].nbBits == maxNbBits)
-+ n--;
-+ huffNode[n + 1].nbBits--;
-+ rankLast[1] = n + 1;
-+ totalCost++;
-+ continue;
-+ }
-+ huffNode[rankLast[1] + 1].nbBits--;
-+ rankLast[1]++;
-+ totalCost++;
-+ }
-+ }
-+ } /* there are several too large elements (at least >= 2) */
-+
-+ return maxNbBits;
-+}
-+
-+typedef struct {
-+ U32 base;
-+ U32 curr;
-+} rankPos;
-+
-+static void HUF_sort(nodeElt *huffNode, const U32 *count, U32 maxSymbolValue)
-+{
-+ rankPos rank[32];
-+ U32 n;
-+
-+ memset(rank, 0, sizeof(rank));
-+ for (n = 0; n <= maxSymbolValue; n++) {
-+ U32 r = BIT_highbit32(count[n] + 1);
-+ rank[r].base++;
-+ }
-+ for (n = 30; n > 0; n--)
-+ rank[n - 1].base += rank[n].base;
-+ for (n = 0; n < 32; n++)
-+ rank[n].curr = rank[n].base;
-+ for (n = 0; n <= maxSymbolValue; n++) {
-+ U32 const c = count[n];
-+ U32 const r = BIT_highbit32(c + 1) + 1;
-+ U32 pos = rank[r].curr++;
-+ while ((pos > rank[r].base) && (c > huffNode[pos - 1].count))
-+ huffNode[pos] = huffNode[pos - 1], pos--;
-+ huffNode[pos].count = c;
-+ huffNode[pos].byte = (BYTE)n;
-+ }
-+}
-+
-+/** HUF_buildCTable_wksp() :
-+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
-+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
-+ */
-+#define STARTNODE (HUF_SYMBOLVALUE_MAX + 1)
-+typedef nodeElt huffNodeTable[2 * HUF_SYMBOLVALUE_MAX + 1 + 1];
-+size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize)
-+{
-+ nodeElt *const huffNode0 = (nodeElt *)workSpace;
-+ nodeElt *const huffNode = huffNode0 + 1;
-+ U32 n, nonNullRank;
-+ int lowS, lowN;
-+ U16 nodeNb = STARTNODE;
-+ U32 nodeRoot;
-+
-+ /* safety checks */
-+ if (wkspSize < sizeof(huffNodeTable))
-+ return ERROR(GENERIC); /* workSpace is not large enough */
-+ if (maxNbBits == 0)
-+ maxNbBits = HUF_TABLELOG_DEFAULT;
-+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
-+ return ERROR(GENERIC);
-+ memset(huffNode0, 0, sizeof(huffNodeTable));
-+
-+ /* sort, decreasing order */
-+ HUF_sort(huffNode, count, maxSymbolValue);
-+
-+ /* init for parents */
-+ nonNullRank = maxSymbolValue;
-+ while (huffNode[nonNullRank].count == 0)
-+ nonNullRank--;
-+ lowS = nonNullRank;
-+ nodeRoot = nodeNb + lowS - 1;
-+ lowN = nodeNb;
-+ huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count;
-+ huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb;
-+ nodeNb++;
-+ lowS -= 2;
-+ for (n = nodeNb; n <= nodeRoot; n++)
-+ huffNode[n].count = (U32)(1U << 30);
-+ huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */
-+
-+ /* create parents */
-+ while (nodeNb <= nodeRoot) {
-+ U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
-+ U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
-+ huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
-+ huffNode[n1].parent = huffNode[n2].parent = nodeNb;
-+ nodeNb++;
-+ }
-+
-+ /* distribute weights (unlimited tree height) */
-+ huffNode[nodeRoot].nbBits = 0;
-+ for (n = nodeRoot - 1; n >= STARTNODE; n--)
-+ huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1;
-+ for (n = 0; n <= nonNullRank; n++)
-+ huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1;
-+
-+ /* enforce maxTableLog */
-+ maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
-+
-+ /* fill result into tree (val, nbBits) */
-+ {
-+ U16 nbPerRank[HUF_TABLELOG_MAX + 1] = {0};
-+ U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0};
-+ if (maxNbBits > HUF_TABLELOG_MAX)
-+ return ERROR(GENERIC); /* check fit into table */
-+ for (n = 0; n <= nonNullRank; n++)
-+ nbPerRank[huffNode[n].nbBits]++;
-+ /* determine stating value per rank */
-+ {
-+ U16 min = 0;
-+ for (n = maxNbBits; n > 0; n--) {
-+ valPerRank[n] = min; /* get starting value within each rank */
-+ min += nbPerRank[n];
-+ min >>= 1;
-+ }
-+ }
-+ for (n = 0; n <= maxSymbolValue; n++)
-+ tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
-+ for (n = 0; n <= maxSymbolValue; n++)
-+ tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
-+ }
-+
-+ return maxNbBits;
-+}
-+
-+static size_t HUF_estimateCompressedSize(HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue)
-+{
-+ size_t nbBits = 0;
-+ int s;
-+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
-+ nbBits += CTable[s].nbBits * count[s];
-+ }
-+ return nbBits >> 3;
-+}
-+
-+static int HUF_validateCTable(const HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue)
-+{
-+ int bad = 0;
-+ int s;
-+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
-+ bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
-+ }
-+ return !bad;
-+}
-+
-+static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt *CTable)
-+{
-+ BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
-+}
-+
-+size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
-+
-+#define HUF_FLUSHBITS(s) BIT_flushBits(s)
-+
-+#define HUF_FLUSHBITS_1(stream) \
-+ if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \
-+ HUF_FLUSHBITS(stream)
-+
-+#define HUF_FLUSHBITS_2(stream) \
-+ if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 4 + 7) \
-+ HUF_FLUSHBITS(stream)
-+
-+size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
-+{
-+ const BYTE *ip = (const BYTE *)src;
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *const oend = ostart + dstSize;
-+ BYTE *op = ostart;
-+ size_t n;
-+ BIT_CStream_t bitC;
-+
-+ /* init */
-+ if (dstSize < 8)
-+ return 0; /* not enough space to compress */
-+ {
-+ size_t const initErr = BIT_initCStream(&bitC, op, oend - op);
-+ if (HUF_isError(initErr))
-+ return 0;
-+ }
-+
-+ n = srcSize & ~3; /* join to mod 4 */
-+ switch (srcSize & 3) {
-+ case 3: HUF_encodeSymbol(&bitC, ip[n + 2], CTable); HUF_FLUSHBITS_2(&bitC);
-+ case 2: HUF_encodeSymbol(&bitC, ip[n + 1], CTable); HUF_FLUSHBITS_1(&bitC);
-+ case 1: HUF_encodeSymbol(&bitC, ip[n + 0], CTable); HUF_FLUSHBITS(&bitC);
-+ case 0:
-+ default:;
-+ }
-+
-+ for (; n > 0; n -= 4) { /* note : n&3==0 at this stage */
-+ HUF_encodeSymbol(&bitC, ip[n - 1], CTable);
-+ HUF_FLUSHBITS_1(&bitC);
-+ HUF_encodeSymbol(&bitC, ip[n - 2], CTable);
-+ HUF_FLUSHBITS_2(&bitC);
-+ HUF_encodeSymbol(&bitC, ip[n - 3], CTable);
-+ HUF_FLUSHBITS_1(&bitC);
-+ HUF_encodeSymbol(&bitC, ip[n - 4], CTable);
-+ HUF_FLUSHBITS(&bitC);
-+ }
-+
-+ return BIT_closeCStream(&bitC);
-+}
-+
-+size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
-+{
-+ size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */
-+ const BYTE *ip = (const BYTE *)src;
-+ const BYTE *const iend = ip + srcSize;
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *const oend = ostart + dstSize;
-+ BYTE *op = ostart;
-+
-+ if (dstSize < 6 + 1 + 1 + 1 + 8)
-+ return 0; /* minimum space to compress successfully */
-+ if (srcSize < 12)
-+ return 0; /* no saving possible : too small input */
-+ op += 6; /* jumpTable */
-+
-+ {
-+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
-+ if (cSize == 0)
-+ return 0;
-+ ZSTD_writeLE16(ostart, (U16)cSize);
-+ op += cSize;
-+ }
-+
-+ ip += segmentSize;
-+ {
-+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
-+ if (cSize == 0)
-+ return 0;
-+ ZSTD_writeLE16(ostart + 2, (U16)cSize);
-+ op += cSize;
-+ }
-+
-+ ip += segmentSize;
-+ {
-+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
-+ if (cSize == 0)
-+ return 0;
-+ ZSTD_writeLE16(ostart + 4, (U16)cSize);
-+ op += cSize;
-+ }
-+
-+ ip += segmentSize;
-+ {
-+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, iend - ip, CTable));
-+ if (cSize == 0)
-+ return 0;
-+ op += cSize;
-+ }
-+
-+ return op - ostart;
-+}
-+
-+static size_t HUF_compressCTable_internal(BYTE *const ostart, BYTE *op, BYTE *const oend, const void *src, size_t srcSize, unsigned singleStream,
-+ const HUF_CElt *CTable)
-+{
-+ size_t const cSize =
-+ singleStream ? HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
-+ if (HUF_isError(cSize)) {
-+ return cSize;
-+ }
-+ if (cSize == 0) {
-+ return 0;
-+ } /* uncompressible */
-+ op += cSize;
-+ /* check compressibility */
-+ if ((size_t)(op - ostart) >= srcSize - 1) {
-+ return 0;
-+ }
-+ return op - ostart;
-+}
-+
-+/* `workSpace` must a table of at least 1024 unsigned */
-+static size_t HUF_compress_internal(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog,
-+ unsigned singleStream, void *workSpace, size_t wkspSize, HUF_CElt *oldHufTable, HUF_repeat *repeat, int preferRepeat)
-+{
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *const oend = ostart + dstSize;
-+ BYTE *op = ostart;
-+
-+ U32 *count;
-+ size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
-+ HUF_CElt *CTable;
-+ size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
-+
-+ /* checks & inits */
-+ if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize)
-+ return ERROR(GENERIC);
-+ if (!srcSize)
-+ return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
-+ if (!dstSize)
-+ return 0; /* cannot fit within dst budget */
-+ if (srcSize > HUF_BLOCKSIZE_MAX)
-+ return ERROR(srcSize_wrong); /* curr block size limit */
-+ if (huffLog > HUF_TABLELOG_MAX)
-+ return ERROR(tableLog_tooLarge);
-+ if (!maxSymbolValue)
-+ maxSymbolValue = HUF_SYMBOLVALUE_MAX;
-+ if (!huffLog)
-+ huffLog = HUF_TABLELOG_DEFAULT;
-+
-+ count = (U32 *)workSpace;
-+ workSpace = (BYTE *)workSpace + countSize;
-+ wkspSize -= countSize;
-+ CTable = (HUF_CElt *)workSpace;
-+ workSpace = (BYTE *)workSpace + CTableSize;
-+ wkspSize -= CTableSize;
-+
-+ /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
-+ if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
-+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
-+ }
-+
-+ /* Scan input and build symbol stats */
-+ {
-+ CHECK_V_F(largest, FSE_count_wksp(count, &maxSymbolValue, (const BYTE *)src, srcSize, (U32 *)workSpace));
-+ if (largest == srcSize) {
-+ *ostart = ((const BYTE *)src)[0];
-+ return 1;
-+ } /* single symbol, rle */
-+ if (largest <= (srcSize >> 7) + 1)
-+ return 0; /* Fast heuristic : not compressible enough */
-+ }
-+
-+ /* Check validity of previous table */
-+ if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
-+ *repeat = HUF_repeat_none;
-+ }
-+ /* Heuristic : use existing table for small inputs */
-+ if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
-+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
-+ }
-+
-+ /* Build Huffman Tree */
-+ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
-+ {
-+ CHECK_V_F(maxBits, HUF_buildCTable_wksp(CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize));
-+ huffLog = (U32)maxBits;
-+ /* Zero the unused symbols so we can check it for validity */
-+ memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
-+ }
-+
-+ /* Write table description header */
-+ {
-+ CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, CTable, maxSymbolValue, huffLog, workSpace, wkspSize));
-+ /* Check if using the previous table will be beneficial */
-+ if (repeat && *repeat != HUF_repeat_none) {
-+ size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
-+ size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
-+ if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
-+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
-+ }
-+ }
-+ /* Use the new table */
-+ if (hSize + 12ul >= srcSize) {
-+ return 0;
-+ }
-+ op += hSize;
-+ if (repeat) {
-+ *repeat = HUF_repeat_none;
-+ }
-+ if (oldHufTable) {
-+ memcpy(oldHufTable, CTable, CTableSize);
-+ } /* Save the new table */
-+ }
-+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
-+}
-+
-+size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
-+ size_t wkspSize)
-+{
-+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
-+}
-+
-+size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
-+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
-+{
-+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat,
-+ preferRepeat);
-+}
-+
-+size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
-+ size_t wkspSize)
-+{
-+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
-+}
-+
-+size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
-+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
-+{
-+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat,
-+ preferRepeat);
-+}
-diff --git a/lib/zstd/huf_decompress.c b/lib/zstd/huf_decompress.c
-new file mode 100644
-index 0000000..6526482
---- /dev/null
-+++ b/lib/zstd/huf_decompress.c
-@@ -0,0 +1,960 @@
-+/*
-+ * Huffman decoder, part of New Generation Entropy library
-+ * Copyright (C) 2013-2016, Yann Collet.
-+ *
-+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are
-+ * met:
-+ *
-+ * * Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ * * Redistributions in binary form must reproduce the above
-+ * copyright notice, this list of conditions and the following disclaimer
-+ * in the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ *
-+ * You can contact the author at :
-+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-+ */
-+
-+/* **************************************************************
-+* Compiler specifics
-+****************************************************************/
-+#define FORCE_INLINE static __always_inline
-+
-+/* **************************************************************
-+* Dependencies
-+****************************************************************/
-+#include "bitstream.h" /* BIT_* */
-+#include "fse.h" /* header compression */
-+#include "huf.h"
-+#include <linux/compiler.h>
-+#include <linux/kernel.h>
-+#include <linux/string.h> /* memcpy, memset */
-+
-+/* **************************************************************
-+* Error Management
-+****************************************************************/
-+#define HUF_STATIC_ASSERT(c) \
-+ { \
-+ enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
-+ } /* use only *after* variable declarations */
-+
-+/*-***************************/
-+/* generic DTableDesc */
-+/*-***************************/
-+
-+typedef struct {
-+ BYTE maxTableLog;
-+ BYTE tableType;
-+ BYTE tableLog;
-+ BYTE reserved;
-+} DTableDesc;
-+
-+static DTableDesc HUF_getDTableDesc(const HUF_DTable *table)
-+{
-+ DTableDesc dtd;
-+ memcpy(&dtd, table, sizeof(dtd));
-+ return dtd;
-+}
-+
-+/*-***************************/
-+/* single-symbol decoding */
-+/*-***************************/
-+
-+typedef struct {
-+ BYTE byte;
-+ BYTE nbBits;
-+} HUF_DEltX2; /* single-symbol decoding */
-+
-+size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
-+{
-+ U32 tableLog = 0;
-+ U32 nbSymbols = 0;
-+ size_t iSize;
-+ void *const dtPtr = DTable + 1;
-+ HUF_DEltX2 *const dt = (HUF_DEltX2 *)dtPtr;
-+
-+ U32 *rankVal;
-+ BYTE *huffWeight;
-+ size_t spaceUsed32 = 0;
-+
-+ rankVal = (U32 *)workspace + spaceUsed32;
-+ spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1;
-+ huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
-+
-+ if ((spaceUsed32 << 2) > workspaceSize)
-+ return ERROR(tableLog_tooLarge);
-+ workspace = (U32 *)workspace + spaceUsed32;
-+ workspaceSize -= (spaceUsed32 << 2);
-+
-+ HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
-+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
-+
-+ iSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize);
-+ if (HUF_isError(iSize))
-+ return iSize;
-+
-+ /* Table header */
-+ {
-+ DTableDesc dtd = HUF_getDTableDesc(DTable);
-+ if (tableLog > (U32)(dtd.maxTableLog + 1))
-+ return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */
-+ dtd.tableType = 0;
-+ dtd.tableLog = (BYTE)tableLog;
-+ memcpy(DTable, &dtd, sizeof(dtd));
-+ }
-+
-+ /* Calculate starting value for each rank */
-+ {
-+ U32 n, nextRankStart = 0;
-+ for (n = 1; n < tableLog + 1; n++) {
-+ U32 const curr = nextRankStart;
-+ nextRankStart += (rankVal[n] << (n - 1));
-+ rankVal[n] = curr;
-+ }
-+ }
-+
-+ /* fill DTable */
-+ {
-+ U32 n;
-+ for (n = 0; n < nbSymbols; n++) {
-+ U32 const w = huffWeight[n];
-+ U32 const length = (1 << w) >> 1;
-+ U32 u;
-+ HUF_DEltX2 D;
-+ D.byte = (BYTE)n;
-+ D.nbBits = (BYTE)(tableLog + 1 - w);
-+ for (u = rankVal[w]; u < rankVal[w] + length; u++)
-+ dt[u] = D;
-+ rankVal[w] += length;
-+ }
-+ }
-+
-+ return iSize;
-+}
-+
-+static BYTE HUF_decodeSymbolX2(BIT_DStream_t *Dstream, const HUF_DEltX2 *dt, const U32 dtLog)
-+{
-+ size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
-+ BYTE const c = dt[val].byte;
-+ BIT_skipBits(Dstream, dt[val].nbBits);
-+ return c;
-+}
-+
-+#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
-+
-+#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
-+ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
-+ HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
-+
-+#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
-+ if (ZSTD_64bits()) \
-+ HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
-+
-+FORCE_INLINE size_t HUF_decodeStreamX2(BYTE *p, BIT_DStream_t *const bitDPtr, BYTE *const pEnd, const HUF_DEltX2 *const dt, const U32 dtLog)
-+{
-+ BYTE *const pStart = p;
-+
-+ /* up to 4 symbols at a time */
-+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd - 4)) {
-+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
-+ HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
-+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
-+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
-+ }
-+
-+ /* closer to the end */
-+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd))
-+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
-+
-+ /* no more data to retrieve from bitstream, hence no need to reload */
-+ while (p < pEnd)
-+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
-+
-+ return pEnd - pStart;
-+}
-+
-+static size_t HUF_decompress1X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ BYTE *op = (BYTE *)dst;
-+ BYTE *const oend = op + dstSize;
-+ const void *dtPtr = DTable + 1;
-+ const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr;
-+ BIT_DStream_t bitD;
-+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
-+ U32 const dtLog = dtd.tableLog;
-+
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+
-+ HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog);
-+
-+ /* check */
-+ if (!BIT_endOfDStream(&bitD))
-+ return ERROR(corruption_detected);
-+
-+ return dstSize;
-+}
-+
-+size_t HUF_decompress1X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ DTableDesc dtd = HUF_getDTableDesc(DTable);
-+ if (dtd.tableType != 0)
-+ return ERROR(GENERIC);
-+ return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
-+}
-+
-+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
-+{
-+ const BYTE *ip = (const BYTE *)cSrc;
-+
-+ size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize);
-+ if (HUF_isError(hSize))
-+ return hSize;
-+ if (hSize >= cSrcSize)
-+ return ERROR(srcSize_wrong);
-+ ip += hSize;
-+ cSrcSize -= hSize;
-+
-+ return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
-+}
-+
-+static size_t HUF_decompress4X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ /* Check */
-+ if (cSrcSize < 10)
-+ return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
-+
-+ {
-+ const BYTE *const istart = (const BYTE *)cSrc;
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *const oend = ostart + dstSize;
-+ const void *const dtPtr = DTable + 1;
-+ const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr;
-+
-+ /* Init */
-+ BIT_DStream_t bitD1;
-+ BIT_DStream_t bitD2;
-+ BIT_DStream_t bitD3;
-+ BIT_DStream_t bitD4;
-+ size_t const length1 = ZSTD_readLE16(istart);
-+ size_t const length2 = ZSTD_readLE16(istart + 2);
-+ size_t const length3 = ZSTD_readLE16(istart + 4);
-+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
-+ const BYTE *const istart1 = istart + 6; /* jumpTable */
-+ const BYTE *const istart2 = istart1 + length1;
-+ const BYTE *const istart3 = istart2 + length2;
-+ const BYTE *const istart4 = istart3 + length3;
-+ const size_t segmentSize = (dstSize + 3) / 4;
-+ BYTE *const opStart2 = ostart + segmentSize;
-+ BYTE *const opStart3 = opStart2 + segmentSize;
-+ BYTE *const opStart4 = opStart3 + segmentSize;
-+ BYTE *op1 = ostart;
-+ BYTE *op2 = opStart2;
-+ BYTE *op3 = opStart3;
-+ BYTE *op4 = opStart4;
-+ U32 endSignal;
-+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
-+ U32 const dtLog = dtd.tableLog;
-+
-+ if (length4 > cSrcSize)
-+ return ERROR(corruption_detected); /* overflow */
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+
-+ /* 16-32 symbols per loop (4-8 symbols per stream) */
-+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
-+ for (; (endSignal == BIT_DStream_unfinished) && (op4 < (oend - 7));) {
-+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
-+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
-+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
-+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
-+ HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
-+ HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
-+ HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
-+ HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
-+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
-+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
-+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
-+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
-+ HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
-+ HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
-+ HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
-+ HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
-+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
-+ }
-+
-+ /* check corruption */
-+ if (op1 > opStart2)
-+ return ERROR(corruption_detected);
-+ if (op2 > opStart3)
-+ return ERROR(corruption_detected);
-+ if (op3 > opStart4)
-+ return ERROR(corruption_detected);
-+ /* note : op4 supposed already verified within main loop */
-+
-+ /* finish bitStreams one by one */
-+ HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
-+ HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
-+ HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
-+ HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
-+
-+ /* check */
-+ endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
-+ if (!endSignal)
-+ return ERROR(corruption_detected);
-+
-+ /* decoded size */
-+ return dstSize;
-+ }
-+}
-+
-+size_t HUF_decompress4X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ DTableDesc dtd = HUF_getDTableDesc(DTable);
-+ if (dtd.tableType != 0)
-+ return ERROR(GENERIC);
-+ return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
-+}
-+
-+size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
-+{
-+ const BYTE *ip = (const BYTE *)cSrc;
-+
-+ size_t const hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize);
-+ if (HUF_isError(hSize))
-+ return hSize;
-+ if (hSize >= cSrcSize)
-+ return ERROR(srcSize_wrong);
-+ ip += hSize;
-+ cSrcSize -= hSize;
-+
-+ return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
-+}
-+
-+/* *************************/
-+/* double-symbols decoding */
-+/* *************************/
-+typedef struct {
-+ U16 sequence;
-+ BYTE nbBits;
-+ BYTE length;
-+} HUF_DEltX4; /* double-symbols decoding */
-+
-+typedef struct {
-+ BYTE symbol;
-+ BYTE weight;
-+} sortedSymbol_t;
-+
-+/* HUF_fillDTableX4Level2() :
-+ * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
-+static void HUF_fillDTableX4Level2(HUF_DEltX4 *DTable, U32 sizeLog, const U32 consumed, const U32 *rankValOrigin, const int minWeight,
-+ const sortedSymbol_t *sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq)
-+{
-+ HUF_DEltX4 DElt;
-+ U32 rankVal[HUF_TABLELOG_MAX + 1];
-+
-+ /* get pre-calculated rankVal */
-+ memcpy(rankVal, rankValOrigin, sizeof(rankVal));
-+
-+ /* fill skipped values */
-+ if (minWeight > 1) {
-+ U32 i, skipSize = rankVal[minWeight];
-+ ZSTD_writeLE16(&(DElt.sequence), baseSeq);
-+ DElt.nbBits = (BYTE)(consumed);
-+ DElt.length = 1;
-+ for (i = 0; i < skipSize; i++)
-+ DTable[i] = DElt;
-+ }
-+
-+ /* fill DTable */
-+ {
-+ U32 s;
-+ for (s = 0; s < sortedListSize; s++) { /* note : sortedSymbols already skipped */
-+ const U32 symbol = sortedSymbols[s].symbol;
-+ const U32 weight = sortedSymbols[s].weight;
-+ const U32 nbBits = nbBitsBaseline - weight;
-+ const U32 length = 1 << (sizeLog - nbBits);
-+ const U32 start = rankVal[weight];
-+ U32 i = start;
-+ const U32 end = start + length;
-+
-+ ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
-+ DElt.nbBits = (BYTE)(nbBits + consumed);
-+ DElt.length = 2;
-+ do {
-+ DTable[i++] = DElt;
-+ } while (i < end); /* since length >= 1 */
-+
-+ rankVal[weight] += length;
-+ }
-+ }
-+}
-+
-+typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
-+typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
-+
-+static void HUF_fillDTableX4(HUF_DEltX4 *DTable, const U32 targetLog, const sortedSymbol_t *sortedList, const U32 sortedListSize, const U32 *rankStart,
-+ rankVal_t rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline)
-+{
-+ U32 rankVal[HUF_TABLELOG_MAX + 1];
-+ const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
-+ const U32 minBits = nbBitsBaseline - maxWeight;
-+ U32 s;
-+
-+ memcpy(rankVal, rankValOrigin, sizeof(rankVal));
-+
-+ /* fill DTable */
-+ for (s = 0; s < sortedListSize; s++) {
-+ const U16 symbol = sortedList[s].symbol;
-+ const U32 weight = sortedList[s].weight;
-+ const U32 nbBits = nbBitsBaseline - weight;
-+ const U32 start = rankVal[weight];
-+ const U32 length = 1 << (targetLog - nbBits);
-+
-+ if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */
-+ U32 sortedRank;
-+ int minWeight = nbBits + scaleLog;
-+ if (minWeight < 1)
-+ minWeight = 1;
-+ sortedRank = rankStart[minWeight];
-+ HUF_fillDTableX4Level2(DTable + start, targetLog - nbBits, nbBits, rankValOrigin[nbBits], minWeight, sortedList + sortedRank,
-+ sortedListSize - sortedRank, nbBitsBaseline, symbol);
-+ } else {
-+ HUF_DEltX4 DElt;
-+ ZSTD_writeLE16(&(DElt.sequence), symbol);
-+ DElt.nbBits = (BYTE)(nbBits);
-+ DElt.length = 1;
-+ {
-+ U32 const end = start + length;
-+ U32 u;
-+ for (u = start; u < end; u++)
-+ DTable[u] = DElt;
-+ }
-+ }
-+ rankVal[weight] += length;
-+ }
-+}
-+
-+size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
-+{
-+ U32 tableLog, maxW, sizeOfSort, nbSymbols;
-+ DTableDesc dtd = HUF_getDTableDesc(DTable);
-+ U32 const maxTableLog = dtd.maxTableLog;
-+ size_t iSize;
-+ void *dtPtr = DTable + 1; /* force compiler to avoid strict-aliasing */
-+ HUF_DEltX4 *const dt = (HUF_DEltX4 *)dtPtr;
-+ U32 *rankStart;
-+
-+ rankValCol_t *rankVal;
-+ U32 *rankStats;
-+ U32 *rankStart0;
-+ sortedSymbol_t *sortedSymbol;
-+ BYTE *weightList;
-+ size_t spaceUsed32 = 0;
-+
-+ HUF_STATIC_ASSERT((sizeof(rankValCol_t) & 3) == 0);
-+
-+ rankVal = (rankValCol_t *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2;
-+ rankStats = (U32 *)workspace + spaceUsed32;
-+ spaceUsed32 += HUF_TABLELOG_MAX + 1;
-+ rankStart0 = (U32 *)workspace + spaceUsed32;
-+ spaceUsed32 += HUF_TABLELOG_MAX + 2;
-+ sortedSymbol = (sortedSymbol_t *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2;
-+ weightList = (BYTE *)((U32 *)workspace + spaceUsed32);
-+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
-+
-+ if ((spaceUsed32 << 2) > workspaceSize)
-+ return ERROR(tableLog_tooLarge);
-+ workspace = (U32 *)workspace + spaceUsed32;
-+ workspaceSize -= (spaceUsed32 << 2);
-+
-+ rankStart = rankStart0 + 1;
-+ memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1));
-+
-+ HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
-+ if (maxTableLog > HUF_TABLELOG_MAX)
-+ return ERROR(tableLog_tooLarge);
-+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
-+
-+ iSize = HUF_readStats_wksp(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize);
-+ if (HUF_isError(iSize))
-+ return iSize;
-+
-+ /* check result */
-+ if (tableLog > maxTableLog)
-+ return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
-+
-+ /* find maxWeight */
-+ for (maxW = tableLog; rankStats[maxW] == 0; maxW--) {
-+ } /* necessarily finds a solution before 0 */
-+
-+ /* Get start index of each weight */
-+ {
-+ U32 w, nextRankStart = 0;
-+ for (w = 1; w < maxW + 1; w++) {
-+ U32 curr = nextRankStart;
-+ nextRankStart += rankStats[w];
-+ rankStart[w] = curr;
-+ }
-+ rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
-+ sizeOfSort = nextRankStart;
-+ }
-+
-+ /* sort symbols by weight */
-+ {
-+ U32 s;
-+ for (s = 0; s < nbSymbols; s++) {
-+ U32 const w = weightList[s];
-+ U32 const r = rankStart[w]++;
-+ sortedSymbol[r].symbol = (BYTE)s;
-+ sortedSymbol[r].weight = (BYTE)w;
-+ }
-+ rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
-+ }
-+
-+ /* Build rankVal */
-+ {
-+ U32 *const rankVal0 = rankVal[0];
-+ {
-+ int const rescale = (maxTableLog - tableLog) - 1; /* tableLog <= maxTableLog */
-+ U32 nextRankVal = 0;
-+ U32 w;
-+ for (w = 1; w < maxW + 1; w++) {
-+ U32 curr = nextRankVal;
-+ nextRankVal += rankStats[w] << (w + rescale);
-+ rankVal0[w] = curr;
-+ }
-+ }
-+ {
-+ U32 const minBits = tableLog + 1 - maxW;
-+ U32 consumed;
-+ for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
-+ U32 *const rankValPtr = rankVal[consumed];
-+ U32 w;
-+ for (w = 1; w < maxW + 1; w++) {
-+ rankValPtr[w] = rankVal0[w] >> consumed;
-+ }
-+ }
-+ }
-+ }
-+
-+ HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog + 1);
-+
-+ dtd.tableLog = (BYTE)maxTableLog;
-+ dtd.tableType = 1;
-+ memcpy(DTable, &dtd, sizeof(dtd));
-+ return iSize;
-+}
-+
-+static U32 HUF_decodeSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog)
-+{
-+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
-+ memcpy(op, dt + val, 2);
-+ BIT_skipBits(DStream, dt[val].nbBits);
-+ return dt[val].length;
-+}
-+
-+static U32 HUF_decodeLastSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog)
-+{
-+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
-+ memcpy(op, dt + val, 1);
-+ if (dt[val].length == 1)
-+ BIT_skipBits(DStream, dt[val].nbBits);
-+ else {
-+ if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) {
-+ BIT_skipBits(DStream, dt[val].nbBits);
-+ if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8))
-+ /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
-+ DStream->bitsConsumed = (sizeof(DStream->bitContainer) * 8);
-+ }
-+ }
-+ return 1;
-+}
-+
-+#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
-+
-+#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
-+ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
-+ ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
-+
-+#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
-+ if (ZSTD_64bits()) \
-+ ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
-+
-+FORCE_INLINE size_t HUF_decodeStreamX4(BYTE *p, BIT_DStream_t *bitDPtr, BYTE *const pEnd, const HUF_DEltX4 *const dt, const U32 dtLog)
-+{
-+ BYTE *const pStart = p;
-+
-+ /* up to 8 symbols at a time */
-+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd - (sizeof(bitDPtr->bitContainer) - 1))) {
-+ HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
-+ HUF_DECODE_SYMBOLX4_1(p, bitDPtr);
-+ HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
-+ HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
-+ }
-+
-+ /* closer to end : up to 2 symbols at a time */
-+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd - 2))
-+ HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
-+
-+ while (p <= pEnd - 2)
-+ HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
-+
-+ if (p < pEnd)
-+ p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
-+
-+ return p - pStart;
-+}
-+
-+static size_t HUF_decompress1X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ BIT_DStream_t bitD;
-+
-+ /* Init */
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+
-+ /* decode */
-+ {
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *const oend = ostart + dstSize;
-+ const void *const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */
-+ const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr;
-+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
-+ HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog);
-+ }
-+
-+ /* check */
-+ if (!BIT_endOfDStream(&bitD))
-+ return ERROR(corruption_detected);
-+
-+ /* decoded size */
-+ return dstSize;
-+}
-+
-+size_t HUF_decompress1X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ DTableDesc dtd = HUF_getDTableDesc(DTable);
-+ if (dtd.tableType != 1)
-+ return ERROR(GENERIC);
-+ return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
-+}
-+
-+size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
-+{
-+ const BYTE *ip = (const BYTE *)cSrc;
-+
-+ size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize);
-+ if (HUF_isError(hSize))
-+ return hSize;
-+ if (hSize >= cSrcSize)
-+ return ERROR(srcSize_wrong);
-+ ip += hSize;
-+ cSrcSize -= hSize;
-+
-+ return HUF_decompress1X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
-+}
-+
-+static size_t HUF_decompress4X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ if (cSrcSize < 10)
-+ return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
-+
-+ {
-+ const BYTE *const istart = (const BYTE *)cSrc;
-+ BYTE *const ostart = (BYTE *)dst;
-+ BYTE *const oend = ostart + dstSize;
-+ const void *const dtPtr = DTable + 1;
-+ const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr;
-+
-+ /* Init */
-+ BIT_DStream_t bitD1;
-+ BIT_DStream_t bitD2;
-+ BIT_DStream_t bitD3;
-+ BIT_DStream_t bitD4;
-+ size_t const length1 = ZSTD_readLE16(istart);
-+ size_t const length2 = ZSTD_readLE16(istart + 2);
-+ size_t const length3 = ZSTD_readLE16(istart + 4);
-+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
-+ const BYTE *const istart1 = istart + 6; /* jumpTable */
-+ const BYTE *const istart2 = istart1 + length1;
-+ const BYTE *const istart3 = istart2 + length2;
-+ const BYTE *const istart4 = istart3 + length3;
-+ size_t const segmentSize = (dstSize + 3) / 4;
-+ BYTE *const opStart2 = ostart + segmentSize;
-+ BYTE *const opStart3 = opStart2 + segmentSize;
-+ BYTE *const opStart4 = opStart3 + segmentSize;
-+ BYTE *op1 = ostart;
-+ BYTE *op2 = opStart2;
-+ BYTE *op3 = opStart3;
-+ BYTE *op4 = opStart4;
-+ U32 endSignal;
-+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
-+ U32 const dtLog = dtd.tableLog;
-+
-+ if (length4 > cSrcSize)
-+ return ERROR(corruption_detected); /* overflow */
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+ {
-+ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
-+ if (HUF_isError(errorCode))
-+ return errorCode;
-+ }
-+
-+ /* 16-32 symbols per loop (4-8 symbols per stream) */
-+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
-+ for (; (endSignal == BIT_DStream_unfinished) & (op4 < (oend - (sizeof(bitD4.bitContainer) - 1)));) {
-+ HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
-+ HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
-+ HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
-+ HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
-+ HUF_DECODE_SYMBOLX4_1(op1, &bitD1);
-+ HUF_DECODE_SYMBOLX4_1(op2, &bitD2);
-+ HUF_DECODE_SYMBOLX4_1(op3, &bitD3);
-+ HUF_DECODE_SYMBOLX4_1(op4, &bitD4);
-+ HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
-+ HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
-+ HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
-+ HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
-+ HUF_DECODE_SYMBOLX4_0(op1, &bitD1);
-+ HUF_DECODE_SYMBOLX4_0(op2, &bitD2);
-+ HUF_DECODE_SYMBOLX4_0(op3, &bitD3);
-+ HUF_DECODE_SYMBOLX4_0(op4, &bitD4);
-+
-+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
-+ }
-+
-+ /* check corruption */
-+ if (op1 > opStart2)
-+ return ERROR(corruption_detected);
-+ if (op2 > opStart3)
-+ return ERROR(corruption_detected);
-+ if (op3 > opStart4)
-+ return ERROR(corruption_detected);
-+ /* note : op4 already verified within main loop */
-+
-+ /* finish bitStreams one by one */
-+ HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
-+ HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
-+ HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
-+ HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
-+
-+ /* check */
-+ {
-+ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
-+ if (!endCheck)
-+ return ERROR(corruption_detected);
-+ }
-+
-+ /* decoded size */
-+ return dstSize;
-+ }
-+}
-+
-+size_t HUF_decompress4X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ DTableDesc dtd = HUF_getDTableDesc(DTable);
-+ if (dtd.tableType != 1)
-+ return ERROR(GENERIC);
-+ return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
-+}
-+
-+size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
-+{
-+ const BYTE *ip = (const BYTE *)cSrc;
-+
-+ size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize);
-+ if (HUF_isError(hSize))
-+ return hSize;
-+ if (hSize >= cSrcSize)
-+ return ERROR(srcSize_wrong);
-+ ip += hSize;
-+ cSrcSize -= hSize;
-+
-+ return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
-+}
-+
-+/* ********************************/
-+/* Generic decompression selector */
-+/* ********************************/
-+
-+size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
-+ return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
-+ : HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
-+}
-+
-+size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
-+{
-+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
-+ return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
-+ : HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
-+}
-+
-+typedef struct {
-+ U32 tableTime;
-+ U32 decode256Time;
-+} algo_time_t;
-+static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = {
-+ /* single, double, quad */
-+ {{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */
-+ {{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */
-+ {{38, 130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */
-+ {{448, 128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */
-+ {{556, 128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */
-+ {{714, 128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */
-+ {{883, 128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */
-+ {{897, 128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */
-+ {{926, 128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */
-+ {{947, 128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */
-+ {{1107, 128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */
-+ {{1177, 128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */
-+ {{1242, 128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */
-+ {{1349, 128}, {2644, 106}, {5260, 106}}, /* Q ==13 : 81-87% */
-+ {{1455, 128}, {2422, 124}, {4174, 124}}, /* Q ==14 : 87-93% */
-+ {{722, 128}, {1891, 145}, {1936, 146}}, /* Q ==15 : 93-99% */
-+};
-+
-+/** HUF_selectDecoder() :
-+* Tells which decoder is likely to decode faster,
-+* based on a set of pre-determined metrics.
-+* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
-+* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
-+U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize)
-+{
-+ /* decoder timing evaluation */
-+ U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
-+ U32 const D256 = (U32)(dstSize >> 8);
-+ U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
-+ U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
-+ DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */
-+
-+ return DTime1 < DTime0;
-+}
-+
-+typedef size_t (*decompressionAlgo)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize);
-+
-+size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
-+{
-+ /* validation checks */
-+ if (dstSize == 0)
-+ return ERROR(dstSize_tooSmall);
-+ if (cSrcSize > dstSize)
-+ return ERROR(corruption_detected); /* invalid */
-+ if (cSrcSize == dstSize) {
-+ memcpy(dst, cSrc, dstSize);
-+ return dstSize;
-+ } /* not compressed */
-+ if (cSrcSize == 1) {
-+ memset(dst, *(const BYTE *)cSrc, dstSize);
-+ return dstSize;
-+ } /* RLE */
-+
-+ {
-+ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-+ return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize)
-+ : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize);
-+ }
-+}
-+
-+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
-+{
-+ /* validation checks */
-+ if (dstSize == 0)
-+ return ERROR(dstSize_tooSmall);
-+ if ((cSrcSize >= dstSize) || (cSrcSize <= 1))
-+ return ERROR(corruption_detected); /* invalid */
-+
-+ {
-+ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-+ return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize)
-+ : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize);
-+ }
-+}
-+
-+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
-+{
-+ /* validation checks */
-+ if (dstSize == 0)
-+ return ERROR(dstSize_tooSmall);
-+ if (cSrcSize > dstSize)
-+ return ERROR(corruption_detected); /* invalid */
-+ if (cSrcSize == dstSize) {
-+ memcpy(dst, cSrc, dstSize);
-+ return dstSize;
-+ } /* not compressed */
-+ if (cSrcSize == 1) {
-+ memset(dst, *(const BYTE *)cSrc, dstSize);
-+ return dstSize;
-+ } /* RLE */
-+
-+ {
-+ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-+ return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize)
-+ : HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize);
-+ }
-+}
-diff --git a/lib/zstd/mem.h b/lib/zstd/mem.h
-new file mode 100644
-index 0000000..3a0f34c
---- /dev/null
-+++ b/lib/zstd/mem.h
-@@ -0,0 +1,151 @@
-+/**
-+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This source code is licensed under the BSD-style license found in the
-+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
-+ * An additional grant of patent rights can be found in the PATENTS file in the
-+ * same directory.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ */
-+
-+#ifndef MEM_H_MODULE
-+#define MEM_H_MODULE
-+
-+/*-****************************************
-+* Dependencies
-+******************************************/
-+#include <asm/unaligned.h>
-+#include <linux/string.h> /* memcpy */
-+#include <linux/types.h> /* size_t, ptrdiff_t */
-+
-+/*-****************************************
-+* Compiler specifics
-+******************************************/
-+#define ZSTD_STATIC static __inline __attribute__((unused))
-+
-+/*-**************************************************************
-+* Basic Types
-+*****************************************************************/
-+typedef uint8_t BYTE;
-+typedef uint16_t U16;
-+typedef int16_t S16;
-+typedef uint32_t U32;
-+typedef int32_t S32;
-+typedef uint64_t U64;
-+typedef int64_t S64;
-+typedef ptrdiff_t iPtrDiff;
-+typedef uintptr_t uPtrDiff;
-+
-+/*-**************************************************************
-+* Memory I/O
-+*****************************************************************/
-+ZSTD_STATIC unsigned ZSTD_32bits(void) { return sizeof(size_t) == 4; }
-+ZSTD_STATIC unsigned ZSTD_64bits(void) { return sizeof(size_t) == 8; }
-+
-+#if defined(__LITTLE_ENDIAN)
-+#define ZSTD_LITTLE_ENDIAN 1
-+#else
-+#define ZSTD_LITTLE_ENDIAN 0
-+#endif
-+
-+ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) { return ZSTD_LITTLE_ENDIAN; }
-+
-+ZSTD_STATIC U16 ZSTD_read16(const void *memPtr) { return get_unaligned((const U16 *)memPtr); }
-+
-+ZSTD_STATIC U32 ZSTD_read32(const void *memPtr) { return get_unaligned((const U32 *)memPtr); }
-+
-+ZSTD_STATIC U64 ZSTD_read64(const void *memPtr) { return get_unaligned((const U64 *)memPtr); }
-+
-+ZSTD_STATIC size_t ZSTD_readST(const void *memPtr) { return get_unaligned((const size_t *)memPtr); }
-+
-+ZSTD_STATIC void ZSTD_write16(void *memPtr, U16 value) { put_unaligned(value, (U16 *)memPtr); }
-+
-+ZSTD_STATIC void ZSTD_write32(void *memPtr, U32 value) { put_unaligned(value, (U32 *)memPtr); }
-+
-+ZSTD_STATIC void ZSTD_write64(void *memPtr, U64 value) { put_unaligned(value, (U64 *)memPtr); }
-+
-+/*=== Little endian r/w ===*/
-+
-+ZSTD_STATIC U16 ZSTD_readLE16(const void *memPtr) { return get_unaligned_le16(memPtr); }
-+
-+ZSTD_STATIC void ZSTD_writeLE16(void *memPtr, U16 val) { put_unaligned_le16(val, memPtr); }
-+
-+ZSTD_STATIC U32 ZSTD_readLE24(const void *memPtr) { return ZSTD_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); }
-+
-+ZSTD_STATIC void ZSTD_writeLE24(void *memPtr, U32 val)
-+{
-+ ZSTD_writeLE16(memPtr, (U16)val);
-+ ((BYTE *)memPtr)[2] = (BYTE)(val >> 16);
-+}
-+
-+ZSTD_STATIC U32 ZSTD_readLE32(const void *memPtr) { return get_unaligned_le32(memPtr); }
-+
-+ZSTD_STATIC void ZSTD_writeLE32(void *memPtr, U32 val32) { put_unaligned_le32(val32, memPtr); }
-+
-+ZSTD_STATIC U64 ZSTD_readLE64(const void *memPtr) { return get_unaligned_le64(memPtr); }
-+
-+ZSTD_STATIC void ZSTD_writeLE64(void *memPtr, U64 val64) { put_unaligned_le64(val64, memPtr); }
-+
-+ZSTD_STATIC size_t ZSTD_readLEST(const void *memPtr)
-+{
-+ if (ZSTD_32bits())
-+ return (size_t)ZSTD_readLE32(memPtr);
-+ else
-+ return (size_t)ZSTD_readLE64(memPtr);
-+}
-+
-+ZSTD_STATIC void ZSTD_writeLEST(void *memPtr, size_t val)
-+{
-+ if (ZSTD_32bits())
-+ ZSTD_writeLE32(memPtr, (U32)val);
-+ else
-+ ZSTD_writeLE64(memPtr, (U64)val);
-+}
-+
-+/*=== Big endian r/w ===*/
-+
-+ZSTD_STATIC U32 ZSTD_readBE32(const void *memPtr) { return get_unaligned_be32(memPtr); }
-+
-+ZSTD_STATIC void ZSTD_writeBE32(void *memPtr, U32 val32) { put_unaligned_be32(val32, memPtr); }
-+
-+ZSTD_STATIC U64 ZSTD_readBE64(const void *memPtr) { return get_unaligned_be64(memPtr); }
-+
-+ZSTD_STATIC void ZSTD_writeBE64(void *memPtr, U64 val64) { put_unaligned_be64(val64, memPtr); }
-+
-+ZSTD_STATIC size_t ZSTD_readBEST(const void *memPtr)
-+{
-+ if (ZSTD_32bits())
-+ return (size_t)ZSTD_readBE32(memPtr);
-+ else
-+ return (size_t)ZSTD_readBE64(memPtr);
-+}
-+
-+ZSTD_STATIC void ZSTD_writeBEST(void *memPtr, size_t val)
-+{
-+ if (ZSTD_32bits())
-+ ZSTD_writeBE32(memPtr, (U32)val);
-+ else
-+ ZSTD_writeBE64(memPtr, (U64)val);
-+}
-+
-+/* function safe only for comparisons */
-+ZSTD_STATIC U32 ZSTD_readMINMATCH(const void *memPtr, U32 length)
-+{
-+ switch (length) {
-+ default:
-+ case 4: return ZSTD_read32(memPtr);
-+ case 3:
-+ if (ZSTD_isLittleEndian())
-+ return ZSTD_read32(memPtr) << 8;
-+ else
-+ return ZSTD_read32(memPtr) >> 8;
-+ }
-+}
-+
-+#endif /* MEM_H_MODULE */
-diff --git a/lib/zstd/zstd_common.c b/lib/zstd/zstd_common.c
-new file mode 100644
-index 0000000..a282624
---- /dev/null
-+++ b/lib/zstd/zstd_common.c
-@@ -0,0 +1,75 @@
-+/**
-+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This source code is licensed under the BSD-style license found in the
-+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
-+ * An additional grant of patent rights can be found in the PATENTS file in the
-+ * same directory.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ */
-+
-+/*-*************************************
-+* Dependencies
-+***************************************/
-+#include "error_private.h"
-+#include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
-+#include <linux/kernel.h>
-+
-+/*=**************************************************************
-+* Custom allocator
-+****************************************************************/
-+
-+#define stack_push(stack, size) \
-+ ({ \
-+ void *const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \
-+ (stack)->ptr = (char *)ptr + (size); \
-+ (stack)->ptr <= (stack)->end ? ptr : NULL; \
-+ })
-+
-+ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize)
-+{
-+ ZSTD_customMem stackMem = {ZSTD_stackAlloc, ZSTD_stackFree, workspace};
-+ ZSTD_stack *stack = (ZSTD_stack *)workspace;
-+ /* Verify preconditions */
-+ if (!workspace || workspaceSize < sizeof(ZSTD_stack) || workspace != ZSTD_PTR_ALIGN(workspace)) {
-+ ZSTD_customMem error = {NULL, NULL, NULL};
-+ return error;
-+ }
-+ /* Initialize the stack */
-+ stack->ptr = workspace;
-+ stack->end = (char *)workspace + workspaceSize;
-+ stack_push(stack, sizeof(ZSTD_stack));
-+ return stackMem;
-+}
-+
-+void *ZSTD_stackAllocAll(void *opaque, size_t *size)
-+{
-+ ZSTD_stack *stack = (ZSTD_stack *)opaque;
-+ *size = (BYTE const *)stack->end - (BYTE *)ZSTD_PTR_ALIGN(stack->ptr);
-+ return stack_push(stack, *size);
-+}
-+
-+void *ZSTD_stackAlloc(void *opaque, size_t size)
-+{
-+ ZSTD_stack *stack = (ZSTD_stack *)opaque;
-+ return stack_push(stack, size);
-+}
-+void ZSTD_stackFree(void *opaque, void *address)
-+{
-+ (void)opaque;
-+ (void)address;
-+}
-+
-+void *ZSTD_malloc(size_t size, ZSTD_customMem customMem) { return customMem.customAlloc(customMem.opaque, size); }
-+
-+void ZSTD_free(void *ptr, ZSTD_customMem customMem)
-+{
-+ if (ptr != NULL)
-+ customMem.customFree(customMem.opaque, ptr);
-+}
-diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h
-new file mode 100644
-index 0000000..1a79fab
---- /dev/null
-+++ b/lib/zstd/zstd_internal.h
-@@ -0,0 +1,263 @@
-+/**
-+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This source code is licensed under the BSD-style license found in the
-+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
-+ * An additional grant of patent rights can be found in the PATENTS file in the
-+ * same directory.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ */
-+
-+#ifndef ZSTD_CCOMMON_H_MODULE
-+#define ZSTD_CCOMMON_H_MODULE
-+
-+/*-*******************************************************
-+* Compiler specifics
-+*********************************************************/
-+#define FORCE_INLINE static __always_inline
-+#define FORCE_NOINLINE static noinline
-+
-+/*-*************************************
-+* Dependencies
-+***************************************/
-+#include "error_private.h"
-+#include "mem.h"
-+#include <linux/compiler.h>
-+#include <linux/kernel.h>
-+#include <linux/xxhash.h>
-+#include <linux/zstd.h>
-+
-+/*-*************************************
-+* shared macros
-+***************************************/
-+#define MIN(a, b) ((a) < (b) ? (a) : (b))
-+#define MAX(a, b) ((a) > (b) ? (a) : (b))
-+#define CHECK_F(f) \
-+ { \
-+ size_t const errcod = f; \
-+ if (ERR_isError(errcod)) \
-+ return errcod; \
-+ } /* check and Forward error code */
-+#define CHECK_E(f, e) \
-+ { \
-+ size_t const errcod = f; \
-+ if (ERR_isError(errcod)) \
-+ return ERROR(e); \
-+ } /* check and send Error code */
-+#define ZSTD_STATIC_ASSERT(c) \
-+ { \
-+ enum { ZSTD_static_assert = 1 / (int)(!!(c)) }; \
-+ }
-+
-+/*-*************************************
-+* Common constants
-+***************************************/
-+#define ZSTD_OPT_NUM (1 << 12)
-+#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */
-+
-+#define ZSTD_REP_NUM 3 /* number of repcodes */
-+#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */
-+#define ZSTD_REP_MOVE (ZSTD_REP_NUM - 1)
-+#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM)
-+static const U32 repStartValue[ZSTD_REP_NUM] = {1, 4, 8};
-+
-+#define KB *(1 << 10)
-+#define MB *(1 << 20)
-+#define GB *(1U << 30)
-+
-+#define BIT7 128
-+#define BIT6 64
-+#define BIT5 32
-+#define BIT4 16
-+#define BIT1 2
-+#define BIT0 1
-+
-+#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
-+static const size_t ZSTD_fcs_fieldSize[4] = {0, 2, 4, 8};
-+static const size_t ZSTD_did_fieldSize[4] = {0, 1, 2, 4};
-+
-+#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
-+static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
-+typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
-+
-+#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
-+#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
-+
-+#define HufLog 12
-+typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
-+
-+#define LONGNBSEQ 0x7F00
-+
-+#define MINMATCH 3
-+#define EQUAL_READ32 4
-+
-+#define Litbits 8
-+#define MaxLit ((1 << Litbits) - 1)
-+#define MaxML 52
-+#define MaxLL 35
-+#define MaxOff 28
-+#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
-+#define MLFSELog 9
-+#define LLFSELog 9
-+#define OffFSELog 8
-+
-+static const U32 LL_bits[MaxLL + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
-+static const S16 LL_defaultNorm[MaxLL + 1] = {4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1};
-+#define LL_DEFAULTNORMLOG 6 /* for static allocation */
-+static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
-+
-+static const U32 ML_bits[MaxML + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-+ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
-+static const S16 ML_defaultNorm[MaxML + 1] = {1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1};
-+#define ML_DEFAULTNORMLOG 6 /* for static allocation */
-+static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
-+
-+static const S16 OF_defaultNorm[MaxOff + 1] = {1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1};
-+#define OF_DEFAULTNORMLOG 5 /* for static allocation */
-+static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
-+
-+/*-*******************************************
-+* Shared functions to include for inlining
-+*********************************************/
-+ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) {
-+ memcpy(dst, src, 8);
-+}
-+/*! ZSTD_wildcopy() :
-+* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
-+#define WILDCOPY_OVERLENGTH 8
-+ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
-+{
-+ const BYTE* ip = (const BYTE*)src;
-+ BYTE* op = (BYTE*)dst;
-+ BYTE* const oend = op + length;
-+ /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
-+ * Avoid the bad case where the loop only runs once by handling the
-+ * special case separately. This doesn't trigger the bug because it
-+ * doesn't involve pointer/integer overflow.
-+ */
-+ if (length <= 8)
-+ return ZSTD_copy8(dst, src);
-+ do {
-+ ZSTD_copy8(op, ip);
-+ op += 8;
-+ ip += 8;
-+ } while (op < oend);
-+}
-+
-+/*-*******************************************
-+* Private interfaces
-+*********************************************/
-+typedef struct ZSTD_stats_s ZSTD_stats_t;
-+
-+typedef struct {
-+ U32 off;
-+ U32 len;
-+} ZSTD_match_t;
-+
-+typedef struct {
-+ U32 price;
-+ U32 off;
-+ U32 mlen;
-+ U32 litlen;
-+ U32 rep[ZSTD_REP_NUM];
-+} ZSTD_optimal_t;
-+
-+typedef struct seqDef_s {
-+ U32 offset;
-+ U16 litLength;
-+ U16 matchLength;
-+} seqDef;
-+
-+typedef struct {
-+ seqDef *sequencesStart;
-+ seqDef *sequences;
-+ BYTE *litStart;
-+ BYTE *lit;
-+ BYTE *llCode;
-+ BYTE *mlCode;
-+ BYTE *ofCode;
-+ U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
-+ U32 longLengthPos;
-+ /* opt */
-+ ZSTD_optimal_t *priceTable;
-+ ZSTD_match_t *matchTable;
-+ U32 *matchLengthFreq;
-+ U32 *litLengthFreq;
-+ U32 *litFreq;
-+ U32 *offCodeFreq;
-+ U32 matchLengthSum;
-+ U32 matchSum;
-+ U32 litLengthSum;
-+ U32 litSum;
-+ U32 offCodeSum;
-+ U32 log2matchLengthSum;
-+ U32 log2matchSum;
-+ U32 log2litLengthSum;
-+ U32 log2litSum;
-+ U32 log2offCodeSum;
-+ U32 factor;
-+ U32 staticPrices;
-+ U32 cachedPrice;
-+ U32 cachedLitLength;
-+ const BYTE *cachedLiterals;
-+} seqStore_t;
-+
-+const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx);
-+void ZSTD_seqToCodes(const seqStore_t *seqStorePtr);
-+int ZSTD_isSkipFrame(ZSTD_DCtx *dctx);
-+
-+/*= Custom memory allocation functions */
-+typedef void *(*ZSTD_allocFunction)(void *opaque, size_t size);
-+typedef void (*ZSTD_freeFunction)(void *opaque, void *address);
-+typedef struct {
-+ ZSTD_allocFunction customAlloc;
-+ ZSTD_freeFunction customFree;
-+ void *opaque;
-+} ZSTD_customMem;
-+
-+void *ZSTD_malloc(size_t size, ZSTD_customMem customMem);
-+void ZSTD_free(void *ptr, ZSTD_customMem customMem);
-+
-+/*====== stack allocation ======*/
-+
-+typedef struct {
-+ void *ptr;
-+ const void *end;
-+} ZSTD_stack;
-+
-+#define ZSTD_ALIGN(x) ALIGN(x, sizeof(size_t))
-+#define ZSTD_PTR_ALIGN(p) PTR_ALIGN(p, sizeof(size_t))
-+
-+ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize);
-+
-+void *ZSTD_stackAllocAll(void *opaque, size_t *size);
-+void *ZSTD_stackAlloc(void *opaque, size_t size);
-+void ZSTD_stackFree(void *opaque, void *address);
-+
-+/*====== common function ======*/
-+
-+ZSTD_STATIC U32 ZSTD_highbit32(U32 val) { return 31 - __builtin_clz(val); }
-+
-+/* hidden functions */
-+
-+/* ZSTD_invalidateRepCodes() :
-+ * ensures next compression will not use repcodes from previous block.
-+ * Note : only works with regular variant;
-+ * do not use with extDict variant ! */
-+void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx);
-+
-+size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx);
-+size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx);
-+size_t ZSTD_freeCDict(ZSTD_CDict *cdict);
-+size_t ZSTD_freeDDict(ZSTD_DDict *cdict);
-+size_t ZSTD_freeCStream(ZSTD_CStream *zcs);
-+size_t ZSTD_freeDStream(ZSTD_DStream *zds);
-+
-+#endif /* ZSTD_CCOMMON_H_MODULE */
-diff --git a/lib/zstd/zstd_opt.h b/lib/zstd/zstd_opt.h
-new file mode 100644
-index 0000000..55e1b4c
---- /dev/null
-+++ b/lib/zstd/zstd_opt.h
-@@ -0,0 +1,1014 @@
-+/**
-+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This source code is licensed under the BSD-style license found in the
-+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
-+ * An additional grant of patent rights can be found in the PATENTS file in the
-+ * same directory.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it under
-+ * the terms of the GNU General Public License version 2 as published by the
-+ * Free Software Foundation. This program is dual-licensed; you may select
-+ * either version 2 of the GNU General Public License ("GPL") or BSD license
-+ * ("BSD").
-+ */
-+
-+/* Note : this file is intended to be included within zstd_compress.c */
-+
-+#ifndef ZSTD_OPT_H_91842398743
-+#define ZSTD_OPT_H_91842398743
-+
-+#define ZSTD_LITFREQ_ADD 2
-+#define ZSTD_FREQ_DIV 4
-+#define ZSTD_MAX_PRICE (1 << 30)
-+
-+/*-*************************************
-+* Price functions for optimal parser
-+***************************************/
-+FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t *ssPtr)
-+{
-+ ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum + 1);
-+ ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum + 1);
-+ ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum + 1);
-+ ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum + 1);
-+ ssPtr->factor = 1 + ((ssPtr->litSum >> 5) / ssPtr->litLengthSum) + ((ssPtr->litSum << 1) / (ssPtr->litSum + ssPtr->matchSum));
-+}
-+
-+ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t *ssPtr, const BYTE *src, size_t srcSize)
-+{
-+ unsigned u;
-+
-+ ssPtr->cachedLiterals = NULL;
-+ ssPtr->cachedPrice = ssPtr->cachedLitLength = 0;
-+ ssPtr->staticPrices = 0;
-+
-+ if (ssPtr->litLengthSum == 0) {
-+ if (srcSize <= 1024)
-+ ssPtr->staticPrices = 1;
-+
-+ for (u = 0; u <= MaxLit; u++)
-+ ssPtr->litFreq[u] = 0;
-+ for (u = 0; u < srcSize; u++)
-+ ssPtr->litFreq[src[u]]++;
-+
-+ ssPtr->litSum = 0;
-+ ssPtr->litLengthSum = MaxLL + 1;
-+ ssPtr->matchLengthSum = MaxML + 1;
-+ ssPtr->offCodeSum = (MaxOff + 1);
-+ ssPtr->matchSum = (ZSTD_LITFREQ_ADD << Litbits);
-+
-+ for (u = 0; u <= MaxLit; u++) {
-+ ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> ZSTD_FREQ_DIV);
-+ ssPtr->litSum += ssPtr->litFreq[u];
-+ }
-+ for (u = 0; u <= MaxLL; u++)
-+ ssPtr->litLengthFreq[u] = 1;
-+ for (u = 0; u <= MaxML; u++)
-+ ssPtr->matchLengthFreq[u] = 1;
-+ for (u = 0; u <= MaxOff; u++)
-+ ssPtr->offCodeFreq[u] = 1;
-+ } else {
-+ ssPtr->matchLengthSum = 0;
-+ ssPtr->litLengthSum = 0;
-+ ssPtr->offCodeSum = 0;
-+ ssPtr->matchSum = 0;
-+ ssPtr->litSum = 0;
-+
-+ for (u = 0; u <= MaxLit; u++) {
-+ ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> (ZSTD_FREQ_DIV + 1));
-+ ssPtr->litSum += ssPtr->litFreq[u];
-+ }
-+ for (u = 0; u <= MaxLL; u++) {
-+ ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u] >> (ZSTD_FREQ_DIV + 1));
-+ ssPtr->litLengthSum += ssPtr->litLengthFreq[u];
-+ }
-+ for (u = 0; u <= MaxML; u++) {
-+ ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV);
-+ ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u];
-+ ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3);
-+ }
-+ ssPtr->matchSum *= ZSTD_LITFREQ_ADD;
-+ for (u = 0; u <= MaxOff; u++) {
-+ ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV);
-+ ssPtr->offCodeSum += ssPtr->offCodeFreq[u];
-+ }
-+ }
-+
-+ ZSTD_setLog2Prices(ssPtr);
-+}
-+
-+FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t *ssPtr, U32 litLength, const BYTE *literals)
-+{
-+ U32 price, u;
-+
-+ if (ssPtr->staticPrices)
-+ return ZSTD_highbit32((U32)litLength + 1) + (litLength * 6);
-+
-+ if (litLength == 0)
-+ return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0] + 1);
-+
-+ /* literals */
-+ if (ssPtr->cachedLiterals == literals) {
-+ U32 const additional = litLength - ssPtr->cachedLitLength;
-+ const BYTE *literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
-+ price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
-+ for (u = 0; u < additional; u++)
-+ price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]] + 1);
-+ ssPtr->cachedPrice = price;
-+ ssPtr->cachedLitLength = litLength;
-+ } else {
-+ price = litLength * ssPtr->log2litSum;
-+ for (u = 0; u < litLength; u++)
-+ price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]] + 1);
-+
-+ if (litLength >= 12) {
-+ ssPtr->cachedLiterals = literals;
-+ ssPtr->cachedPrice = price;
-+ ssPtr->cachedLitLength = litLength;
-+ }
-+ }
-+
-+ /* literal Length */
-+ {
-+ const BYTE LL_deltaCode = 19;
-+ const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
-+ price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode] + 1);
-+ }
-+
-+ return price;
-+}
-+
-+FORCE_INLINE U32 ZSTD_getPrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength, const int ultra)
-+{
-+ /* offset */
-+ U32 price;
-+ BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1);
-+
-+ if (seqStorePtr->staticPrices)
-+ return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength + 1) + 16 + offCode;
-+
-+ price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode] + 1);
-+ if (!ultra && offCode >= 20)
-+ price += (offCode - 19) * 2;
-+
-+ /* match Length */
-+ {
-+ const BYTE ML_deltaCode = 36;
-+ const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
-+ price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode] + 1);
-+ }
-+
-+ return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
-+}
-+
-+ZSTD_STATIC void ZSTD_updatePrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength)
-+{
-+ U32 u;
-+
-+ /* literals */
-+ seqStorePtr->litSum += litLength * ZSTD_LITFREQ_ADD;
-+ for (u = 0; u < litLength; u++)
-+ seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
-+
-+ /* literal Length */
-+ {
-+ const BYTE LL_deltaCode = 19;
-+ const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
-+ seqStorePtr->litLengthFreq[llCode]++;
-+ seqStorePtr->litLengthSum++;
-+ }
-+
-+ /* match offset */
-+ {
-+ BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1);
-+ seqStorePtr->offCodeSum++;
-+ seqStorePtr->offCodeFreq[offCode]++;
-+ }
-+
-+ /* match Length */
-+ {
-+ const BYTE ML_deltaCode = 36;
-+ const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
-+ seqStorePtr->matchLengthFreq[mlCode]++;
-+ seqStorePtr->matchLengthSum++;
-+ }
-+
-+ ZSTD_setLog2Prices(seqStorePtr);
-+}
-+
-+#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \
-+ { \
-+ while (last_pos < pos) { \
-+ opt[last_pos + 1].price = ZSTD_MAX_PRICE; \
-+ last_pos++; \
-+ } \
-+ opt[pos].mlen = mlen_; \
-+ opt[pos].off = offset_; \
-+ opt[pos].litlen = litlen_; \
-+ opt[pos].price = price_; \
-+ }
-+
-+/* Update hashTable3 up to ip (excluded)
-+ Assumption : always within prefix (i.e. not within extDict) */
-+FORCE_INLINE
-+U32 ZSTD_insertAndFindFirstIndexHash3(ZSTD_CCtx *zc, const BYTE *ip)
-+{
-+ U32 *const hashTable3 = zc->hashTable3;
-+ U32 const hashLog3 = zc->hashLog3;
-+ const BYTE *const base = zc->base;
-+ U32 idx = zc->nextToUpdate3;
-+ const U32 target = zc->nextToUpdate3 = (U32)(ip - base);
-+ const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3);
-+
-+ while (idx < target) {
-+ hashTable3[ZSTD_hash3Ptr(base + idx, hashLog3)] = idx;
-+ idx++;
-+ }
-+
-+ return hashTable3[hash3];
-+}
-+
-+/*-*************************************
-+* Binary Tree search
-+***************************************/
-+static U32 ZSTD_insertBtAndGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, U32 nbCompares, const U32 mls, U32 extDict,
-+ ZSTD_match_t *matches, const U32 minMatchLen)
-+{
-+ const BYTE *const base = zc->base;
-+ const U32 curr = (U32)(ip - base);
-+ const U32 hashLog = zc->params.cParams.hashLog;
-+ const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
-+ U32 *const hashTable = zc->hashTable;
-+ U32 matchIndex = hashTable[h];
-+ U32 *const bt = zc->chainTable;
-+ const U32 btLog = zc->params.cParams.chainLog - 1;
-+ const U32 btMask = (1U << btLog) - 1;
-+ size_t commonLengthSmaller = 0, commonLengthLarger = 0;
-+ const BYTE *const dictBase = zc->dictBase;
-+ const U32 dictLimit = zc->dictLimit;
-+ const BYTE *const dictEnd = dictBase + dictLimit;
-+ const BYTE *const prefixStart = base + dictLimit;
-+ const U32 btLow = btMask >= curr ? 0 : curr - btMask;
-+ const U32 windowLow = zc->lowLimit;
-+ U32 *smallerPtr = bt + 2 * (curr & btMask);
-+ U32 *largerPtr = bt + 2 * (curr & btMask) + 1;
-+ U32 matchEndIdx = curr + 8;
-+ U32 dummy32; /* to be nullified at the end */
-+ U32 mnum = 0;
-+
-+ const U32 minMatch = (mls == 3) ? 3 : 4;
-+ size_t bestLength = minMatchLen - 1;
-+
-+ if (minMatch == 3) { /* HC3 match finder */
-+ U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(zc, ip);
-+ if (matchIndex3 > windowLow && (curr - matchIndex3 < (1 << 18))) {
-+ const BYTE *match;
-+ size_t currMl = 0;
-+ if ((!extDict) || matchIndex3 >= dictLimit) {
-+ match = base + matchIndex3;
-+ if (match[bestLength] == ip[bestLength])
-+ currMl = ZSTD_count(ip, match, iLimit);
-+ } else {
-+ match = dictBase + matchIndex3;
-+ if (ZSTD_readMINMATCH(match, MINMATCH) ==
-+ ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
-+ currMl = ZSTD_count_2segments(ip + MINMATCH, match + MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH;
-+ }
-+
-+ /* save best solution */
-+ if (currMl > bestLength) {
-+ bestLength = currMl;
-+ matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3;
-+ matches[mnum].len = (U32)currMl;
-+ mnum++;
-+ if (currMl > ZSTD_OPT_NUM)
-+ goto update;
-+ if (ip + currMl == iLimit)
-+ goto update; /* best possible, and avoid read overflow*/
-+ }
-+ }
-+ }
-+
-+ hashTable[h] = curr; /* Update Hash Table */
-+
-+ while (nbCompares-- && (matchIndex > windowLow)) {
-+ U32 *nextPtr = bt + 2 * (matchIndex & btMask);
-+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
-+ const BYTE *match;
-+
-+ if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
-+ match = base + matchIndex;
-+ if (match[matchLength] == ip[matchLength]) {
-+ matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iLimit) + 1;
-+ }
-+ } else {
-+ match = dictBase + matchIndex;
-+ matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iLimit, dictEnd, prefixStart);
-+ if (matchIndex + matchLength >= dictLimit)
-+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
-+ }
-+
-+ if (matchLength > bestLength) {
-+ if (matchLength > matchEndIdx - matchIndex)
-+ matchEndIdx = matchIndex + (U32)matchLength;
-+ bestLength = matchLength;
-+ matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex;
-+ matches[mnum].len = (U32)matchLength;
-+ mnum++;
-+ if (matchLength > ZSTD_OPT_NUM)
-+ break;
-+ if (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */
-+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
-+ }
-+
-+ if (match[matchLength] < ip[matchLength]) {
-+ /* match is smaller than curr */
-+ *smallerPtr = matchIndex; /* update smaller idx */
-+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
-+ if (matchIndex <= btLow) {
-+ smallerPtr = &dummy32;
-+ break;
-+ } /* beyond tree size, stop the search */
-+ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
-+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
-+ } else {
-+ /* match is larger than curr */
-+ *largerPtr = matchIndex;
-+ commonLengthLarger = matchLength;
-+ if (matchIndex <= btLow) {
-+ largerPtr = &dummy32;
-+ break;
-+ } /* beyond tree size, stop the search */
-+ largerPtr = nextPtr;
-+ matchIndex = nextPtr[0];
-+ }
-+ }
-+
-+ *smallerPtr = *largerPtr = 0;
-+
-+update:
-+ zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1;
-+ return mnum;
-+}
-+
-+/** Tree updater, providing best match */
-+static U32 ZSTD_BtGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t *matches,
-+ const U32 minMatchLen)
-+{
-+ if (ip < zc->base + zc->nextToUpdate)
-+ return 0; /* skipped area */
-+ ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
-+ return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen);
-+}
-+
-+static U32 ZSTD_BtGetAllMatches_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
-+ const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
-+ ZSTD_match_t *matches, const U32 minMatchLen)
-+{
-+ switch (matchLengthSearch) {
-+ case 3: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
-+ default:
-+ case 4: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
-+ case 5: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
-+ case 7:
-+ case 6: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
-+ }
-+}
-+
-+/** Tree updater, providing best match */
-+static U32 ZSTD_BtGetAllMatches_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls,
-+ ZSTD_match_t *matches, const U32 minMatchLen)
-+{
-+ if (ip < zc->base + zc->nextToUpdate)
-+ return 0; /* skipped area */
-+ ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
-+ return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen);
-+}
-+
-+static U32 ZSTD_BtGetAllMatches_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
-+ const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
-+ ZSTD_match_t *matches, const U32 minMatchLen)
-+{
-+ switch (matchLengthSearch) {
-+ case 3: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
-+ default:
-+ case 4: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
-+ case 5: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
-+ case 7:
-+ case 6: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
-+ }
-+}
-+
-+/*-*******************************
-+* Optimal parser
-+*********************************/
-+FORCE_INLINE
-+void ZSTD_compressBlock_opt_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
-+{
-+ seqStore_t *seqStorePtr = &(ctx->seqStore);
-+ const BYTE *const istart = (const BYTE *)src;
-+ const BYTE *ip = istart;
-+ const BYTE *anchor = istart;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *const ilimit = iend - 8;
-+ const BYTE *const base = ctx->base;
-+ const BYTE *const prefixStart = base + ctx->dictLimit;
-+
-+ const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
-+ const U32 sufficient_len = ctx->params.cParams.targetLength;
-+ const U32 mls = ctx->params.cParams.searchLength;
-+ const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
-+
-+ ZSTD_optimal_t *opt = seqStorePtr->priceTable;
-+ ZSTD_match_t *matches = seqStorePtr->matchTable;
-+ const BYTE *inr;
-+ U32 offset, rep[ZSTD_REP_NUM];
-+
-+ /* init */
-+ ctx->nextToUpdate3 = ctx->nextToUpdate;
-+ ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
-+ ip += (ip == prefixStart);
-+ {
-+ U32 i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ rep[i] = ctx->rep[i];
-+ }
-+
-+ /* Match Loop */
-+ while (ip < ilimit) {
-+ U32 cur, match_num, last_pos, litlen, price;
-+ U32 u, mlen, best_mlen, best_off, litLength;
-+ memset(opt, 0, sizeof(ZSTD_optimal_t));
-+ last_pos = 0;
-+ litlen = (U32)(ip - anchor);
-+
-+ /* check repCode */
-+ {
-+ U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
-+ for (i = (ip == anchor); i < last_i; i++) {
-+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
-+ if ((repCur > 0) && (repCur < (S32)(ip - prefixStart)) &&
-+ (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) {
-+ mlen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repCur, iend) + minMatch;
-+ if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
-+ best_mlen = mlen;
-+ best_off = i;
-+ cur = 0;
-+ last_pos = 1;
-+ goto _storeSequence;
-+ }
-+ best_off = i - (ip == anchor);
-+ do {
-+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
-+ if (mlen > last_pos || price < opt[mlen].price)
-+ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
-+ mlen--;
-+ } while (mlen >= minMatch);
-+ }
-+ }
-+ }
-+
-+ match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch);
-+
-+ if (!last_pos && !match_num) {
-+ ip++;
-+ continue;
-+ }
-+
-+ if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
-+ best_mlen = matches[match_num - 1].len;
-+ best_off = matches[match_num - 1].off;
-+ cur = 0;
-+ last_pos = 1;
-+ goto _storeSequence;
-+ }
-+
-+ /* set prices using matches at position = 0 */
-+ best_mlen = (last_pos) ? last_pos : minMatch;
-+ for (u = 0; u < match_num; u++) {
-+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
-+ best_mlen = matches[u].len;
-+ while (mlen <= best_mlen) {
-+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
-+ if (mlen > last_pos || price < opt[mlen].price)
-+ SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
-+ mlen++;
-+ }
-+ }
-+
-+ if (last_pos < minMatch) {
-+ ip++;
-+ continue;
-+ }
-+
-+ /* initialize opt[0] */
-+ {
-+ U32 i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ opt[0].rep[i] = rep[i];
-+ }
-+ opt[0].mlen = 1;
-+ opt[0].litlen = litlen;
-+
-+ /* check further positions */
-+ for (cur = 1; cur <= last_pos; cur++) {
-+ inr = ip + cur;
-+
-+ if (opt[cur - 1].mlen == 1) {
-+ litlen = opt[cur - 1].litlen + 1;
-+ if (cur > litlen) {
-+ price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen);
-+ } else
-+ price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
-+ } else {
-+ litlen = 1;
-+ price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1);
-+ }
-+
-+ if (cur > last_pos || price <= opt[cur].price)
-+ SET_PRICE(cur, 1, 0, litlen, price);
-+
-+ if (cur == last_pos)
-+ break;
-+
-+ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
-+ continue;
-+
-+ mlen = opt[cur].mlen;
-+ if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
-+ opt[cur].rep[2] = opt[cur - mlen].rep[1];
-+ opt[cur].rep[1] = opt[cur - mlen].rep[0];
-+ opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
-+ } else {
-+ opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
-+ opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
-+ opt[cur].rep[0] =
-+ ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
-+ }
-+
-+ best_mlen = minMatch;
-+ {
-+ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
-+ for (i = (opt[cur].mlen != 1); i < last_i; i++) { /* check rep */
-+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
-+ if ((repCur > 0) && (repCur < (S32)(inr - prefixStart)) &&
-+ (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) {
-+ mlen = (U32)ZSTD_count(inr + minMatch, inr + minMatch - repCur, iend) + minMatch;
-+
-+ if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
-+ best_mlen = mlen;
-+ best_off = i;
-+ last_pos = cur + 1;
-+ goto _storeSequence;
-+ }
-+
-+ best_off = i - (opt[cur].mlen != 1);
-+ if (mlen > best_mlen)
-+ best_mlen = mlen;
-+
-+ do {
-+ if (opt[cur].mlen == 1) {
-+ litlen = opt[cur].litlen;
-+ if (cur > litlen) {
-+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
-+ best_off, mlen - MINMATCH, ultra);
-+ } else
-+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
-+ } else {
-+ litlen = 0;
-+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
-+ }
-+
-+ if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
-+ SET_PRICE(cur + mlen, mlen, i, litlen, price);
-+ mlen--;
-+ } while (mlen >= minMatch);
-+ }
-+ }
-+ }
-+
-+ match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen);
-+
-+ if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
-+ best_mlen = matches[match_num - 1].len;
-+ best_off = matches[match_num - 1].off;
-+ last_pos = cur + 1;
-+ goto _storeSequence;
-+ }
-+
-+ /* set prices using matches at position = cur */
-+ for (u = 0; u < match_num; u++) {
-+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
-+ best_mlen = matches[u].len;
-+
-+ while (mlen <= best_mlen) {
-+ if (opt[cur].mlen == 1) {
-+ litlen = opt[cur].litlen;
-+ if (cur > litlen)
-+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
-+ matches[u].off - 1, mlen - MINMATCH, ultra);
-+ else
-+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
-+ } else {
-+ litlen = 0;
-+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra);
-+ }
-+
-+ if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
-+ SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
-+
-+ mlen++;
-+ }
-+ }
-+ }
-+
-+ best_mlen = opt[last_pos].mlen;
-+ best_off = opt[last_pos].off;
-+ cur = last_pos - best_mlen;
-+
-+ /* store sequence */
-+_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
-+ opt[0].mlen = 1;
-+
-+ while (1) {
-+ mlen = opt[cur].mlen;
-+ offset = opt[cur].off;
-+ opt[cur].mlen = best_mlen;
-+ opt[cur].off = best_off;
-+ best_mlen = mlen;
-+ best_off = offset;
-+ if (mlen > cur)
-+ break;
-+ cur -= mlen;
-+ }
-+
-+ for (u = 0; u <= last_pos;) {
-+ u += opt[u].mlen;
-+ }
-+
-+ for (cur = 0; cur < last_pos;) {
-+ mlen = opt[cur].mlen;
-+ if (mlen == 1) {
-+ ip++;
-+ cur++;
-+ continue;
-+ }
-+ offset = opt[cur].off;
-+ cur += mlen;
-+ litLength = (U32)(ip - anchor);
-+
-+ if (offset > ZSTD_REP_MOVE_OPT) {
-+ rep[2] = rep[1];
-+ rep[1] = rep[0];
-+ rep[0] = offset - ZSTD_REP_MOVE_OPT;
-+ offset--;
-+ } else {
-+ if (offset != 0) {
-+ best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
-+ if (offset != 1)
-+ rep[2] = rep[1];
-+ rep[1] = rep[0];
-+ rep[0] = best_off;
-+ }
-+ if (litLength == 0)
-+ offset--;
-+ }
-+
-+ ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
-+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
-+ anchor = ip = ip + mlen;
-+ }
-+ } /* for (cur=0; cur < last_pos; ) */
-+
-+ /* Save reps for next block */
-+ {
-+ int i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ ctx->repToConfirm[i] = rep[i];
-+ }
-+
-+ /* Last Literals */
-+ {
-+ size_t const lastLLSize = iend - anchor;
-+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
-+ seqStorePtr->lit += lastLLSize;
-+ }
-+}
-+
-+FORCE_INLINE
-+void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
-+{
-+ seqStore_t *seqStorePtr = &(ctx->seqStore);
-+ const BYTE *const istart = (const BYTE *)src;
-+ const BYTE *ip = istart;
-+ const BYTE *anchor = istart;
-+ const BYTE *const iend = istart + srcSize;
-+ const BYTE *const ilimit = iend - 8;
-+ const BYTE *const base = ctx->base;
-+ const U32 lowestIndex = ctx->lowLimit;
-+ const U32 dictLimit = ctx->dictLimit;
-+ const BYTE *const prefixStart = base + dictLimit;
-+ const BYTE *const dictBase = ctx->dictBase;
-+ const BYTE *const dictEnd = dictBase + dictLimit;
-+
-+ const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
-+ const U32 sufficient_len = ctx->params.cParams.targetLength;
-+ const U32 mls = ctx->params.cParams.searchLength;
-+ const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
-+
-+ ZSTD_optimal_t *opt = seqStorePtr->priceTable;
-+ ZSTD_match_t *matches = seqStorePtr->matchTable;
-+ const BYTE *inr;
-+
-+ /* init */
-+ U32 offset, rep[ZSTD_REP_NUM];
-+ {
-+ U32 i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ rep[i] = ctx->rep[i];
-+ }
-+
-+ ctx->nextToUpdate3 = ctx->nextToUpdate;
-+ ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
-+ ip += (ip == prefixStart);
-+
-+ /* Match Loop */
-+ while (ip < ilimit) {
-+ U32 cur, match_num, last_pos, litlen, price;
-+ U32 u, mlen, best_mlen, best_off, litLength;
-+ U32 curr = (U32)(ip - base);
-+ memset(opt, 0, sizeof(ZSTD_optimal_t));
-+ last_pos = 0;
-+ opt[0].litlen = (U32)(ip - anchor);
-+
-+ /* check repCode */
-+ {
-+ U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
-+ for (i = (ip == anchor); i < last_i; i++) {
-+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
-+ const U32 repIndex = (U32)(curr - repCur);
-+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
-+ const BYTE *const repMatch = repBase + repIndex;
-+ if ((repCur > 0 && repCur <= (S32)curr) &&
-+ (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
-+ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
-+ /* repcode detected we should take it */
-+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
-+ mlen = (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
-+
-+ if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
-+ best_mlen = mlen;
-+ best_off = i;
-+ cur = 0;
-+ last_pos = 1;
-+ goto _storeSequence;
-+ }
-+
-+ best_off = i - (ip == anchor);
-+ litlen = opt[0].litlen;
-+ do {
-+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
-+ if (mlen > last_pos || price < opt[mlen].price)
-+ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
-+ mlen--;
-+ } while (mlen >= minMatch);
-+ }
-+ }
-+ }
-+
-+ match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */
-+
-+ if (!last_pos && !match_num) {
-+ ip++;
-+ continue;
-+ }
-+
-+ {
-+ U32 i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ opt[0].rep[i] = rep[i];
-+ }
-+ opt[0].mlen = 1;
-+
-+ if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
-+ best_mlen = matches[match_num - 1].len;
-+ best_off = matches[match_num - 1].off;
-+ cur = 0;
-+ last_pos = 1;
-+ goto _storeSequence;
-+ }
-+
-+ best_mlen = (last_pos) ? last_pos : minMatch;
-+
-+ /* set prices using matches at position = 0 */
-+ for (u = 0; u < match_num; u++) {
-+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
-+ best_mlen = matches[u].len;
-+ litlen = opt[0].litlen;
-+ while (mlen <= best_mlen) {
-+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
-+ if (mlen > last_pos || price < opt[mlen].price)
-+ SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
-+ mlen++;
-+ }
-+ }
-+
-+ if (last_pos < minMatch) {
-+ ip++;
-+ continue;
-+ }
-+
-+ /* check further positions */
-+ for (cur = 1; cur <= last_pos; cur++) {
-+ inr = ip + cur;
-+
-+ if (opt[cur - 1].mlen == 1) {
-+ litlen = opt[cur - 1].litlen + 1;
-+ if (cur > litlen) {
-+ price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen);
-+ } else
-+ price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
-+ } else {
-+ litlen = 1;
-+ price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1);
-+ }
-+
-+ if (cur > last_pos || price <= opt[cur].price)
-+ SET_PRICE(cur, 1, 0, litlen, price);
-+
-+ if (cur == last_pos)
-+ break;
-+
-+ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
-+ continue;
-+
-+ mlen = opt[cur].mlen;
-+ if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
-+ opt[cur].rep[2] = opt[cur - mlen].rep[1];
-+ opt[cur].rep[1] = opt[cur - mlen].rep[0];
-+ opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
-+ } else {
-+ opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
-+ opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
-+ opt[cur].rep[0] =
-+ ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
-+ }
-+
-+ best_mlen = minMatch;
-+ {
-+ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
-+ for (i = (mlen != 1); i < last_i; i++) {
-+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
-+ const U32 repIndex = (U32)(curr + cur - repCur);
-+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
-+ const BYTE *const repMatch = repBase + repIndex;
-+ if ((repCur > 0 && repCur <= (S32)(curr + cur)) &&
-+ (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
-+ && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
-+ /* repcode detected */
-+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
-+ mlen = (U32)ZSTD_count_2segments(inr + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
-+
-+ if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
-+ best_mlen = mlen;
-+ best_off = i;
-+ last_pos = cur + 1;
-+ goto _storeSequence;
-+ }
-+
-+ best_off = i - (opt[cur].mlen != 1);
-+ if (mlen > best_mlen)
-+ best_mlen = mlen;
-+
-+ do {
-+ if (opt[cur].mlen == 1) {
-+ litlen = opt[cur].litlen;
-+ if (cur > litlen) {
-+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
-+ best_off, mlen - MINMATCH, ultra);
-+ } else
-+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
-+ } else {
-+ litlen = 0;
-+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
-+ }
-+
-+ if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
-+ SET_PRICE(cur + mlen, mlen, i, litlen, price);
-+ mlen--;
-+ } while (mlen >= minMatch);
-+ }
-+ }
-+ }
-+
-+ match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
-+
-+ if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
-+ best_mlen = matches[match_num - 1].len;
-+ best_off = matches[match_num - 1].off;
-+ last_pos = cur + 1;
-+ goto _storeSequence;
-+ }
-+
-+ /* set prices using matches at position = cur */
-+ for (u = 0; u < match_num; u++) {
-+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
-+ best_mlen = matches[u].len;
-+
-+ while (mlen <= best_mlen) {
-+ if (opt[cur].mlen == 1) {
-+ litlen = opt[cur].litlen;
-+ if (cur > litlen)
-+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
-+ matches[u].off - 1, mlen - MINMATCH, ultra);
-+ else
-+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
-+ } else {
-+ litlen = 0;
-+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra);
-+ }
-+
-+ if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
-+ SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
-+
-+ mlen++;
-+ }
-+ }
-+ } /* for (cur = 1; cur <= last_pos; cur++) */
-+
-+ best_mlen = opt[last_pos].mlen;
-+ best_off = opt[last_pos].off;
-+ cur = last_pos - best_mlen;
-+
-+ /* store sequence */
-+_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
-+ opt[0].mlen = 1;
-+
-+ while (1) {
-+ mlen = opt[cur].mlen;
-+ offset = opt[cur].off;
-+ opt[cur].mlen = best_mlen;
-+ opt[cur].off = best_off;
-+ best_mlen = mlen;
-+ best_off = offset;
-+ if (mlen > cur)
-+ break;
-+ cur -= mlen;
-+ }
-+
-+ for (u = 0; u <= last_pos;) {
-+ u += opt[u].mlen;
-+ }
-+
-+ for (cur = 0; cur < last_pos;) {
-+ mlen = opt[cur].mlen;
-+ if (mlen == 1) {
-+ ip++;
-+ cur++;
-+ continue;
-+ }
-+ offset = opt[cur].off;
-+ cur += mlen;
-+ litLength = (U32)(ip - anchor);
-+
-+ if (offset > ZSTD_REP_MOVE_OPT) {
-+ rep[2] = rep[1];
-+ rep[1] = rep[0];
-+ rep[0] = offset - ZSTD_REP_MOVE_OPT;
-+ offset--;
-+ } else {
-+ if (offset != 0) {
-+ best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
-+ if (offset != 1)
-+ rep[2] = rep[1];
-+ rep[1] = rep[0];
-+ rep[0] = best_off;
-+ }
-+
-+ if (litLength == 0)
-+ offset--;
-+ }
-+
-+ ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
-+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
-+ anchor = ip = ip + mlen;
-+ }
-+ } /* for (cur=0; cur < last_pos; ) */
-+
-+ /* Save reps for next block */
-+ {
-+ int i;
-+ for (i = 0; i < ZSTD_REP_NUM; i++)
-+ ctx->repToConfirm[i] = rep[i];
-+ }
-+
-+ /* Last Literals */
-+ {
-+ size_t lastLLSize = iend - anchor;
-+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
-+ seqStorePtr->lit += lastLLSize;
-+ }
-+}
-+
-+#endif /* ZSTD_OPT_H_91842398743 */
---
-2.9.3
diff --git a/0003-btrfs-Add-zstd-support.patch b/0003-btrfs-Add-zstd-support.patch
deleted file mode 100644
index edc7839..0000000
--- a/0003-btrfs-Add-zstd-support.patch
+++ /dev/null
@@ -1,740 +0,0 @@
-From 8a9dddfbf6551afea73911e367dd4be64d62b9fd Mon Sep 17 00:00:00 2001
-From: Nick Terrell <terrelln@fb.com>
-Date: Mon, 17 Jul 2017 17:08:39 -0700
-Subject: [PATCH v5 3/5] btrfs: Add zstd support
-
-Add zstd compression and decompression support to BtrFS. zstd at its
-fastest level compresses almost as well as zlib, while offering much
-faster compression and decompression, approaching lzo speeds.
-
-I benchmarked btrfs with zstd compression against no compression, lzo
-compression, and zlib compression. I benchmarked two scenarios. Copying
-a set of files to btrfs, and then reading the files. Copying a tarball
-to btrfs, extracting it to btrfs, and then reading the extracted files.
-After every operation, I call `sync` and include the sync time.
-Between every pair of operations I unmount and remount the filesystem
-to avoid caching. The benchmark files can be found in the upstream
-zstd source repository under
-`contrib/linux-kernel/{btrfs-benchmark.sh,btrfs-extract-benchmark.sh}`
-[1] [2].
-
-I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
-The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
-16 GB of RAM, and a SSD.
-
-The first compression benchmark is copying 10 copies of the unzipped
-Silesia corpus [3] into a BtrFS filesystem mounted with
-`-o compress-force=Method`. The decompression benchmark times how long
-it takes to `tar` all 10 copies into `/dev/null`. The compression ratio is
-measured by comparing the output of `df` and `du`. See the benchmark file
-[1] for details. I benchmarked multiple zstd compression levels, although
-the patch uses zstd level 1.
-
-| Method | Ratio | Compression MB/s | Decompression speed |
-|---------|-------|------------------|---------------------|
-| None | 0.99 | 504 | 686 |
-| lzo | 1.66 | 398 | 442 |
-| zlib | 2.58 | 65 | 241 |
-| zstd 1 | 2.57 | 260 | 383 |
-| zstd 3 | 2.71 | 174 | 408 |
-| zstd 6 | 2.87 | 70 | 398 |
-| zstd 9 | 2.92 | 43 | 406 |
-| zstd 12 | 2.93 | 21 | 408 |
-| zstd 15 | 3.01 | 11 | 354 |
-
-The next benchmark first copies `linux-4.11.6.tar` [4] to btrfs. Then it
-measures the compression ratio, extracts the tar, and deletes the tar.
-Then it measures the compression ratio again, and `tar`s the extracted
-files into `/dev/null`. See the benchmark file [2] for details.
-
-| Method | Tar Ratio | Extract Ratio | Copy (s) | Extract (s)| Read (s) |
-|--------|-----------|---------------|----------|------------|----------|
-| None | 0.97 | 0.78 | 0.981 | 5.501 | 8.807 |
-| lzo | 2.06 | 1.38 | 1.631 | 8.458 | 8.585 |
-| zlib | 3.40 | 1.86 | 7.750 | 21.544 | 11.744 |
-| zstd 1 | 3.57 | 1.85 | 2.579 | 11.479 | 9.389 |
-
-[1] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-benchmark.sh
-[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-extract-benchmark.sh
-[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
-[4] https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.11.6.tar.xz
-
-zstd source repository: https://github.com/facebook/zstd
-
-Signed-off-by: Nick Terrell <terrelln@fb.com>
----
-v2 -> v3:
-- Port upstream BtrFS commits e1ddce71d6, 389a6cfc2a, and 6acafd1eff
-- Change default compression level for BtrFS to 3
-
-v3 -> v4:
-- Add missing includes, which fixes the aarch64 build
-- Fix minor linter warnings
-
- fs/btrfs/Kconfig | 2 +
- fs/btrfs/Makefile | 2 +-
- fs/btrfs/compression.c | 1 +
- fs/btrfs/compression.h | 6 +-
- fs/btrfs/ctree.h | 1 +
- fs/btrfs/disk-io.c | 2 +
- fs/btrfs/ioctl.c | 6 +-
- fs/btrfs/props.c | 6 +
- fs/btrfs/super.c | 12 +-
- fs/btrfs/sysfs.c | 2 +
- fs/btrfs/zstd.c | 432 +++++++++++++++++++++++++++++++++++++++++++++
- include/uapi/linux/btrfs.h | 8 +-
- 12 files changed, 468 insertions(+), 12 deletions(-)
- create mode 100644 fs/btrfs/zstd.c
-
-diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
-index 80e9c18..a26c63b 100644
---- a/fs/btrfs/Kconfig
-+++ b/fs/btrfs/Kconfig
-@@ -6,6 +6,8 @@ config BTRFS_FS
- select ZLIB_DEFLATE
- select LZO_COMPRESS
- select LZO_DECOMPRESS
-+ select ZSTD_COMPRESS
-+ select ZSTD_DECOMPRESS
- select RAID6_PQ
- select XOR_BLOCKS
- select SRCU
-diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile
-index 128ce17..962a95a 100644
---- a/fs/btrfs/Makefile
-+++ b/fs/btrfs/Makefile
-@@ -6,7 +6,7 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \
- transaction.o inode.o file.o tree-defrag.o \
- extent_map.o sysfs.o struct-funcs.o xattr.o ordered-data.o \
- extent_io.o volumes.o async-thread.o ioctl.o locking.o orphan.o \
-- export.o tree-log.o free-space-cache.o zlib.o lzo.o \
-+ export.o tree-log.o free-space-cache.o zlib.o lzo.o zstd.o \
- compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \
- reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
- uuid-tree.o props.o hash.o free-space-tree.o
-diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
-index d2ef9ac..4ff42d1 100644
---- a/fs/btrfs/compression.c
-+++ b/fs/btrfs/compression.c
-@@ -704,6 +704,7 @@ static struct {
- static const struct btrfs_compress_op * const btrfs_compress_op[] = {
- &btrfs_zlib_compress,
- &btrfs_lzo_compress,
-+ &btrfs_zstd_compress,
- };
-
- void __init btrfs_init_compress(void)
-diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
-index 87f6d33..2269e00 100644
---- a/fs/btrfs/compression.h
-+++ b/fs/btrfs/compression.h
-@@ -99,8 +99,9 @@ enum btrfs_compression_type {
- BTRFS_COMPRESS_NONE = 0,
- BTRFS_COMPRESS_ZLIB = 1,
- BTRFS_COMPRESS_LZO = 2,
-- BTRFS_COMPRESS_TYPES = 2,
-- BTRFS_COMPRESS_LAST = 3,
-+ BTRFS_COMPRESS_ZSTD = 3,
-+ BTRFS_COMPRESS_TYPES = 3,
-+ BTRFS_COMPRESS_LAST = 4,
- };
-
- struct btrfs_compress_op {
-@@ -128,5 +129,6 @@ struct btrfs_compress_op {
-
- extern const struct btrfs_compress_op btrfs_zlib_compress;
- extern const struct btrfs_compress_op btrfs_lzo_compress;
-+extern const struct btrfs_compress_op btrfs_zstd_compress;
-
- #endif
-diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
-index 3f3eb7b..845d77c 100644
---- a/fs/btrfs/ctree.h
-+++ b/fs/btrfs/ctree.h
-@@ -270,6 +270,7 @@ struct btrfs_super_block {
- BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
- BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \
- BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \
-+ BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD | \
- BTRFS_FEATURE_INCOMPAT_RAID56 | \
- BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
- BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
-diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
-index 080e2eb..04632f4 100644
---- a/fs/btrfs/disk-io.c
-+++ b/fs/btrfs/disk-io.c
-@@ -2828,6 +2828,8 @@ int open_ctree(struct super_block *sb,
- features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
- if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
- features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
-+ else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
-+ features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
-
- if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
- btrfs_info(fs_info, "has skinny extents");
-diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
-index fa1b78c..b9963d9 100644
---- a/fs/btrfs/ioctl.c
-+++ b/fs/btrfs/ioctl.c
-@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg)
-
- if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
- comp = "lzo";
-- else
-+ else if (fs_info->compress_type == BTRFS_COMPRESS_ZLIB)
- comp = "zlib";
-+ else
-+ comp = "zstd";
- ret = btrfs_set_prop(inode, "btrfs.compression",
- comp, strlen(comp), 0);
- if (ret)
-@@ -1466,6 +1468,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
-
- if (range->compress_type == BTRFS_COMPRESS_LZO) {
- btrfs_set_fs_incompat(fs_info, COMPRESS_LZO);
-+ } else if (range->compress_type == BTRFS_COMPRESS_ZSTD) {
-+ btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD);
- }
-
- ret = defrag_count;
-diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
-index 4b23ae5..20631e9 100644
---- a/fs/btrfs/props.c
-+++ b/fs/btrfs/props.c
-@@ -390,6 +390,8 @@ static int prop_compression_validate(const char *value, size_t len)
- return 0;
- else if (!strncmp("zlib", value, len))
- return 0;
-+ else if (!strncmp("zstd", value, len))
-+ return 0;
-
- return -EINVAL;
- }
-@@ -412,6 +414,8 @@ static int prop_compression_apply(struct inode *inode,
- type = BTRFS_COMPRESS_LZO;
- else if (!strncmp("zlib", value, len))
- type = BTRFS_COMPRESS_ZLIB;
-+ else if (!strncmp("zstd", value, len))
-+ type = BTRFS_COMPRESS_ZSTD;
- else
- return -EINVAL;
-
-@@ -429,6 +433,8 @@ static const char *prop_compression_extract(struct inode *inode)
- return "zlib";
- case BTRFS_COMPRESS_LZO:
- return "lzo";
-+ case BTRFS_COMPRESS_ZSTD:
-+ return "zstd";
- }
-
- return NULL;
-diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
-index 12540b6..c370dea 100644
---- a/fs/btrfs/super.c
-+++ b/fs/btrfs/super.c
-@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
- btrfs_clear_opt(info->mount_opt, NODATASUM);
- btrfs_set_fs_incompat(info, COMPRESS_LZO);
- no_compress = 0;
-+ } else if (strcmp(args[0].from, "zstd") == 0) {
-+ compress_type = "zstd";
-+ info->compress_type = BTRFS_COMPRESS_ZSTD;
-+ btrfs_set_opt(info->mount_opt, COMPRESS);
-+ btrfs_clear_opt(info->mount_opt, NODATACOW);
-+ btrfs_clear_opt(info->mount_opt, NODATASUM);
-+ btrfs_set_fs_incompat(info, COMPRESS_ZSTD);
-+ no_compress = 0;
- } else if (strncmp(args[0].from, "no", 2) == 0) {
- compress_type = "no";
- btrfs_clear_opt(info->mount_opt, COMPRESS);
-@@ -1227,8 +1235,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
- if (btrfs_test_opt(info, COMPRESS)) {
- if (info->compress_type == BTRFS_COMPRESS_ZLIB)
- compress_type = "zlib";
-- else
-+ else if (info->compress_type == BTRFS_COMPRESS_LZO)
- compress_type = "lzo";
-+ else
-+ compress_type = "zstd";
- if (btrfs_test_opt(info, FORCE_COMPRESS))
- seq_printf(seq, ",compress-force=%s", compress_type);
- else
-diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
-index c2d5f35..2b6d37c 100644
---- a/fs/btrfs/sysfs.c
-+++ b/fs/btrfs/sysfs.c
-@@ -200,6 +200,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(mixed_backref, MIXED_BACKREF);
- BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL);
- BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS);
- BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO);
-+BTRFS_FEAT_ATTR_INCOMPAT(compress_zstd, COMPRESS_ZSTD);
- BTRFS_FEAT_ATTR_INCOMPAT(big_metadata, BIG_METADATA);
- BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF);
- BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56);
-@@ -212,6 +213,7 @@ static struct attribute *btrfs_supported_feature_attrs[] = {
- BTRFS_FEAT_ATTR_PTR(default_subvol),
- BTRFS_FEAT_ATTR_PTR(mixed_groups),
- BTRFS_FEAT_ATTR_PTR(compress_lzo),
-+ BTRFS_FEAT_ATTR_PTR(compress_zstd),
- BTRFS_FEAT_ATTR_PTR(big_metadata),
- BTRFS_FEAT_ATTR_PTR(extended_iref),
- BTRFS_FEAT_ATTR_PTR(raid56),
-diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c
-new file mode 100644
-index 0000000..607ce47
---- /dev/null
-+++ b/fs/btrfs/zstd.c
-@@ -0,0 +1,432 @@
-+/*
-+ * Copyright (c) 2016-present, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This program is free software; you can redistribute it and/or
-+ * modify it under the terms of the GNU General Public
-+ * License v2 as published by the Free Software Foundation.
-+ *
-+ * This program is distributed in the hope that it will be useful,
-+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ * General Public License for more details.
-+ */
-+#include <linux/bio.h>
-+#include <linux/err.h>
-+#include <linux/init.h>
-+#include <linux/kernel.h>
-+#include <linux/mm.h>
-+#include <linux/pagemap.h>
-+#include <linux/refcount.h>
-+#include <linux/sched.h>
-+#include <linux/slab.h>
-+#include <linux/zstd.h>
-+#include "compression.h"
-+
-+#define ZSTD_BTRFS_MAX_WINDOWLOG 17
-+#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
-+#define ZSTD_BTRFS_DEFAULT_LEVEL 3
-+
-+static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
-+{
-+ ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL,
-+ src_len, 0);
-+
-+ if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
-+ params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
-+ WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
-+ return params;
-+}
-+
-+struct workspace {
-+ void *mem;
-+ size_t size;
-+ char *buf;
-+ struct list_head list;
-+};
-+
-+static void zstd_free_workspace(struct list_head *ws)
-+{
-+ struct workspace *workspace = list_entry(ws, struct workspace, list);
-+
-+ kvfree(workspace->mem);
-+ kfree(workspace->buf);
-+ kfree(workspace);
-+}
-+
-+static struct list_head *zstd_alloc_workspace(void)
-+{
-+ ZSTD_parameters params =
-+ zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
-+ struct workspace *workspace;
-+
-+ workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
-+ if (!workspace)
-+ return ERR_PTR(-ENOMEM);
-+
-+ workspace->size = max_t(size_t,
-+ ZSTD_CStreamWorkspaceBound(params.cParams),
-+ ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
-+ workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
-+ workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
-+ if (!workspace->mem || !workspace->buf)
-+ goto fail;
-+
-+ INIT_LIST_HEAD(&workspace->list);
-+
-+ return &workspace->list;
-+fail:
-+ zstd_free_workspace(&workspace->list);
-+ return ERR_PTR(-ENOMEM);
-+}
-+
-+static int zstd_compress_pages(struct list_head *ws,
-+ struct address_space *mapping,
-+ u64 start,
-+ struct page **pages,
-+ unsigned long *out_pages,
-+ unsigned long *total_in,
-+ unsigned long *total_out)
-+{
-+ struct workspace *workspace = list_entry(ws, struct workspace, list);
-+ ZSTD_CStream *stream;
-+ int ret = 0;
-+ int nr_pages = 0;
-+ struct page *in_page = NULL; /* The current page to read */
-+ struct page *out_page = NULL; /* The current page to write to */
-+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
-+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
-+ unsigned long tot_in = 0;
-+ unsigned long tot_out = 0;
-+ unsigned long len = *total_out;
-+ const unsigned long nr_dest_pages = *out_pages;
-+ unsigned long max_out = nr_dest_pages * PAGE_SIZE;
-+ ZSTD_parameters params = zstd_get_btrfs_parameters(len);
-+
-+ *out_pages = 0;
-+ *total_out = 0;
-+ *total_in = 0;
-+
-+ /* Initialize the stream */
-+ stream = ZSTD_initCStream(params, len, workspace->mem,
-+ workspace->size);
-+ if (!stream) {
-+ pr_warn("BTRFS: ZSTD_initCStream failed\n");
-+ ret = -EIO;
-+ goto out;
-+ }
-+
-+ /* map in the first page of input data */
-+ in_page = find_get_page(mapping, start >> PAGE_SHIFT);
-+ in_buf.src = kmap(in_page);
-+ in_buf.pos = 0;
-+ in_buf.size = min_t(size_t, len, PAGE_SIZE);
-+
-+
-+ /* Allocate and map in the output buffer */
-+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
-+ if (out_page == NULL) {
-+ ret = -ENOMEM;
-+ goto out;
-+ }
-+ pages[nr_pages++] = out_page;
-+ out_buf.dst = kmap(out_page);
-+ out_buf.pos = 0;
-+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
-+
-+ while (1) {
-+ size_t ret2;
-+
-+ ret2 = ZSTD_compressStream(stream, &out_buf, &in_buf);
-+ if (ZSTD_isError(ret2)) {
-+ pr_debug("BTRFS: ZSTD_compressStream returned %d\n",
-+ ZSTD_getErrorCode(ret2));
-+ ret = -EIO;
-+ goto out;
-+ }
-+
-+ /* Check to see if we are making it bigger */
-+ if (tot_in + in_buf.pos > 8192 &&
-+ tot_in + in_buf.pos <
-+ tot_out + out_buf.pos) {
-+ ret = -E2BIG;
-+ goto out;
-+ }
-+
-+ /* We've reached the end of our output range */
-+ if (out_buf.pos >= max_out) {
-+ tot_out += out_buf.pos;
-+ ret = -E2BIG;
-+ goto out;
-+ }
-+
-+ /* Check if we need more output space */
-+ if (out_buf.pos == out_buf.size) {
-+ tot_out += PAGE_SIZE;
-+ max_out -= PAGE_SIZE;
-+ kunmap(out_page);
-+ if (nr_pages == nr_dest_pages) {
-+ out_page = NULL;
-+ ret = -E2BIG;
-+ goto out;
-+ }
-+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
-+ if (out_page == NULL) {
-+ ret = -ENOMEM;
-+ goto out;
-+ }
-+ pages[nr_pages++] = out_page;
-+ out_buf.dst = kmap(out_page);
-+ out_buf.pos = 0;
-+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
-+ }
-+
-+ /* We've reached the end of the input */
-+ if (in_buf.pos >= len) {
-+ tot_in += in_buf.pos;
-+ break;
-+ }
-+
-+ /* Check if we need more input */
-+ if (in_buf.pos == in_buf.size) {
-+ tot_in += PAGE_SIZE;
-+ kunmap(in_page);
-+ put_page(in_page);
-+
-+ start += PAGE_SIZE;
-+ len -= PAGE_SIZE;
-+ in_page = find_get_page(mapping, start >> PAGE_SHIFT);
-+ in_buf.src = kmap(in_page);
-+ in_buf.pos = 0;
-+ in_buf.size = min_t(size_t, len, PAGE_SIZE);
-+ }
-+ }
-+ while (1) {
-+ size_t ret2;
-+
-+ ret2 = ZSTD_endStream(stream, &out_buf);
-+ if (ZSTD_isError(ret2)) {
-+ pr_debug("BTRFS: ZSTD_endStream returned %d\n",
-+ ZSTD_getErrorCode(ret2));
-+ ret = -EIO;
-+ goto out;
-+ }
-+ if (ret2 == 0) {
-+ tot_out += out_buf.pos;
-+ break;
-+ }
-+ if (out_buf.pos >= max_out) {
-+ tot_out += out_buf.pos;
-+ ret = -E2BIG;
-+ goto out;
-+ }
-+
-+ tot_out += PAGE_SIZE;
-+ max_out -= PAGE_SIZE;
-+ kunmap(out_page);
-+ if (nr_pages == nr_dest_pages) {
-+ out_page = NULL;
-+ ret = -E2BIG;
-+ goto out;
-+ }
-+ out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
-+ if (out_page == NULL) {
-+ ret = -ENOMEM;
-+ goto out;
-+ }
-+ pages[nr_pages++] = out_page;
-+ out_buf.dst = kmap(out_page);
-+ out_buf.pos = 0;
-+ out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
-+ }
-+
-+ if (tot_out >= tot_in) {
-+ ret = -E2BIG;
-+ goto out;
-+ }
-+
-+ ret = 0;
-+ *total_in = tot_in;
-+ *total_out = tot_out;
-+out:
-+ *out_pages = nr_pages;
-+ /* Cleanup */
-+ if (in_page) {
-+ kunmap(in_page);
-+ put_page(in_page);
-+ }
-+ if (out_page)
-+ kunmap(out_page);
-+ return ret;
-+}
-+
-+static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
-+{
-+ struct workspace *workspace = list_entry(ws, struct workspace, list);
-+ struct page **pages_in = cb->compressed_pages;
-+ u64 disk_start = cb->start;
-+ struct bio *orig_bio = cb->orig_bio;
-+ size_t srclen = cb->compressed_len;
-+ ZSTD_DStream *stream;
-+ int ret = 0;
-+ unsigned long page_in_index = 0;
-+ unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
-+ unsigned long buf_start;
-+ unsigned long total_out = 0;
-+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
-+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
-+
-+ stream = ZSTD_initDStream(
-+ ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
-+ if (!stream) {
-+ pr_debug("BTRFS: ZSTD_initDStream failed\n");
-+ ret = -EIO;
-+ goto done;
-+ }
-+
-+ in_buf.src = kmap(pages_in[page_in_index]);
-+ in_buf.pos = 0;
-+ in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
-+
-+ out_buf.dst = workspace->buf;
-+ out_buf.pos = 0;
-+ out_buf.size = PAGE_SIZE;
-+
-+ while (1) {
-+ size_t ret2;
-+
-+ ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf);
-+ if (ZSTD_isError(ret2)) {
-+ pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
-+ ZSTD_getErrorCode(ret2));
-+ ret = -EIO;
-+ goto done;
-+ }
-+ buf_start = total_out;
-+ total_out += out_buf.pos;
-+ out_buf.pos = 0;
-+
-+ ret = btrfs_decompress_buf2page(out_buf.dst, buf_start,
-+ total_out, disk_start, orig_bio);
-+ if (ret == 0)
-+ break;
-+
-+ if (in_buf.pos >= srclen)
-+ break;
-+
-+ /* Check if we've hit the end of a frame */
-+ if (ret2 == 0)
-+ break;
-+
-+ if (in_buf.pos == in_buf.size) {
-+ kunmap(pages_in[page_in_index++]);
-+ if (page_in_index >= total_pages_in) {
-+ in_buf.src = NULL;
-+ ret = -EIO;
-+ goto done;
-+ }
-+ srclen -= PAGE_SIZE;
-+ in_buf.src = kmap(pages_in[page_in_index]);
-+ in_buf.pos = 0;
-+ in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
-+ }
-+ }
-+ ret = 0;
-+ zero_fill_bio(orig_bio);
-+done:
-+ if (in_buf.src)
-+ kunmap(pages_in[page_in_index]);
-+ return ret;
-+}
-+
-+static int zstd_decompress(struct list_head *ws, unsigned char *data_in,
-+ struct page *dest_page,
-+ unsigned long start_byte,
-+ size_t srclen, size_t destlen)
-+{
-+ struct workspace *workspace = list_entry(ws, struct workspace, list);
-+ ZSTD_DStream *stream;
-+ int ret = 0;
-+ size_t ret2;
-+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
-+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
-+ unsigned long total_out = 0;
-+ unsigned long pg_offset = 0;
-+ char *kaddr;
-+
-+ stream = ZSTD_initDStream(
-+ ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
-+ if (!stream) {
-+ pr_warn("BTRFS: ZSTD_initDStream failed\n");
-+ ret = -EIO;
-+ goto finish;
-+ }
-+
-+ destlen = min_t(size_t, destlen, PAGE_SIZE);
-+
-+ in_buf.src = data_in;
-+ in_buf.pos = 0;
-+ in_buf.size = srclen;
-+
-+ out_buf.dst = workspace->buf;
-+ out_buf.pos = 0;
-+ out_buf.size = PAGE_SIZE;
-+
-+ ret2 = 1;
-+ while (pg_offset < destlen && in_buf.pos < in_buf.size) {
-+ unsigned long buf_start;
-+ unsigned long buf_offset;
-+ unsigned long bytes;
-+
-+ /* Check if the frame is over and we still need more input */
-+ if (ret2 == 0) {
-+ pr_debug("BTRFS: ZSTD_decompressStream ended early\n");
-+ ret = -EIO;
-+ goto finish;
-+ }
-+ ret2 = ZSTD_decompressStream(stream, &out_buf, &in_buf);
-+ if (ZSTD_isError(ret2)) {
-+ pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
-+ ZSTD_getErrorCode(ret2));
-+ ret = -EIO;
-+ goto finish;
-+ }
-+
-+ buf_start = total_out;
-+ total_out += out_buf.pos;
-+ out_buf.pos = 0;
-+
-+ if (total_out <= start_byte)
-+ continue;
-+
-+ if (total_out > start_byte && buf_start < start_byte)
-+ buf_offset = start_byte - buf_start;
-+ else
-+ buf_offset = 0;
-+
-+ bytes = min_t(unsigned long, destlen - pg_offset,
-+ out_buf.size - buf_offset);
-+
-+ kaddr = kmap_atomic(dest_page);
-+ memcpy(kaddr + pg_offset, out_buf.dst + buf_offset, bytes);
-+ kunmap_atomic(kaddr);
-+
-+ pg_offset += bytes;
-+ }
-+ ret = 0;
-+finish:
-+ if (pg_offset < destlen) {
-+ kaddr = kmap_atomic(dest_page);
-+ memset(kaddr + pg_offset, 0, destlen - pg_offset);
-+ kunmap_atomic(kaddr);
-+ }
-+ return ret;
-+}
-+
-+const struct btrfs_compress_op btrfs_zstd_compress = {
-+ .alloc_workspace = zstd_alloc_workspace,
-+ .free_workspace = zstd_free_workspace,
-+ .compress_pages = zstd_compress_pages,
-+ .decompress_bio = zstd_decompress_bio,
-+ .decompress = zstd_decompress,
-+};
-diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
-index 9aa74f3..378230c 100644
---- a/include/uapi/linux/btrfs.h
-+++ b/include/uapi/linux/btrfs.h
-@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args {
- #define BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL (1ULL << 1)
- #define BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS (1ULL << 2)
- #define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO (1ULL << 3)
--/*
-- * some patches floated around with a second compression method
-- * lets save that incompat here for when they do get in
-- * Note we don't actually support it, we're just reserving the
-- * number
-- */
--#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4)
-+#define BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD (1ULL << 4)
-
- /*
- * older kernels tried to do bigger metadata blocks, but the
---
-2.9.3
diff --git a/0003-kbuild-LLVMLinux-Add-support-for-generating-LLVM-bit.patch b/0003-kbuild-LLVMLinux-Add-support-for-generating-LLVM-bit.patch
deleted file mode 100644
index f84f1f2..0000000
--- a/0003-kbuild-LLVMLinux-Add-support-for-generating-LLVM-bit.patch
+++ /dev/null
@@ -1,94 +0,0 @@
-From 282c44dbde5c83338bd94bffe8a17390cc22b6fa Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:08:21 +0100
-Subject: [PATCH 03/30] kbuild, LLVMLinux: Add support for generating LLVM
- bitcode files
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Add rules to kbuild in order to generate LLVM bitcode files with the .ll
-extension when using clang.
-
- # from c code
- CC=clang make kernel/pid.ll
-
- # from asm code
- CC=clang make arch/x86/kernel/preempt.ll
-
-Author: Vinicius Tinti <viniciustinti@gmail.com>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
-Signed-off-by: Behan Webster <behanw@converseincode.com>
----
- .gitignore | 1 +
- Makefile | 8 +++++++-
- scripts/Makefile.build | 14 ++++++++++++++
- 3 files changed, 22 insertions(+), 1 deletion(-)
-
-diff --git a/.gitignore b/.gitignore
-index c2ed4ecb0acd..0c39aa20b6ba 100644
---- a/.gitignore
-+++ b/.gitignore
-@@ -33,6 +33,7 @@
- *.lzo
- *.patch
- *.gcno
-+*.ll
- modules.builtin
- Module.symvers
- *.dwo
-diff --git a/Makefile b/Makefile
-index e3e8522f0322..0e76a1ea9cae 100644
---- a/Makefile
-+++ b/Makefile
-@@ -1377,7 +1377,9 @@ help:
- @echo ' firmware_install- Install all firmware to INSTALL_FW_PATH'
- @echo ' (default: $$(INSTALL_MOD_PATH)/lib/firmware)'
- @echo ' dir/ - Build all files in dir and below'
-- @echo ' dir/file.[ois] - Build specified target only'
-+ @echo ' dir/file.[oisS] - Build specified target only'
-+ @echo ' dir/file.ll - Build the LLVM bitcode file'
-+ @echo ' (requires compiler support for LLVM bitcode generation)'
- @echo ' dir/file.lst - Build specified mixed source/assembly target only'
- @echo ' (requires a recent binutils and recent build (System.map))'
- @echo ' dir/file.ko - Build module including final link'
-@@ -1665,6 +1667,10 @@ endif
- $(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
- %.symtypes: %.c prepare scripts FORCE
- $(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
-+%.ll: %.c prepare scripts FORCE
-+ $(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
-+%.ll: %.S prepare scripts FORCE
-+ $(Q)$(MAKE) $(build)=$(build-dir) $(target-dir)$(notdir $@)
-
- # Modules
- /: prepare scripts FORCE
-diff --git a/scripts/Makefile.build b/scripts/Makefile.build
-index d883116ebaa4..e5a28da2e6fa 100644
---- a/scripts/Makefile.build
-+++ b/scripts/Makefile.build
-@@ -177,6 +177,20 @@ cmd_cc_symtypes_c = \
- $(obj)/%.symtypes : $(src)/%.c FORCE
- $(call cmd,cc_symtypes_c)
-
-+# LLVM bitcode
-+# Generate .ll files from .s and .c
-+quiet_cmd_cc_ll_c = CC $(quiet_modtag) $@
-+ cmd_cc_ll_c = $(CC) $(c_flags) -emit-llvm -S -o $@ $<
-+
-+$(obj)/%.ll: $(src)/%.c FORCE
-+ $(call if_changed_dep,cc_ll_c)
-+
-+quiet_cmd_as_ll_S = CPP $(quiet_modtag) $@
-+ cmd_as_ll_S = $(CPP) $(a_flags) -o $@ $<
-+
-+$(obj)/%.ll: $(src)/%.S FORCE
-+ $(call if_changed_dep,as_ll_S)
-+
- # C (.c) files
- # The C file is compiled and updated dependency information is generated.
- # (See cmd_cc_o_c + relevant part of rule_cc_o_c)
---
-2.11.0
-
diff --git a/0004-kbuild-LLVMLinux-Make-asm-offset-generation-work-wit.patch b/0004-kbuild-LLVMLinux-Make-asm-offset-generation-work-wit.patch
deleted file mode 100644
index ac4b7b1..0000000
--- a/0004-kbuild-LLVMLinux-Make-asm-offset-generation-work-wit.patch
+++ /dev/null
@@ -1,121 +0,0 @@
-From d2699a2c0021c7837bc871a34b82419a0e178fa2 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:18:44 +0100
-Subject: [PATCH 04/30] kbuild, LLVMLinux: Make asm-offset generation work with
- clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-When using clang with -fno-integerated-as, clang will use the gnu assembler
-instead of the integrated assembler. However clang will still perform asm
-error checking before sending the inline assembly language to gas.
-
-The generation of asm-offsets from within C code is dependent on gcc's blind
-passing of whatever is in asm() through to gas. Arbirary text is
-passed through which is then modified by a sed script into the appropriate .h
-and .S code. Since the arbitrary text is not valid assembly language, clang fails.
-
-This can be fixed by making the arbitrary text into an ASM comment and then
-updating the sed scripts accordingly to work as expected.
-
-This solution works for both gcc and clang.
-
-Signed-off-by: Behan Webster <behanw@converseincode.com>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
-Reviewed-by: Mark Charlebois <charlebm@gmail.com>
-Reviewed-by: Jan-Simon Möller <dl9pf@gmx.de>
-Cc: Jan Moskyto Matejka <mq@suse.cz>
-Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
-Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
-Cc: "H. Peter Anvin" <hpa@linux.intel.com>
-Cc: Tom Gundersen <teg@jklm.no>
-Cc: Masahiro Yamada <yamada.m@jp.panasonic.com>
-Cc: Arnd Bergmann <arnd@arndb.de>
----
- Kbuild | 25 +++++++++++++++++++++++++
- include/linux/kbuild.h | 6 +++---
- scripts/mod/Makefile | 8 ++++----
- 3 files changed, 32 insertions(+), 7 deletions(-)
-
-diff --git a/Kbuild b/Kbuild
-index 3d0ae152af7c..0a4022da6b6e 100644
---- a/Kbuild
-+++ b/Kbuild
-@@ -75,6 +75,31 @@ offsets-file := include/generated/asm-offsets.h
- always += $(offsets-file)
- targets += arch/$(SRCARCH)/kernel/asm-offsets.s
-
-+# Default sed regexp - multiline due to syntax constraints
-+define sed-y
-+ "/^@->/{s:@->#\(.*\):/* \1 */:; \
-+ s:^@->\([^ ]*\) [\$$#]*\([-0-9]*\) \(.*\):#define \1 \2 /* \3 */:; \
-+ s:^@->\([^ ]*\) [\$$#]*\([^ ]*\) \(.*\):#define \1 \2 /* \3 */:; \
-+ s:@->::; p;}"
-+endef
-+
-+quiet_cmd_offsets = GEN $@
-+define cmd_offsets
-+ (set -e; \
-+ echo "#ifndef __ASM_OFFSETS_H__"; \
-+ echo "#define __ASM_OFFSETS_H__"; \
-+ echo "/*"; \
-+ echo " * DO NOT MODIFY."; \
-+ echo " *"; \
-+ echo " * This file was generated by Kbuild"; \
-+ echo " *"; \
-+ echo " */"; \
-+ echo ""; \
-+ sed -ne $(sed-y) $<; \
-+ echo ""; \
-+ echo "#endif" ) > $@
-+endef
-+
- # We use internal kbuild rules to avoid the "is up to date" message from make
- arch/$(SRCARCH)/kernel/asm-offsets.s: arch/$(SRCARCH)/kernel/asm-offsets.c \
- $(obj)/$(timeconst-file) $(obj)/$(bounds-file) FORCE
-diff --git a/include/linux/kbuild.h b/include/linux/kbuild.h
-index 22a72198c14b..75fa2c3e0e1d 100644
---- a/include/linux/kbuild.h
-+++ b/include/linux/kbuild.h
-@@ -2,14 +2,14 @@
- #define __LINUX_KBUILD_H
-
- #define DEFINE(sym, val) \
-- asm volatile("\n->" #sym " %0 " #val : : "i" (val))
-+ asm volatile("\n@->" #sym " %0 " #val : : "i" (val))
-
--#define BLANK() asm volatile("\n->" : : )
-+#define BLANK() asm volatile("\n@->" : : )
-
- #define OFFSET(sym, str, mem) \
- DEFINE(sym, offsetof(struct str, mem))
-
- #define COMMENT(x) \
-- asm volatile("\n->#" x)
-+ asm volatile("\n@->#" x)
-
- #endif
-diff --git a/scripts/mod/Makefile b/scripts/mod/Makefile
-index 19d9bcadc0cc..c3db2fa31052 100644
---- a/scripts/mod/Makefile
-+++ b/scripts/mod/Makefile
-@@ -8,10 +8,10 @@ modpost-objs := modpost.o file2alias.o sumversion.o
- devicetable-offsets-file := devicetable-offsets.h
-
- define sed-y
-- "/^->/{s:->#\(.*\):/* \1 */:; \
-- s:^->\([^ ]*\) [\$$#]*\([-0-9]*\) \(.*\):#define \1 \2 /* \3 */:; \
-- s:^->\([^ ]*\) [\$$#]*\([^ ]*\) \(.*\):#define \1 \2 /* \3 */:; \
-- s:->::; p;}"
-+ "/^@->/{s:@->#\(.*\):/* \1 */:; \
-+ s:^@->\([^ ]*\) [\$$#]*\([-0-9]*\) \(.*\):#define \1 \2 /* \3 */:; \
-+ s:^@->\([^ ]*\) [\$$#]*\([^ ]*\) \(.*\):#define \1 \2 /* \3 */:; \
-+ s:@->::; p;}"
- endef
-
- quiet_cmd_offsets = GEN $@
---
-2.11.0
-
diff --git a/0004-squashfs-Add-zstd-support.patch b/0004-squashfs-Add-zstd-support.patch
deleted file mode 100644
index 36cdf71..0000000
--- a/0004-squashfs-Add-zstd-support.patch
+++ /dev/null
@@ -1,306 +0,0 @@
-From 46bf8f6d30d6ddf2446c110f122482b5e5e16933 Mon Sep 17 00:00:00 2001
-From: Sean Purcell <me@seanp.xyz>
-Date: Mon, 17 Jul 2017 17:08:59 -0700
-Subject: [PATCH v5 4/5] squashfs: Add zstd support
-
-Add zstd compression and decompression support to SquashFS. zstd is a
-great fit for SquashFS because it can compress at ratios approaching xz,
-while decompressing twice as fast as zlib. For SquashFS in particular,
-it can decompress as fast as lzo and lz4. It also has the flexibility
-to turn down the compression ratio for faster compression times.
-
-The compression benchmark is run on the file tree from the SquashFS archive
-found in ubuntu-16.10-desktop-amd64.iso [1]. It uses `mksquashfs` with the
-default block size (128 KB) and and various compression algorithms/levels.
-xz and zstd are also benchmarked with 256 KB blocks. The decompression
-benchmark times how long it takes to `tar` the file tree into `/dev/null`.
-See the benchmark file in the upstream zstd source repository located under
-`contrib/linux-kernel/squashfs-benchmark.sh` [2] for details.
-
-I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
-The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
-16 GB of RAM, and a SSD.
-
-| Method | Ratio | Compression MB/s | Decompression MB/s |
-|----------------|-------|------------------|--------------------|
-| gzip | 2.92 | 15 | 128 |
-| lzo | 2.64 | 9.5 | 217 |
-| lz4 | 2.12 | 94 | 218 |
-| xz | 3.43 | 5.5 | 35 |
-| xz 256 KB | 3.53 | 5.4 | 40 |
-| zstd 1 | 2.71 | 96 | 210 |
-| zstd 5 | 2.93 | 69 | 198 |
-| zstd 10 | 3.01 | 41 | 225 |
-| zstd 15 | 3.13 | 11.4 | 224 |
-| zstd 16 256 KB | 3.24 | 8.1 | 210 |
-
-This patch was written by Sean Purcell <me@seanp.xyz>, but I will be
-taking over the submission process.
-
-[1] http://releases.ubuntu.com/16.10/
-[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh
-
-zstd source repository: https://github.com/facebook/zstd
-
-Signed-off-by: Sean Purcell <me@seanp.xyz>
-Signed-off-by: Nick Terrell <terrelln@fb.com>
----
-v3 -> v4:
-- Fix minor linter warnings
-
-v4 -> v5:
-- Fix ZSTD_DStream initialization code in squashfs
-- Fix patch documentation to reflect that Sean Purcell is the author
-
- fs/squashfs/Kconfig | 14 +++++
- fs/squashfs/Makefile | 1 +
- fs/squashfs/decompressor.c | 7 +++
- fs/squashfs/decompressor.h | 4 ++
- fs/squashfs/squashfs_fs.h | 1 +
- fs/squashfs/zstd_wrapper.c | 151 +++++++++++++++++++++++++++++++++++++++++++++
- 6 files changed, 178 insertions(+)
- create mode 100644 fs/squashfs/zstd_wrapper.c
-
-diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
-index ffb093e..1adb334 100644
---- a/fs/squashfs/Kconfig
-+++ b/fs/squashfs/Kconfig
-@@ -165,6 +165,20 @@ config SQUASHFS_XZ
-
- If unsure, say N.
-
-+config SQUASHFS_ZSTD
-+ bool "Include support for ZSTD compressed file systems"
-+ depends on SQUASHFS
-+ select ZSTD_DECOMPRESS
-+ help
-+ Saying Y here includes support for reading Squashfs file systems
-+ compressed with ZSTD compression. ZSTD gives better compression than
-+ the default ZLIB compression, while using less CPU.
-+
-+ ZSTD is not the standard compression used in Squashfs and so most
-+ file systems will be readable without selecting this option.
-+
-+ If unsure, say N.
-+
- config SQUASHFS_4K_DEVBLK_SIZE
- bool "Use 4K device block size?"
- depends on SQUASHFS
-diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile
-index 246a6f3..6655631 100644
---- a/fs/squashfs/Makefile
-+++ b/fs/squashfs/Makefile
-@@ -15,3 +15,4 @@ squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o
- squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o
- squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o
- squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o
-+squashfs-$(CONFIG_SQUASHFS_ZSTD) += zstd_wrapper.o
-diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c
-index d2bc136..8366398 100644
---- a/fs/squashfs/decompressor.c
-+++ b/fs/squashfs/decompressor.c
-@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = {
- };
- #endif
-
-+#ifndef CONFIG_SQUASHFS_ZSTD
-+static const struct squashfs_decompressor squashfs_zstd_comp_ops = {
-+ NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0
-+};
-+#endif
-+
- static const struct squashfs_decompressor squashfs_unknown_comp_ops = {
- NULL, NULL, NULL, NULL, 0, "unknown", 0
- };
-@@ -75,6 +81,7 @@ static const struct squashfs_decompressor *decompressor[] = {
- &squashfs_lzo_comp_ops,
- &squashfs_xz_comp_ops,
- &squashfs_lzma_unsupported_comp_ops,
-+ &squashfs_zstd_comp_ops,
- &squashfs_unknown_comp_ops
- };
-
-diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h
-index a25713c..0f5a8e4 100644
---- a/fs/squashfs/decompressor.h
-+++ b/fs/squashfs/decompressor.h
-@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops;
- extern const struct squashfs_decompressor squashfs_zlib_comp_ops;
- #endif
-
-+#ifdef CONFIG_SQUASHFS_ZSTD
-+extern const struct squashfs_decompressor squashfs_zstd_comp_ops;
-+#endif
-+
- #endif
-diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h
-index 506f4ba..24d12fd 100644
---- a/fs/squashfs/squashfs_fs.h
-+++ b/fs/squashfs/squashfs_fs.h
-@@ -241,6 +241,7 @@ struct meta_index {
- #define LZO_COMPRESSION 3
- #define XZ_COMPRESSION 4
- #define LZ4_COMPRESSION 5
-+#define ZSTD_COMPRESSION 6
-
- struct squashfs_super_block {
- __le32 s_magic;
-diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c
-new file mode 100644
-index 0000000..eeaabf8
---- /dev/null
-+++ b/fs/squashfs/zstd_wrapper.c
-@@ -0,0 +1,151 @@
-+/*
-+ * Squashfs - a compressed read only filesystem for Linux
-+ *
-+ * Copyright (c) 2016-present, Facebook, Inc.
-+ * All rights reserved.
-+ *
-+ * This program is free software; you can redistribute it and/or
-+ * modify it under the terms of the GNU General Public License
-+ * as published by the Free Software Foundation; either version 2,
-+ * or (at your option) any later version.
-+ *
-+ * This program is distributed in the hope that it will be useful,
-+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-+ * GNU General Public License for more details.
-+ *
-+ * zstd_wrapper.c
-+ */
-+
-+#include <linux/mutex.h>
-+#include <linux/buffer_head.h>
-+#include <linux/slab.h>
-+#include <linux/zstd.h>
-+#include <linux/vmalloc.h>
-+
-+#include "squashfs_fs.h"
-+#include "squashfs_fs_sb.h"
-+#include "squashfs.h"
-+#include "decompressor.h"
-+#include "page_actor.h"
-+
-+struct workspace {
-+ void *mem;
-+ size_t mem_size;
-+ size_t window_size;
-+};
-+
-+static void *zstd_init(struct squashfs_sb_info *msblk, void *buff)
-+{
-+ struct workspace *wksp = kmalloc(sizeof(*wksp), GFP_KERNEL);
-+
-+ if (wksp == NULL)
-+ goto failed;
-+ wksp->window_size = max_t(size_t,
-+ msblk->block_size, SQUASHFS_METADATA_SIZE);
-+ wksp->mem_size = ZSTD_DStreamWorkspaceBound(wksp->window_size);
-+ wksp->mem = vmalloc(wksp->mem_size);
-+ if (wksp->mem == NULL)
-+ goto failed;
-+
-+ return wksp;
-+
-+failed:
-+ ERROR("Failed to allocate zstd workspace\n");
-+ kfree(wksp);
-+ return ERR_PTR(-ENOMEM);
-+}
-+
-+
-+static void zstd_free(void *strm)
-+{
-+ struct workspace *wksp = strm;
-+
-+ if (wksp)
-+ vfree(wksp->mem);
-+ kfree(wksp);
-+}
-+
-+
-+static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm,
-+ struct buffer_head **bh, int b, int offset, int length,
-+ struct squashfs_page_actor *output)
-+{
-+ struct workspace *wksp = strm;
-+ ZSTD_DStream *stream;
-+ size_t total_out = 0;
-+ size_t zstd_err;
-+ int k = 0;
-+ ZSTD_inBuffer in_buf = { NULL, 0, 0 };
-+ ZSTD_outBuffer out_buf = { NULL, 0, 0 };
-+
-+ stream = ZSTD_initDStream(wksp->window_size, wksp->mem, wksp->mem_size);
-+
-+ if (!stream) {
-+ ERROR("Failed to initialize zstd decompressor\n");
-+ goto out;
-+ }
-+
-+ out_buf.size = PAGE_SIZE;
-+ out_buf.dst = squashfs_first_page(output);
-+
-+ do {
-+ if (in_buf.pos == in_buf.size && k < b) {
-+ int avail = min(length, msblk->devblksize - offset);
-+
-+ length -= avail;
-+ in_buf.src = bh[k]->b_data + offset;
-+ in_buf.size = avail;
-+ in_buf.pos = 0;
-+ offset = 0;
-+ }
-+
-+ if (out_buf.pos == out_buf.size) {
-+ out_buf.dst = squashfs_next_page(output);
-+ if (out_buf.dst == NULL) {
-+ /* Shouldn't run out of pages
-+ * before stream is done.
-+ */
-+ squashfs_finish_page(output);
-+ goto out;
-+ }
-+ out_buf.pos = 0;
-+ out_buf.size = PAGE_SIZE;
-+ }
-+
-+ total_out -= out_buf.pos;
-+ zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf);
-+ total_out += out_buf.pos; /* add the additional data produced */
-+
-+ if (in_buf.pos == in_buf.size && k < b)
-+ put_bh(bh[k++]);
-+ } while (zstd_err != 0 && !ZSTD_isError(zstd_err));
-+
-+ squashfs_finish_page(output);
-+
-+ if (ZSTD_isError(zstd_err)) {
-+ ERROR("zstd decompression error: %d\n",
-+ (int)ZSTD_getErrorCode(zstd_err));
-+ goto out;
-+ }
-+
-+ if (k < b)
-+ goto out;
-+
-+ return (int)total_out;
-+
-+out:
-+ for (; k < b; k++)
-+ put_bh(bh[k]);
-+
-+ return -EIO;
-+}
-+
-+const struct squashfs_decompressor squashfs_zstd_comp_ops = {
-+ .init = zstd_init,
-+ .free = zstd_free,
-+ .decompress = zstd_uncompress,
-+ .id = ZSTD_COMPRESSION,
-+ .name = "zstd",
-+ .supported = 1
-+};
---
-2.9.3
diff --git a/0005-crypto-Add-zstd-support.patch b/0005-crypto-Add-zstd-support.patch
deleted file mode 100644
index 9c29f0b..0000000
--- a/0005-crypto-Add-zstd-support.patch
+++ /dev/null
@@ -1,393 +0,0 @@
---- linux-4.13/crypto/Kconfig.0134~ 2017-09-03 22:56:17.000000000 +0200
-+++ linux-4.13/crypto/Kconfig 2017-09-04 14:35:24.681180391 +0200
-@@ -1662,6 +1662,15 @@ config CRYPTO_LZ4HC
- help
- This is the LZ4 high compression mode algorithm.
-
-+config CRYPTO_ZSTD
-+ tristate "Zstd compression algorithm"
-+ select CRYPTO_ALGAPI
-+ select CRYPTO_ACOMP2
-+ select ZSTD_COMPRESS
-+ select ZSTD_DECOMPRESS
-+ help
-+ This is the zstd algorithm.
-+
- comment "Random Number Generation"
-
- config CRYPTO_ANSI_CPRNG
---- linux-4.13/crypto/Makefile.0134~ 2017-09-04 14:35:24.681180391 +0200
-+++ linux-4.13/crypto/Makefile 2017-09-04 14:38:40.919589876 +0200
-@@ -138,6 +138,7 @@ ecdh_generic-y := ecc.o
- ecdh_generic-y += ecdh.o
- ecdh_generic-y += ecdh_helper.o
- obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o
-+obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o
-
- #
- # generic algorithms and the async_tx api
---- linux-4.13/crypto/testmgr.c.0134~ 2017-09-03 22:56:17.000000000 +0200
-+++ linux-4.13/crypto/testmgr.c 2017-09-04 14:35:24.682180389 +0200
-@@ -3603,6 +3603,16 @@ static const struct alg_test_desc alg_te
- .decomp = __VECS(zlib_deflate_decomp_tv_template)
- }
- }
-+ }, {
-+ .alg = "zstd",
-+ .test = alg_test_comp,
-+ .fips_allowed = 1,
-+ .suite = {
-+ .comp = {
-+ .comp = __VECS(zstd_comp_tv_template),
-+ .decomp = __VECS(zstd_decomp_tv_template)
-+ }
-+ }
- }
- };
-
---- linux-4.13/crypto/testmgr.h.0134~ 2017-09-03 22:56:17.000000000 +0200
-+++ linux-4.13/crypto/testmgr.h 2017-09-04 14:35:24.684180383 +0200
-@@ -34638,4 +34638,75 @@ static const struct comp_testvec lz4hc_d
- },
- };
-
-+static const struct comp_testvec zstd_comp_tv_template[] = {
-+ {
-+ .inlen = 68,
-+ .outlen = 39,
-+ .input = "The algorithm is zstd. "
-+ "The algorithm is zstd. "
-+ "The algorithm is zstd.",
-+ .output = "\x28\xb5\x2f\xfd\x00\x50\xf5\x00\x00\xb8\x54\x68\x65"
-+ "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73"
-+ "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01"
-+ ,
-+ },
-+ {
-+ .inlen = 244,
-+ .outlen = 151,
-+ .input = "zstd, short for Zstandard, is a fast lossless "
-+ "compression algorithm, targeting real-time "
-+ "compression scenarios at zlib-level and better "
-+ "compression ratios. The zstd compression library "
-+ "provides in-memory compression and decompression "
-+ "functions.",
-+ .output = "\x28\xb5\x2f\xfd\x00\x50\x75\x04\x00\x42\x4b\x1e\x17"
-+ "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32"
-+ "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f"
-+ "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad"
-+ "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60"
-+ "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86"
-+ "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90"
-+ "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64"
-+ "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30"
-+ "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc"
-+ "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e"
-+ "\x20\xa9\x0e\x82\xb9\x43\x45\x01",
-+ },
-+};
-+
-+static const struct comp_testvec zstd_decomp_tv_template[] = {
-+ {
-+ .inlen = 43,
-+ .outlen = 68,
-+ .input = "\x28\xb5\x2f\xfd\x04\x50\xf5\x00\x00\xb8\x54\x68\x65"
-+ "\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d\x20\x69\x73"
-+ "\x20\x7a\x73\x74\x64\x2e\x20\x01\x00\x55\x73\x36\x01"
-+ "\x6b\xf4\x13\x35",
-+ .output = "The algorithm is zstd. "
-+ "The algorithm is zstd. "
-+ "The algorithm is zstd.",
-+ },
-+ {
-+ .inlen = 155,
-+ .outlen = 244,
-+ .input = "\x28\xb5\x2f\xfd\x04\x50\x75\x04\x00\x42\x4b\x1e\x17"
-+ "\x90\x81\x31\x00\xf2\x2f\xe4\x36\xc9\xef\x92\x88\x32"
-+ "\xc9\xf2\x24\x94\xd8\x68\x9a\x0f\x00\x0c\xc4\x31\x6f"
-+ "\x0d\x0c\x38\xac\x5c\x48\x03\xcd\x63\x67\xc0\xf3\xad"
-+ "\x4e\x90\xaa\x78\xa0\xa4\xc5\x99\xda\x2f\xb6\x24\x60"
-+ "\xe2\x79\x4b\xaa\xb6\x6b\x85\x0b\xc9\xc6\x04\x66\x86"
-+ "\xe2\xcc\xe2\x25\x3f\x4f\x09\xcd\xb8\x9d\xdb\xc1\x90"
-+ "\xa9\x11\xbc\x35\x44\x69\x2d\x9c\x64\x4f\x13\x31\x64"
-+ "\xcc\xfb\x4d\x95\x93\x86\x7f\x33\x7f\x1a\xef\xe9\x30"
-+ "\xf9\x67\xa1\x94\x0a\x69\x0f\x60\xcd\xc3\xab\x99\xdc"
-+ "\x42\xed\x97\x05\x00\x33\xc3\x15\x95\x3a\x06\xa0\x0e"
-+ "\x20\xa9\x0e\x82\xb9\x43\x45\x01\xaa\x6d\xda\x0d",
-+ .output = "zstd, short for Zstandard, is a fast lossless "
-+ "compression algorithm, targeting real-time "
-+ "compression scenarios at zlib-level and better "
-+ "compression ratios. The zstd compression library "
-+ "provides in-memory compression and decompression "
-+ "functions.",
-+ },
-+};
- #endif /* _CRYPTO_TESTMGR_H */
---- linux-4.13/crypto/zstd.c.0134~ 2017-09-04 14:35:24.685180380 +0200
-+++ linux-4.13/crypto/zstd.c 2017-09-04 14:35:24.685180380 +0200
-@@ -0,0 +1,265 @@
-+/*
-+ * Cryptographic API.
-+ *
-+ * Copyright (c) 2017-present, Facebook, Inc.
-+ *
-+ * This program is free software; you can redistribute it and/or modify it
-+ * under the terms of the GNU General Public License version 2 as published by
-+ * the Free Software Foundation.
-+ *
-+ * This program is distributed in the hope that it will be useful, but WITHOUT
-+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-+ * more details.
-+ */
-+#include <linux/crypto.h>
-+#include <linux/init.h>
-+#include <linux/interrupt.h>
-+#include <linux/mm.h>
-+#include <linux/module.h>
-+#include <linux/net.h>
-+#include <linux/vmalloc.h>
-+#include <linux/zstd.h>
-+#include <crypto/internal/scompress.h>
-+
-+
-+#define ZSTD_DEF_LEVEL 3
-+
-+struct zstd_ctx {
-+ ZSTD_CCtx *cctx;
-+ ZSTD_DCtx *dctx;
-+ void *cwksp;
-+ void *dwksp;
-+};
-+
-+static ZSTD_parameters zstd_params(void)
-+{
-+ return ZSTD_getParams(ZSTD_DEF_LEVEL, 0, 0);
-+}
-+
-+static int zstd_comp_init(struct zstd_ctx *ctx)
-+{
-+ int ret = 0;
-+ const ZSTD_parameters params = zstd_params();
-+ const size_t wksp_size = ZSTD_CCtxWorkspaceBound(params.cParams);
-+
-+ ctx->cwksp = vzalloc(wksp_size);
-+ if (!ctx->cwksp) {
-+ ret = -ENOMEM;
-+ goto out;
-+ }
-+
-+ ctx->cctx = ZSTD_initCCtx(ctx->cwksp, wksp_size);
-+ if (!ctx->cctx) {
-+ ret = -EINVAL;
-+ goto out_free;
-+ }
-+out:
-+ return ret;
-+out_free:
-+ vfree(ctx->cwksp);
-+ goto out;
-+}
-+
-+static int zstd_decomp_init(struct zstd_ctx *ctx)
-+{
-+ int ret = 0;
-+ const size_t wksp_size = ZSTD_DCtxWorkspaceBound();
-+
-+ ctx->dwksp = vzalloc(wksp_size);
-+ if (!ctx->dwksp) {
-+ ret = -ENOMEM;
-+ goto out;
-+ }
-+
-+ ctx->dctx = ZSTD_initDCtx(ctx->dwksp, wksp_size);
-+ if (!ctx->dctx) {
-+ ret = -EINVAL;
-+ goto out_free;
-+ }
-+out:
-+ return ret;
-+out_free:
-+ vfree(ctx->dwksp);
-+ goto out;
-+}
-+
-+static void zstd_comp_exit(struct zstd_ctx *ctx)
-+{
-+ vfree(ctx->cwksp);
-+ ctx->cwksp = NULL;
-+ ctx->cctx = NULL;
-+}
-+
-+static void zstd_decomp_exit(struct zstd_ctx *ctx)
-+{
-+ vfree(ctx->dwksp);
-+ ctx->dwksp = NULL;
-+ ctx->dctx = NULL;
-+}
-+
-+static int __zstd_init(void *ctx)
-+{
-+ int ret;
-+
-+ ret = zstd_comp_init(ctx);
-+ if (ret)
-+ return ret;
-+ ret = zstd_decomp_init(ctx);
-+ if (ret)
-+ zstd_comp_exit(ctx);
-+ return ret;
-+}
-+
-+static void *zstd_alloc_ctx(struct crypto_scomp *tfm)
-+{
-+ int ret;
-+ struct zstd_ctx *ctx;
-+
-+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
-+ if (!ctx)
-+ return ERR_PTR(-ENOMEM);
-+
-+ ret = __zstd_init(ctx);
-+ if (ret) {
-+ kfree(ctx);
-+ return ERR_PTR(ret);
-+ }
-+
-+ return ctx;
-+}
-+
-+static int zstd_init(struct crypto_tfm *tfm)
-+{
-+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
-+
-+ return __zstd_init(ctx);
-+}
-+
-+static void __zstd_exit(void *ctx)
-+{
-+ zstd_comp_exit(ctx);
-+ zstd_decomp_exit(ctx);
-+}
-+
-+static void zstd_free_ctx(struct crypto_scomp *tfm, void *ctx)
-+{
-+ __zstd_exit(ctx);
-+ kzfree(ctx);
-+}
-+
-+static void zstd_exit(struct crypto_tfm *tfm)
-+{
-+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
-+
-+ __zstd_exit(ctx);
-+}
-+
-+static int __zstd_compress(const u8 *src, unsigned int slen,
-+ u8 *dst, unsigned int *dlen, void *ctx)
-+{
-+ size_t out_len;
-+ struct zstd_ctx *zctx = ctx;
-+ const ZSTD_parameters params = zstd_params();
-+
-+ out_len = ZSTD_compressCCtx(zctx->cctx, dst, *dlen, src, slen, params);
-+ if (ZSTD_isError(out_len))
-+ return -EINVAL;
-+ *dlen = out_len;
-+ return 0;
-+}
-+
-+static int zstd_compress(struct crypto_tfm *tfm, const u8 *src,
-+ unsigned int slen, u8 *dst, unsigned int *dlen)
-+{
-+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
-+
-+ return __zstd_compress(src, slen, dst, dlen, ctx);
-+}
-+
-+static int zstd_scompress(struct crypto_scomp *tfm, const u8 *src,
-+ unsigned int slen, u8 *dst, unsigned int *dlen,
-+ void *ctx)
-+{
-+ return __zstd_compress(src, slen, dst, dlen, ctx);
-+}
-+
-+static int __zstd_decompress(const u8 *src, unsigned int slen,
-+ u8 *dst, unsigned int *dlen, void *ctx)
-+{
-+ size_t out_len;
-+ struct zstd_ctx *zctx = ctx;
-+
-+ out_len = ZSTD_decompressDCtx(zctx->dctx, dst, *dlen, src, slen);
-+ if (ZSTD_isError(out_len))
-+ return -EINVAL;
-+ *dlen = out_len;
-+ return 0;
-+}
-+
-+static int zstd_decompress(struct crypto_tfm *tfm, const u8 *src,
-+ unsigned int slen, u8 *dst, unsigned int *dlen)
-+{
-+ struct zstd_ctx *ctx = crypto_tfm_ctx(tfm);
-+
-+ return __zstd_decompress(src, slen, dst, dlen, ctx);
-+}
-+
-+static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src,
-+ unsigned int slen, u8 *dst, unsigned int *dlen,
-+ void *ctx)
-+{
-+ return __zstd_decompress(src, slen, dst, dlen, ctx);
-+}
-+
-+static struct crypto_alg alg = {
-+ .cra_name = "zstd",
-+ .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
-+ .cra_ctxsize = sizeof(struct zstd_ctx),
-+ .cra_module = THIS_MODULE,
-+ .cra_init = zstd_init,
-+ .cra_exit = zstd_exit,
-+ .cra_u = { .compress = {
-+ .coa_compress = zstd_compress,
-+ .coa_decompress = zstd_decompress } }
-+};
-+
-+static struct scomp_alg scomp = {
-+ .alloc_ctx = zstd_alloc_ctx,
-+ .free_ctx = zstd_free_ctx,
-+ .compress = zstd_scompress,
-+ .decompress = zstd_sdecompress,
-+ .base = {
-+ .cra_name = "zstd",
-+ .cra_driver_name = "zstd-scomp",
-+ .cra_module = THIS_MODULE,
-+ }
-+};
-+
-+static int __init zstd_mod_init(void)
-+{
-+ int ret;
-+
-+ ret = crypto_register_alg(&alg);
-+ if (ret)
-+ return ret;
-+
-+ ret = crypto_register_scomp(&scomp);
-+ if (ret)
-+ crypto_unregister_alg(&alg);
-+
-+ return ret;
-+}
-+
-+static void __exit zstd_mod_fini(void)
-+{
-+ crypto_unregister_alg(&alg);
-+ crypto_unregister_scomp(&scomp);
-+}
-+
-+module_init(zstd_mod_init);
-+module_exit(zstd_mod_fini);
-+
-+MODULE_LICENSE("GPL");
-+MODULE_DESCRIPTION("Zstd Compression Algorithm");
-+MODULE_ALIAS_CRYPTO("zstd");
diff --git a/0005-md-sysfs-LLVMLinux-Remove-nested-function-from-bcach.patch b/0005-md-sysfs-LLVMLinux-Remove-nested-function-from-bcach.patch
deleted file mode 100644
index a9b3e20..0000000
--- a/0005-md-sysfs-LLVMLinux-Remove-nested-function-from-bcach.patch
+++ /dev/null
@@ -1,65 +0,0 @@
-From 36def6a1b04411fb3b0ae97871c4308a59d6350a Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:20:49 +0100
-Subject: [PATCH 05/30] md, sysfs, LLVMLinux: Remove nested function from
- bcache sysfs
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Replace the use of nested functions where a normal function will suffice.
-
-Nested functions are not liked by upstream kernel developers in general. Their
-use breaks the use of clang as a compiler, and doesn't make the code any
-better.
-
-This code now works for both gcc and clang.
-
-Signed-off-by: Behan Webster <behanw@converseincode.com>
-Reviewed-by: Mark Charlebois <charlebm@gmail.com>
-Suggested-by: Arnd Bergmann <arnd@arndb.de>
-Cc: Arnd Bergmann <arnd@arndb.de>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- drivers/md/bcache/sysfs.c | 10 ++++++----
- 1 file changed, 6 insertions(+), 4 deletions(-)
-
-diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c
-index b3ff57d61dde..53d8baa741fb 100644
---- a/drivers/md/bcache/sysfs.c
-+++ b/drivers/md/bcache/sysfs.c
-@@ -731,6 +731,11 @@ static struct attribute *bch_cache_set_internal_files[] = {
- };
- KTYPE(bch_cache_set_internal);
-
-+static int __bch_cache_cmp(const void *l, const void *r)
-+{
-+ return *((uint16_t *) r) - *((uint16_t *) l);
-+}
-+
- SHOW(__bch_cache)
- {
- struct cache *ca = container_of(kobj, struct cache, kobj);
-@@ -755,9 +760,6 @@ SHOW(__bch_cache)
- CACHE_REPLACEMENT(&ca->sb));
-
- if (attr == &sysfs_priority_stats) {
-- int cmp(const void *l, const void *r)
-- { return *((uint16_t *) r) - *((uint16_t *) l); }
--
- struct bucket *b;
- size_t n = ca->sb.nbuckets, i;
- size_t unused = 0, available = 0, dirty = 0, meta = 0;
-@@ -786,7 +788,7 @@ SHOW(__bch_cache)
- p[i] = ca->buckets[i].prio;
- mutex_unlock(&ca->set->bucket_lock);
-
-- sort(p, n, sizeof(uint16_t), cmp, NULL);
-+ sort(p, n, sizeof(uint16_t), __bch_cache_cmp, NULL);
-
- while (n &&
- !cached[n - 1])
---
-2.11.0
-
diff --git a/0006-apparmor-LLVMLinux-Remove-VLAIS.patch b/0006-apparmor-LLVMLinux-Remove-VLAIS.patch
deleted file mode 100644
index 9fde18d..0000000
--- a/0006-apparmor-LLVMLinux-Remove-VLAIS.patch
+++ /dev/null
@@ -1,108 +0,0 @@
-From de33aa01ff91fb63a9a22f362135b9aeb56b5ccc Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:22:50 +0100
-Subject: [PATCH 06/30] apparmor, LLVMLinux: Remove VLAIS
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Replaced the use of a Variable Length Array In Struct (VLAIS) with a C99
-compliant equivalent. This patch instead allocates the appropriate amount of
-memory using a char array using the SHASH_DESC_ON_STACK macro.
-
-The new code can be compiled with both gcc and clang.
-
-Based on an older version by Vinicius Tinti <viniciustinti@gmail.com>
-
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
-Cc: Vinícius Tinti <viniciustinti@gmail.com>
-Cc: Jan-Simon Möller <dl9pf@gmx.de>
-Cc: Mark Charlebois <charlebm@gmail.com>
-Cc: Behan Webster <behanw@converseincode.com>
----
- security/apparmor/crypto.c | 36 +++++++++++++++++-------------------
- 1 file changed, 17 insertions(+), 19 deletions(-)
-
-diff --git a/security/apparmor/crypto.c b/security/apparmor/crypto.c
-index de8dc78b6144..198c10b2f46c 100644
---- a/security/apparmor/crypto.c
-+++ b/security/apparmor/crypto.c
-@@ -31,10 +31,6 @@ unsigned int aa_hash_size(void)
-
- char *aa_calc_hash(void *data, size_t len)
- {
-- struct {
-- struct shash_desc shash;
-- char ctx[crypto_shash_descsize(apparmor_tfm)];
-- } desc;
- char *hash = NULL;
- int error = -ENOMEM;
-
-@@ -45,16 +41,19 @@ char *aa_calc_hash(void *data, size_t len)
- if (!hash)
- goto fail;
-
-- desc.shash.tfm = apparmor_tfm;
-- desc.shash.flags = 0;
-+ SHASH_DESC_ON_STACK(shash, apparmor_tfm);
-+ if (!shash)
-+ goto fail;
-+ shash->tfm = apparmor_tfm;
-+ shash->flags = 0;
-
-- error = crypto_shash_init(&desc.shash);
-+ error = crypto_shash_init(shash);
- if (error)
- goto fail;
-- error = crypto_shash_update(&desc.shash, (u8 *) data, len);
-+ error = crypto_shash_update(shash, (u8 *) data, len);
- if (error)
- goto fail;
-- error = crypto_shash_final(&desc.shash, hash);
-+ error = crypto_shash_final(shash, hash);
- if (error)
- goto fail;
-
-@@ -69,10 +68,6 @@ char *aa_calc_hash(void *data, size_t len)
- int aa_calc_profile_hash(struct aa_profile *profile, u32 version, void *start,
- size_t len)
- {
-- struct {
-- struct shash_desc shash;
-- char ctx[crypto_shash_descsize(apparmor_tfm)];
-- } desc;
- int error = -ENOMEM;
- __le32 le32_version = cpu_to_le32(version);
-
-@@ -86,19 +81,22 @@ int aa_calc_profile_hash(struct aa_profile *profile, u32 version, void *start,
- if (!profile->hash)
- goto fail;
-
-- desc.shash.tfm = apparmor_tfm;
-- desc.shash.flags = 0;
-+ SHASH_DESC_ON_STACK(shash, apparmor_tfm);
-+ if (!shash)
-+ goto fail;
-+ shash->tfm = apparmor_tfm;
-+ shash->flags = 0;
-
-- error = crypto_shash_init(&desc.shash);
-+ error = crypto_shash_init(shash);
- if (error)
- goto fail;
-- error = crypto_shash_update(&desc.shash, (u8 *) &le32_version, 4);
-+ error = crypto_shash_update(shash, (u8 *) &le32_version, 4);
- if (error)
- goto fail;
-- error = crypto_shash_update(&desc.shash, (u8 *) start, len);
-+ error = crypto_shash_update(shash, (u8 *) start, len);
- if (error)
- goto fail;
-- error = crypto_shash_final(&desc.shash, profile->hash);
-+ error = crypto_shash_final(shash, profile->hash);
- if (error)
- goto fail;
-
---
-2.11.0
-
diff --git a/0007-exofs-LLVMLinux-Remove-VLAIS-from-exofs-FIXME-Check-.patch b/0007-exofs-LLVMLinux-Remove-VLAIS-from-exofs-FIXME-Check-.patch
deleted file mode 100644
index 9f8fffd..0000000
--- a/0007-exofs-LLVMLinux-Remove-VLAIS-from-exofs-FIXME-Check-.patch
+++ /dev/null
@@ -1,91 +0,0 @@
-From 1cbc1d31554f7e80ab76442c8e5d72007ba4d16b Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:23:43 +0100
-Subject: [PATCH 07/30] exofs, LLVMLinux: Remove VLAIS from exofs FIXME Check
- __weak and BROKEN fragment
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-This makes it possible to build exofs with clang.
-
-Signed-off-by: Behan Webster <behanw@converseincode.com>
-Suggested-by: Arnd Bergmann <arnd@arndb.de>
-Cc: Arnd Bergmann <arnd@arndb.de>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- fs/exofs/Kconfig | 2 +-
- fs/exofs/common.h | 2 +-
- fs/exofs/super.c | 21 +++++++++------------
- 3 files changed, 11 insertions(+), 14 deletions(-)
-
-diff --git a/fs/exofs/Kconfig b/fs/exofs/Kconfig
-index 86194b2f799d..492746f9e61b 100644
---- a/fs/exofs/Kconfig
-+++ b/fs/exofs/Kconfig
-@@ -1,6 +1,6 @@
- config EXOFS_FS
- tristate "exofs: OSD based file system support"
-- depends on SCSI_OSD_ULD
-+ depends on SCSI_OSD_ULD && BROKEN
- help
- EXOFS is a file system that uses an OSD storage device,
- as its backing storage.
-diff --git a/fs/exofs/common.h b/fs/exofs/common.h
-index 7d88ef566213..9fdf02333c20 100644
---- a/fs/exofs/common.h
-+++ b/fs/exofs/common.h
-@@ -183,7 +183,7 @@ struct exofs_fcb {
- #define EXOFS_INO_ATTR_SIZE sizeof(struct exofs_fcb)
-
- /* This is the Attribute the fcb is stored in */
--static const struct __weak osd_attr g_attr_inode_data = ATTR_DEF(
-+static const struct osd_attr g_attr_inode_data __maybe_unused = ATTR_DEF(
- EXOFS_APAGE_FS_DATA,
- EXOFS_ATTR_INODE_DATA,
- EXOFS_INO_ATTR_SIZE);
-diff --git a/fs/exofs/super.c b/fs/exofs/super.c
-index 1076a4233b39..410572994dad 100644
---- a/fs/exofs/super.c
-+++ b/fs/exofs/super.c
-@@ -546,27 +546,24 @@ static int exofs_devs_2_odi(struct exofs_dt_device_info *dt_dev,
- static int __alloc_dev_table(struct exofs_sb_info *sbi, unsigned numdevs,
- struct exofs_dev **peds)
- {
-- struct __alloc_ore_devs_and_exofs_devs {
-- /* Twice bigger table: See exofs_init_comps() and comment at
-- * exofs_read_lookup_dev_table()
-- */
-- struct ore_dev *oreds[numdevs * 2 - 1];
-- struct exofs_dev eds[numdevs];
-- } *aoded;
- struct exofs_dev *eds;
- unsigned i;
-
-- aoded = kzalloc(sizeof(*aoded), GFP_KERNEL);
-- if (unlikely(!aoded)) {
-+ /* Twice bigger table: See exofs_init_comps() and comment at
-+ * exofs_read_lookup_dev_table()
-+ * XXX: why -1?
-+ */
-+ sbi->oc.ods = kzalloc(sizeof(struct ore_dev) * (numdevs * 2 - 1) +
-+ sizeof(struct exofs_dev) * numdevs, GFP_KERNEL);
-+ if (unlikely(!sbi->oc.ods)) {
- EXOFS_ERR("ERROR: failed allocating Device array[%d]\n",
- numdevs);
- return -ENOMEM;
- }
-
-- sbi->oc.ods = aoded->oreds;
-- *peds = eds = aoded->eds;
-+ *peds = eds = (void *)sbi->oc.ods[numdevs * 2 - 1];
- for (i = 0; i < numdevs; ++i)
-- aoded->oreds[i] = &eds[i].ored;
-+ sbi->oc.ods[i] = &eds[i].ored;
- return 0;
- }
-
---
-2.11.0
-
diff --git a/0011-Kbuild-LLVMLinux-Use-Oz-instead-of-Os-when-using-cla.patch b/0011-Kbuild-LLVMLinux-Use-Oz-instead-of-Os-when-using-cla.patch
deleted file mode 100644
index 62b4b90..0000000
--- a/0011-Kbuild-LLVMLinux-Use-Oz-instead-of-Os-when-using-cla.patch
+++ /dev/null
@@ -1,35 +0,0 @@
-From 638df6b609af6cfacc3134137c7b513113a1676a Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:32:12 +0100
-Subject: [PATCH 11/30] Kbuild, LLVMLinux: Use -Oz instead of -Os when using
- clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-This generates smaller resulting object code when compiled with clang.
-
-Signed-off-by: Behan Webster <behanw@converseincode.com>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- Makefile | 3 ++-
- 1 file changed, 2 insertions(+), 1 deletion(-)
-
-diff --git a/Makefile b/Makefile
-index 0e76a1ea9cae..a15d91aebe52 100644
---- a/Makefile
-+++ b/Makefile
-@@ -651,7 +651,8 @@ KBUILD_CFLAGS += $(call cc-option,-fdata-sections,)
- endif
-
- ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE
--KBUILD_CFLAGS += -Os $(call cc-disable-warning,maybe-uninitialized,)
-+KBUILD_CFLAGS += $(call cc-option,-Oz,-Os)
-+KBUILD_CFLAGS += $(call cc-disable-warning,maybe-uninitialized,)
- else
- ifdef CONFIG_PROFILE_ALL_BRANCHES
- KBUILD_CFLAGS += -O2 $(call cc-disable-warning,maybe-uninitialized,)
---
-2.11.0
-
diff --git a/0012-WORKAROUND-x86-boot-LLVMLinux-Work-around-clang-PR39.patch b/0012-WORKAROUND-x86-boot-LLVMLinux-Work-around-clang-PR39.patch
deleted file mode 100644
index 769ff06..0000000
--- a/0012-WORKAROUND-x86-boot-LLVMLinux-Work-around-clang-PR39.patch
+++ /dev/null
@@ -1,71 +0,0 @@
-From 0b2388ba1b93f5fed9746a6f24a83e02c5548a62 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:33:18 +0100
-Subject: [PATCH 12/30] WORKAROUND x86, boot, LLVMLinux: Work around clang
- PR3997
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Clang's intrinsics ignore -mregparm=3 when they fall back to calling the
-out-of-line implementations. Putting the args on the stack when memcpy()
-expects them in registers is not a recipe for a happy kernel.
-
-This bites with -m32 too, so clang is presumably catastrophically
-broken for the i386 kernel until this is fixed, unless I'm missing
-something.
-
-For information/testing only; do not apply. With this, I can use
-'clang -m16' to build all the kernel's 16-bit code and get a successful
-boot.
-
-Not-signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
-Forward-ported-by: Jan-Simon Möller <dl9pf@gmx.de>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- arch/x86/boot/memory.c | 7 ++++++-
- arch/x86/boot/string.h | 2 ++
- 2 files changed, 8 insertions(+), 1 deletion(-)
-
-diff --git a/arch/x86/boot/memory.c b/arch/x86/boot/memory.c
-index db75d07c3645..0a61ad084927 100644
---- a/arch/x86/boot/memory.c
-+++ b/arch/x86/boot/memory.c
-@@ -63,8 +63,13 @@ static int detect_memory_e820(void)
- count = 0;
- break;
- }
--
-+#ifdef __clang__
-+ /* PR18415, PR3997 */
-+ memcpy(desc, &buf, sizeof(*desc));
-+ desc++;
-+#else
- *desc++ = buf;
-+#endif
- count++;
- } while (ireg.ebx && count < ARRAY_SIZE(boot_params.e820_map));
-
-diff --git a/arch/x86/boot/string.h b/arch/x86/boot/string.h
-index 113588ddb43f..d03f4e21a7f3 100644
---- a/arch/x86/boot/string.h
-+++ b/arch/x86/boot/string.h
-@@ -10,6 +10,7 @@ void *memcpy(void *dst, const void *src, size_t len);
- void *memset(void *dst, int c, size_t len);
- int memcmp(const void *s1, const void *s2, size_t len);
-
-+#ifndef __clang__ /* PR3997, PR18415 */
- /*
- * Access builtin version by default. If one needs to use optimized version,
- * do "undef memcpy" in .c file and link against right string.c
-@@ -17,6 +18,7 @@ int memcmp(const void *s1, const void *s2, size_t len);
- #define memcpy(d,s,l) __builtin_memcpy(d,s,l)
- #define memset(d,c,l) __builtin_memset(d,c,l)
- #define memcmp __builtin_memcmp
-+#endif
-
- extern int strcmp(const char *str1, const char *str2);
- extern int strncmp(const char *cs, const char *ct, size_t count);
---
-2.11.0
-
diff --git a/0013-DO-NOT-UPSTREAM-xen-LLVMLinux-Remove-VLAIS-from-xen-.patch b/0013-DO-NOT-UPSTREAM-xen-LLVMLinux-Remove-VLAIS-from-xen-.patch
deleted file mode 100644
index c78c351..0000000
--- a/0013-DO-NOT-UPSTREAM-xen-LLVMLinux-Remove-VLAIS-from-xen-.patch
+++ /dev/null
@@ -1,76 +0,0 @@
-From e1be0e294634148b3b538d303bc6291b2d6cf801 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Vin=C3=ADcius=20Tinti?= <viniciustinti@gmail.com>
-Date: Sun, 14 Dec 2014 00:19:00 -0200
-Subject: [PATCH 13/30] DO-NOT-UPSTREAM xen, LLVMLinux: Remove VLAIS from xen
- mmu
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Signed-off-by: Vinícius Tinti <viniciustinti@gmail.com>
----
- arch/x86/xen/mmu.c | 35 ++++++++++++++++++-----------------
- 1 file changed, 18 insertions(+), 17 deletions(-)
-
-diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
-index f6740b5b1738..5adf2b65f1fb 100644
---- a/arch/x86/xen/mmu.c
-+++ b/arch/x86/xen/mmu.c
-@@ -1375,36 +1375,37 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
- struct mm_struct *mm, unsigned long start,
- unsigned long end)
- {
-- struct {
-- struct mmuext_op op;
--#ifdef CONFIG_SMP
-- DECLARE_BITMAP(mask, num_processors);
--#else
-- DECLARE_BITMAP(mask, NR_CPUS);
--#endif
-- } *args;
- struct multicall_space mcs;
-+ struct mmuext_op *op;
-+ struct cpumask *mask;
-
- trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
-
- if (cpumask_empty(cpus))
- return; /* nothing to do */
-
-- mcs = xen_mc_entry(sizeof(*args));
-- args = mcs.args;
-- args->op.arg2.vcpumask = to_cpumask(args->mask);
-+#ifdef CONFIG_SMP
-+ mcs = xen_mc_entry(sizeof(struct mmuext_op) + BITS_TO_LONGS(num_processors)*sizeof(unsigned long));
-+#else
-+ mcs = xen_mc_entry(sizeof(struct mmuext_op) + BITS_TO_LONGS(NR_CPUS)*sizeof(unsigned long));
-+#endif
-+ /* Extract fields */
-+ op = mcs.args;
-+ mask = to_cpumask(mcs.args + sizeof(struct mmuext_op));
-+
-+ op->arg2.vcpumask = mask;
-
- /* Remove us, and any offline CPUS. */
-- cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
-- cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
-+ cpumask_and(mask, cpus, cpu_online_mask);
-+ cpumask_clear_cpu(smp_processor_id(), mask);
-
-- args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
-+ op->cmd = MMUEXT_TLB_FLUSH_MULTI;
- if (end != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
-- args->op.cmd = MMUEXT_INVLPG_MULTI;
-- args->op.arg1.linear_addr = start;
-+ op->cmd = MMUEXT_INVLPG_MULTI;
-+ op->arg1.linear_addr = start;
- }
-
-- MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
-+ MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
-
- xen_mc_issue(PARAVIRT_LAZY_MMU);
- }
---
-2.11.0
-
diff --git a/0014-DO-NOT-UPSTREAM-arm-LLVMLinux-Provide-__aeabi_-symbo.patch b/0014-DO-NOT-UPSTREAM-arm-LLVMLinux-Provide-__aeabi_-symbo.patch
deleted file mode 100644
index 912d448..0000000
--- a/0014-DO-NOT-UPSTREAM-arm-LLVMLinux-Provide-__aeabi_-symbo.patch
+++ /dev/null
@@ -1,72 +0,0 @@
-From a4e5575ace4bd519270b1ff89a3e0ac069798966 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?=
- <Bernhard.Rosenkranzer@linaro.org>
-Date: Wed, 30 Mar 2016 00:57:50 +0200
-Subject: [PATCH 14/30] DO-NOT-UPSTREAM arm: LLVMLinux: Provide __aeabi_*
- symbols which are needed for clang
-
-These symbols are required when compiling the Linux kernel for arch ARM with
-clang.
-
-KernelVersion:
-Author: Mark Charlebois <charlebm@gmail.com>
-Signed-off-by: Mark Charlebois <charlebm@gmail.com>
-Not-signed-off-by: Behan Webster <behanw@converseincode.com>
----
- arch/arm/lib/Makefile | 4 ++++
- arch/arm/lib/eabi.c | 28 ++++++++++++++++++++++++++++
- 2 files changed, 32 insertions(+)
- create mode 100644 arch/arm/lib/eabi.c
-
-diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile
-index 27f4d96258a2..53f1c85178a8 100644
---- a/arch/arm/lib/Makefile
-+++ b/arch/arm/lib/Makefile
-@@ -18,6 +18,10 @@ lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \
- mmu-y := clear_user.o copy_page.o getuser.o putuser.o \
- copy_from_user.o copy_to_user.o
-
-+ifeq ($(COMPILER),clang)
-+lib-y += eabi.o
-+endif
-+
- # using lib_ here won't override already available weak symbols
- obj-$(CONFIG_UACCESS_WITH_MEMCPY) += uaccess_with_memcpy.o
-
-diff --git a/arch/arm/lib/eabi.c b/arch/arm/lib/eabi.c
-new file mode 100644
-index 000000000000..636a403c9579
---- /dev/null
-+++ b/arch/arm/lib/eabi.c
-@@ -0,0 +1,28 @@
-+/*
-+ * EABI routines
-+ *
-+ * Copyright (C) 2012 Mark Charlebois
-+ */
-+
-+#include <linux/types.h>
-+#include <linux/string.h>
-+#include <linux/ctype.h>
-+#include <linux/export.h>
-+
-+void __aeabi_memcpy(void *dest, const void *src, size_t n)
-+{
-+ (void)memcpy(dest, src, n);
-+}
-+EXPORT_SYMBOL(__aeabi_memcpy);
-+
-+void __aeabi_memmove(void *dest, const void *src, size_t n)
-+{
-+ (void)memmove(dest, src, n);
-+}
-+EXPORT_SYMBOL(__aeabi_memmove);
-+
-+void __aeabi_memset(void *s, size_t n, int c)
-+{
-+ (void)memset(s, c, n);
-+}
-+EXPORT_SYMBOL(__aeabi_memset);
---
-2.11.0
-
diff --git a/0015-DO-NOT-UPSTREAM-arm-firmware-LLVMLinux-replace-naked.patch b/0015-DO-NOT-UPSTREAM-arm-firmware-LLVMLinux-replace-naked.patch
deleted file mode 100644
index ab71d53..0000000
--- a/0015-DO-NOT-UPSTREAM-arm-firmware-LLVMLinux-replace-naked.patch
+++ /dev/null
@@ -1,47 +0,0 @@
-From 5068a19bab6aff3cc63d2cde3e3ffff26e0fe9c5 Mon Sep 17 00:00:00 2001
-From: Behan Webster <behanw@converseincode.com>
-Date: Fri, 30 May 2014 11:35:50 -0700
-Subject: [PATCH 15/30] DO-NOT-UPSTREAM arm, firmware, LLVMLinux: replace naked
- function with hard coded parameters
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-As documented in GCC naked functions should only use Basic asm syntax. The
-Extended asm or mixture of Basic asm and "C" code is not guarantee. Currently
-this works because it was hard coded to follow and check GCC behavior for
-arguments and register placement.
-
-By replacing the naked function with an inline one it is possible to have the
-same result in a more portable and reliable way. Register placement checks
-are not needed since there is no hard code.
-
-Not-signed-off-by: Behan Webster <behanw@converseincode.com>
-Not-signed-off-by: Vinícius Tinti <viniciustinti@gmail.com>
----
- arch/arm/firmware/trusted_foundations.c | 5 +----
- 1 file changed, 1 insertion(+), 4 deletions(-)
-
-diff --git a/arch/arm/firmware/trusted_foundations.c b/arch/arm/firmware/trusted_foundations.c
-index 3fb1b5a1dce9..f319c3fa0fe5 100644
---- a/arch/arm/firmware/trusted_foundations.c
-+++ b/arch/arm/firmware/trusted_foundations.c
-@@ -31,14 +31,11 @@
-
- static unsigned long cpu_boot_addr;
-
--static void __naked tf_generic_smc(u32 type, u32 arg1, u32 arg2)
-+static inline void tf_generic_smc(u32 type, u32 arg1, u32 arg2)
- {
- asm volatile(
- ".arch_extension sec\n\t"
- "stmfd sp!, {r4 - r11, lr}\n\t"
-- __asmeq("%0", "r0")
-- __asmeq("%1", "r1")
-- __asmeq("%2", "r2")
- "mov r3, #0\n\t"
- "mov r4, #0\n\t"
- "smc #0\n\t"
---
-2.11.0
-
diff --git a/0016-arm-LLVMLinux-Remove-unreachable-from-naked-function.patch b/0016-arm-LLVMLinux-Remove-unreachable-from-naked-function.patch
deleted file mode 100644
index 93f7a75..0000000
--- a/0016-arm-LLVMLinux-Remove-unreachable-from-naked-function.patch
+++ /dev/null
@@ -1,28 +0,0 @@
-From 21b07dfa5426577f604743948a45481794e7a241 Mon Sep 17 00:00:00 2001
-From: Behan Webster <behanw@converseincode.com>
-Date: Tue, 23 Sep 2014 22:43:30 -0700
-Subject: [PATCH 16/30] arm, LLVMLinux: Remove unreachable from naked function
-
-Signed-off-by: Behan Webster <behanw@converseincode.com>
-Suggested-by: Arnd Bergmann <arnd@arndb.de>
-Cc: Arnd Bergmann <arnd@arndb.de>
----
- drivers/bus/arm-cci.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/drivers/bus/arm-cci.c b/drivers/bus/arm-cci.c
-index c49da15d9790..49c257d572f0 100644
---- a/drivers/bus/arm-cci.c
-+++ b/drivers/bus/arm-cci.c
-@@ -2101,7 +2101,7 @@ asmlinkage void __naked cci_enable_port_for_self(void)
- [sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)),
- [offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) );
-
-- unreachable();
-+ // unreachable(); // not allowed in naked function
- }
-
- /**
---
-2.11.0
-
diff --git a/0017-MIPS-LLVMLinux-Fix-a-cast-to-type-not-present-in-uni.patch b/0017-MIPS-LLVMLinux-Fix-a-cast-to-type-not-present-in-uni.patch
deleted file mode 100644
index 577cf49..0000000
--- a/0017-MIPS-LLVMLinux-Fix-a-cast-to-type-not-present-in-uni.patch
+++ /dev/null
@@ -1,36 +0,0 @@
-From 0ad5200769177645876d71c2375f4659e49a1bcf Mon Sep 17 00:00:00 2001
-From: Toma Tabacu <toma.tabacu@imgtec.com>
-Date: Mon, 8 Dec 2014 14:55:09 +0000
-Subject: [PATCH 17/30] MIPS: LLVMLinux: Fix a 'cast to type not present in
- union' error.
-
-Replace a 16-bit unsigned int member bitfield with an unsigned short member
-variable in the 'mips16e_instruction' union.
-
-This follows the example set by the 'mips_instruction' union, which doesn't use
-bitfields for its generic union members.
-
-This error gets reported by clang, but not by gcc.
-The changed code can be compiled successfully by both gcc and clang.
-
-Signed-off-by: Toma Tabacu <toma.tabacu@imgtec.com>
----
- arch/mips/include/uapi/asm/inst.h | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/arch/mips/include/uapi/asm/inst.h b/arch/mips/include/uapi/asm/inst.h
-index 77429d1622b3..248e718180a0 100644
---- a/arch/mips/include/uapi/asm/inst.h
-+++ b/arch/mips/include/uapi/asm/inst.h
-@@ -1071,7 +1071,7 @@ union mips_instruction {
- };
-
- union mips16e_instruction {
-- unsigned int full : 16;
-+ unsigned short full;
- struct m16e_rr rr;
- struct m16e_jal jal;
- struct m16e_i64 i64;
---
-2.11.0
-
diff --git a/0018-MIPS-LLVMLinux-Fix-an-inline-asm-input-output-type-m.patch b/0018-MIPS-LLVMLinux-Fix-an-inline-asm-input-output-type-m.patch
deleted file mode 100644
index 3183812..0000000
--- a/0018-MIPS-LLVMLinux-Fix-an-inline-asm-input-output-type-m.patch
+++ /dev/null
@@ -1,41 +0,0 @@
-From a4bf1a8f23047a1c91ca2d3db64892af92c2436d Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@lindev.ch>
-Date: Wed, 1 Feb 2017 16:13:16 +0100
-Subject: [PATCH 18/30] MIPS: LLVMLinux: Fix an 'inline asm input/output type
- mismatch' error.
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Change the type of csum_ipv6_magic's 'proto' argument from unsigned short to
-__u32.
-
-This fixes a type mismatch between the 'htonl(proto)' inline asm input, which
-is __u32, and the 'proto' output, which is unsigned short.
-
-This error gets reported by clang, but not by gcc.
-The changed code can be compiled successfully by both gcc and clang.
-
-Signed-off-by: Toma Tabacu <toma.tabacu@imgtec.com>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- arch/mips/include/asm/checksum.h | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/arch/mips/include/asm/checksum.h b/arch/mips/include/asm/checksum.h
-index c8b574f7e0cc..f963fff3be29 100644
---- a/arch/mips/include/asm/checksum.h
-+++ b/arch/mips/include/asm/checksum.h
-@@ -217,7 +217,7 @@ static inline __sum16 ip_compute_csum(const void *buff, int len)
- #define _HAVE_ARCH_IPV6_CSUM
- static __inline__ __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
- const struct in6_addr *daddr,
-- __u32 len, __u8 proto,
-+ __u32 len, __u32 proto,
- __wsum sum)
- {
- __wsum tmp;
---
-2.11.0
-
diff --git a/0019-MIPS-LLVMLinux-Silence-variable-self-assignment-warn.patch b/0019-MIPS-LLVMLinux-Silence-variable-self-assignment-warn.patch
deleted file mode 100644
index 89b8430..0000000
--- a/0019-MIPS-LLVMLinux-Silence-variable-self-assignment-warn.patch
+++ /dev/null
@@ -1,123 +0,0 @@
-From 8bd652c08ac8662f3ce461c99517fab21b75a1cd Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?=
- <Bernhard.Rosenkranzer@linaro.org>
-Date: Wed, 30 Mar 2016 01:02:19 +0200
-Subject: [PATCH 19/30] MIPS: LLVMLinux: Silence variable self-assignment
- warnings.
-
-Replace variable self-assignments with explanatory comments.
-
-This silences a bunch of -Wself-assign warnings reported by clang.
-The changed code can be compiled without warnings by both gcc and clang.
-
-Signed-off-by: Toma Tabacu <toma.tabacu@imgtec.com>
----
- arch/mips/math-emu/dp_add.c | 5 +++++
- arch/mips/math-emu/dp_sub.c | 5 +++++
- arch/mips/math-emu/sp_add.c | 5 +++++
- arch/mips/math-emu/sp_sub.c | 5 +++++
- 4 files changed, 20 insertions(+)
-
-diff --git a/arch/mips/math-emu/dp_add.c b/arch/mips/math-emu/dp_add.c
-index 8954ef031f84..34081484db6c 100644
---- a/arch/mips/math-emu/dp_add.c
-+++ b/arch/mips/math-emu/dp_add.c
-@@ -151,6 +151,8 @@ union ieee754dp ieee754dp_add(union ieee754dp x, union ieee754dp y)
- * leaving result in xm, xs and xe.
- */
- xm = xm + ym;
-+ /* xe does not change */
-+ /* xs does not change */
-
- if (xm >> (DP_FBITS + 1 + 3)) { /* carry out */
- xm = XDPSRS1(xm);
-@@ -159,8 +161,11 @@ union ieee754dp ieee754dp_add(union ieee754dp x, union ieee754dp y)
- } else {
- if (xm >= ym) {
- xm = xm - ym;
-+ /* xe does not change */
-+ /* xs does not change */
- } else {
- xm = ym - xm;
-+ /* xe does not change */
- xs = ys;
- }
- if (xm == 0)
-diff --git a/arch/mips/math-emu/dp_sub.c b/arch/mips/math-emu/dp_sub.c
-index fc17a781b9ae..22fac9097e7f 100644
---- a/arch/mips/math-emu/dp_sub.c
-+++ b/arch/mips/math-emu/dp_sub.c
-@@ -154,6 +154,8 @@ union ieee754dp ieee754dp_sub(union ieee754dp x, union ieee754dp y)
- /* generate 28 bit result of adding two 27 bit numbers
- */
- xm = xm + ym;
-+ /* xe does not change */
-+ /* xs does not change */
-
- if (xm >> (DP_FBITS + 1 + 3)) { /* carry out */
- xm = XDPSRS1(xm); /* shift preserving sticky */
-@@ -162,8 +164,11 @@ union ieee754dp ieee754dp_sub(union ieee754dp x, union ieee754dp y)
- } else {
- if (xm >= ym) {
- xm = xm - ym;
-+ /* xe does not change */
-+ /* xs does not change */
- } else {
- xm = ym - xm;
-+ /* xe does not change */
- xs = ys;
- }
- if (xm == 0) {
-diff --git a/arch/mips/math-emu/sp_add.c b/arch/mips/math-emu/sp_add.c
-index c55c0c00bca8..91566a8bb8f6 100644
---- a/arch/mips/math-emu/sp_add.c
-+++ b/arch/mips/math-emu/sp_add.c
-@@ -151,6 +151,8 @@ union ieee754sp ieee754sp_add(union ieee754sp x, union ieee754sp y)
- * leaving result in xm, xs and xe.
- */
- xm = xm + ym;
-+ /* xe does not change */
-+ /* xs does not change */
-
- if (xm >> (SP_FBITS + 1 + 3)) { /* carry out */
- SPXSRSX1();
-@@ -158,8 +160,11 @@ union ieee754sp ieee754sp_add(union ieee754sp x, union ieee754sp y)
- } else {
- if (xm >= ym) {
- xm = xm - ym;
-+ /* xe does not change */
-+ /* xs does not change */
- } else {
- xm = ym - xm;
-+ /* xe does not change */
- xs = ys;
- }
- if (xm == 0)
-diff --git a/arch/mips/math-emu/sp_sub.c b/arch/mips/math-emu/sp_sub.c
-index dc998ed47295..0bacd49b9815 100644
---- a/arch/mips/math-emu/sp_sub.c
-+++ b/arch/mips/math-emu/sp_sub.c
-@@ -151,6 +151,8 @@ union ieee754sp ieee754sp_sub(union ieee754sp x, union ieee754sp y)
- /* generate 28 bit result of adding two 27 bit numbers
- */
- xm = xm + ym;
-+ /* xe does not change */
-+ /* xs does not change */
-
- if (xm >> (SP_FBITS + 1 + 3)) { /* carry out */
- SPXSRSX1(); /* shift preserving sticky */
-@@ -158,8 +160,11 @@ union ieee754sp ieee754sp_sub(union ieee754sp x, union ieee754sp y)
- } else {
- if (xm >= ym) {
- xm = xm - ym;
-+ /* xe does not change */
-+ /* xs does not change */
- } else {
- xm = ym - xm;
-+ /* xe does not change */
- xs = ys;
- }
- if (xm == 0) {
---
-2.11.0
-
diff --git a/0020-MIPS-LLVMLinux-Silence-unicode-warnings-when-preproc.patch b/0020-MIPS-LLVMLinux-Silence-unicode-warnings-when-preproc.patch
deleted file mode 100644
index 785878d..0000000
--- a/0020-MIPS-LLVMLinux-Silence-unicode-warnings-when-preproc.patch
+++ /dev/null
@@ -1,44 +0,0 @@
-From 706cb8d4c962002068f26b0acc69642d25c74f76 Mon Sep 17 00:00:00 2001
-From: Toma Tabacu <toma.tabacu@imgtec.com>
-Date: Mon, 8 Dec 2014 15:05:17 +0000
-Subject: [PATCH 20/30] MIPS: LLVMLinux: Silence unicode warnings when
- preprocessing assembly.
-
-Differentiate the 'u' GAS .macro argument from the '\u' C preprocessor unicode
-escape sequence by renaming it to '_u'.
-
-When the 'u' argument is evaluated, we end up writing '\u', which causes
-clang to emit -Wunicode warnings.
-
-This silences a couple of -Wunicode warnings reported by clang.
-The changed code can be preprocessed without warnings by both gcc and clang.
-
-Signed-off-by: Toma Tabacu <toma.tabacu@imgtec.com>
----
- arch/mips/include/asm/asmmacro.h | 8 ++++----
- 1 file changed, 4 insertions(+), 4 deletions(-)
-
-diff --git a/arch/mips/include/asm/asmmacro.h b/arch/mips/include/asm/asmmacro.h
-index 83054f79f72a..6588350df089 100644
---- a/arch/mips/include/asm/asmmacro.h
-+++ b/arch/mips/include/asm/asmmacro.h
-@@ -225,12 +225,12 @@
- .word 0x41600021 | (\reg << 16)
- .endm
-
-- .macro MFTR rt=0, rd=0, u=0, sel=0
-- .word 0x41000000 | (\rt << 16) | (\rd << 11) | (\u << 5) | (\sel)
-+ .macro MFTR rt=0, rd=0, _u=0, sel=0
-+ .word 0x41000000 | (\rt << 16) | (\rd << 11) | (\_u << 5) | (\sel)
- .endm
-
-- .macro MTTR rt=0, rd=0, u=0, sel=0
-- .word 0x41800000 | (\rt << 16) | (\rd << 11) | (\u << 5) | (\sel)
-+ .macro MTTR rt=0, rd=0, _u=0, sel=0
-+ .word 0x41800000 | (\rt << 16) | (\rd << 11) | (\_u << 5) | (\sel)
- .endm
-
- #ifdef TOOLCHAIN_SUPPORTS_MSA
---
-2.11.0
-
diff --git a/0021-Don-t-use-attributes-error-and-warning-with-clang.patch b/0021-Don-t-use-attributes-error-and-warning-with-clang.patch
deleted file mode 100644
index 99c7108..0000000
--- a/0021-Don-t-use-attributes-error-and-warning-with-clang.patch
+++ /dev/null
@@ -1,30 +0,0 @@
-From e476573e2f1b83f6a7cdf4059f344a4c89a55be4 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?=
- <Bernhard.Rosenkranzer@linaro.org>
-Date: Tue, 5 Apr 2016 23:30:17 +0200
-Subject: [PATCH 21/30] Don't use attributes error and warning with clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Signed-off-by: Bernhard Rosenkränzer <Bernhard.Rosenkranzer@linaro.org>
----
- include/linux/compiler-gcc.h | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
-index 0444b1336268..85bf55a08b3f 100644
---- a/include/linux/compiler-gcc.h
-+++ b/include/linux/compiler-gcc.h
-@@ -181,7 +181,7 @@
-
- #define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__)
-
--#ifndef __CHECKER__
-+#if !defined(__CHECKER__) && !defined(__clang__)
- # define __compiletime_warning(message) __attribute__((warning(message)))
- # define __compiletime_error(message) __attribute__((error(message)))
- #endif /* __CHECKER__ */
---
-2.11.0
-
diff --git a/0022-Fix-undefined-references-to-acpi_idle_driver-on-aarc.patch b/0022-Fix-undefined-references-to-acpi_idle_driver-on-aarc.patch
deleted file mode 100644
index f8268c8..0000000
--- a/0022-Fix-undefined-references-to-acpi_idle_driver-on-aarc.patch
+++ /dev/null
@@ -1,51 +0,0 @@
-From 61e4d47a66ba32eed9e83257f36e82d795bf4c9e Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:41:15 +0100
-Subject: [PATCH 22/30] Fix undefined references to acpi_idle_driver on aarch64
- with clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-gcc removes the reference to acpi_idle_driver in processor_driver.c when
-building without CONFIG_ACPI_PROCESSOR_IDLE - it thinks the code is
-unreachable. clang doesn't, creating an undefined reference.
-
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- drivers/acpi/processor_driver.c | 2 ++
- include/acpi/processor.h | 2 +-
- 2 files changed, 3 insertions(+), 1 deletion(-)
-
-diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c
-index 9d5f0c7ed3f7..aafe719e6bb8 100644
---- a/drivers/acpi/processor_driver.c
-+++ b/drivers/acpi/processor_driver.c
-@@ -239,8 +239,10 @@ static int __acpi_processor_start(struct acpi_device *device)
- if (result && !IS_ENABLED(CONFIG_ACPI_CPU_FREQ_PSS))
- dev_warn(&device->dev, "CPPC data invalid or not present\n");
-
-+#ifdef CONFIG_ACPI_PROCESSOR_IDLE
- if (!cpuidle_get_driver() || cpuidle_get_driver() == &acpi_idle_driver)
- acpi_processor_power_init(pr);
-+#endif
-
- result = acpi_pss_perf_init(pr, device);
- if (result)
-diff --git a/include/acpi/processor.h b/include/acpi/processor.h
-index c1ba00fc4888..06848b600926 100644
---- a/include/acpi/processor.h
-+++ b/include/acpi/processor.h
-@@ -387,8 +387,8 @@ static inline void acpi_processor_throttling_init(void) {}
- #endif /* CONFIG_ACPI_CPU_FREQ_PSS */
-
- /* in processor_idle.c */
--extern struct cpuidle_driver acpi_idle_driver;
- #ifdef CONFIG_ACPI_PROCESSOR_IDLE
-+extern struct cpuidle_driver acpi_idle_driver;
- int acpi_processor_power_init(struct acpi_processor *pr);
- int acpi_processor_power_exit(struct acpi_processor *pr);
- int acpi_processor_power_state_has_changed(struct acpi_processor *pr);
---
-2.11.0
-
diff --git a/0023-HACK-firmware-LLVMLinux-fix-EFI-libstub-with-clang.patch b/0023-HACK-firmware-LLVMLinux-fix-EFI-libstub-with-clang.patch
deleted file mode 100644
index 327a516..0000000
--- a/0023-HACK-firmware-LLVMLinux-fix-EFI-libstub-with-clang.patch
+++ /dev/null
@@ -1,46 +0,0 @@
-From 746104bee67aca342c77887648efc471215dd6d8 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:42:02 +0100
-Subject: [PATCH 23/30] HACK: firmware, LLVMLinux: fix EFI libstub with clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Without any extra guidance, clang will generate libstub with either
-absolute or relative ELF relocations. Use the right combination of
--fpic and -fno-pic on different files to avoid this.
-
-Signed-off-by: Greg Hackmann <ghackmann@google.com>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- drivers/firmware/efi/libstub/Makefile | 6 ++++++
- 1 file changed, 6 insertions(+)
-
-diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
-index f7425960f6a5..abe4d91bcc89 100644
---- a/drivers/firmware/efi/libstub/Makefile
-+++ b/drivers/firmware/efi/libstub/Makefile
-@@ -11,6 +11,9 @@ cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ -O2 \
- -mno-mmx -mno-sse
-
- cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS))
-+ifeq ($(cc-name),clang)
-+cflags-$(CONFIG_ARM64) += -fpic
-+endif
- cflags-$(CONFIG_ARM) := $(subst -pg,,$(KBUILD_CFLAGS)) \
- -fno-builtin -fpic -mno-single-pic-base
-
-@@ -42,6 +45,9 @@ lib-$(CONFIG_EFI_ARMSTUB) += arm-stub.o fdt.o string.o random.o \
- lib-$(CONFIG_ARM) += arm32-stub.o
- lib-$(CONFIG_ARM64) += arm64-stub.o
- CFLAGS_arm64-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
-+ifeq ($(cc-name),clang)
-+CFLAGS_arm64-stub.o += -fno-pic
-+endif
-
- #
- # arm64 puts the stub in the kernel proper, which will unnecessarily retain all
---
-2.11.0
-
diff --git a/0024-aarch64-crypto-LLVMLinux-Fix-inline-assembly-for-cla.patch b/0024-aarch64-crypto-LLVMLinux-Fix-inline-assembly-for-cla.patch
deleted file mode 100644
index ba06cf6..0000000
--- a/0024-aarch64-crypto-LLVMLinux-Fix-inline-assembly-for-cla.patch
+++ /dev/null
@@ -1,46 +0,0 @@
-From 87257c567c10ef01b28988ef6caf890949f33593 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:43:25 +0100
-Subject: [PATCH 24/30] aarch64: crypto: LLVMLinux: Fix inline assembly for
- clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Fix inline assembly to be compatible with both gcc and clang
-
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- arch/arm64/crypto/sha1-ce-glue.c | 2 +-
- arch/arm64/crypto/sha2-ce-glue.c | 2 +-
- 2 files changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c
-index aefda9868627..c71e94ba0e43 100644
---- a/arch/arm64/crypto/sha1-ce-glue.c
-+++ b/arch/arm64/crypto/sha1-ce-glue.c
-@@ -18,7 +18,7 @@
- #include <linux/module.h>
-
- #define ASM_EXPORT(sym, val) \
-- asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val));
-+ asm(".globl " #sym "; .set " #sym ", %c0" :: "I"(val));
-
- MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions");
- MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
-diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c
-index 7cd587564a41..381b5fb2dcb2 100644
---- a/arch/arm64/crypto/sha2-ce-glue.c
-+++ b/arch/arm64/crypto/sha2-ce-glue.c
-@@ -18,7 +18,7 @@
- #include <linux/module.h>
-
- #define ASM_EXPORT(sym, val) \
-- asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val));
-+ asm(".globl " #sym "; .set " #sym ", %c0" :: "I"(val));
-
- MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions");
- MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
---
-2.11.0
-
diff --git a/0025-aarch64-LLVMLinux-Make-spin_lock_prefetch-asm-code-c.patch b/0025-aarch64-LLVMLinux-Make-spin_lock_prefetch-asm-code-c.patch
deleted file mode 100644
index 82c1b2c..0000000
--- a/0025-aarch64-LLVMLinux-Make-spin_lock_prefetch-asm-code-c.patch
+++ /dev/null
@@ -1,34 +0,0 @@
-From f0d4542049a4041f34ec8c66a16d02d23a272599 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:44:18 +0100
-Subject: [PATCH 25/30] aarch64: LLVMLinux: Make spin_lock_prefetch asm code
- compatible with clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-This version is accepted by both gcc and clang.
-
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- arch/arm64/include/asm/processor.h | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
-index c97b8bd2acba..edc9fe9bb7c6 100644
---- a/arch/arm64/include/asm/processor.h
-+++ b/arch/arm64/include/asm/processor.h
-@@ -178,8 +178,8 @@ static inline void prefetchw(const void *ptr)
- static inline void spin_lock_prefetch(const void *ptr)
- {
- asm volatile(ARM64_LSE_ATOMIC_INSN(
-- "prfm pstl1strm, %a0",
-- "nop") : : "p" (ptr));
-+ "prfm pstl1strm, [%x0]",
-+ "nop") : : "r" (ptr));
- }
-
- #define HAVE_ARCH_PICK_MMAP_LAYOUT
---
-2.11.0
-
diff --git a/0026-LLVMLinux-Don-t-use-attribute-externally_visible-whe.patch b/0026-LLVMLinux-Don-t-use-attribute-externally_visible-whe.patch
deleted file mode 100644
index 43548de..0000000
--- a/0026-LLVMLinux-Don-t-use-attribute-externally_visible-whe.patch
+++ /dev/null
@@ -1,33 +0,0 @@
-From d94e33448ec3cd7807600c83e46db3c96084be76 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:45:31 +0100
-Subject: [PATCH 26/30] LLVMLinux: Don't use attribute externally_visible when
- building with clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-As of 4.0, clang doesn't support attribute externally_visible (PR16683).
-Don't use the attribute when building with clang.
-
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- include/linux/compiler-gcc.h | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
-index 85bf55a08b3f..44dc519ca5e2 100644
---- a/include/linux/compiler-gcc.h
-+++ b/include/linux/compiler-gcc.h
-@@ -211,7 +211,7 @@
-
- #endif /* GCC_VERSION >= 40500 */
-
--#if GCC_VERSION >= 40600
-+#if GCC_VERSION >= 40600 && !defined(__clang__)
- /*
- * When used with Link Time Optimization, gcc can optimize away C functions or
- * variables which are referenced only from assembly code. __visible tells the
---
-2.11.0
-
diff --git a/0027-x86-crypto-LLVMLinux-Fix-building-x86_64-AES-extensi.patch b/0027-x86-crypto-LLVMLinux-Fix-building-x86_64-AES-extensi.patch
deleted file mode 100644
index e301ca0..0000000
--- a/0027-x86-crypto-LLVMLinux-Fix-building-x86_64-AES-extensi.patch
+++ /dev/null
@@ -1,32 +0,0 @@
-From 68c9b1821b70a7c58a66be42b0e18cf0a0ede537 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:46:28 +0100
-Subject: [PATCH 27/30] x86, crypto, LLVMLinux: Fix building x86_64 AES
- extensions with clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-This is a workaround for PR24494
-
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- arch/x86/crypto/aes_ctrby8_avx-x86_64.S | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
-index a916c4a61165..7a71553b23dc 100644
---- a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
-+++ b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
-@@ -93,7 +93,7 @@
-
- #define tmp %r10
- #define DDQ(i) CONCAT(ddq_add_,i)
--#define XMM(i) CONCAT(%xmm, i)
-+#define XMM(i) CONCAT(%xmm,i)
- #define DDQ_DATA 0
- #define XDATA 1
- #define KEY_128 1
---
-2.11.0
-
diff --git a/0028-x86-LLVMLinux-Qualify-mul-as-mulq-to-make-clang-happ.patch b/0028-x86-LLVMLinux-Qualify-mul-as-mulq-to-make-clang-happ.patch
deleted file mode 100644
index cacd635..0000000
--- a/0028-x86-LLVMLinux-Qualify-mul-as-mulq-to-make-clang-happ.patch
+++ /dev/null
@@ -1,33 +0,0 @@
-From d582ae8c61d52a3a7c3841c87ac48942c2b556ef Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 00:47:09 +0100
-Subject: [PATCH 28/30] x86, LLVMLinux: Qualify mul as mulq to make clang happy
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- arch/x86/lib/kaslr.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/arch/x86/lib/kaslr.c b/arch/x86/lib/kaslr.c
-index 121f59c6ee54..9915ff924a5e 100644
---- a/arch/x86/lib/kaslr.c
-+++ b/arch/x86/lib/kaslr.c
-@@ -79,7 +79,11 @@ unsigned long kaslr_get_random_long(const char *purpose)
- }
-
- /* Circular multiply for better bit diffusion */
-- asm("mul %3"
-+#ifdef CONFIG_X86_64
-+ asm("mulq %3"
-+#else
-+ asm("mull %3"
-+#endif
- : "=a" (random), "=d" (raw)
- : "a" (random), "rm" (mix_const));
- random += raw;
---
-2.11.0
-
diff --git a/0029-kbuild-LLVMLinux-Add-Werror-to-cc-option-in-order-to.patch b/0029-kbuild-LLVMLinux-Add-Werror-to-cc-option-in-order-to.patch
deleted file mode 100644
index fc62b87..0000000
--- a/0029-kbuild-LLVMLinux-Add-Werror-to-cc-option-in-order-to.patch
+++ /dev/null
@@ -1,55 +0,0 @@
-From 8de733e8e653892cae10581b816e58f8ad29846e Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 05:09:42 +0100
-Subject: [PATCH 29/30] kbuild, LLVMLinux: Add -Werror to cc-option in order to
- support clang
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Clang will warn about unknown warnings but will not return false
-unless -Werror is set. GCC will return false if an unknown
-warning is passed.
-
-Adding -Werror make both compiler behave the same.
-
-Signed-off-by: Mark Charlebois <charlebm@gmail.com>
-Signed-off-by: Behan Webster <behanw@converseincode.com>
-Forward-ported-by: Bernhard Rosenkränzer <bero@linaro.org>
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
-Reviewed-by: Jan-Simon Möller <dl9pf@gmx.de>
----
- scripts/Kbuild.include | 6 +++---
- 1 file changed, 3 insertions(+), 3 deletions(-)
-
-diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
-index d6ca649cb0e9..a70fd26204de 100644
---- a/scripts/Kbuild.include
-+++ b/scripts/Kbuild.include
-@@ -116,12 +116,12 @@ CC_OPTION_CFLAGS = $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS))
- # Usage: cflags-y += $(call cc-option,-march=winchip-c6,-march=i586)
-
- cc-option = $(call try-run,\
-- $(CC) $(KBUILD_CPPFLAGS) $(CC_OPTION_CFLAGS) $(1) -c -x c /dev/null -o "$$TMP",$(1),$(2))
-+ $(CC) -Werror $(KBUILD_CPPFLAGS) $(CC_OPTION_CFLAGS) $(1) -c -x c /dev/null -o "$$TMP",$(1),$(2))
-
- # cc-option-yn
- # Usage: flag := $(call cc-option-yn,-march=winchip-c6)
- cc-option-yn = $(call try-run,\
-- $(CC) $(KBUILD_CPPFLAGS) $(CC_OPTION_CFLAGS) $(1) -c -x c /dev/null -o "$$TMP",y,n)
-+ $(CC) -Werror $(KBUILD_CPPFLAGS) $(CC_OPTION_CFLAGS) $(1) -c -x c /dev/null -o "$$TMP",y,n)
-
- # cc-option-align
- # Prefix align with either -falign or -malign
-@@ -131,7 +131,7 @@ cc-option-align = $(subst -functions=0,,\
- # cc-disable-warning
- # Usage: cflags-y += $(call cc-disable-warning,unused-but-set-variable)
- cc-disable-warning = $(call try-run,\
-- $(CC) $(KBUILD_CPPFLAGS) $(CC_OPTION_CFLAGS) -W$(strip $(1)) -c -x c /dev/null -o "$$TMP",-Wno-$(strip $(1)))
-+ $(CC) -Werror $(KBUILD_CPPFLAGS) $(CC_OPTION_CFLAGS) -W$(strip $(1)) -c -x c /dev/null -o "$$TMP",-Wno-$(strip $(1)))
-
- # cc-name
- # Expands to either gcc or clang
---
-2.11.0
-
diff --git a/0030-x86-kbuild-LLVMLinux-Check-for-compiler-support-of-f.patch b/0030-x86-kbuild-LLVMLinux-Check-for-compiler-support-of-f.patch
deleted file mode 100644
index 79fc95a..0000000
--- a/0030-x86-kbuild-LLVMLinux-Check-for-compiler-support-of-f.patch
+++ /dev/null
@@ -1,37 +0,0 @@
-From 5747a302540b1b3b1fe66caf7aa724b8bdc8cc6f Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= <bero@linaro.org>
-Date: Thu, 23 Feb 2017 15:03:10 +0100
-Subject: [PATCH 30/30] x86, kbuild, LLVMLinux: Check for compiler support of
- -falign-*=1
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-As of 4.0, clang doesn't support -falign-loops=1 -falign-jumps=1.
-Don't add those flags unconditionally.
-
-Signed-off-by: Bernhard Rosenkränzer <bero@linaro.org>
----
- arch/x86/Makefile | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/arch/x86/Makefile b/arch/x86/Makefile
-index 2d449337a360..0a62ebada700 100644
---- a/arch/x86/Makefile
-+++ b/arch/x86/Makefile
-@@ -88,10 +88,10 @@ else
- KBUILD_CFLAGS += -m64
-
- # Align jump targets to 1 byte, not the default 16 bytes:
-- KBUILD_CFLAGS += -falign-jumps=1
-+ KBUILD_CFLAGS += $(call cc-option,-falign-jumps=1)
-
- # Pack loops tightly as well:
-- KBUILD_CFLAGS += -falign-loops=1
-+ KBUILD_CFLAGS += $(call cc-option,-falign-loops=1)
-
- # Don't autogenerate traditional x87 instructions
- KBUILD_CFLAGS += $(call cc-option,-mno-80387)
---
-2.11.0
-
diff --git a/01-revert-6602f080cb28745259e2fab1a4cf55eeb5894f93.patch b/01-revert-6602f080cb28745259e2fab1a4cf55eeb5894f93.patch
new file mode 100644
index 0000000..d527b7c
--- /dev/null
+++ b/01-revert-6602f080cb28745259e2fab1a4cf55eeb5894f93.patch
@@ -0,0 +1,166 @@
+diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c
+index 4ed21dad6a8e..6049d3766c64 100644
+--- a/drivers/net/wireless/ath/ath9k/hif_usb.c
++++ b/drivers/net/wireless/ath/ath9k/hif_usb.c
+@@ -643,9 +643,9 @@ static void ath9k_hif_usb_rx_stream(struct hif_device_usb *hif_dev,
+
+ static void ath9k_hif_usb_rx_cb(struct urb *urb)
+ {
+- struct rx_buf *rx_buf = (struct rx_buf *)urb->context;
+- struct hif_device_usb *hif_dev = rx_buf->hif_dev;
+- struct sk_buff *skb = rx_buf->skb;
++ struct sk_buff *skb = (struct sk_buff *) urb->context;
++ struct hif_device_usb *hif_dev =
++ usb_get_intfdata(usb_ifnum_to_if(urb->dev, 0));
+ int ret;
+
+ if (!skb)
+@@ -685,15 +685,14 @@ static void ath9k_hif_usb_rx_cb(struct urb *urb)
+ return;
+ free:
+ kfree_skb(skb);
+- kfree(rx_buf);
+ }
+
+ static void ath9k_hif_usb_reg_in_cb(struct urb *urb)
+ {
+- struct rx_buf *rx_buf = (struct rx_buf *)urb->context;
+- struct hif_device_usb *hif_dev = rx_buf->hif_dev;
+- struct sk_buff *skb = rx_buf->skb;
++ struct sk_buff *skb = (struct sk_buff *) urb->context;
+ struct sk_buff *nskb;
++ struct hif_device_usb *hif_dev =
++ usb_get_intfdata(usb_ifnum_to_if(urb->dev, 0));
+ int ret;
+
+ if (!skb)
+@@ -751,7 +750,6 @@ static void ath9k_hif_usb_reg_in_cb(struct urb *urb)
+ return;
+ free:
+ kfree_skb(skb);
+- kfree(rx_buf);
+ urb->context = NULL;
+ }
+
+@@ -797,7 +795,7 @@ static int ath9k_hif_usb_alloc_tx_urbs(struct hif_device_usb *hif_dev)
+ init_usb_anchor(&hif_dev->mgmt_submitted);
+
+ for (i = 0; i < MAX_TX_URB_NUM; i++) {
+- tx_buf = kzalloc(sizeof(*tx_buf), GFP_KERNEL);
++ tx_buf = kzalloc(sizeof(struct tx_buf), GFP_KERNEL);
+ if (!tx_buf)
+ goto err;
+
+@@ -834,9 +832,8 @@ static void ath9k_hif_usb_dealloc_rx_urbs(struct hif_device_usb *hif_dev)
+
+ static int ath9k_hif_usb_alloc_rx_urbs(struct hif_device_usb *hif_dev)
+ {
+- struct rx_buf *rx_buf = NULL;
+- struct sk_buff *skb = NULL;
+ struct urb *urb = NULL;
++ struct sk_buff *skb = NULL;
+ int i, ret;
+
+ init_usb_anchor(&hif_dev->rx_submitted);
+@@ -844,12 +841,6 @@ static int ath9k_hif_usb_alloc_rx_urbs(struct hif_device_usb *hif_dev)
+
+ for (i = 0; i < MAX_RX_URB_NUM; i++) {
+
+- rx_buf = kzalloc(sizeof(*rx_buf), GFP_KERNEL);
+- if (!rx_buf) {
+- ret = -ENOMEM;
+- goto err_rxb;
+- }
+-
+ /* Allocate URB */
+ urb = usb_alloc_urb(0, GFP_KERNEL);
+ if (urb == NULL) {
+@@ -864,14 +855,11 @@ static int ath9k_hif_usb_alloc_rx_urbs(struct hif_device_usb *hif_dev)
+ goto err_skb;
+ }
+
+- rx_buf->hif_dev = hif_dev;
+- rx_buf->skb = skb;
+-
+ usb_fill_bulk_urb(urb, hif_dev->udev,
+ usb_rcvbulkpipe(hif_dev->udev,
+ USB_WLAN_RX_PIPE),
+ skb->data, MAX_RX_BUF_SIZE,
+- ath9k_hif_usb_rx_cb, rx_buf);
++ ath9k_hif_usb_rx_cb, skb);
+
+ /* Anchor URB */
+ usb_anchor_urb(urb, &hif_dev->rx_submitted);
+@@ -897,8 +885,6 @@ static int ath9k_hif_usb_alloc_rx_urbs(struct hif_device_usb *hif_dev)
+ err_skb:
+ usb_free_urb(urb);
+ err_urb:
+- kfree(rx_buf);
+-err_rxb:
+ ath9k_hif_usb_dealloc_rx_urbs(hif_dev);
+ return ret;
+ }
+@@ -910,21 +896,14 @@ static void ath9k_hif_usb_dealloc_reg_in_urbs(struct hif_device_usb *hif_dev)
+
+ static int ath9k_hif_usb_alloc_reg_in_urbs(struct hif_device_usb *hif_dev)
+ {
+- struct rx_buf *rx_buf = NULL;
+- struct sk_buff *skb = NULL;
+ struct urb *urb = NULL;
++ struct sk_buff *skb = NULL;
+ int i, ret;
+
+ init_usb_anchor(&hif_dev->reg_in_submitted);
+
+ for (i = 0; i < MAX_REG_IN_URB_NUM; i++) {
+
+- rx_buf = kzalloc(sizeof(*rx_buf), GFP_KERNEL);
+- if (!rx_buf) {
+- ret = -ENOMEM;
+- goto err_rxb;
+- }
+-
+ /* Allocate URB */
+ urb = usb_alloc_urb(0, GFP_KERNEL);
+ if (urb == NULL) {
+@@ -939,14 +918,11 @@ static int ath9k_hif_usb_alloc_reg_in_urbs(struct hif_device_usb *hif_dev)
+ goto err_skb;
+ }
+
+- rx_buf->hif_dev = hif_dev;
+- rx_buf->skb = skb;
+-
+ usb_fill_int_urb(urb, hif_dev->udev,
+ usb_rcvintpipe(hif_dev->udev,
+ USB_REG_IN_PIPE),
+ skb->data, MAX_REG_IN_BUF_SIZE,
+- ath9k_hif_usb_reg_in_cb, rx_buf, 1);
++ ath9k_hif_usb_reg_in_cb, skb, 1);
+
+ /* Anchor URB */
+ usb_anchor_urb(urb, &hif_dev->reg_in_submitted);
+@@ -972,8 +948,6 @@ static int ath9k_hif_usb_alloc_reg_in_urbs(struct hif_device_usb *hif_dev)
+ err_skb:
+ usb_free_urb(urb);
+ err_urb:
+- kfree(rx_buf);
+-err_rxb:
+ ath9k_hif_usb_dealloc_reg_in_urbs(hif_dev);
+ return ret;
+ }
+diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.h b/drivers/net/wireless/ath/ath9k/hif_usb.h
+index 5985aa15ca93..a94e7e1c86e9 100644
+--- a/drivers/net/wireless/ath/ath9k/hif_usb.h
++++ b/drivers/net/wireless/ath/ath9k/hif_usb.h
+@@ -86,11 +86,6 @@ struct tx_buf {
+ struct list_head list;
+ };
+
+-struct rx_buf {
+- struct sk_buff *skb;
+- struct hif_device_usb *hif_dev;
+-};
+-
+ #define HIF_USB_TX_STOP BIT(0)
+ #define HIF_USB_TX_FLUSH BIT(1)
+
diff --git a/0107-intel_idle-tweak-cpuidle-cstates.patch b/0107-intel_idle-tweak-cpuidle-cstates.patch
index 984d4f6..4804923 100644
--- a/0107-intel_idle-tweak-cpuidle-cstates.patch
+++ b/0107-intel_idle-tweak-cpuidle-cstates.patch
@@ -1,32 +1,16 @@
-From 790db86f51a23533d457b361bb61e2845b6de6b8 Mon Sep 17 00:00:00 2001
-From: Arjan van de Ven <arjan@linux.intel.com>
-Date: Sat, 19 Mar 2016 21:32:19 -0400
-Subject: [PATCH 107/126] intel_idle: tweak cpuidle cstates
-
-Increase target_residency in cpuidle cstate
-
-Tune intel_idle to be a bit less agressive;
-Clear linux is cleaner in hygiene (wakupes) than the average linux,
-so we can afford changing these in a way that increases
-performance while keeping power efficiency
----
- drivers/idle/intel_idle.c | 46 +++++++++++++++++++++++-----------------------
- 1 file changed, 23 insertions(+), 23 deletions(-)
-
-diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
-index f0b06b14e782..24609fbb1010 100644
---- a/drivers/idle/intel_idle.c
-+++ b/drivers/idle/intel_idle.c
-@@ -466,7 +466,7 @@ static struct cpuidle_state hsw_cstates[] = {
+diff -up linux-5.6/drivers/idle/intel_idle.c.23~ linux-5.6/drivers/idle/intel_idle.c
+--- linux-5.6/drivers/idle/intel_idle.c.23~ 2020-03-30 00:25:41.000000000 +0200
++++ linux-5.6/drivers/idle/intel_idle.c 2020-04-01 01:18:28.274248595 +0200
+@@ -462,7 +462,7 @@ static struct cpuidle_state hsw_cstates[
.desc = "MWAIT 0x01",
- .flags = MWAIT2flg(0x01),
+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
.exit_latency = 10,
- .target_residency = 20,
+ .target_residency = 120,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -474,7 +474,7 @@ static struct cpuidle_state hsw_cstates[] = {
+@@ -470,7 +470,7 @@ static struct cpuidle_state hsw_cstates[
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 33,
@@ -35,7 +19,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -482,7 +482,7 @@ static struct cpuidle_state hsw_cstates[] = {
+@@ -478,7 +478,7 @@ static struct cpuidle_state hsw_cstates[
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 133,
@@ -44,7 +28,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -490,7 +490,7 @@ static struct cpuidle_state hsw_cstates[] = {
+@@ -486,7 +486,7 @@ static struct cpuidle_state hsw_cstates[
.desc = "MWAIT 0x32",
.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 166,
@@ -53,7 +37,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -498,7 +498,7 @@ static struct cpuidle_state hsw_cstates[] = {
+@@ -494,7 +494,7 @@ static struct cpuidle_state hsw_cstates[
.desc = "MWAIT 0x40",
.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 300,
@@ -62,7 +46,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -506,7 +506,7 @@ static struct cpuidle_state hsw_cstates[] = {
+@@ -502,7 +502,7 @@ static struct cpuidle_state hsw_cstates[
.desc = "MWAIT 0x50",
.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 600,
@@ -71,7 +55,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -514,7 +514,7 @@ static struct cpuidle_state hsw_cstates[] = {
+@@ -510,7 +510,7 @@ static struct cpuidle_state hsw_cstates[
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 2600,
@@ -80,16 +64,16 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -534,7 +534,7 @@ static struct cpuidle_state bdw_cstates[] = {
+@@ -530,7 +530,7 @@ static struct cpuidle_state bdw_cstates[
.desc = "MWAIT 0x01",
- .flags = MWAIT2flg(0x01),
+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
.exit_latency = 10,
- .target_residency = 20,
+ .target_residency = 120,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -542,7 +542,7 @@ static struct cpuidle_state bdw_cstates[] = {
+@@ -538,7 +538,7 @@ static struct cpuidle_state bdw_cstates[
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 40,
@@ -98,7 +82,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -550,7 +550,7 @@ static struct cpuidle_state bdw_cstates[] = {
+@@ -546,7 +546,7 @@ static struct cpuidle_state bdw_cstates[
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 133,
@@ -107,7 +91,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -558,7 +558,7 @@ static struct cpuidle_state bdw_cstates[] = {
+@@ -554,7 +554,7 @@ static struct cpuidle_state bdw_cstates[
.desc = "MWAIT 0x32",
.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 166,
@@ -116,7 +100,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -566,7 +566,7 @@ static struct cpuidle_state bdw_cstates[] = {
+@@ -562,7 +562,7 @@ static struct cpuidle_state bdw_cstates[
.desc = "MWAIT 0x40",
.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 300,
@@ -125,7 +109,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -574,7 +574,7 @@ static struct cpuidle_state bdw_cstates[] = {
+@@ -570,7 +570,7 @@ static struct cpuidle_state bdw_cstates[
.desc = "MWAIT 0x50",
.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 600,
@@ -134,7 +118,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -582,7 +582,7 @@ static struct cpuidle_state bdw_cstates[] = {
+@@ -578,7 +578,7 @@ static struct cpuidle_state bdw_cstates[
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 2600,
@@ -143,16 +127,16 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -603,7 +603,7 @@ static struct cpuidle_state skl_cstates[] = {
+@@ -599,7 +599,7 @@ static struct cpuidle_state skl_cstates[
.desc = "MWAIT 0x01",
- .flags = MWAIT2flg(0x01),
+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
.exit_latency = 10,
- .target_residency = 20,
+ .target_residency = 120,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -611,7 +611,7 @@ static struct cpuidle_state skl_cstates[] = {
+@@ -607,7 +607,7 @@ static struct cpuidle_state skl_cstates[
.desc = "MWAIT 0x10",
.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 70,
@@ -161,7 +145,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -619,7 +619,7 @@ static struct cpuidle_state skl_cstates[] = {
+@@ -615,7 +615,7 @@ static struct cpuidle_state skl_cstates[
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 85,
@@ -170,7 +154,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -627,7 +627,7 @@ static struct cpuidle_state skl_cstates[] = {
+@@ -623,7 +623,7 @@ static struct cpuidle_state skl_cstates[
.desc = "MWAIT 0x33",
.flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 124,
@@ -179,7 +163,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -635,7 +635,7 @@ static struct cpuidle_state skl_cstates[] = {
+@@ -631,7 +631,7 @@ static struct cpuidle_state skl_cstates[
.desc = "MWAIT 0x40",
.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 200,
@@ -188,7 +172,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -643,7 +643,7 @@ static struct cpuidle_state skl_cstates[] = {
+@@ -639,7 +639,7 @@ static struct cpuidle_state skl_cstates[
.desc = "MWAIT 0x50",
.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 480,
@@ -197,7 +181,7 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -651,7 +651,7 @@ static struct cpuidle_state skl_cstates[] = {
+@@ -647,7 +647,7 @@ static struct cpuidle_state skl_cstates[
.desc = "MWAIT 0x60",
.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 890,
@@ -206,16 +190,16 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -672,7 +672,7 @@ static struct cpuidle_state skx_cstates[] = {
+@@ -668,7 +668,7 @@ static struct cpuidle_state skx_cstates[
.desc = "MWAIT 0x01",
- .flags = MWAIT2flg(0x01),
+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
.exit_latency = 10,
- .target_residency = 20,
+ .target_residency = 1000,
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
-@@ -680,7 +680,7 @@ static struct cpuidle_state skx_cstates[] = {
+@@ -676,7 +676,7 @@ static struct cpuidle_state skx_cstates[
.desc = "MWAIT 0x20",
.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 133,
@@ -224,5 +208,3 @@ index f0b06b14e782..24609fbb1010 100644
.enter = &intel_idle,
.enter_s2idle = intel_idle_s2idle, },
{
---
-2.15.0
diff --git a/2-0001-x86-Don-t-declare-__force_order-in-kaslr_64.c.patch b/2-0001-x86-Don-t-declare-__force_order-in-kaslr_64.c.patch
new file mode 100644
index 0000000..b809432
--- /dev/null
+++ b/2-0001-x86-Don-t-declare-__force_order-in-kaslr_64.c.patch
@@ -0,0 +1,37 @@
+From c8c26194cf5a344cd53763eaaf16c3ab609736f4 Mon Sep 17 00:00:00 2001
+From: "H.J. Lu" <hjl.tools@gmail.com>
+Date: Thu, 16 Jan 2020 12:46:51 -0800
+Subject: [PATCH] x86: Don't declare __force_order in kaslr_64.c
+
+GCC 10 changed the default to -fno-common, which leads to
+
+ LD arch/x86/boot/compressed/vmlinux
+ld: arch/x86/boot/compressed/pgtable_64.o:(.bss+0x0): multiple definition of `__force_order'; arch/x86/boot/compressed/kaslr_64.o:(.bss+0x0): first defined here
+make[2]: *** [arch/x86/boot/compressed/Makefile:119: arch/x86/boot/compressed/vmlinux] Error 1
+
+Since __force_order is already provided in pgtable_64.c, there is no
+need to declare __force_order in kaslr_64.c.
+
+Signed-off-by: H.J. Lu <hjl.tools@gmail.com>
+Signed-off-by: Yu-cheng Yu <yu-cheng.yu@intel.com>
+---
+ arch/x86/boot/compressed/kaslr_64.c | 3 ---
+ 1 file changed, 3 deletions(-)
+
+diff --git a/arch/x86/boot/compressed/kaslr_64.c b/arch/x86/boot/compressed/kaslr_64.c
+index 748456c365f4..9557c5a15b91 100644
+--- a/arch/x86/boot/compressed/kaslr_64.c
++++ b/arch/x86/boot/compressed/kaslr_64.c
+@@ -29,9 +29,6 @@
+ #define __PAGE_OFFSET __PAGE_OFFSET_BASE
+ #include "../../mm/ident_map.c"
+
+-/* Used by pgtable.h asm code to force instruction serialization. */
+-unsigned long __force_order;
+-
+ /* Used to track our page table allocation area. */
+ struct alloc_pgt_data {
+ unsigned char *pgt_buf;
+--
+2.24.1
+
diff --git a/7a8b64d17e35810dc3176fe61208b45c15d25402.patch b/7a8b64d17e35810dc3176fe61208b45c15d25402.patch
new file mode 100644
index 0000000..efa63b9
--- /dev/null
+++ b/7a8b64d17e35810dc3176fe61208b45c15d25402.patch
@@ -0,0 +1,77 @@
+commit 7a8b64d17e35810dc3176fe61208b45c15d25402
+Author: Rob Herring <robh@kernel.org>
+Date: Thu Feb 6 14:02:30 2020 +0000
+
+ of/address: use range parser for of_dma_get_range
+
+ of_dma_get_range() does the same ranges parsing as
+ of_pci_range_parser_one(), so let's refactor of_dma_get_range() to use
+ it instead.
+
+ This commit is no functional change. Subsequent commits will parse more
+ than the 1st dma-ranges entry.
+
+ Signed-off-by: Rob Herring <robh@kernel.org>
+
+diff --git a/drivers/of/address.c b/drivers/of/address.c
+index 6d33f849f11440..a2c45812a50e52 100644
+--- a/drivers/of/address.c
++++ b/drivers/of/address.c
+@@ -939,10 +939,11 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
+ {
+ struct device_node *node = of_node_get(np);
+ const __be32 *ranges = NULL;
+- int len, naddr, nsize, pna;
++ int len;
+ int ret = 0;
+ bool found_dma_ranges = false;
+- u64 dmaaddr;
++ struct of_range_parser parser;
++ struct of_range range;
+
+ while (node) {
+ ranges = of_get_property(node, "dma-ranges", &len);
+@@ -967,33 +968,20 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
+ goto out;
+ }
+
+- naddr = of_bus_n_addr_cells(node);
+- nsize = of_bus_n_size_cells(node);
+- pna = of_n_addr_cells(node);
+- if ((len / sizeof(__be32)) % (pna + naddr + nsize)) {
+- ret = -EINVAL;
+- goto out;
+- }
++ of_dma_range_parser_init(&parser, node);
++
++ for_each_of_range(&parser, &range) {
++ pr_debug("dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
++ range.bus_addr, range.cpu_addr, range.size);
++
++ *dma_addr = range.bus_addr;
++ *paddr = range.cpu_addr;
++ *size = range.size;
+
+- /* dma-ranges format:
+- * DMA addr : naddr cells
+- * CPU addr : pna cells
+- * size : nsize cells
+- */
+- dmaaddr = of_read_number(ranges, naddr);
+- *paddr = of_translate_dma_address(node, ranges + naddr);
+- if (*paddr == OF_BAD_ADDR) {
+- pr_err("translation of DMA address(%llx) to CPU address failed node(%pOF)\n",
+- dmaaddr, np);
+- ret = -EINVAL;
+ goto out;
+ }
+- *dma_addr = dmaaddr;
+-
+- *size = of_read_number(ranges + naddr + pna, nsize);
+
+- pr_debug("dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
+- *dma_addr, *paddr, *size);
++ pr_err("translation of DMA ranges failed on node(%pOF)\n", np);
+
+ out:
+ of_node_put(node);
diff --git a/9d55bebd9816903b821a403a69a94190442ac043.patch b/9d55bebd9816903b821a403a69a94190442ac043.patch
new file mode 100644
index 0000000..b3ead93
--- /dev/null
+++ b/9d55bebd9816903b821a403a69a94190442ac043.patch
@@ -0,0 +1,68 @@
+commit 9d55bebd9816903b821a403a69a94190442ac043
+Author: Rob Herring <robh@kernel.org>
+Date: Thu Feb 6 14:26:40 2020 +0000
+
+ of/address: Support multiple 'dma-ranges' entries
+
+ Currently, the DMA offset and mask for a device are set based only on the
+ first 'dma-ranges' entry. We should really be using all the entries. The
+ kernel doesn't yet support multiple offsets and sizes, so the best we can
+ do is to find the biggest size for a single offset. The algorithm is
+ copied from acpi_dma_get_range().
+
+ If there's different offsets from the first entry, then we warn and
+ continue. It really should be an error, but this will likely break
+ existing DTs.
+
+ Signed-off-by: Rob Herring <robh@kernel.org>
+
+diff --git a/drivers/of/address.c b/drivers/of/address.c
+index a2c45812a50e52..8eea3f6e29a441 100644
+--- a/drivers/of/address.c
++++ b/drivers/of/address.c
+@@ -944,6 +944,7 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
+ bool found_dma_ranges = false;
+ struct of_range_parser parser;
+ struct of_range range;
++ u64 dma_start = U64_MAX, dma_end = 0, dma_offset = 0;
+
+ while (node) {
+ ranges = of_get_property(node, "dma-ranges", &len);
+@@ -974,14 +975,33 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
+ pr_debug("dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
+ range.bus_addr, range.cpu_addr, range.size);
+
+- *dma_addr = range.bus_addr;
+- *paddr = range.cpu_addr;
+- *size = range.size;
++ if (dma_offset && range.cpu_addr - range.bus_addr != dma_offset) {
++ pr_warn("Can't handle multiple dma-ranges with different offsets on node(%pOF)\n", node);
++ /* Don't error out as we'd break some existing DTs */
++ continue;
++ }
++ dma_offset = range.cpu_addr - range.bus_addr;
++
++ /* Take lower and upper limits */
++ if (range.bus_addr < dma_start)
++ dma_start = range.bus_addr;
++ if (range.bus_addr + range.size > dma_end)
++ dma_end = range.bus_addr + range.size;
++ }
+
++ if (dma_start >= dma_end) {
++ ret = -EINVAL;
++ pr_debug("Invalid DMA ranges configuration on node(%pOF)\n",
++ node);
+ goto out;
+ }
+
+- pr_err("translation of DMA ranges failed on node(%pOF)\n", np);
++ *dma_addr = dma_start;
++ *size = dma_end - dma_start;
++ *paddr = dma_start + dma_offset;
++
++ pr_debug("final: dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
++ *dma_addr, *paddr, *size);
+
+ out:
+ of_node_put(node);
diff --git a/android-enable-building-ashmem-and-binder-as-modules.patch b/android-enable-building-ashmem-and-binder-as-modules.patch
index 115ec80..05ad8f8 100644
--- a/android-enable-building-ashmem-and-binder-as-modules.patch
+++ b/android-enable-building-ashmem-and-binder-as-modules.patch
@@ -1,30 +1,18 @@
-From: Ben Hutchings <ben@decadent.org.uk>
-Date: Fri, 22 Jun 2018 17:27:00 +0100
-Subject: android: Enable building ashmem and binder as modules
-Bug-Debian: https://bugs.debian.org/901492
-
-We want to enable use of the Android ashmem and binder drivers to
-support Anbox, but they should not be built-in as that would waste
-resources and increase security attack surface on systems that don't
-need them.
-
-- Add a MODULE_LICENSE declaration to ashmem
-- Change the Makefiles to build each driver as an object with the
- "_linux" suffix (which is what Anbox expects)
-- Change config symbol types to tristate
----
- drivers/android/Kconfig | 2 +-
- drivers/android/Makefile | 7 ++++---
- drivers/android/binder_alloc.c | 2 +-
- drivers/staging/android/Kconfig | 2 +-
- drivers/staging/android/Makefile | 3 ++-
- drivers/staging/android/ashmem.c | 3 +++
- 6 files changed, 12 insertions(+), 7 deletions(-)
-
-diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig
-index 6fdf2abe4598..329ec9941fbf 100644
---- a/drivers/android/Kconfig
-+++ b/drivers/android/Kconfig
+diff -up linux-5.6-rc3/drivers/android/binder_alloc.c.22~ linux-5.6-rc3/drivers/android/binder_alloc.c
+--- linux-5.6-rc3/drivers/android/binder_alloc.c.22~ 2020-02-24 01:17:42.000000000 +0100
++++ linux-5.6-rc3/drivers/android/binder_alloc.c 2020-02-24 23:26:47.054727484 +0100
+@@ -38,7 +38,7 @@ enum {
+ };
+ static uint32_t binder_alloc_debug_mask = BINDER_DEBUG_USER_ERROR;
+
+-module_param_named(debug_mask, binder_alloc_debug_mask,
++module_param_named(alloc_debug_mask, binder_alloc_debug_mask,
+ uint, 0644);
+
+ #define binder_alloc_debug(mask, x...) \
+diff -up linux-5.6-rc3/drivers/android/Kconfig.22~ linux-5.6-rc3/drivers/android/Kconfig
+--- linux-5.6-rc3/drivers/android/Kconfig.22~ 2020-02-24 01:17:42.000000000 +0100
++++ linux-5.6-rc3/drivers/android/Kconfig 2020-02-24 23:26:47.054727484 +0100
@@ -9,7 +9,7 @@ config ANDROID
if ANDROID
@@ -34,10 +22,9 @@ index 6fdf2abe4598..329ec9941fbf 100644
depends on MMU
default n
---help---
-diff --git a/drivers/android/Makefile b/drivers/android/Makefile
-index c9d3d0c99c25..55411d9a9c2a 100644
---- a/drivers/android/Makefile
-+++ b/drivers/android/Makefile
+diff -up linux-5.6-rc3/drivers/android/Makefile.22~ linux-5.6-rc3/drivers/android/Makefile
+--- linux-5.6-rc3/drivers/android/Makefile.22~ 2020-02-24 01:17:42.000000000 +0100
++++ linux-5.6-rc3/drivers/android/Makefile 2020-02-24 23:26:47.054727484 +0100
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: GPL-2.0-only
ccflags-y += -I$(src) # needed for trace events
@@ -49,23 +36,26 @@ index c9d3d0c99c25..55411d9a9c2a 100644
+binder_linux-y := binder.o binder_alloc.o
+binder_linux-$(CONFIG_ANDROID_BINDERFS) += binderfs.o
+binder_linux-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o
-diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
-index ce5603c2291c..099fef731148 100644
---- a/drivers/android/binder_alloc.c
-+++ b/drivers/android/binder_alloc.c
-@@ -37,7 +37,7 @@ enum {
- };
- static uint32_t binder_alloc_debug_mask = BINDER_DEBUG_USER_ERROR;
-
--module_param_named(debug_mask, binder_alloc_debug_mask,
-+module_param_named(alloc_debug_mask, binder_alloc_debug_mask,
- uint, 0644);
+diff -up linux-5.6-rc3/drivers/staging/android/ashmem.c.22~ linux-5.6-rc3/drivers/staging/android/ashmem.c
+--- linux-5.6-rc3/drivers/staging/android/ashmem.c.22~ 2020-02-24 01:17:42.000000000 +0100
++++ linux-5.6-rc3/drivers/staging/android/ashmem.c 2020-02-24 23:26:47.054727484 +0100
+@@ -24,6 +24,7 @@
+ #include <linux/bitops.h>
+ #include <linux/mutex.h>
+ #include <linux/shmem_fs.h>
++#include <linux/module.h>
+ #include "ashmem.h"
- #define binder_alloc_debug(mask, x...) \
-diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig
-index d6d605d5cbde..afe9085369a3 100644
---- a/drivers/staging/android/Kconfig
-+++ b/drivers/staging/android/Kconfig
+ #define ASHMEM_NAME_PREFIX "dev/ashmem/"
+@@ -953,3 +954,5 @@ out:
+ return ret;
+ }
+ device_initcall(ashmem_init);
++
++MODULE_LICENSE("GPL v2");
+diff -up linux-5.6-rc3/drivers/staging/android/Kconfig.22~ linux-5.6-rc3/drivers/staging/android/Kconfig
+--- linux-5.6-rc3/drivers/staging/android/Kconfig.22~ 2020-02-24 01:17:42.000000000 +0100
++++ linux-5.6-rc3/drivers/staging/android/Kconfig 2020-02-24 23:26:47.054727484 +0100
@@ -4,7 +4,7 @@ menu "Android"
if ANDROID
@@ -75,36 +65,14 @@ index d6d605d5cbde..afe9085369a3 100644
depends on SHMEM
help
The ashmem subsystem is a new shared memory allocator, similar to
-diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile
-index 14bd9c6ce10d..c202aa4f8323 100644
---- a/drivers/staging/android/Makefile
-+++ b/drivers/staging/android/Makefile
-@@ -3,5 +3,6 @@ ccflags-y += -I$(src) # needed for trace events
+diff -up linux-5.6-rc3/drivers/staging/android/Makefile.22~ linux-5.6-rc3/drivers/staging/android/Makefile
+--- linux-5.6-rc3/drivers/staging/android/Makefile.22~ 2020-02-24 23:26:47.054727484 +0100
++++ linux-5.6-rc3/drivers/staging/android/Makefile 2020-02-24 23:27:56.058734039 +0100
+@@ -3,4 +3,6 @@ ccflags-y += -I$(src) # needed for tra
obj-y += ion/
-obj-$(CONFIG_ASHMEM) += ashmem.o
+obj-$(CONFIG_ASHMEM) += ashmem_linux.o
+ashmem_linux-y += ashmem.o
- obj-$(CONFIG_ANDROID_VSOC) += vsoc.o
-diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c
-index 74d497d39c5a..fbbe467b0ba8 100644
---- a/drivers/staging/android/ashmem.c
-+++ b/drivers/staging/android/ashmem.c
-@@ -24,6 +24,7 @@
- #include <linux/bitops.h>
- #include <linux/mutex.h>
- #include <linux/shmem_fs.h>
-+#include <linux/module.h>
- #include "ashmem.h"
-
- #define ASHMEM_NAME_PREFIX "dev/ashmem/"
-@@ -925,3 +926,5 @@ static int __init ashmem_init(void)
- return ret;
- }
- device_initcall(ashmem_init);
+
-+MODULE_LICENSE("GPL v2");
---
-2.20.1
-
diff --git a/arm64-common.config b/arm64-common.config
index c444a48..3a6539a 100644
--- a/arm64-common.config
+++ b/arm64-common.config
@@ -1029,3 +1029,9 @@ CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT=m
CONFIG_ARM64_PTDUMP_CORE=y
CONFIG_ARM64_PTDUMP_DEBUGFS=y
CONFIG_ARM64_RELOC_TEST=m
+CONFIG_GOOGLE_FIRMWARE=y
+CONFIG_GOOGLE_COREBOOT_TABLE=m
+CONFIG_GOOGLE_FRAMEBUFFER_COREBOOT=m
+CONFIG_GOOGLE_MEMCONSOLE_COREBOOT=m
+CONFIG_GOOGLE_VPD=m
+CONFIG_IMX_DSP=m
diff --git a/btrfs-zstd-backport-to-4.12-api.patch b/btrfs-zstd-backport-to-4.12-api.patch
deleted file mode 100644
index e52717e..0000000
--- a/btrfs-zstd-backport-to-4.12-api.patch
+++ /dev/null
@@ -1,17 +0,0 @@
---- linux-4.12/fs/btrfs/zstd.c.omv~ 2017-08-17 21:12:35.674612608 +0200
-+++ linux-4.12/fs/btrfs/zstd.c 2017-08-17 21:13:29.883512195 +0200
-@@ -260,13 +260,9 @@ out:
- return ret;
- }
-
--static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
-+static int zstd_decompress_bio(struct list_head *ws, struct page **pages_in, u64 disk_start, struct bio *orig_bio, size_t srclen)
- {
- struct workspace *workspace = list_entry(ws, struct workspace, list);
-- struct page **pages_in = cb->compressed_pages;
-- u64 disk_start = cb->start;
-- struct bio *orig_bio = cb->orig_bio;
-- size_t srclen = cb->compressed_len;
- ZSTD_DStream *stream;
- int ret = 0;
- unsigned long page_in_index = 0;
diff --git a/common-desktop.config b/common-desktop.config
index e1d0ebf..185ad4c 100644
--- a/common-desktop.config
+++ b/common-desktop.config
@@ -30,3 +30,5 @@ CONFIG_UKSM=y
CONFIG_PREEMPT_VOLUNTARY=y
# CONFIG_PREEMPT is not set
CONFIG_PREEMPT_COUNT=y
+CONFIG_HZ_1000=y
+CONFIG_HZ=1000
diff --git a/common-server.config b/common-server.config
index a6d6012..7f739c0 100644
--- a/common-server.config
+++ b/common-server.config
@@ -1,5 +1,3 @@
-# CONFIG_SCHED_MUQSS is not set
-
#
# RCU Subsystem
#
@@ -23,6 +21,7 @@ CONFIG_INLINE_READ_UNLOCK_IRQ=y
CONFIG_INLINE_WRITE_UNLOCK=y
CONFIG_INLINE_WRITE_UNLOCK_IRQ=y
+
#
# Processor type and features
#
@@ -31,3 +30,7 @@ CONFIG_PREEMPT_NONE=y
# CONFIG_PREEMPT is not set
# CONFIG_PREEMPT_COUNT is not set
CONFIG_SLAB=y
+CONFIG_HZ_300=y
+CONFIG_HZ=300
+# CONFIG_HZ_250 is not set
+# CONFIG_HZ_100 is not set
diff --git a/common.config b/common.config
index 57e65c7..28c1ebe 100644
--- a/common.config
+++ b/common.config
@@ -1,7 +1,7 @@
# CONFIG_KERNEL_GZIP is not set
CONFIG_KERNEL_ZSTD=y
CONFIG_DEFAULT_HOSTNAME="omv"
-CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3=y
+# CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 is not set
# CONFIG_USELIB is not set
CONFIG_GENERIC_IRQ_CHIP=y
CONFIG_IRQ_SIM=y
@@ -31,13 +31,14 @@ CONFIG_CGROUP_HUGETLB=y
CONFIG_CGROUP_DEVICE=y
CONFIG_CGROUP_PERF=y
CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_FREEZER=y
CONFIG_HAVE_GCC_PLUGINS=y
CONFIG_GCC_PLUGINS=y
CONFIG_SOCK_CGROUP_DATA=y
CONFIG_USER_NS=y
CONFIG_CHECKPOINT_RESTORE=y
CONFIG_SCHED_AUTOGROUP=y
-CONFIG_EXPERT=y
+# CONFIG_EXPERT is not set
# CONFIG_SYSFS_SYSCALL is not set
# CONFIG_SYSFS_DEPRECATED is not set
CONFIG_SYSCTL_SYSCALL=y
@@ -131,6 +132,7 @@ CONFIG_ACPI_PCI_SLOT=y
CONFIG_ACPI_HOTPLUG_MEMORY=y
CONFIG_ACPI_CUSTOM_METHOD=m
CONFIG_ACPI_NFIT=m
+CONFIG_ACPI_HMAT=m
CONFIG_DPTF_POWER=m
CONFIG_ACPI_WATCHDOG=y
CONFIG_ACPI_EXTLOG=m
@@ -220,7 +222,7 @@ CONFIG_FW_CFG_SYSFS=m
CONFIG_FW_CFG_SYSFS_CMDLINE=y
CONFIG_FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER=y
CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y
-# CONFIG_EFI_VARS is not set
+CONFIG_EFI_VARS=m
CONFIG_EFI_CAPSULE_LOADER=m
CONFIG_EFI_TEST=m
CONFIG_APPLE_PROPERTIES=y
@@ -365,6 +367,8 @@ CONFIG_NET_IPVTI=m
CONFIG_NET_UDP_TUNNEL=m
CONFIG_NET_FOU=m
CONFIG_NET_FOU_IP_TUNNELS=y
+CONFIG_WIREGUARD=m
+# CONFIG_WIREGUARD_DEBUG is not set
CONFIG_INET_AH=m
CONFIG_INET_ESP=m
CONFIG_INET_ESP_OFFLOAD=m
@@ -393,6 +397,7 @@ CONFIG_TCP_CONG_YEAH=m
CONFIG_TCP_CONG_ILLINOIS=m
CONFIG_TCP_CONG_DCTCP=m
CONFIG_TCP_CONG_BBR=m
+CONFIG_MPTCP=y
CONFIG_IPV6=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_IPV6_ROUTE_INFO=y
@@ -829,6 +834,7 @@ CONFIG_IEEE802154_SOCKET=m
CONFIG_IEEE802154_6LOWPAN=m
CONFIG_MAC802154=m
CONFIG_NET_SCH_CBQ=m
+CONFIG_NET_SCH_ETS=m
CONFIG_NET_SCH_CAKE=m
CONFIG_NET_SCH_HTB=m
CONFIG_NET_SCH_HFSC=m
@@ -851,6 +857,7 @@ CONFIG_NET_SCH_QFQ=m
CONFIG_NET_SCH_CODEL=m
CONFIG_NET_SCH_FQ_CODEL=y
CONFIG_NET_SCH_FQ=m
+CONFIG_NET_SCH_FQ_PIE=m
CONFIG_NET_SCH_HHF=m
CONFIG_NET_SCH_PIE=m
CONFIG_NET_SCH_INGRESS=m
@@ -1421,7 +1428,7 @@ CONFIG_IDE_GD_ATAPI=y
CONFIG_BLK_DEV_IDECS=m
CONFIG_BLK_DEV_DELKIN=m
CONFIG_BLK_DEV_IDECD=m
-CONFIG_BLK_DEV_IDECD_VERBOSE_ERRORS=y
+# CONFIG_BLK_DEV_IDECD_VERBOSE_ERRORS is not set
CONFIG_BLK_DEV_IDETAPE=m
CONFIG_BLK_DEV_IDEACPI=y
CONFIG_IDE_TASK_IOCTL=y
@@ -1892,6 +1899,7 @@ CONFIG_BE2NET_SKYHAWK=y
CONFIG_PCMCIA_FMVJ18X=m
CONFIG_HP100=m
CONFIG_HINIC=m
+CONFIG_IGC=m
CONFIG_E100=m
CONFIG_E1000=m
CONFIG_E1000E=m
@@ -2219,6 +2227,8 @@ CONFIG_LIBIPW=m
CONFIG_IWLEGACY=m
CONFIG_IWL4965=m
CONFIG_IWL3945=m
+CONFIG_RTL8723DE=m
+CONFIG_RTL8821CE=m
#
# iwl3945 / iwl4965 Debugging Options
@@ -4333,8 +4343,8 @@ CONFIG_CHASH=m
# CONFIG_CHASH_STATS is not set
# CONFIG_CHASH_SELFTEST is not set
CONFIG_DRM_NOUVEAU=m
-CONFIG_NOUVEAU_DEBUG=5
-CONFIG_NOUVEAU_DEBUG_DEFAULT=3
+CONFIG_NOUVEAU_DEBUG=3
+CONFIG_NOUVEAU_DEBUG_DEFAULT=1
# CONFIG_NOUVEAU_DEBUG_MMU is not set
CONFIG_DRM_NOUVEAU_BACKLIGHT=y
CONFIG_DRM_I915=m
@@ -4519,7 +4529,7 @@ CONFIG_SOUND=m
CONFIG_SOUND_OSS_CORE=y
# CONFIG_SOUND_OSS_CORE_PRECLAIM is not set
CONFIG_SND=m
-CONFIG_SND_DEBUG_VERBOSE=y
+# CONFIG_SND_DEBUG_VERBOSE is not set
CONFIG_SND_TIMER=m
CONFIG_SND_PCM=m
CONFIG_SND_PCM_ELD=y
@@ -4914,6 +4924,7 @@ CONFIG_HID_BELKIN=m
CONFIG_HID_BETOP_FF=m
CONFIG_HID_CHERRY=m
CONFIG_HID_CHICONY=m
+CONFIG_HID_CORSAIR=m
CONFIG_HID_PRODIKEYS=m
CONFIG_HID_CMEDIA=m
CONFIG_HID_CP2112=m
@@ -6955,6 +6966,9 @@ CONFIG_842_DECOMPRESS=y
CONFIG_LZ4_COMPRESS=m
CONFIG_LZ4HC_COMPRESS=m
CONFIG_ZSTD_COMPRESS=y
+CONFIG_RD_ZSTD=y
+CONFIG_DECOMPRESS_ZSTD=y
+CONFIG_INITRAMFS_COMPRESSION_ZSTD=y
# CONFIG_XZ_DEC_POWERPC is not set
# CONFIG_XZ_DEC_IA64 is not set
# CONFIG_XZ_DEC_ARM is not set
@@ -7037,8 +7051,10 @@ CONFIG_TRACING_EVENTS_GPIO=y
# CONFIG_RUNTIME_TESTING_MENU is not set
CONFIG_MEMTEST=y
CONFIG_IO_STRICT_DEVMEM=y
+# CONFIG_EARLY_PRINTK is not set
# CONFIG_EARLY_PRINTK_DBGP is not set
-CONFIG_EARLY_PRINTK_USB_XDBC=y
+# CONFIG_EARLY_PRINTK_USB_XDBC is not set
+# CONFIG_X86_VERBOSE_BOOTUP is not set
# CONFIG_IOMMU_DEBUG is not set
# CONFIG_DEBUG_BOOT_PARAMS is not set
# CONFIG_UNWINDER_GUESS is not set
diff --git a/export-symbols-needed-by-android-drivers.patch b/export-symbols-needed-by-android-drivers.patch
index ab7ab0f..437a37e 100644
--- a/export-symbols-needed-by-android-drivers.patch
+++ b/export-symbols-needed-by-android-drivers.patch
@@ -22,9 +22,10 @@ Export the currently un-exported symbols they depend on.
security/security.c | 4 ++++
9 files changed, 17 insertions(+)
---- a/fs/file.c
-+++ b/fs/file.c
-@@ -409,6 +409,7 @@ struct files_struct *get_files_struct(st
+diff -uraN linux-5.7.8/fs/file.c omv-linux-5.7.8/fs/file.c
+--- linux-5.7.8/fs/file.c 2020-07-09 09:39:40.000000000 +0200
++++ omv-linux-5.7.8/fs/file.c 2020-07-13 15:14:49.188263989 +0200
+@@ -409,6 +409,7 @@
return files;
}
@@ -32,7 +33,7 @@ Export the currently un-exported symbols they depend on.
void put_files_struct(struct files_struct *files)
{
-@@ -421,6 +422,7 @@ void put_files_struct(struct files_struc
+@@ -421,6 +422,7 @@
kmem_cache_free(files_cachep, files);
}
}
@@ -40,7 +41,7 @@ Export the currently un-exported symbols they depend on.
void reset_files_struct(struct files_struct *files)
{
-@@ -533,6 +535,7 @@ out:
+@@ -534,6 +536,7 @@
spin_unlock(&files->file_lock);
return error;
}
@@ -48,7 +49,7 @@ Export the currently un-exported symbols they depend on.
static int alloc_fd(unsigned start, unsigned flags)
{
-@@ -606,6 +609,7 @@ void __fd_install(struct files_struct *f
+@@ -612,6 +615,7 @@
rcu_assign_pointer(fdt->fd[fd], file);
rcu_read_unlock_sched();
}
@@ -56,7 +57,7 @@ Export the currently un-exported symbols they depend on.
void fd_install(unsigned int fd, struct file *file)
{
-@@ -668,6 +672,7 @@ out_unlock:
+@@ -676,6 +680,7 @@
*res = NULL;
return -ENOENT;
}
@@ -64,9 +65,10 @@ Export the currently un-exported symbols they depend on.
void do_close_on_exec(struct files_struct *files)
{
---- a/kernel/fork.c
-+++ b/kernel/fork.c
-@@ -1085,6 +1085,7 @@ void mmput_async(struct mm_struct *mm)
+diff -uraN linux-5.7.8/kernel/fork.c omv-linux-5.7.8/kernel/fork.c
+--- linux-5.7.8/kernel/fork.c 2020-07-09 09:39:40.000000000 +0200
++++ omv-linux-5.7.8/kernel/fork.c 2020-07-13 15:15:57.177994389 +0200
+@@ -1123,6 +1123,7 @@
schedule_work(&mm->async_put_work);
}
}
@@ -74,9 +76,10 @@ Export the currently un-exported symbols they depend on.
#endif
/**
---- a/kernel/sched/core.c
-+++ b/kernel/sched/core.c
-@@ -3943,6 +3943,7 @@ int can_nice(const struct task_struct *p
+diff -uraN linux-5.7.8/kernel/sched/core.c omv-linux-5.7.8/kernel/sched/core.c
+--- linux-5.7.8/kernel/sched/core.c 2020-07-09 09:39:40.000000000 +0200
++++ omv-linux-5.7.8/kernel/sched/core.c 2020-07-13 15:16:48.193591744 +0200
+@@ -4572,6 +4572,7 @@
return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
capable(CAP_SYS_NICE));
}
@@ -84,9 +87,10 @@ Export the currently un-exported symbols they depend on.
#ifdef __ARCH_WANT_SYS_NICE
---- a/kernel/signal.c
-+++ b/kernel/signal.c
-@@ -1353,6 +1353,7 @@ struct sighand_struct *__lock_task_sigha
+diff -uraN linux-5.7.8/kernel/signal.c omv-linux-5.7.8/kernel/signal.c
+--- linux-5.7.8/kernel/signal.c 2020-07-09 09:39:40.000000000 +0200
++++ omv-linux-5.7.8/kernel/signal.c 2020-07-13 15:18:04.854196194 +0200
+@@ -1396,6 +1396,7 @@
return sighand;
}
@@ -94,19 +98,21 @@ Export the currently un-exported symbols they depend on.
/*
* send signal info to all the members of a group
---- a/kernel/task_work.c
-+++ b/kernel/task_work.c
-@@ -40,6 +40,7 @@ task_work_add(struct task_struct *task,
- set_notify_resume(task);
+diff -uraN linux-5.7.8/kernel/task_work.c omv-linux-5.7.8/kernel/task_work.c
+--- linux-5.7.8/kernel/task_work.c 2020-07-09 09:39:40.000000000 +0200
++++ omv-linux-5.7.8/kernel/task_work.c 2020-07-13 15:18:57.298331300 +0200
+@@ -52,6 +52,7 @@
+
return 0;
}
+EXPORT_SYMBOL(task_work_add);
/**
* task_work_cancel - cancel a pending work added by task_work_add()
---- a/mm/memory.c
-+++ b/mm/memory.c
-@@ -1364,6 +1364,7 @@ void zap_page_range(struct vm_area_struc
+diff -uraN linux-5.7.8/mm/memory.c omv-linux-5.7.8/mm/memory.c
+--- linux-5.7.8/mm/memory.c 2020-07-09 09:39:40.000000000 +0200
++++ omv-linux-5.7.8/mm/memory.c 2020-07-13 15:19:48.003475257 +0200
+@@ -1370,6 +1370,7 @@
mmu_notifier_invalidate_range_end(&range);
tlb_finish_mmu(&tlb, start, range.end);
}
@@ -114,9 +120,10 @@ Export the currently un-exported symbols they depend on.
/**
* zap_page_range_single - remove user pages in a given range
---- a/mm/shmem.c
-+++ b/mm/shmem.c
-@@ -3992,6 +3992,7 @@ int shmem_zero_setup(struct vm_area_stru
+diff -uraN linux-5.7.8/mm/shmem.c omv-linux-5.7.8/mm/shmem.c
+--- linux-5.7.8/mm/shmem.c 2020-07-09 09:39:40.000000000 +0200
++++ omv-linux-5.7.8/mm/shmem.c 2020-07-13 15:20:27.829371644 +0200
+@@ -4185,6 +4185,7 @@
return 0;
}
@@ -124,9 +131,10 @@ Export the currently un-exported symbols they depend on.
/**
* shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
---- a/mm/vmalloc.c
-+++ b/mm/vmalloc.c
-@@ -1295,6 +1295,7 @@ int map_kernel_range_noflush(unsigned lo
+diff -uraN linux-5.7.8/mm/vmalloc.c omv-linux-5.7.8/mm/vmalloc.c
+--- linux-5.7.8/mm/vmalloc.c 2020-07-09 09:39:40.000000000 +0200
++++ omv-linux-5.7.8/mm/vmalloc.c 2020-07-13 15:22:00.544919570 +0200
+@@ -2011,6 +2011,7 @@
{
return vmap_page_range_noflush(addr, addr + size, prot, pages);
}
@@ -134,7 +142,7 @@ Export the currently un-exported symbols they depend on.
/**
* unmap_kernel_range_noflush - unmap kernel VM area
-@@ -1435,6 +1436,7 @@ struct vm_struct *get_vm_area(unsigned l
+@@ -2161,6 +2162,7 @@
NUMA_NO_NODE, GFP_KERNEL,
__builtin_return_address(0));
}
@@ -142,9 +150,10 @@ Export the currently un-exported symbols they depend on.
struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
const void *caller)
---- a/security/security.c
-+++ b/security/security.c
-@@ -247,24 +247,28 @@ int security_binder_set_context_mgr(stru
+diff -uraN linux-5.7.8/security/security.c omv-linux-5.7.8/security/security.c
+--- linux-5.7.8/security/security.c 2020-07-09 09:39:40.000000000 +0200
++++ omv-linux-5.7.8/security/security.c 2020-07-13 15:23:59.601954504 +0200
+@@ -725,24 +725,28 @@
{
return call_int_hook(binder_set_context_mgr, 0, mgr);
}
diff --git a/extra-wifi-drivers-compile.patch b/extra-wifi-drivers-compile.patch
new file mode 100644
index 0000000..e948733
--- /dev/null
+++ b/extra-wifi-drivers-compile.patch
@@ -0,0 +1,50 @@
+diff -up linux-5.5/drivers/net/wireless/rtl8723de/include/ieee80211.h.omv~ linux-5.5/drivers/net/wireless/rtl8723de/include/ieee80211.h
+--- linux-5.5/drivers/net/wireless/rtl8723de/include/ieee80211.h.omv~ 2020-03-01 04:31:59.060616017 +0100
++++ linux-5.5/drivers/net/wireless/rtl8723de/include/ieee80211.h 2020-03-01 04:32:21.404617002 +0100
+@@ -1481,18 +1481,18 @@ enum ieee80211_state {
+ (((Addr[2]) & 0xff) == 0xff) && (((Addr[3]) & 0xff) == 0xff) && (((Addr[4]) & 0xff) == 0xff) && \
+ (((Addr[5]) & 0xff) == 0xff))
+ #else
+-extern __inline int is_multicast_mac_addr(const u8 *addr)
++static inline int is_multicast_mac_addr(const u8 *addr)
+ {
+ return (addr[0] != 0xff) && (0x01 & addr[0]);
+ }
+
+-extern __inline int is_broadcast_mac_addr(const u8 *addr)
++static inline int is_broadcast_mac_addr(const u8 *addr)
+ {
+ return ((addr[0] == 0xff) && (addr[1] == 0xff) && (addr[2] == 0xff) && \
+ (addr[3] == 0xff) && (addr[4] == 0xff) && (addr[5] == 0xff));
+ }
+
+-extern __inline int is_zero_mac_addr(const u8 *addr)
++static inline int is_zero_mac_addr(const u8 *addr)
+ {
+ return ((addr[0] == 0x00) && (addr[1] == 0x00) && (addr[2] == 0x00) && \
+ (addr[3] == 0x00) && (addr[4] == 0x00) && (addr[5] == 0x00));
+diff -up linux-5.5/drivers/net/wireless/rtl8821ce/include/ieee80211.h.omv~ linux-5.5/drivers/net/wireless/rtl8821ce/include/ieee80211.h
+--- linux-5.5/drivers/net/wireless/rtl8821ce/include/ieee80211.h.omv~ 2020-03-01 04:41:48.093641990 +0100
++++ linux-5.5/drivers/net/wireless/rtl8821ce/include/ieee80211.h 2020-03-01 04:42:05.069642739 +0100
+@@ -1581,18 +1581,18 @@ enum ieee80211_state {
+ (((Addr[2]) & 0xff) == 0xff) && (((Addr[3]) & 0xff) == 0xff) && (((Addr[4]) & 0xff) == 0xff) && \
+ (((Addr[5]) & 0xff) == 0xff))
+ #else
+-extern __inline int is_multicast_mac_addr(const u8 *addr)
++static inline int is_multicast_mac_addr(const u8 *addr)
+ {
+ return (addr[0] != 0xff) && (0x01 & addr[0]);
+ }
+
+-extern __inline int is_broadcast_mac_addr(const u8 *addr)
++static inline int is_broadcast_mac_addr(const u8 *addr)
+ {
+ return ((addr[0] == 0xff) && (addr[1] == 0xff) && (addr[2] == 0xff) && \
+ (addr[3] == 0xff) && (addr[4] == 0xff) && (addr[5] == 0xff));
+ }
+
+-extern __inline int is_zero_mac_addr(const u8 *addr)
++static inline int is_zero_mac_addr(const u8 *addr)
+ {
+ return ((addr[0] == 0x00) && (addr[1] == 0x00) && (addr[2] == 0x00) && \
+ (addr[3] == 0x00) && (addr[4] == 0x00) && (addr[5] == 0x00));
diff --git a/extra-wifi-drivers-port-to-5.6.patch b/extra-wifi-drivers-port-to-5.6.patch
new file mode 100644
index 0000000..adcce3b
--- /dev/null
+++ b/extra-wifi-drivers-port-to-5.6.patch
@@ -0,0 +1,412 @@
+diff -up linux-5.6-rc4/drivers/net/wireless/rtl8723de/os_dep/linux/rtw_proc.c.omv~ linux-5.6-rc4/drivers/net/wireless/rtl8723de/os_dep/linux/rtw_proc.c
+--- linux-5.6-rc4/drivers/net/wireless/rtl8723de/os_dep/linux/rtw_proc.c.omv~ 2020-03-02 19:01:13.164067081 +0100
++++ linux-5.6-rc4/drivers/net/wireless/rtl8723de/os_dep/linux/rtw_proc.c 2020-03-02 23:08:55.704439445 +0100
+@@ -69,12 +69,12 @@ inline struct proc_dir_entry *rtw_proc_c
+ }
+
+ inline struct proc_dir_entry *rtw_proc_create_entry(const char *name, struct proc_dir_entry *parent,
+- const struct file_operations *fops, void * data)
++ const struct proc_ops *ops, void * data)
+ {
+ struct proc_dir_entry *entry;
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26))
+- entry = proc_create_data(name, S_IFREG | S_IRUGO | S_IWUGO, parent, fops, data);
++ entry = proc_create_data(name, S_IFREG | S_IRUGO | S_IWUGO, parent, ops, data);
+ #else
+ entry = create_proc_entry(name, S_IFREG | S_IRUGO | S_IWUGO, parent);
+ if (entry) {
+@@ -219,22 +219,20 @@ static ssize_t rtw_drv_proc_write(struct
+ return -EROFS;
+ }
+
+-static const struct file_operations rtw_drv_proc_seq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_drv_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = seq_release,
+- .write = rtw_drv_proc_write,
++static const struct proc_ops rtw_drv_proc_seq_ops = {
++ .proc_open = rtw_drv_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = seq_release,
++ .proc_write = rtw_drv_proc_write,
+ };
+
+-static const struct file_operations rtw_drv_proc_sseq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_drv_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = single_release,
+- .write = rtw_drv_proc_write,
++static const struct proc_ops rtw_drv_proc_sseq_ops = {
++ .proc_open = rtw_drv_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = single_release,
++ .proc_write = rtw_drv_proc_write,
+ };
+
+ int rtw_drv_proc_init(void)
+@@ -257,9 +255,9 @@ int rtw_drv_proc_init(void)
+
+ for (i = 0; i < drv_proc_hdls_num; i++) {
+ if (drv_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+- entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_seq_fops, (void *)i);
++ entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_seq_ops, (void *)i);
+ else if (drv_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ)
+- entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_sseq_fops, (void *)i);
++ entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_sseq_ops, (void *)i);
+ else
+ entry = NULL;
+
+@@ -2335,22 +2333,20 @@ static ssize_t rtw_adapter_proc_write(st
+ return -EROFS;
+ }
+
+-static const struct file_operations rtw_adapter_proc_seq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_adapter_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = seq_release,
+- .write = rtw_adapter_proc_write,
++static const struct proc_ops rtw_adapter_proc_seq_ops = {
++ .proc_open = rtw_adapter_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = seq_release,
++ .proc_write = rtw_adapter_proc_write,
+ };
+
+-static const struct file_operations rtw_adapter_proc_sseq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_adapter_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = single_release,
+- .write = rtw_adapter_proc_write,
++static const struct proc_ops rtw_adapter_proc_sseq_ops = {
++ .proc_open = rtw_adapter_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = single_release,
++ .proc_write = rtw_adapter_proc_write,
+ };
+
+ int proc_get_odm_dbg_comp(struct seq_file *m, void *v)
+@@ -2670,22 +2666,20 @@ static ssize_t rtw_odm_proc_write(struct
+ return -EROFS;
+ }
+
+-static const struct file_operations rtw_odm_proc_seq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_odm_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = seq_release,
+- .write = rtw_odm_proc_write,
++static const struct proc_ops rtw_odm_proc_seq_ops = {
++ .proc_open = rtw_odm_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = seq_release,
++ .proc_write = rtw_odm_proc_write,
+ };
+
+-static const struct file_operations rtw_odm_proc_sseq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_odm_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = single_release,
+- .write = rtw_odm_proc_write,
++static const struct proc_ops rtw_odm_proc_sseq_ops = {
++ .proc_open = rtw_odm_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = single_release,
++ .proc_write = rtw_odm_proc_write,
+ };
+
+ struct proc_dir_entry *rtw_odm_proc_init(struct net_device *dev)
+@@ -2715,9 +2709,9 @@ struct proc_dir_entry *rtw_odm_proc_init
+
+ for (i = 0; i < odm_proc_hdls_num; i++) {
+ if (odm_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+- entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_seq_fops, (void *)i);
++ entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_seq_ops, (void *)i);
+ else if (odm_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ)
+- entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_sseq_fops, (void *)i);
++ entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_sseq_ops, (void *)i);
+ else
+ entry = NULL;
+
+@@ -2809,22 +2803,20 @@ static ssize_t rtw_mcc_proc_write(struct
+ return -EROFS;
+ }
+
+-static const struct file_operations rtw_mcc_proc_seq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_mcc_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = seq_release,
+- .write = rtw_mcc_proc_write,
++static const struct file_operations rtw_mcc_proc_seq_ops = {
++ .proc_open = rtw_mcc_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = seq_release,
++ .proc_write = rtw_mcc_proc_write,
+ };
+
+-static const struct file_operations rtw_mcc_proc_sseq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_mcc_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = single_release,
+- .write = rtw_mcc_proc_write,
++static const struct file_operations rtw_mcc_proc_sseq_ops = {
++ .proc_open = rtw_mcc_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = single_release,
++ .proc_write = rtw_mcc_proc_write,
+ };
+
+ struct proc_dir_entry *rtw_mcc_proc_init(struct net_device *dev)
+@@ -2854,9 +2846,9 @@ struct proc_dir_entry *rtw_mcc_proc_init
+
+ for (i = 0; i < mcc_proc_hdls_num; i++) {
+ if (mcc_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+- entry = rtw_proc_create_entry(mcc_proc_hdls[i].name, dir_mcc, &rtw_mcc_proc_seq_fops, (void *)i);
++ entry = rtw_proc_create_entry(mcc_proc_hdls[i].name, dir_mcc, &rtw_mcc_proc_seq_ops, (void *)i);
+ else if (mcc_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ)
+- entry = rtw_proc_create_entry(mcc_proc_hdls[i].name, dir_mcc, &rtw_mcc_proc_sseq_fops, (void *)i);
++ entry = rtw_proc_create_entry(mcc_proc_hdls[i].name, dir_mcc, &rtw_mcc_proc_sseq_ops, (void *)i);
+ else
+ entry = NULL;
+
+@@ -2920,9 +2912,9 @@ struct proc_dir_entry *rtw_adapter_proc_
+
+ for (i = 0; i < adapter_proc_hdls_num; i++) {
+ if (adapter_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+- entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_seq_fops, (void *)i);
++ entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_seq_ops, (void *)i);
+ else if (adapter_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ)
+- entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_sseq_fops, (void *)i);
++ entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_sseq_ops, (void *)i);
+ else
+ entry = NULL;
+
+diff -up linux-5.6-rc4/drivers/net/wireless/rtl8821ce/os_dep/linux/rtw_proc.c.omv~ linux-5.6-rc4/drivers/net/wireless/rtl8821ce/os_dep/linux/rtw_proc.c
+--- linux-5.6-rc4/drivers/net/wireless/rtl8821ce/os_dep/linux/rtw_proc.c.omv~ 2020-03-02 23:04:39.749450254 +0100
++++ linux-5.6-rc4/drivers/net/wireless/rtl8821ce/os_dep/linux/rtw_proc.c 2020-03-02 23:08:18.209441028 +0100
+@@ -62,12 +62,12 @@ inline struct proc_dir_entry *rtw_proc_c
+ }
+
+ inline struct proc_dir_entry *rtw_proc_create_entry(const char *name, struct proc_dir_entry *parent,
+- const struct file_operations *fops, void * data)
++ const struct proc_ops *ops, void * data)
+ {
+ struct proc_dir_entry *entry;
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26))
+- entry = proc_create_data(name, S_IFREG | S_IRUGO | S_IWUGO, parent, fops, data);
++ entry = proc_create_data(name, S_IFREG | S_IRUGO | S_IWUGO, parent, ops, data);
+ #else
+ entry = create_proc_entry(name, S_IFREG | S_IRUGO | S_IWUGO, parent);
+ if (entry) {
+@@ -238,22 +238,20 @@ static ssize_t rtw_drv_proc_write(struct
+ return -EROFS;
+ }
+
+-static const struct file_operations rtw_drv_proc_seq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_drv_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = seq_release,
+- .write = rtw_drv_proc_write,
++static const struct proc_ops rtw_drv_proc_seq_ops = {
++ .proc_open = rtw_drv_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = seq_release,
++ .proc_write = rtw_drv_proc_write,
+ };
+
+-static const struct file_operations rtw_drv_proc_sseq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_drv_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = single_release,
+- .write = rtw_drv_proc_write,
++static const struct proc_ops rtw_drv_proc_sseq_ops = {
++ .proc_open = rtw_drv_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = single_release,
++ .proc_write = rtw_drv_proc_write,
+ };
+
+ int rtw_drv_proc_init(void)
+@@ -276,9 +274,9 @@ int rtw_drv_proc_init(void)
+
+ for (i = 0; i < drv_proc_hdls_num; i++) {
+ if (drv_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+- entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_seq_fops, (void *)i);
++ entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_seq_ops, (void *)i);
+ else if (drv_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ)
+- entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_sseq_fops, (void *)i);
++ entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_sseq_ops, (void *)i);
+ else
+ entry = NULL;
+
+@@ -3899,22 +3897,20 @@ static ssize_t rtw_adapter_proc_write(st
+ return -EROFS;
+ }
+
+-static const struct file_operations rtw_adapter_proc_seq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_adapter_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = seq_release,
+- .write = rtw_adapter_proc_write,
++static const struct proc_ops rtw_adapter_proc_seq_ops = {
++ .proc_open = rtw_adapter_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = seq_release,
++ .proc_write = rtw_adapter_proc_write,
+ };
+
+-static const struct file_operations rtw_adapter_proc_sseq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_adapter_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = single_release,
+- .write = rtw_adapter_proc_write,
++static const struct proc_ops rtw_adapter_proc_sseq_ops = {
++ .proc_open = rtw_adapter_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = single_release,
++ .proc_write = rtw_adapter_proc_write,
+ };
+
+ int proc_get_odm_adaptivity(struct seq_file *m, void *v)
+@@ -4068,22 +4064,20 @@ static ssize_t rtw_odm_proc_write(struct
+ return -EROFS;
+ }
+
+-static const struct file_operations rtw_odm_proc_seq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_odm_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = seq_release,
+- .write = rtw_odm_proc_write,
++static const struct proc_ops rtw_odm_proc_seq_ops = {
++ .proc_open = rtw_odm_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = seq_release,
++ .proc_write = rtw_odm_proc_write,
+ };
+
+-static const struct file_operations rtw_odm_proc_sseq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_odm_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = single_release,
+- .write = rtw_odm_proc_write,
++static const struct proc_ops rtw_odm_proc_sseq_ops = {
++ .proc_open = rtw_odm_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = single_release,
++ .proc_write = rtw_odm_proc_write,
+ };
+
+ struct proc_dir_entry *rtw_odm_proc_init(struct net_device *dev)
+@@ -4113,9 +4107,9 @@ struct proc_dir_entry *rtw_odm_proc_init
+
+ for (i = 0; i < odm_proc_hdls_num; i++) {
+ if (odm_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+- entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_seq_fops, (void *)i);
++ entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_seq_ops, (void *)i);
+ else if (odm_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ)
+- entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_sseq_fops, (void *)i);
++ entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_sseq_ops, (void *)i);
+ else
+ entry = NULL;
+
+@@ -4209,22 +4203,20 @@ static ssize_t rtw_mcc_proc_write(struct
+ return -EROFS;
+ }
+
+-static const struct file_operations rtw_mcc_proc_seq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_mcc_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = seq_release,
+- .write = rtw_mcc_proc_write,
++static const struct proc_ops rtw_mcc_proc_seq_ops = {
++ .proc_open = rtw_mcc_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = seq_release,
++ .proc_write = rtw_mcc_proc_write,
+ };
+
+-static const struct file_operations rtw_mcc_proc_sseq_fops = {
+- .owner = THIS_MODULE,
+- .open = rtw_mcc_proc_open,
+- .read = seq_read,
+- .llseek = seq_lseek,
+- .release = single_release,
+- .write = rtw_mcc_proc_write,
++static const struct proc_ops rtw_mcc_proc_sseq_ops = {
++ .proc_open = rtw_mcc_proc_open,
++ .proc_read = seq_read,
++ .proc_lseek = seq_lseek,
++ .proc_release = single_release,
++ .proc_write = rtw_mcc_proc_write,
+ };
+
+ struct proc_dir_entry *rtw_mcc_proc_init(struct net_device *dev)
+@@ -4254,9 +4246,9 @@ struct proc_dir_entry *rtw_mcc_proc_init
+
+ for (i = 0; i < mcc_proc_hdls_num; i++) {
+ if (mcc_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+- entry = rtw_proc_create_entry(mcc_proc_hdls[i].name, dir_mcc, &rtw_mcc_proc_seq_fops, (void *)i);
++ entry = rtw_proc_create_entry(mcc_proc_hdls[i].name, dir_mcc, &rtw_mcc_proc_seq_ops, (void *)i);
+ else if (mcc_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ)
+- entry = rtw_proc_create_entry(mcc_proc_hdls[i].name, dir_mcc, &rtw_mcc_proc_sseq_fops, (void *)i);
++ entry = rtw_proc_create_entry(mcc_proc_hdls[i].name, dir_mcc, &rtw_mcc_proc_sseq_ops, (void *)i);
+ else
+ entry = NULL;
+
+@@ -4319,9 +4311,9 @@ struct proc_dir_entry *rtw_adapter_proc_
+
+ for (i = 0; i < adapter_proc_hdls_num; i++) {
+ if (adapter_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+- entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_seq_fops, (void *)i);
++ entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_seq_ops, (void *)i);
+ else if (adapter_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ)
+- entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_sseq_fops, (void *)i);
++ entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_sseq_ops, (void *)i);
+ else
+ entry = NULL;
+
diff --git a/kernel-5.6-kvm-gcc10.patch b/kernel-5.6-kvm-gcc10.patch
new file mode 100644
index 0000000..bb29126
--- /dev/null
+++ b/kernel-5.6-kvm-gcc10.patch
@@ -0,0 +1,12 @@
+diff -up linux-5.6/virt/kvm/kvm_main.c.omv~ linux-5.6/virt/kvm/kvm_main.c
+--- linux-5.6/virt/kvm/kvm_main.c.omv~ 2020-04-17 21:04:40.240584475 +0200
++++ linux-5.6/virt/kvm/kvm_main.c 2020-04-17 21:04:43.150676715 +0200
+@@ -2219,7 +2219,7 @@ static int __kvm_gfn_to_hva_cache_init(s
+ gfn_t start_gfn = gpa >> PAGE_SHIFT;
+ gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
+ gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
+- gfn_t nr_pages_avail;
++ gfn_t nr_pages_avail = 0;
+
+ /* Update ghc->generation before performing any error checks. */
+ ghc->generation = slots->generation;
diff --git a/kernel-5.7-fewer-conditions-for-ARM64_PTR_AUTH.patch b/kernel-5.7-fewer-conditions-for-ARM64_PTR_AUTH.patch
new file mode 100644
index 0000000..63ed69a
--- /dev/null
+++ b/kernel-5.7-fewer-conditions-for-ARM64_PTR_AUTH.patch
@@ -0,0 +1,27 @@
+This is wrong and should never go upstream.
+
+It is needed nevertheless: Without this patch, the config builder doesn't emit
+any value for ARM64_PTR_AUTH because the preconditions aren't met.
+Then on "real" make, it knows more about the toolchain, sees it can be
+enabled, and "make syncconfig" hangs prompting for the value.
+
+Given our toolchain is known good (we'll always have binutils >= 2.33.1 and
+current versions of gcc and clang), inside our package disabling those
+preconditions doesn't do any harm.
+
+diff -up linux-5.7/arch/arm64/Kconfig.omv~ linux-5.7/arch/arm64/Kconfig
+--- linux-5.7/arch/arm64/Kconfig.omv~ 2020-07-25 23:31:02.229462957 +0000
++++ linux-5.7/arch/arm64/Kconfig 2020-07-25 23:31:12.161907200 +0000
+@@ -1502,12 +1502,6 @@ config ARM64_PTR_AUTH
+ bool "Enable support for pointer authentication"
+ default y
+ depends on !KVM || ARM64_VHE
+- depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC
+- # GCC 9.1 and later inserts a .note.gnu.property section note for PAC
+- # which is only understood by binutils starting with version 2.33.1.
+- depends on !CC_IS_GCC || GCC_VERSION < 90100 || LD_VERSION >= 233010000
+- depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE
+- depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS)
+ help
+ Pointer authentication (part of the ARMv8.3 Extensions) provides
+ instructions for signing and authenticating pointers against secret
diff --git a/kernel-5.8-nouveau-write-combining-only-on-x86.patch b/kernel-5.8-nouveau-write-combining-only-on-x86.patch
new file mode 100644
index 0000000..41aa007
--- /dev/null
+++ b/kernel-5.8-nouveau-write-combining-only-on-x86.patch
@@ -0,0 +1,72 @@
+commit bb14834cef399ae99771f0f33fe6b57a68165370
+Author: Bernhard Rosenkränzer <bero@lindev.ch>
+Date: Tue Jul 21 15:45:49 2020 +0200
+
+ nouveau: avoid write-combine mappings for VRAM and IO space
+
+ NOUPSTREAM
+
+ Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+ Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
+ Signed-off-by: Graeme Gregory <graeme.gregory@linaro.org>
+ Signed-off-by: Bernhard Rosenkränzer <bero@lindev.ch>
+ [daniel.thompson@linaro.org:
+ Updated to undo changes to the bar2 mapping in v4.15-rc1, see
+ dfcbd5506817 ("drm/nouveau/imem/nv50: map bar2 write-combined")
+ and b807270cbd87 ("drm/nouveau/imem/nv40: map bar2
+ write-combined") for more details]
+ Signed-off-by: Daniel Thompson <daniel.thompson@linaro.org>
+
+diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
+index c40f127de3d0b2..525b2dc1ed91af 100644
+--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
++++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
+@@ -662,9 +662,14 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
+ case TTM_PL_VRAM:
+ man->flags = TTM_MEMTYPE_FLAG_FIXED |
+ TTM_MEMTYPE_FLAG_MAPPABLE;
++#ifdef CONFIG_X86
+ man->available_caching = TTM_PL_FLAG_UNCACHED |
+ TTM_PL_FLAG_WC;
+ man->default_caching = TTM_PL_FLAG_WC;
++#else
++ man->available_caching = TTM_PL_FLAG_UNCACHED;
++ man->default_caching = TTM_PL_FLAG_UNCACHED;
++#endif
+
+ if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
+ /* Some BARs do not support being ioremapped WC */
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c
+index 086c118488ef5f..d7e4c0e325afb1 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv40.c
+@@ -253,8 +253,13 @@ nv40_instmem_new(struct nvkm_device *device, int index,
+ else
+ bar = 3;
+
++#ifdef CONFIG_X86
+ imem->iomem = ioremap_wc(device->func->resource_addr(device, bar),
+ device->func->resource_size(device, bar));
++#else
++ imem->iomem = ioremap(device->func->resource_addr(device, bar),
++ device->func->resource_size(device, bar));
++#endif
+ if (!imem->iomem) {
+ nvkm_error(&imem->base.subdev, "unable to map PRAMIN BAR\n");
+ return -EFAULT;
+diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
+index 02c4eb28cef44d..80a28164c15041 100644
+--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
++++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
+@@ -171,7 +171,11 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
+
+ /* Make the mapping visible to the host. */
+ iobj->bar = bar;
++#ifdef CONFIG_X86
+ iobj->map = ioremap_wc(device->func->resource_addr(device, 3) +
++#else
++ iobj->map = ioremap(device->func->resource_addr(device, 3) +
++#endif
+ (u32)iobj->bar->addr, size);
+ if (!iobj->map) {
+ nvkm_warn(subdev, "PRAMIN ioremap failed\n");
diff --git a/kernel-release.spec b/kernel-release.spec
index c1cba9e..fe33307 100644
--- a/kernel-release.spec
+++ b/kernel-release.spec
@@ -4,8 +4,8 @@
#end
%define _disable_ld_no_undefined 1
-# (tpg) try to speed up things
-%global optflags %{optflags} -O3
+## STOP: Adding weird and unsupported upstream kernel C/LD flags of any sort
+## yes , including ftlo . O3 and whatever else
# (crazy) , well that new way of doing buil-id symlinks
# does not seems to work, see:
@@ -17,22 +17,22 @@
# This is the place where you set kernel version i.e 4.5.0
# compose tar.xz name and release
%define kernelversion 5
-%define patchlevel 5
-%define sublevel 2
+%define patchlevel 7
+%define sublevel 11
%define relc %{nil}
# Only ever wrong on x.0 releases...
%define previous %{kernelversion}.%(echo $((%{patchlevel}-1)))
%define buildrel %{kversion}-%{buildrpmrel}
-%define rpmtag %{disttag}
+%define rpmtag %{disttag}
# IMPORTANT
# This is the place where you set release version %{version}-1omv2015
%if 0%{relc}
%define rpmrel 0.rc%{relc}.1
-%define tar_ver %{kernelversion}.%{patchlevel}-rc%{relc}
+%define tar_ver %{kernelversion}.%{patchlevel}-rc%{relc}
%else
-%define rpmrel 1
+%define rpmrel 2
%define tar_ver %{kernelversion}.%{patchlevel}
%endif
%define buildrpmrel %{rpmrel}%{rpmtag}
@@ -65,7 +65,7 @@
# Build defines
%bcond_with build_doc
%ifarch %{ix86} %{x86_64}
-%bcond_with uksm
+%bcond_without uksm
%else
%bcond_with uksm
%endif
@@ -73,7 +73,12 @@
%bcond_without build_devel
%bcond_with build_debug
%bcond_with clang
-%bcond_with bootsplash
+## enabled it runs dracut -f --regenerate-all
+## we *should* enable that, is bc we keep or can keep lots
+## kernel around and the initrd is created using sys libs, sys configs,
+## *systemd* service & apps etc. IOW, a old initrd may have old files, libs, etc
+## changed since last rebuild and may result in either broken boot, or very hard to debug bugs.
+%bcond_with dracut_all_initrd
# (tpg) enable patches from ClearLinux
%bcond_without clr
%if %mdvver > 3000000
@@ -82,7 +87,7 @@
%bcond_with cross_headers
%endif
-%global cross_header_archs aarch64-linux armv7hnl-linux i686-linux x86_64-linux x32-linux riscv32-linux riscv64-linux aarch64-linuxmusl armv7hnl-linuxmusl i686-linuxmusl x86_64-linuxmusl x32-linuxmusl riscv32-linuxmusl riscv64-linuxmusl aarch64-android armv7l-android armv8l-android
+%global cross_header_archs aarch64-linux armv7hnl-linux i686-linux x86_64-linux x32-linux riscv32-linux riscv64-linux aarch64-linuxmusl armv7hnl-linuxmusl i686-linuxmusl x86_64-linuxmusl x32-linuxmusl riscv32-linuxmusl riscv64-linuxmusl aarch64-android armv7l-android armv8l-android x86_64-android aarch64-linuxuclibc armv7hnl-linuxuclibc i686-linuxuclibc x86_64-linuxuclibc x32-linuxuclibc riscv32-linuxuclibc riscv64-linuxuclibc
%global long_cross_header_archs %(
for i in %{cross_header_archs}; do
CPU=$(echo $i |cut -d- -f1)
@@ -120,7 +125,6 @@
# unfortunately kmod does not support Zstandard for now, so kernel modules
# compressed with zstd will not bo loaded and system will fail
# https://github.com/facebook/zstd/issues/1121
-# Currently only supported on x86
%ifarch %{ix86} %{x86_64}
%bcond_without build_modzstd
# compress modules with XZ
@@ -222,84 +226,29 @@ Source51: cpupower.config
%if 0%{sublevel}
Source90: https://cdn.kernel.org/pub/linux/kernel/v%(echo %{version}|cut -d. -f1).x/patch-%{version}.xz
%endif
+Patch1: linux-5.6-fix-disassembler-4args-detection.patch
Patch2: die-floppy-die.patch
Patch3: 0001-Add-support-for-Acer-Predator-macro-keys.patch
Patch4: linux-4.7-intel-dvi-duallink.patch
-Patch5: linux-4.8.1-buildfix.patch
+Patch5: kernel-5.6-kvm-gcc10.patch
Patch6: linux-5.2.9-riscv-compile.patch
# Work around rpm dependency generator screaming about
# error: Illegal char ']' (0x5d) in: 1.2.1[50983]_custom
# caused by aacraid versioning ("1.2.1[50983]-custom")
Patch7: aacraid-dont-freak-out-dependency-generator.patch
-
-%if %{with clang}
-# Patches to make it build with clang
-Patch1000: 0001-kbuild-LLVMLinux-Set-compiler-flags-for-clang.patch
-Patch1001: 0002-fs-LLVMLinux-Remove-warning-from-COMPATIBLE_IOCTL.patch
-Patch1002: 0003-kbuild-LLVMLinux-Add-support-for-generating-LLVM-bit.patch
-Patch1003: 0004-kbuild-LLVMLinux-Make-asm-offset-generation-work-wit.patch
-Patch1004: 0005-md-sysfs-LLVMLinux-Remove-nested-function-from-bcach.patch
-Patch1005: 0006-apparmor-LLVMLinux-Remove-VLAIS.patch
-Patch1006: 0007-exofs-LLVMLinux-Remove-VLAIS-from-exofs-FIXME-Check-.patch
-Patch1007: 0008-md-raid10-LLVMLinux-Remove-VLAIS-from-raid10-driver.patch
-Patch1008: 0009-fs-nfs-LLVMLinux-Remove-VLAIS-from-nfs.patch
-Patch1009: 0010-net-wimax-i2400-LLVMLinux-Remove-VLAIS-from-wimax-i2.patch
-Patch1010: 0011-Kbuild-LLVMLinux-Use-Oz-instead-of-Os-when-using-cla.patch
-Patch1011: 0012-WORKAROUND-x86-boot-LLVMLinux-Work-around-clang-PR39.patch
-Patch1012: 0013-DO-NOT-UPSTREAM-xen-LLVMLinux-Remove-VLAIS-from-xen-.patch
-Patch1013: 0014-DO-NOT-UPSTREAM-arm-LLVMLinux-Provide-__aeabi_-symbo.patch
-Patch1014: 0015-DO-NOT-UPSTREAM-arm-firmware-LLVMLinux-replace-naked.patch
-Patch1015: 0016-arm-LLVMLinux-Remove-unreachable-from-naked-function.patch
-Patch1016: 0017-MIPS-LLVMLinux-Fix-a-cast-to-type-not-present-in-uni.patch
-Patch1017: 0018-MIPS-LLVMLinux-Fix-an-inline-asm-input-output-type-m.patch
-Patch1018: 0019-MIPS-LLVMLinux-Silence-variable-self-assignment-warn.patch
-Patch1019: 0020-MIPS-LLVMLinux-Silence-unicode-warnings-when-preproc.patch
-Patch1020: 0021-Don-t-use-attributes-error-and-warning-with-clang.patch
-Patch1021: 0022-Fix-undefined-references-to-acpi_idle_driver-on-aarc.patch
-Patch1022: 0023-HACK-firmware-LLVMLinux-fix-EFI-libstub-with-clang.patch
-Patch1023: 0024-aarch64-crypto-LLVMLinux-Fix-inline-assembly-for-cla.patch
-Patch1024: 0025-aarch64-LLVMLinux-Make-spin_lock_prefetch-asm-code-c.patch
-Patch1025: 0026-LLVMLinux-Don-t-use-attribute-externally_visible-whe.patch
-Patch1026: 0027-x86-crypto-LLVMLinux-Fix-building-x86_64-AES-extensi.patch
-Patch1027: 0028-x86-LLVMLinux-Qualify-mul-as-mulq-to-make-clang-happ.patch
-Patch1028: 0029-kbuild-LLVMLinux-Add-Werror-to-cc-option-in-order-to.patch
-Patch1029: 0030-x86-kbuild-LLVMLinux-Check-for-compiler-support-of-f.patch
-#Patch1030: 0031-x86-cmpxchg-break.patch
-Patch1031: 0001-Fix-for-compilation-with-clang.patch
-%endif
-
-# Bootsplash system
-# (tpg) disable it for now 2018-11-07
-%if %{with bootsplash}
-# https://lkml.org/lkml/2017/10/25/346
-# https://patchwork.kernel.org/patch/10172665/, rebased
-Patch100: RFC-v3-01-13-bootsplash-Initial-implementation-showing-black-screen.patch
-# https://patchwork.kernel.org/patch/10172669/
-Patch101: RFC-v3-02-13-bootsplash-Add-file-reading-and-picture-rendering.patch
-# https://patchwork.kernel.org/patch/10172715/
-Patch102: RFC-v3-03-13-bootsplash-Flush-framebuffer-after-drawing.patch
-# https://patchwork.kernel.org/patch/10172699/
-Patch103: RFC-v3-04-13-bootsplash-Add-corner-positioning.patch
-# https://patchwork.kernel.org/patch/10172667/
-Patch104: RFC-v3-05-13-bootsplash-Add-animation-support.patch
-# https://patchwork.kernel.org/patch/10172605/, rebased
-Patch105: RFC-v3-06-13-vt-Redraw-bootsplash-fully-on-console_unblank.patch
-# https://patchwork.kernel.org/patch/10172599/
-Patch106: RFC-v3-07-13-vt-Add-keyboard-hook-to-disable-bootsplash.patch
-# https://patchwork.kernel.org/patch/10172603/
-Patch107: RFC-v3-08-13-sysrq-Disable-bootsplash-on-SAK.patch
-# https://patchwork.kernel.org/patch/10172601/
-Patch108: RFC-v3-09-13-fbcon-Disable-bootsplash-on-oops.patch
-# https://patchwork.kernel.org/patch/10172663/
-Patch109: RFC-v3-10-13-Documentation-Add-bootsplash-main-documentation.patch
-# https://patchwork.kernel.org/patch/10172685/
-Patch110: RFC-v3-11-13-bootsplash-sysfs-entries-to-load-and-unload-files.patch
-# https://patchwork.kernel.org/patch/10172597/
-Patch111: RFC-v3-12-13-tools-bootsplash-Add-a-basic-splash-file-creation-tool.patch
-# https://patchwork.kernel.org/patch/10172661/
-# Contains git binary patch -- needs to be applied with git apply instead of autopatch -p1
-Source112: RFC-v3-13-13-tools-bootsplash-Add-script-and-data-to-create-sample-file.patch
-%endif
+# Make uClibc-ng happy
+Patch8: socket.h-include-bitsperlong.h.patch
+# Make Nouveau work on SynQuacer (and probably all other non-x86 boards)
+Patch9: kernel-5.8-nouveau-write-combining-only-on-x86.patch
+Patch10: kernel-5.7-fewer-conditions-for-ARM64_PTR_AUTH.patch
+
+# FIXME git bisect shows upstream commit
+# 7a8b64d17e35810dc3176fe61208b45c15d25402 breaks
+# booting SynQuacer from USB flash drives.
+# 9d55bebd9816903b821a403a69a94190442ac043 builds on
+# 7a8b64d17e35810dc3176fe61208b45c15d25402.
+Source100: 7a8b64d17e35810dc3176fe61208b45c15d25402.patch
+Source101: 9d55bebd9816903b821a403a69a94190442ac043.patch
# Patches to VirtualBox and other external modules are
# pulled in as Source: rather than Patch: because it's arch specific
@@ -308,20 +257,16 @@ Source112: RFC-v3-13-13-tools-bootsplash-Add-script-and-data-to-create-sample-fi
# (tpg) The Ultra Kernel Same Page Deduplication
# (tpg) http://kerneldedup.org/en/projects/uksm/download/
# (tpg) sources can be found here https://github.com/dolohow/uksm
+# (crazy) each new patch has to be checked for GPL violation on ksm.h
+# it cannot be re-licensed to GPL3 by random patches.
%if %{with uksm}
# brokes armx builds
-Patch120: https://raw.githubusercontent.com/dolohow/uksm/master/v5.x/uksm-5.4.patch
-# Sometimes other people are ahead of upstream porting to new releases...
-# No UKSM for 5.5 yet... :/
-#Patch120: https://github.com/sirlucjan/kernel-patches/raw/master/5.1/uksm-pf/0001-uksm-5.1-initial-submission.patch
-#Patch121: https://github.com/sirlucjan/kernel-patches/raw/master/5.1/uksm-pf-fix/0001-uksm-5.1-apply-52d1e606ee733.patch
+Patch120: https://raw.githubusercontent.com/dolohow/uksm/master/v5.x/uksm-5.7.patch
%endif
%if %{with build_modzstd}
-# https://patchwork.kernel.org/patch/10003007/
-Patch126: v2-1-2-lib-Add-support-for-ZSTD-compressed-kernel.patch
-# https://patchwork.kernel.org/patch/10003011/
-Patch127: v2-2-2-x86-Add-support-for-ZSTD-compressed-kernel.patch
+# v4 -> https://lkml.org/lkml/2020/4/1/29
+Patch126: https://gitweb.frugalware.org/frugalware-current/raw/master/source/base/kernel/support-kernel-and-ramfs-comp-and-decomp-with-zstd.patch
%endif
### Additional hardware support
@@ -342,20 +287,23 @@ Patch146: saa716x-4.15.patch
Patch147: saa716x-linux-4.19.patch
Patch148: saa716x-5.4.patch
+# Additional WiFi drivers taken from the Endless kernel
+# git clone https://github.com/endlessm/linux.git
+# cd linux
+# tar cf extra-wifi-drivers-`date +%Y%m%d`.tar drivers/net/wireless/rtl8*
+# zstd -19 extra-wifi-drivers*.tar
+Source200: extra-wifi-drivers-20200301.tar.zst
+Patch201: extra-wifi-drivers-compile.patch
+Patch202: extra-wifi-drivers-port-to-5.6.patch
+
# Lima driver for ARM Mali graphics chips
# Generated from https://gitlab.freedesktop.org/lima/linux.git
# using git diff v5.1..lima/lima-5.1
# Currently no patch necessary
-# NOT YET
-#Patch250: 4.14-C11.patch
-
-# VirtualBox shared folders support
-# https://patchwork.kernel.org/patch/10906949/
-# For newer versions, check
-# https://patchwork.kernel.org/project/linux-fsdevel/list/?submitter=582
-Patch300: v19-fs-Add-VirtualBox-guest-shared-folder-vboxsf-support.diff
-Source300: virtualbox-kernel-5.3.patch
+# VirtualBox patches -- added as Source: rather than Patch:
+# because they need to be applied after stuff from the
+# virtualbox-kernel-module-sources package is copied around
Source301: vbox-6.1-fix-build-on-znver1-hosts.patch
# Better support for newer x86 processors
@@ -405,11 +353,6 @@ Patch800: Unknow-SSD-HFM128GDHTNG-8310B-QUIRK_NO_APST.patch
# Restore ACPI loglevels to sane values
Patch801: https://gitweb.frugalware.org/wip_kernel/raw/86234abea5e625043153f6b8295642fd9f42bff0/source/base/kernel/acpi-use-kern_warning_even_when_error.patch
Patch802: https://gitweb.frugalware.org/wip_kernel/raw/23f5e50042768b823e18613151cc81b4c0cf6e22/source/base/kernel/fix-acpi_dbg_level.patch
-# (tpg) enable MuQSS CPU scheduler
-# FIXME re-enable when ported to 5.3
-Patch803: http://ck.kolivas.org/patches/muqss/5.0/5.4/0001-MultiQueue-Skiplist-Scheduler-v0.196.patch
-# (bero) And make it compatible with modular binder
-Patch804: MuQSS-export-can_nice-for-binder.patch
# (crazy) need to know what function() breaks on nvme failures
Patch809: nvme-pci-more-info.patch
# ( crazy ) this one is adding be_silent mod parameter to acer-wmi
@@ -419,6 +362,8 @@ Patch809: nvme-pci-more-info.patch
# until is implemented / fixed.
#Patch810: acer-wmi-silence-unknow-functions-messages.patch
Patch810: linux-5.4.5-fix-build.patch
+Patch812: linux-5.5-corsair-strafe-quirks.patch
+Patch814: http://crazy.dev.frugalware.org/smpboot-no-stack-protector-for-gcc10.patch
# Defines for the things that are needed for all the kernels
#
@@ -465,9 +410,9 @@ BuildRequires: flex
BuildRequires: bison
BuildRequires: binutils
BuildRequires: hostname
-BuildRequires: gcc >= 7.2.1_2017.11-3
-BuildRequires: gcc-plugin-devel >= 7.2.1_2017.11-3
-BuildRequires: gcc-c++ >= 7.2.1_2017.11-3
+BuildRequires: gcc
+BuildRequires: gcc-c++
+BuildRequires: pkgconfig(libcap)
BuildRequires: pkgconfig(libssl)
BuildRequires: diffutils
# For git apply
@@ -485,6 +430,10 @@ BuildRequires: numa-devel
BuildRequires: pkgconfig(libpci)
%endif
+%if %{with build_turbostat}
+BuildRequires: pkgconfig(libpcap)
+%endif
+
# for docs
%if %{with build_doc}
BuildRequires: xmlto
@@ -499,17 +448,15 @@ BuildRequires: asciidoc
BuildRequires: pkgconfig(audit)
BuildRequires: binutils-devel
BuildRequires: bison
-# BuildRequires: docbook-style-xsl
BuildRequires: flex
-# BuildRequires: gettext
-# BuildRequires: gtk2-devel
BuildRequires: pkgconfig(libunwind)
BuildRequires: pkgconfig(libnewt)
BuildRequires: perl-devel
-# BuildRequires: perl(ExtUtils::Embed)
BuildRequires: pkgconfig(gtk+-2.0)
BuildRequires: pkgconfig(python)
BuildRequires: pkgconfig(zlib)
+# (tpg) needed for bfd
+BuildRequires: binutils-devel
%endif
%ifarch %{arm}
@@ -524,8 +471,8 @@ Suggests: microcode-intel
# get compiler error messages on failures)
%if %mdvver >= 3000000
%ifarch %{x86_64}
-BuildRequires: virtualbox-kernel-module-sources
-BuildRequires: virtualbox-guest-kernel-module-sources
+BuildRequires: virtualbox-kernel-module-sources >= 6.1.10
+BuildRequires: virtualbox-guest-kernel-module-sources >= 6.1.10
%endif
%endif
@@ -552,7 +499,7 @@ Requires(pre): %requires3 %requires4 \
Requires: %requires5 \
Obsoletes: %kobsoletes1 %kobsoletes2 %kobsoletes3 \
Conflicts: %kconflicts1 %kconflicts2 %kconflicts3 \
-Conflicts: %kconflicts4 %kconflicts5 \
+Conflicts: %kconflicts4 %kconflicts5 \
Conflicts: %{kname}-%{1}-latest <= %{kversion}-%{rpmrel} \
Obsoletes: %{kname}-%{1}-latest <= %{kversion}-%{rpmrel} \
Provides: installonlypkg(kernel) \
@@ -636,7 +583,6 @@ needs debugging info from the kernel, this package may help. \
\
%post -n %{kname}-%{1} -f kernel_files.%{1}-post \
%posttrans -n %{kname}-%{1} -f kernel_files.%{1}-posttrans \
-%preun -n %{kname}-%{1} -f kernel_files.%{1}-preun \
%postun -n %{kname}-%{1} -f kernel_files.%{1}-postun \
\
%if %{with build_devel} \
@@ -678,7 +624,7 @@ voluntary preempt, CFS cpu scheduler and BFQ i/o scheduler, ONDEMAND governor.
%define summary_server Linux Kernel for server use with i686 & 64GB RAM
%define info_server This kernel is compiled for server use, single or \
multiple i686 processor(s)/core(s) and up to 64GB RAM using PAE, using \
-no preempt, HZ_100, CFS cpu scheduler and BFQ i/o scheduler, PERFORMANCE governor.
+no preempt, HZ_300, CFS cpu scheduler and BFQ i/o scheduler, PERFORMANCE governor.
%else
%define summary_server Linux Kernel for server use with %{_arch}
%define info_server This kernel is compiled for server use, single or \
@@ -780,15 +726,6 @@ Conflicts: %{_lib}cpufreq-devel
This package contains the development files for cpupower.
%endif
-%package -n bootsplash-packer
-Summary: Tool for packing bootsplash images
-Group: System/Kernel and hardware
-Version: %{kversion}
-Release: %{rpmrel}
-
-%description -n bootsplash-packer
-Tool for packing bootsplash images.
-
%if %{with build_x86_energy_perf_policy}
%package -n x86_energy_perf_policy
Version: %{kversion}
@@ -811,6 +748,34 @@ Group: System/Kernel and hardware
Tool to report processor frequency and idle statistics.
%endif
+%define bpf_major 0
+%define libbpf %mklibname bpf %{bpf_major}
+%define libbpfdevel %mklibname bpf -d
+
+%package -n bpftool
+Summary: Inspection and simple manipulation of eBPF programs and maps
+Group: System/Kernel and hardware
+
+%description -n bpftool
+This package contains the bpftool, which allows inspection and simple
+manipulation of eBPF programs and maps.
+
+%package -n %{libbpf}
+Summary: The bpf library from kernel source
+Group: System/Libraries
+
+%description -n %{libbpf}
+This package contains the kernel source bpf library.
+
+%package -n %{libbpfdevel}
+Summary: Developement files for the bpf library from kernel source
+Group: Development/Kernel
+Requires: %{libbpf} = %{EVRD}
+
+%description -n %{libbpfdevel}
+This package includes libraries and header files needed for development
+of applications which use bpf library from kernel sour
+
%package headers
Version: %{kversion}
Release: %{rpmrel}
@@ -871,20 +836,19 @@ done
# End packages - here begins build stage
#
%prep
-%setup -q -n linux-%{tar_ver} -a 140
+%setup -q -n linux-%{tar_ver} -a 140 -a 200
cp %{S:6} %{S:7} %{S:8} %{S:9} %{S:10} %{S:11} %{S:12} %{S:13} kernel/configs/
%if 0%{sublevel}
[ -e .git ] || git init
xzcat %{SOURCE90} |git apply - || git apply %{SOURCE90}
rm -rf .git
%endif
-%if %mdvver > 3000000
%autopatch -p1
-%else
-%autopatch -p1
-%endif
-%if %{with bootsplash}
-git apply %{SOURCE112}
+
+%ifarch %{aarch64}
+# FIXME SynQuacer workaround
+patch -p1 -R <%{S:101}
+patch -p1 -R <%{S:100}
%endif
# merge SAA716x DVB driver from extra tarball
@@ -892,6 +856,14 @@ sed -i -e '/saa7164/isource "drivers/media/pci/saa716x/Kconfig"' drivers/media/p
sed -i -e '/saa7164/iobj-$(CONFIG_SAA716X_CORE) += saa716x/' drivers/media/pci/Makefile
find drivers/media/tuners drivers/media/dvb-frontends -name "*.c" -o -name "*.h" |xargs sed -i -e 's,"dvb_frontend.h",<media/dvb_frontend.h>,g'
+# Merge RTL8723DE and RTL8821CE drivers
+cd drivers/net/wireless
+sed -i -e '/quantenna\/Kconfig/asource "drivers/net/wireless/rtl8821ce/Kconfig' Kconfig
+sed -i -e '/quantenna\/Kconfig/asource "drivers/net/wireless/rtl8723de/Kconfig' Kconfig
+sed -i -e '/QUANTENNA/aobj-$(CONFIG_RTL8821CE) += rtl8821ce/' Makefile
+sed -i -e '/QUANTENNA/aobj-$(CONFIG_RTL8723DE) += rtl8723de/' Makefile
+cd -
+
%if %{with build_debug}
%define debug --debug
%else
@@ -910,9 +882,9 @@ LC_ALL=C sed -i -e "s/^SUBLEVEL.*/SUBLEVEL = %{sublevel}/" Makefile
# There is an in-kernel version of vboxvideo -- unfortunately
# it doesn't seem to work properly with vbox just yet
# Let's replace it with the one that comes with VB for now
-rm -rf drivers/staging/vboxvideo
-cp -a $(ls --sort=time -1d /usr/src/vboxadditions-*|head -n1)/vboxvideo drivers/staging
-cat >drivers/staging/vboxvideo/Kconfig <<'EOF'
+rm -rf drivers/gpu/drm/vboxvideo
+cp -a $(ls --sort=time -1d /usr/src/vboxadditions-*|head -n1)/vboxvideo drivers/gpu/drm/
+cat >drivers/gpu/drm/vboxvideo/Kconfig <<'EOF'
config DRM_VBOXVIDEO
tristate "Virtual Box Graphics Card"
depends on DRM && X86 && PCI
@@ -929,14 +901,14 @@ config DRM_VBOXVIDEO
driver as a module and add support for these devices via drm/kms
interfaces.
EOF
-sed -i -e 's,\$(KBUILD_EXTMOD),drivers/gpu/drm/vboxvideo,g' drivers/staging/vboxvideo/Makefile*
-sed -i -e "s,^KERN_DIR.*,KERN_DIR := $(pwd)," drivers/staging/vboxvideo/Makefile*
+sed -i -e 's,\$(KBUILD_EXTMOD),drivers/gpu/drm/vboxvideo,g' drivers/gpu/drm/vboxvideo/Makefile*
+sed -i -e "s,^KERN_DIR.*,KERN_DIR := $(pwd)," drivers/gpu/drm/vboxvideo/Makefile*
%endif
# 800x600 is too small to be useful -- even calamares doesn't
# fit into that anymore (this fix is needed for both the in-kernel
# version and the vbox version of the driver)
-sed -i -e 's|800, 600|1024, 768|g' drivers/staging/vboxvideo/vbox_mode.c
+sed -i -e 's|800, 600|1024, 768|g' drivers/gpu/drm/vboxvideo/vbox_mode.c
# VirtualBox shared folders now come in through patch 300
# === VirtualBox host modules ===
@@ -960,7 +932,6 @@ cp -a $(ls --sort=time -1d /usr/src/virtualbox-*|head -n1)/vboxpci drivers/pci/
sed -i -e 's,\$(KBUILD_EXTMOD),drivers/pci/vboxpci,g' drivers/pci/vboxpci/Makefile*
sed -i -e "s,^KERN_DIR.*,KERN_DIR := $(pwd)," drivers/pci/vboxpci/Makefile*
echo 'obj-m += vboxpci/' >>drivers/pci/Makefile
-#patch -p1 -z .300a~ -b <%{S:300}
patch -p1 -z .301a~ -b <%{S:301}
%endif
%endif
@@ -1001,12 +972,6 @@ CreateConfig() {
type="$2"
rm -f .config
-%if %{with clang}
- CLANG_EXTRAS=clang-workarounds
-%else
- CLANG_EXTRAS=""
-%endif
-
%if %{with build_modxz}
sed -i -e "s/^# CONFIG_KERNEL_XZ is not set/CONFIG_KERNEL_XZ=y/g" kernel/configs/common.config
%endif
@@ -1028,7 +993,7 @@ sed -i -e "s/^# CONFIG_RD_ZSTD is not set/CONFIG_RD_ZSTD=y/g" kernel/configs/com
;;
esac
- for i in common common-${type} ${arch}-common ${arch}-${type} $CLANG_EXTRAS; do
+ for i in common common-${type} ${arch}-common ${arch}-${type}; do
[ -e kernel/configs/$i.config ] && CONFIGS="$CONFIGS $i.config"
done
if [ "$arch" = "znver1" ]; then
@@ -1064,9 +1029,9 @@ BuildKernel() {
# (tpg) build with gcc, as kernel is not yet ready for LLVM/clang
%ifarch %{x86_64}
%if %{with clang}
- %kmake all CC=clang CXX=clang++ CFLAGS="$CFLAGS -flto"
+ %kmake all CC=clang CXX=clang++ CFLAGS="$CFLAGS"
%else
- %kmake all CC=gcc CXX=g++ CFLAGS="$CFLAGS -flto"
+ %kmake all CC=gcc CXX=g++ CFLAGS="$CFLAGS"
%endif
%else
%if %{with clang}
@@ -1170,9 +1135,6 @@ SaveDevel() {
# Needed for external dvb tree (#41418)
cp -fR drivers/media/dvb-frontends/lgdt330x.h $TempDevelRoot/drivers/media/dvb-frontends/
-# add acpica header files, needed for fglrx build
- cp -fR drivers/acpi/acpica/*.h $TempDevelRoot/drivers/acpi/acpica/
-
# orc unwinder needs theese
cp -fR tools/build/Build{,.include} $TempDevelRoot/tools/build
cp -fR tools/build/fixdep.c $TempDevelRoot/tools/build
@@ -1181,6 +1143,10 @@ SaveDevel() {
cp -fR tools/objtool/* $TempDevelRoot/tools/objtool
cp -fR tools/scripts/utilities.mak $TempDevelRoot/tools/scripts
+# Make clean fails on the include statements in the Makefiles - and the drivers aren't relevant for -devel
+ rm -rf $TempDevelRoot/drivers/net/wireless/rtl8*
+ sed -i -e '/rtl8.*/d' $TempDevelRoot/drivers/net/wireless/{Makefile,Kconfig}
+
for i in alpha arc avr32 blackfin c6x cris csky frv h8300 hexagon ia64 m32r m68k m68knommu metag microblaze \
mips mn10300 nds32 nios2 openrisc parisc powerpc s390 score sh sparc tile unicore32 xtensa; do
rm -rf $TempDevelRoot/arch/$i
@@ -1337,54 +1303,19 @@ cat kernel_exclude_debug_files.$kernel_flavour >> $kernel_files
### Create kernel Post script
cat > $kernel_files-post <<EOF
-# create initrd/grub.cfg for installed kernel first.
+%if %{with dracut_all_initrd}
+[ -x /sbin/dracut ] && /sbin/dracut -f --regenerate-all
+%endif
/sbin/depmod -a %{kversion}-$kernel_flavour-%{buildrpmrel}
-/usr/bin/dracut -f --kver %{kversion}-$kernel_flavour-%{buildrpmrel}
+[ -x /sbin/dracut ] && /sbin/dracut -f --kver %{kversion}-$kernel_flavour-%{buildrpmrel}
-# try rebuild all other initrd's , however that may take a while with lots
-# kernels installed
-cd /boot > /dev/null
-
-for i in $(ls vmlinuz-[0-9]*| sed 's/.*vmlinuz-//g')
-do
- if [[ vmlinuz-$i =~ vmlinuz-%{kversion}-$kernel_flavour-%{buildrpmrel} ]]; then
- # we just create this
- continue
- fi
- if [[ -e "initrd-$i.img" ]]; then
- ## if exist ignore
- continue
- fi
- /sbin/depmod -a "$i"
- /usr/bin/dracut -f --kver "$i"
-done
## cleanup some werid symlinks we never used anyway
rm -rf vmlinuz-{server,desktop} initrd0.img initrd-{server,desktop}
# run update-grub2
-/usr/sbin/update-grub2
-
-# (crazy) only half the story , need grub patches , OM scripts ( including ARM ) removed suport for systemd-boot
-# and so on .. we hit a limit here with lots kernels installed.
-# also half of that is not used bc missing grub part support. Also we produce ofc broken symlinks and ducplicate
-# 'wath-should-be-machine-id' too. I cannot see why we need that anyway.
-
-#/usr/bin/kernel-install add %{kversion}-$kernel_flavour-%{buildrpmrel} /boot/vmlinuz-%{kversion}-$kernel_flavour-%{buildrpmrel}
-#cd /boot > /dev/null
-#if [ -L vmlinuz-$kernel_flavour ]; then
-# rm -f vmlinuz-$kernel_flavour
-#fi
-#ln -sf vmlinuz-%{kversion}-$kernel_flavour-%{buildrpmrel} vmlinuz-$kernel_flavour
-#if [ -L initrd-$kernel_flavour.img ]; then
-# rm -f initrd-$kernel_flavour.img
-#fi
-#ln -sf initrd-%{kversion}-$kernel_flavour-%{buildrpmrel}.img initrd-$kernel_flavour.img
-#if [ -e initrd-%{kversion}-$kernel_flavour-%{buildrpmrel}.img ]; then
-# ln -sf vmlinuz-%{kversion}-$kernel_flavour-%{buildrpmrel} vmlinuz
-# ln -sf initrd-%{kversion}-$kernel_flavour-%{buildrpmrel}.img initrd.img
-#fi
+[ -x /usr/sbin/update-grub2 ] && /usr/sbin/update-grub2
cd - > /dev/null
%if %{with build_devel}
@@ -1399,11 +1330,11 @@ EOF
### Create kernel Posttrans script
cat > $kernel_files-posttrans <<EOF
-if [ -x /usr/sbin/dkms_autoinstaller -a -d /usr/src/linux-%{kversion}-$kernel_flavour-%{buildrpmrel} ]; then
+if [ -x /usr/sbin/dkms_autoinstaller ] && [ -d /usr/src/linux-%{kversion}-$kernel_flavour-%{buildrpmrel} ]; then
/usr/sbin/dkms_autoinstaller start %{kversion}-$kernel_flavour-%{buildrpmrel}
fi
-if [ -x %{_sbindir}/dkms -a -e %{_unitdir}/dkms.service -a -d /usr/src/linux-%{kversion}-$kernel_flavour-%{buildrpmrel} ]; then
+if [ -x %{_sbindir}/dkms ] && [ -e %{_unitdir}/dkms.service ] && [ -d /usr/src/linux-%{kversion}-$kernel_flavour-%{buildrpmrel} ]; then
/bin/systemctl --quiet restart dkms.service
/bin/systemctl --quiet try-restart fedora-loadmodules.service
%{_sbindir}/dkms autoinstall --verbose --kernelver %{kversion}-$kernel_flavour-%{buildrpmrel}
@@ -1411,10 +1342,10 @@ fi
EOF
-### Create kernel Preun script on the fly
-cat > $kernel_files-preun <<EOF
+### Create kernel Postun script on the fly
+cat > $kernel_files-postun <<EOF
-rm -rf /lib/modules/%{kversion}-$kernel_flavour-%{buildrpmrel}/modules.{alias{,.bin},builtin.bin,dep{,.bin},devname,softdep,symbols{,.bin}}
+rm -rf /lib/modules/%{kversion}-$kernel_flavour-%{buildrpmrel}/modules.{alias{,.bin},builtin.bin,dep{,.bin},devname,softdep,symbols{,.bin}} ||:
cd /boot > /dev/null
if [ -e vmlinuz-%{kversion}-$kernel_flavour-%{buildrpmrel} ]; then
@@ -1422,26 +1353,20 @@ if [ -e vmlinuz-%{kversion}-$kernel_flavour-%{buildrpmrel} ]; then
fi
if [ -e initrd-%{kversion}-$kernel_flavour-%{buildrpmrel}.img ]; then
- rm -rf initrd-%{kversion}-$kernel_flavour-%{buildrpmrel}.img
+ rm -rf initrd-%{kversion}-$kernel_flavour-%{buildrpmrel}.img
fi
-#/usr/bin/kernel-install remove %{kversion}-$kernel_flavour-%{buildrpmrel}
-#cd /boot > /dev/null
-## (crazy) we dont use ( nor have support in grub to look ) for initrd-fooname or vmlinuz-fooname
-## so that never worked anyway.
-#if [ -L vmlinuz-$kernel_flavour ]; then
-# if [ "$(readlink vmlinuz-$kernel_flavour)" = "vmlinuz-%{kversion}-$kernel_flavour-%{buildrpmrel}" ]; then
-# rm -f vmlinuz-$kernel_flavour
-# fi
-#fi
-#if [ -L initrd-$kernel_flavour.img ]; then
-# if [ "$(readlink initrd-$kernel_flavour.img)" = "initrd-%{kversion}-$kernel_flavour-%{buildrpmrel}.img" ]; then
-# rm -f initrd-$kernel_flavour.img
-# fi
-#fi
-
cd - > /dev/null
+
+rm -rf /lib/modules/%{kversion}-$kernel_flavour-%{buildrpmrel} >/dev/null
+if [ -d /var/lib/dkms ]; then
+ rm -f /var/lib/dkms/*/kernel-%{kversion}-$devel_flavour-%{buildrpmrel}-%{_target_cpu} >/dev/null
+ rm -rf /var/lib/dkms/*/*/%{kversion}-$devel_flavour-%{buildrpmrel} >/dev/null
+ rm -f /var/lib/dkms-binary/*/kernel-%{kversion}-$devel_flavour-%{buildrpmrel}-%{_target_cpu} >/dev/null
+ rm -rf /var/lib/dkms-binary/*/*/%{kversion}-$devel_flavour-%{buildrpmrel} >/dev/null
+fi
+
%if %{with build_devel}
if [ -L /lib/modules/%{kversion}-$kernel_flavour-%{buildrpmrel}/build ]; then
rm -f /lib/modules/%{kversion}-$kernel_flavour-%{buildrpmrel}/build
@@ -1452,18 +1377,6 @@ fi
%endif
exit 0
EOF
-
-### Create kernel Postun script on the fly
-cat > $kernel_files-postun <<EOF
-rm -f /boot/initrd-%{kversion}-$kernel_flavour-%{buildrpmrel}.img
-rm -rf /lib/modules/%{kversion}-$kernel_flavour-%{buildrpmrel} >/dev/null
-if [ -d /var/lib/dkms ]; then
- rm -f /var/lib/dkms/*/kernel-%{kversion}-$devel_flavour-%{buildrpmrel}-%{_target_cpu} >/dev/null
- rm -rf /var/lib/dkms/*/*/%{kversion}-$devel_flavour-%{buildrpmrel} >/dev/null
- rm -f /var/lib/dkms-binary/*/kernel-%{kversion}-$devel_flavour-%{buildrpmrel}-%{_target_cpu} >/dev/null
- rm -rf /var/lib/dkms-binary/*/*/%{kversion}-$devel_flavour-%{buildrpmrel} >/dev/null
-fi
-EOF
}
CreateKernel() {
@@ -1567,16 +1480,12 @@ sed -ri "s|^(EXTRAVERSION =).*|\1 -%{rpmrel}|" Makefile
%if %{with build_cpupower}
# make sure version-gen.sh is executable.
chmod +x tools/power/cpupower/utils/version-gen.sh
-%kmake -C tools/power/cpupower CPUFREQ_BENCH=false LDFLAGS="%{optflags}"
-%endif
-
-%if %{with bootsplash}
-%kmake -C tools/bootsplash LDFLAGS="%{optflags}"
+%kmake -C tools/power/cpupower CPUFREQ_BENCH=false
%endif
%ifarch %{ix86} %{x86_64}
%if %{with build_x86_energy_perf_policy}
-%kmake -C tools/power/x86/x86_energy_perf_policy CC=clang LDFLAGS="%{optflags} -Wl,--build-id=none"
+%kmake -C tools/power/x86/x86_energy_perf_policy CC=clang LDFLAGS="-Wl,--build-id=none"
%endif
%if %{with build_turbostat}
@@ -1584,6 +1493,11 @@ chmod +x tools/power/cpupower/utils/version-gen.sh
%endif
%endif
+%kmake -C tools/lib/bpf CC=clang libbpf.a libbpf.pc libbpf.so.0.0.8
+cd tools/bpf/bpftool
+%kmake CC=clang bpftool
+cd -
+
############################################################
### Linker end3 > Check point to build for omv or rosa ###
############################################################
@@ -1649,17 +1563,17 @@ sed -ri "s|^(EXTRAVERSION =).*|\1 -%{rpmrel}|" Makefile
%if %{with build_perf}
# perf tool binary and supporting scripts/binaries
-make -C tools/perf -s CC=%{__cc} V=1 DESTDIR=%{buildroot} WERROR=0 HAVE_CPLUS_DEMANGLE=1 prefix=%{_prefix} install
+make -C tools/perf -s CC=%{__cc} DESTDIR=%{buildroot} WERROR=0 HAVE_CPLUS_DEMANGLE=1 prefix=%{_prefix} install
# perf man pages (note: implicit rpm magic compresses them later)
-make -C tools/perf -s CC=%{__cc} V=1 DESTDIR=%{buildroot} WERROR=0 HAVE_CPLUS_DEMANGLE=1 prefix=%{_prefix} install-man
+make -C tools/perf -s CC=%{__cc} DESTDIR=%{buildroot} WERROR=0 HAVE_CPLUS_DEMANGLE=1 prefix=%{_prefix} install-man
%endif
############################################################
### Linker start4 > Check point to build for omv or rosa ###
############################################################
%if %{with build_cpupower}
-%{make_build} -C tools/power/cpupower DESTDIR=%{buildroot} libdir=%{_libdir} mandir=%{_mandir} CPUFREQ_BENCH=false CC=%{__cc} LDFLAGS="%{optflags}" install
+%make_install -C tools/power/cpupower DESTDIR=%{buildroot} libdir=%{_libdir} mandir=%{_mandir} CPUFREQ_BENCH=false CC=%{__cc} install
rm -f %{buildroot}%{_libdir}/*.{a,la}
%find_lang cpupower
@@ -1669,22 +1583,21 @@ install -m644 %{SOURCE50} %{buildroot}%{_unitdir}/cpupower.service
install -m644 %{SOURCE51} %{buildroot}%{_sysconfdir}/sysconfig/cpupower
%endif
-%if %{with bootsplash}
-mkdir -p %{buildroot}%{_bindir}
-install -m755 tools/bootsplash/bootsplash-packer %{buildroot}%{_bindir}/
-%endif
-
%ifarch %{ix86} %{x86_64}
%if %{with build_x86_energy_perf_policy}
mkdir -p %{buildroot}%{_bindir} %{buildroot}%{_mandir}/man8
-%kmake -C tools/power/x86/x86_energy_perf_policy install DESTDIR="%{buildroot}"
+%make_install -C tools/power/x86/x86_energy_perf_policy install DESTDIR="%{buildroot}"
%endif
%if %{with build_turbostat}
mkdir -p %{buildroot}%{_bindir} %{buildroot}%{_mandir}/man8
-%kmake -C tools/power/x86/turbostat install DESTDIR="%{buildroot}"
+%make_install -C tools/power/x86/turbostat install DESTDIR="%{buildroot}"
%endif
%endif
+# install bpftool and libbpf
+%make_install -C tools/lib/bpf install install_headers DESTDIR=%{buildroot} prefix=%{_prefix} libdir=%{_libdir}
+%make_install -C tools/bpf/bpftool install DESTDIR=%{buildroot} prefix=%{_prefix} bash_compdir=%{_sysconfdir}/bash_completion.d/ mandir=%{_mandir}
+
# Create directories infastructure
%if %{with build_source}
install -d %{target_source}
@@ -1842,11 +1755,6 @@ cd -
%{_includedir}/cpufreq.h
%endif
-%if %{with bootsplash}
-%files -n bootsplash-packer
-%{_bindir}/bootsplash-packer
-%endif
-
%ifarch %{ix86} %{x86_64}
%if %{with build_x86_energy_perf_policy}
%files -n x86_energy_perf_policy
@@ -1860,3 +1768,17 @@ cd -
%{_mandir}/man8/turbostat.8*
%endif
%endif
+
+%files -n bpftool
+%{_sbindir}/bpftool
+%{_sysconfdir}/bash_completion.d/bpftool
+
+%files -n %{libbpf}
+%{_libdir}/libbpf.so.%{bpf_major}*
+
+%files -n %{libbpfdevel}
+%{_libdir}/libbpf.a
+%{_libdir}/libbpf.so
+%{_libdir}/pkgconfig/*.pc
+%dir %{_includedir}/bpf
+%{_includedir}/bpf/*.h
diff --git a/linux-4.7-intel-dvi-duallink.patch b/linux-4.7-intel-dvi-duallink.patch
index 99a0d13..a9951f8 100644
--- a/linux-4.7-intel-dvi-duallink.patch
+++ b/linux-4.7-intel-dvi-duallink.patch
@@ -1,19 +1,19 @@
-diff -up linux-5.1/drivers/gpu/drm/i915/display/intel_hdmi.c.3~ linux-5.1/drivers/gpu/drm/i915/display/intel_hdmi.c
---- linux-5.1/drivers/gpu/drm/i915/display/intel_hdmi.c.3~ 2019-05-07 17:17:24.996151319 +0200
-+++ linux-5.1/drivers/gpu/drm/i915/display/intel_hdmi.c 2019-05-07 17:19:03.873570987 +0200
-@@ -1584,13 +1584,13 @@ intel_hdmi_mode_valid(struct drm_connect
+diff -up linux-5.7-rc1/drivers/gpu/drm/i915/display/intel_hdmi.c.4~ linux-5.7-rc1/drivers/gpu/drm/i915/display/intel_hdmi.c
+--- linux-5.7-rc1/drivers/gpu/drm/i915/display/intel_hdmi.c.4~ 2020-04-15 00:04:04.207753898 +0200
++++ linux-5.7-rc1/drivers/gpu/drm/i915/display/intel_hdmi.c 2020-04-15 00:08:26.075745335 +0200
+@@ -2200,13 +2200,13 @@ intel_hdmi_mode_valid(struct drm_connect
clock /= 2;
/* check if we can do 8bpc */
-- status = hdmi_port_clock_valid(hdmi, clock, true, force_dvi);
-+ status = hdmi_port_clock_valid(hdmi, clock, false, force_dvi);
+- status = hdmi_port_clock_valid(hdmi, clock, true, has_hdmi_sink);
++ status = hdmi_port_clock_valid(hdmi, clock, false, has_hdmi_sink);
- if (hdmi->has_hdmi_sink && !force_dvi) {
+ if (has_hdmi_sink) {
/* if we can't do 8bpc we may still be able to do 12bpc */
if (status != MODE_OK && !HAS_GMCH(dev_priv))
status = hdmi_port_clock_valid(hdmi, clock * 3 / 2,
-- true, force_dvi);
-+ false, force_dvi);
+- true, has_hdmi_sink);
++ false, has_hdmi_sink);
/* if we can't do 8,12bpc we may still be able to do 10bpc */
if (status != MODE_OK && INTEL_GEN(dev_priv) >= 11)
diff --git a/linux-4.8.1-buildfix.patch b/linux-4.8.1-buildfix.patch
deleted file mode 100644
index 4c1632b..0000000
--- a/linux-4.8.1-buildfix.patch
+++ /dev/null
@@ -1,33 +0,0 @@
-diff -up linux-5.3-rc1/arch/arm64/kernel/vdso/Makefile.4~ linux-5.3-rc1/arch/arm64/kernel/vdso/Makefile
---- linux-5.3-rc1/arch/arm64/kernel/vdso/Makefile.4~ 2019-07-24 00:26:39.079420412 +0200
-+++ linux-5.3-rc1/arch/arm64/kernel/vdso/Makefile 2019-07-24 00:32:03.196644465 +0200
-@@ -17,7 +17,7 @@ obj-vdso := vgettimeofday.o note.o sigre
- targets := $(obj-vdso) vdso.so vdso.so.dbg
- obj-vdso := $(addprefix $(obj)/, $(obj-vdso))
-
--ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \
-+ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv -fuse-ld=bfd \
- --build-id -n -T
-
- ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18
-diff -up linux-5.3-rc1/arch/x86/entry/vdso/Makefile.4~ linux-5.3-rc1/arch/x86/entry/vdso/Makefile
---- linux-5.3-rc1/arch/x86/entry/vdso/Makefile.4~ 2019-07-21 23:05:38.000000000 +0200
-+++ linux-5.3-rc1/arch/x86/entry/vdso/Makefile 2019-07-24 00:32:22.289564153 +0200
-@@ -53,7 +53,7 @@ targets += $(vdso_img_sodbg) $(vdso_img-
- CPPFLAGS_vdso.lds += -P -C
-
- VDSO_LDFLAGS_vdso.lds = -m elf_x86_64 -soname linux-vdso.so.1 --no-undefined \
-- -z max-page-size=4096
-+ -z max-page-size=4096 -fuse-ld=bfd
-
- $(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE
- $(call if_changed,vdso_and_check)
-@@ -104,7 +104,7 @@ CFLAGS_REMOVE_vvar.o = -pg
-
- CPPFLAGS_vdsox32.lds = $(CPPFLAGS_vdso.lds)
- VDSO_LDFLAGS_vdsox32.lds = -m elf32_x86_64 -soname linux-vdso.so.1 \
-- -z max-page-size=4096
-+ -z max-page-size=4096 -fuse-ld=bfd
-
- # x32-rebranded versions
- vobjx32s-y := $(vobjs-y:.o=-x32.o)
diff --git a/linux-5.5-corsair-strafe-quirks.patch b/linux-5.5-corsair-strafe-quirks.patch
new file mode 100644
index 0000000..56d9ffe
--- /dev/null
+++ b/linux-5.5-corsair-strafe-quirks.patch
@@ -0,0 +1,22 @@
+diff -up linux-5.5/drivers/hid/hid-ids.h.omv~ linux-5.5/drivers/hid/hid-ids.h
+--- linux-5.5/drivers/hid/hid-ids.h.omv~ 2020-02-13 15:44:42.401817937 +0100
++++ linux-5.5/drivers/hid/hid-ids.h 2020-02-13 15:45:29.512814852 +0100
+@@ -310,6 +310,7 @@
+ #define USB_DEVICE_ID_CORSAIR_K70RGB_RAPIDFIRE 0x1b38
+ #define USB_DEVICE_ID_CORSAIR_K65RGB_RAPIDFIRE 0x1b39
+ #define USB_DEVICE_ID_CORSAIR_SCIMITAR_PRO_RGB 0x1b3e
++#define USB_DEVICE_ID_CORSAIR_STRAFE_2 0x1b44
+
+ #define USB_VENDOR_ID_CREATIVELABS 0x041e
+ #define USB_DEVICE_ID_CREATIVE_SB_OMNI_SURROUND_51 0x322c
+diff -up linux-5.5/drivers/hid/hid-quirks.c.omv~ linux-5.5/drivers/hid/hid-quirks.c
+--- linux-5.5/drivers/hid/hid-quirks.c.omv~ 2020-02-13 15:46:19.977811548 +0100
++++ linux-5.5/drivers/hid/hid-quirks.c 2020-02-13 15:46:22.903811356 +0100
+@@ -64,6 +64,7 @@ static const struct hid_device_id hid_qu
+ { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_GLAIVE_RGB), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
+ { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_SCIMITAR_PRO_RGB), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
+ { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_STRAFE), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
++ { HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_STRAFE_2), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
+ { HID_USB_DEVICE(USB_VENDOR_ID_CREATIVELABS, USB_DEVICE_ID_CREATIVE_SB_OMNI_SURROUND_51), HID_QUIRK_NOGET },
+ { HID_USB_DEVICE(USB_VENDOR_ID_DELL, USB_DEVICE_ID_DELL_PIXART_USB_OPTICAL_MOUSE), HID_QUIRK_ALWAYS_POLL },
+ { HID_USB_DEVICE(USB_VENDOR_ID_DMI, USB_DEVICE_ID_DMI_ENC), HID_QUIRK_NOGET },
diff --git a/linux-5.5.tar.sign b/linux-5.5.tar.sign
deleted file mode 100644
index 360859c..0000000
--- a/linux-5.5.tar.sign
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN PGP SIGNATURE-----
-Comment: This signature is for the .tar version of the archive
-Comment: git archive --format tar --prefix=linux-5.5/ v5.5
-Comment: git version 2.25.0
-
-iQIzBAABCAAdFiEEZH8oZUiU471FcZm+ONu9yGCSaT4FAl4uhxEACgkQONu9yGCS
-aT7Mig//e4oxjnoKWzrGpC3EtiaOG/ueY0NJWvXWU66xMMzqRnNT8BOL0HdKkZWB
-xS60lR7vKXKTK2w1yz92Tmeo0bbF77J5+uthCQ0F5pvxHvIXGMLhsS5uRwSsjO49
-T0rRea0n+oyjmZQpECyVDLtgjjtaYLSBPSEEylix+m7yetRONcmD49YoizzU9Wtc
-YdpK0bWES/hwcDQlb9owi6giv5qeCvs69CQ2/dzL9ByeSzFuL5TTYw/9CoyCzvMi
-XLCabg5IVdxwGWePLnKamEQRrLbSXanjBacNZXOb9CxRmsESy1NFPIy+yXwSIKIS
-/rMe4zwTj6UK5Sdn5J1onUtk064+TFRnG6QEv7503sml+QMjUcWeTXpSRruoXHov
-H/vnLQBo+my4qatejzdLQLnLKnZB1SSbNA8LeVwxOFKsMxfuIZ5SdztUKUljAaeV
-ppohOLsVoC8XeECL74ZT3KzxOTXy1KWg6Qq5uUIJi40NKTvXNQoi+8q1URcLAk6+
-3s6w/m4nDcSA5x1wkOS7P22qX2oHOwbBOq14xoaR9y/UdfNiNdRCTpMVZcplFvFl
-hNoVhjArabK9M7ufib29mEAJZhqNjp8Q0kXlHVP9scHXIJTRAWCAlqSonmYrNew9
-jocFQoSaD0DdDTlRSA0RFn/YVRfXhvarwKeSTFtv1sVax9yfLd4=
-=kpiR
------END PGP SIGNATURE-----
diff --git a/linux-5.6-fix-disassembler-4args-detection.patch b/linux-5.6-fix-disassembler-4args-detection.patch
new file mode 100644
index 0000000..36aa382
--- /dev/null
+++ b/linux-5.6-fix-disassembler-4args-detection.patch
@@ -0,0 +1,13 @@
+diff -up linux-5.6/tools/bpf/bpftool/jit_disasm.c.omv~ linux-5.6/tools/bpf/bpftool/jit_disasm.c
+diff -up linux-5.6/tools/build/feature/Makefile.omv~ linux-5.6/tools/build/feature/Makefile
+--- linux-5.6/tools/build/feature/Makefile.omv~ 2020-04-02 06:07:08.488570621 +0200
++++ linux-5.6/tools/build/feature/Makefile 2020-04-02 06:07:19.983631490 +0200
+@@ -224,7 +224,7 @@ $(OUTPUT)test-libbfd.bin:
+ $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl
+
+ $(OUTPUT)test-disassembler-four-args.bin:
+- $(BUILD) -DPACKAGE='"perf"' -lbfd -lopcodes
++ $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl -lopcodes
+
+ $(OUTPUT)test-reallocarray.bin:
+ $(BUILD)
diff --git a/smpboot-no-stack-protector-for-gcc10.patch b/smpboot-no-stack-protector-for-gcc10.patch
new file mode 100644
index 0000000..d7285e5
--- /dev/null
+++ b/smpboot-no-stack-protector-for-gcc10.patch
@@ -0,0 +1,13 @@
+diff -Naur linux-5.5.9/arch/x86/kernel/Makefile linux-5.5.9-p/arch/x86/kernel/Makefile
+--- linux-5.5.9/arch/x86/kernel/Makefile 2020-03-12 07:18:49.000000000 +0100
++++ linux-5.5.9-p/arch/x86/kernel/Makefile 2020-03-16 15:06:33.354637851 +0100
+@@ -11,6 +11,9 @@
+
+ CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE)
+
++dropstackprotector := $(call cc-option, -fno-stack-protector)
++CFLAGS_smpboot.o := $(dropstackprotector)
++
+ ifdef CONFIG_FUNCTION_TRACER
+ # Do not profile debug and lowlevel utilities
+ CFLAGS_REMOVE_tsc.o = -pg
diff --git a/socket.h-include-bitsperlong.h.patch b/socket.h-include-bitsperlong.h.patch
new file mode 100644
index 0000000..d021976
--- /dev/null
+++ b/socket.h-include-bitsperlong.h.patch
@@ -0,0 +1,11 @@
+diff -up linux-5.7/include/uapi/asm-generic/socket.h.omv~ linux-5.7/include/uapi/asm-generic/socket.h
+--- linux-5.7/include/uapi/asm-generic/socket.h.omv~ 2020-06-21 14:40:19.478327294 +0200
++++ linux-5.7/include/uapi/asm-generic/socket.h 2020-06-21 14:41:14.638889566 +0200
+@@ -3,6 +3,7 @@
+ #define __ASM_GENERIC_SOCKET_H
+
+ #include <linux/posix_types.h>
++#include <asm/bitsperlong.h>
+ #include <asm/sockios.h>
+
+ /* For setsockopt(2) */
diff --git a/v2-1-2-lib-Add-support-for-ZSTD-compressed-kernel.patch b/support-kernel-and-ramfs-comp-and-decomp-with-zstd.patch
similarity index 59%
rename from v2-1-2-lib-Add-support-for-ZSTD-compressed-kernel.patch
rename to support-kernel-and-ramfs-comp-and-decomp-with-zstd.patch
index 8acb356..28141f1 100644
--- a/v2-1-2-lib-Add-support-for-ZSTD-compressed-kernel.patch
+++ b/support-kernel-and-ramfs-comp-and-decomp-with-zstd.patch
@@ -1,23 +1,125 @@
-diff -up linux-5.1/include/linux/decompress/unzstd.h.19~ linux-5.1/include/linux/decompress/unzstd.h
---- linux-5.1/include/linux/decompress/unzstd.h.19~ 2019-05-07 17:19:25.014249610 +0200
-+++ linux-5.1/include/linux/decompress/unzstd.h 2019-05-07 17:19:25.014249610 +0200
-@@ -0,0 +1,26 @@
-+/*
-+ * Copyright (C) 2017 Facebook
-+ *
-+ * This program is free software; you can redistribute it and/or
-+ * modify it under the terms of the GNU General Public
-+ * License v2 as published by the Free Software Foundation.
-+ *
-+ * This program is distributed in the hope that it will be useful,
-+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ * General Public License for more details.
-+ *
-+ * You should have received a copy of the GNU General Public License
-+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
-+ */
+diff -Naur linux-5.6.1/arch/x86/boot/compressed/Makefile linux-5.6.1-p/arch/x86/boot/compressed/Makefile
+--- linux-5.6.1/arch/x86/boot/compressed/Makefile 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/arch/x86/boot/compressed/Makefile 2020-04-01 13:29:07.958649103 +0200
+@@ -24,7 +24,7 @@
+ KCOV_INSTRUMENT := n
+
+ targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \
+- vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4
++ vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst
+
+ KBUILD_CFLAGS := -m$(BITS) -O2
+ KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC)
+@@ -145,6 +145,8 @@
+ $(call if_changed,lzo)
+ $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE
+ $(call if_changed,lz4)
++$(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE
++ $(call if_changed,zstd)
+
+ suffix-$(CONFIG_KERNEL_GZIP) := gz
+ suffix-$(CONFIG_KERNEL_BZIP2) := bz2
+@@ -152,6 +154,7 @@
+ suffix-$(CONFIG_KERNEL_XZ) := xz
+ suffix-$(CONFIG_KERNEL_LZO) := lzo
+ suffix-$(CONFIG_KERNEL_LZ4) := lz4
++suffix-$(CONFIG_KERNEL_ZSTD) := zst
+
+ quiet_cmd_mkpiggy = MKPIGGY $@
+ cmd_mkpiggy = $(obj)/mkpiggy $< > $@
+diff -Naur linux-5.6.1/arch/x86/boot/compressed/misc.c linux-5.6.1-p/arch/x86/boot/compressed/misc.c
+--- linux-5.6.1/arch/x86/boot/compressed/misc.c 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/arch/x86/boot/compressed/misc.c 2020-04-01 13:29:07.958649103 +0200
+@@ -77,6 +77,10 @@
+ #ifdef CONFIG_KERNEL_LZ4
+ #include "../../../../lib/decompress_unlz4.c"
+ #endif
+
++#ifdef CONFIG_KERNEL_ZSTD
++#include "../../../../lib/decompress_unzstd.c"
++#endif
+ /*
+ * NOTE: When adding a new decompressor, please update the analysis in
+ * ../header.S.
+diff -Naur linux-5.6.1/arch/x86/boot/header.S linux-5.6.1-p/arch/x86/boot/header.S
+--- linux-5.6.1/arch/x86/boot/header.S 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/arch/x86/boot/header.S 2020-04-01 13:29:03.958623285 +0200
+@@ -536,8 +536,14 @@
+ # the size-dependent part now grows so fast.
+ #
+ # extra_bytes = (uncompressed_size >> 8) + 65536
++#
++# ZSTD compressed data grows by at most 3 bytes per 128K, and only has a 22
++# byte fixed overhead but has a maximum block size of 128K, so it needs a
++# larger margin.
++#
++# extra_bytes = (uncompressed_size >> 8) + 131072
+
+-#define ZO_z_extra_bytes ((ZO_z_output_len >> 8) + 65536)
++#define ZO_z_extra_bytes ((ZO_z_output_len >> 8) + 131072)
+ #if ZO_z_output_len > ZO_z_input_len
+ # define ZO_z_extract_offset (ZO_z_output_len + ZO_z_extra_bytes - \
+ ZO_z_input_len)
+diff -Naur linux-5.6.1/arch/x86/include/asm/boot.h linux-5.6.1-p/arch/x86/include/asm/boot.h
+--- linux-5.6.1/arch/x86/include/asm/boot.h 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/arch/x86/include/asm/boot.h 2020-04-01 13:29:07.958649103 +0200
+@@ -24,9 +24,11 @@
+ # error "Invalid value for CONFIG_PHYSICAL_ALIGN"
+ #endif
+
+-#ifdef CONFIG_KERNEL_BZIP2
++#if defined(CONFIG_KERNEL_BZIP2)
+ # define BOOT_HEAP_SIZE 0x400000
+-#else /* !CONFIG_KERNEL_BZIP2 */
++#elif defined(CONFIG_KERNEL_ZSTD)
++# define BOOT_HEAP_SIZE 0x30000
++#else
+ # define BOOT_HEAP_SIZE 0x10000
+ #endif
+
+diff -Naur linux-5.6.1/arch/x86/Kconfig linux-5.6.1-p/arch/x86/Kconfig
+--- linux-5.6.1/arch/x86/Kconfig 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/arch/x86/Kconfig 2020-04-01 13:29:07.958649103 +0200
+@@ -183,6 +183,7 @@
+ select HAVE_KERNEL_LZMA
+ select HAVE_KERNEL_LZO
+ select HAVE_KERNEL_XZ
++ select HAVE_KERNEL_ZSTD
+ select HAVE_KPROBES
+ select HAVE_KPROBES_ON_FTRACE
+ select HAVE_FUNCTION_ERROR_INJECTION
+diff -Naur linux-5.6.1/Documentation/x86/boot.rst linux-5.6.1-p/Documentation/x86/boot.rst
+--- linux-5.6.1/Documentation/x86/boot.rst 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/Documentation/x86/boot.rst 2020-04-01 13:29:07.958649103 +0200
+@@ -786,9 +786,9 @@
+ uncompressed data should be determined using the standard magic
+ numbers. The currently supported compression formats are gzip
+ (magic numbers 1F 8B or 1F 9E), bzip2 (magic number 42 5A), LZMA
+- (magic number 5D 00), XZ (magic number FD 37), and LZ4 (magic number
+- 02 21). The uncompressed payload is currently always ELF (magic
+- number 7F 45 4C 46).
++ (magic number 5D 00), XZ (magic number FD 37), LZ4 (magic number
++ 02 21) and ZSTD (magic number 28 B5). The uncompressed payload is
++ currently always ELF (magic number 7F 45 4C 46).
+
+ ============ ==============
+ Field name: payload_length
+diff -Naur linux-5.6.1/.gitignore linux-5.6.1-p/.gitignore
+--- linux-5.6.1/.gitignore 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/.gitignore 2020-04-01 13:29:11.158669756 +0200
+@@ -43,6 +43,7 @@
+ *.tab.[ch]
+ *.tar
+ *.xz
++*.zst
+ Module.symvers
+ modules.builtin
+ modules.order
+diff -Naur linux-5.6.1/include/linux/decompress/unzstd.h linux-5.6.1-p/include/linux/decompress/unzstd.h
+--- linux-5.6.1/include/linux/decompress/unzstd.h 1970-01-01 01:00:00.000000000 +0100
++++ linux-5.6.1-p/include/linux/decompress/unzstd.h 2020-04-01 13:28:53.998558979 +0200
+@@ -0,0 +1,11 @@
++/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef LINUX_DECOMPRESS_UNZSTD_H
+#define LINUX_DECOMPRESS_UNZSTD_H
+
@@ -28,10 +130,10 @@ diff -up linux-5.1/include/linux/decompress/unzstd.h.19~ linux-5.1/include/linux
+ long *pos,
+ void (*error_fn)(char *x));
+#endif
-diff -up linux-5.1/init/Kconfig.19~ linux-5.1/init/Kconfig
---- linux-5.1/init/Kconfig.19~ 2019-05-06 02:42:58.000000000 +0200
-+++ linux-5.1/init/Kconfig 2019-05-07 17:19:25.015249595 +0200
-@@ -153,13 +153,16 @@ config HAVE_KERNEL_LZO
+diff -Naur linux-5.6.1/init/Kconfig linux-5.6.1-p/init/Kconfig
+--- linux-5.6.1/init/Kconfig 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/init/Kconfig 2020-04-01 13:28:58.038585064 +0200
+@@ -173,13 +173,16 @@
config HAVE_KERNEL_LZ4
bool
@@ -49,7 +151,7 @@ diff -up linux-5.1/init/Kconfig.19~ linux-5.1/init/Kconfig
help
The linux kernel is a kind of self-extracting executable.
Several compression algorithms are available, which differ
-@@ -238,6 +241,15 @@ config KERNEL_LZ4
+@@ -258,6 +261,16 @@
is about 8% bigger than LZO. But the decompression speed is
faster than LZO.
@@ -60,14 +162,15 @@ diff -up linux-5.1/init/Kconfig.19~ linux-5.1/init/Kconfig
+ ZSTD is a compression algorithm targeting intermediate compression
+ with fast decompression speed. It will compress better than GZIP and
+ decompress around the same speed as LZO, but slower than LZ4. You
-+ will need at least 192 KB RAM or more for booting.
++ will need at least 192 KB RAM or more for booting. The zstd command
++ line tools is required for compression.
+
config KERNEL_UNCOMPRESSED
bool "None"
depends on HAVE_KERNEL_UNCOMPRESSED
-diff -up linux-5.1/lib/decompress.c.19~ linux-5.1/lib/decompress.c
---- linux-5.1/lib/decompress.c.19~ 2019-05-06 02:42:58.000000000 +0200
-+++ linux-5.1/lib/decompress.c 2019-05-07 17:19:25.015249595 +0200
+diff -Naur linux-5.6.1/lib/decompress.c linux-5.6.1-p/lib/decompress.c
+--- linux-5.6.1/lib/decompress.c 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/lib/decompress.c 2020-04-01 13:28:53.998558979 +0200
@@ -13,6 +13,7 @@
#include <linux/decompress/inflate.h>
#include <linux/decompress/unlzo.h>
@@ -86,7 +189,7 @@ diff -up linux-5.1/lib/decompress.c.19~ linux-5.1/lib/decompress.c
struct compress_format {
unsigned char magic[2];
-@@ -52,6 +56,7 @@ static const struct compress_format comp
+@@ -52,6 +56,7 @@
{ {0xfd, 0x37}, "xz", unxz },
{ {0x89, 0x4c}, "lzo", unlzo },
{ {0x02, 0x21}, "lz4", unlz4 },
@@ -94,25 +197,11 @@ diff -up linux-5.1/lib/decompress.c.19~ linux-5.1/lib/decompress.c
{ {0, 0}, NULL, NULL }
};
-diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
---- linux-5.1/lib/decompress_unzstd.c.19~ 2019-05-07 17:19:25.015249595 +0200
-+++ linux-5.1/lib/decompress_unzstd.c 2019-05-07 17:19:25.015249595 +0200
-@@ -0,0 +1,341 @@
-+/*
-+ * Copyright (C) 2017 Facebook
-+ *
-+ * This program is free software; you can redistribute it and/or
-+ * modify it under the terms of the GNU General Public
-+ * License v2 as published by the Free Software Foundation.
-+ *
-+ * This program is distributed in the hope that it will be useful,
-+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
-+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-+ * General Public License for more details.
-+ *
-+ * You should have received a copy of the GNU General Public License
-+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
-+ */
+diff -Naur linux-5.6.1/lib/decompress_unzstd.c linux-5.6.1-p/lib/decompress_unzstd.c
+--- linux-5.6.1/lib/decompress_unzstd.c 1970-01-01 01:00:00.000000000 +0100
++++ linux-5.6.1-p/lib/decompress_unzstd.c 2020-04-01 13:28:53.998558979 +0200
+@@ -0,0 +1,342 @@
++// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Important notes about in-place decompression
@@ -167,32 +256,38 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ * <= 22 + (uncompressed_size >> 15) + 131072
+ */
+
++/*
++ * Preboot environments #include "path/to/decompress_unzstd.c".
++ * All of the source files we depend on must be #included.
++ * zstd's only source dependeny is xxhash, which has no source
++ * dependencies.
++ *
++ * zstd and xxhash avoid declaring themselves as modules
++ * when ZSTD_PREBOOT and XXH_PREBOOT are defined.
++ */
+#ifdef STATIC
-+ /* Preboot environments #include "path/to/decompress_unzstd.c".
-+ * All of the source files we depend on must be #included.
-+ * zstd's only source dependeny is xxhash, which has no source
-+ * dependencies.
-+ *
-+ * zstd and xxhash both avoid declaring themselves as modules
-+ * when PREBOOT is defined.
-+ */
-+# define PREBOOT
-+# include "xxhash.c"
-+# include "zstd/entropy_common.c"
-+# include "zstd/fse_decompress.c"
-+# include "zstd/huf_decompress.c"
-+# include "zstd/zstd_common.c"
-+# include "zstd/decompress.c"
++# define ZSTD_PREBOOT
++# define XXH_PREBOOT
++# include "xxhash.c"
++# include "zstd/entropy_common.c"
++# include "zstd/fse_decompress.c"
++# include "zstd/huf_decompress.c"
++# include "zstd/zstd_common.c"
++# include "zstd/decompress.c"
+#endif
+
+#include <linux/decompress/mm.h>
+#include <linux/kernel.h>
+#include <linux/zstd.h>
+
-+/* 8 MB maximum window size */
-+#define ZSTD_WINDOWSIZE_MAX (1 << 23)
-+/* Size of the input and output buffers in multi-call mdoe */
-+#define ZSTD_IOBUF_SIZE 4096
++/* 128MB is the maximum window size supported by zstd. */
++#define ZSTD_WINDOWSIZE_MAX (1 << ZSTD_WINDOWLOG_MAX)
++/* Size of the input and output buffers in multi-call mode.
++ * Pick a larger size because it isn't used during kernel decompression,
++ * since that is single pass, and we have to allocate a large buffer for
++ * zstd's window anyways. The larger size speeds up initramfs decompression.
++ */
++#define ZSTD_IOBUF_SIZE (1 << 17)
+
+static int INIT handle_zstd_error(size_t ret, void (*error)(char *x))
+{
@@ -220,7 +315,8 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ return -1;
+}
+
-+/* Handle the case where we have the entire input and output in one segment.
++/*
++ * Handle the case where we have the entire input and output in one segment.
+ * We can allocate less memory (no circular buffer for the sliding window),
+ * and avoid some memcpy() calls.
+ */
@@ -239,7 +335,8 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ err = -1;
+ goto out;
+ }
-+ /* Find out how large the frame actually is, there may be junk at
++ /*
++ * Find out how large the frame actually is, there may be junk at
+ * the end of the frame that ZSTD_decompressDCtx() can't handle.
+ */
+ ret = ZSTD_findFrameCompressedSize(in_buf, in_len);
@@ -285,18 +382,20 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ out_len = LONG_MAX; /* no limit */
+
+ if (fill == NULL && flush == NULL)
-+ /* We can decompress faster and with less memory when we have a
++ /*
++ * We can decompress faster and with less memory when we have a
+ * single chunk.
+ */
+ return decompress_single(in_buf, in_len, out_buf, out_len,
+ in_pos, error);
+
-+ /* If in_buf is not provided, we must be using fill(), so allocate
++ /*
++ * If in_buf is not provided, we must be using fill(), so allocate
+ * a large enough buffer. If it is provided, it must be at least
+ * ZSTD_IOBUF_SIZE large.
+ */
+ if (in_buf == NULL) {
-+ in_allocated = malloc(ZSTD_IOBUF_SIZE);
++ in_allocated = large_malloc(ZSTD_IOBUF_SIZE);
+ if (in_allocated == NULL) {
+ error("Out of memory while allocating input buffer");
+ err = -1;
@@ -305,7 +404,7 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ in_buf = in_allocated;
+ in_len = 0;
+ }
-+ /* Read the first chunk, since we need to decode the frame header */
++ /* Read the first chunk, since we need to decode the frame header. */
+ if (fill != NULL)
+ in_len = fill(in_buf, ZSTD_IOBUF_SIZE);
+ if (in_len < 0) {
@@ -313,13 +412,13 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ err = -1;
+ goto out;
+ }
-+ /* Set the first non-empty input buffer */
++ /* Set the first non-empty input buffer. */
+ in.src = in_buf;
+ in.pos = 0;
+ in.size = in_len;
+ /* Allocate the output buffer if we are using flush(). */
+ if (flush != NULL) {
-+ out_allocated = malloc(ZSTD_IOBUF_SIZE);
++ out_allocated = large_malloc(ZSTD_IOBUF_SIZE);
+ if (out_allocated == NULL) {
+ error("Out of memory while allocating output buffer");
+ err = -1;
@@ -328,12 +427,13 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ out_buf = out_allocated;
+ out_len = ZSTD_IOBUF_SIZE;
+ }
-+ /* Set the output buffer */
++ /* Set the output buffer. */
+ out.dst = out_buf;
+ out.pos = 0;
+ out.size = out_len;
+
-+ /* We need to know the window size to allocate the ZSTD_DStream.
++ /*
++ * We need to know the window size to allocate the ZSTD_DStream.
+ * Since we are streaming, we need to allocate a buffer for the sliding
+ * window. The window size varies from 1 KB to ZSTD_WINDOWSIZE_MAX
+ * (8 MB), so it is important to use the actual value so as not to
@@ -354,7 +454,8 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ goto out;
+ }
+
-+ /* Allocate the ZSTD_DStream now that we know how much memory is
++ /*
++ * Allocate the ZSTD_DStream now that we know how much memory is
+ * required.
+ */
+ wksp_size = ZSTD_DStreamWorkspaceBound(params.windowSize);
@@ -365,7 +466,9 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ err = -1;
+ goto out;
+ }
-+ /* Decompression loop:
++
++ /*
++ * Decompression loop:
+ * Read more data if necessary (error if no more data can be read).
+ * Call the decompression function, which returns 0 when finished.
+ * Flush any data produced if using flush().
@@ -373,7 +476,8 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ if (in_pos != NULL)
+ *in_pos = 0;
+ do {
-+ /* If we need to reload data, either we have fill() and can
++ /*
++ * If we need to reload data, either we have fill() and can
+ * try to get more data, or we don't and the input is truncated.
+ */
+ if (in.pos == in.size) {
@@ -388,12 +492,12 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ in.pos = 0;
+ in.size = in_len;
+ }
-+ /* Returns zero when the frame is complete */
++ /* Returns zero when the frame is complete. */
+ ret = ZSTD_decompressStream(dstream, &out, &in);
+ err = handle_zstd_error(ret, error);
+ if (err)
+ goto out;
-+ /* Flush all of the data produced if using flush() */
++ /* Flush all of the data produced if using flush(). */
+ if (flush != NULL && out.pos > 0) {
+ if (out.pos != flush(out.dst, out.pos)) {
+ error("Failed to flush()");
@@ -410,15 +514,15 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ err = 0;
+out:
+ if (in_allocated != NULL)
-+ free(in_allocated);
++ large_free(in_allocated);
+ if (out_allocated != NULL)
-+ free(out_allocated);
++ large_free(out_allocated);
+ if (wksp != NULL)
+ large_free(wksp);
+ return err;
+}
+
-+#ifndef PREBOOT
++#ifndef ZSTD_PREBOOT
+STATIC int INIT unzstd(unsigned char *buf, long len,
+ long (*fill)(void*, unsigned long),
+ long (*flush)(void*, unsigned long),
@@ -439,10 +543,10 @@ diff -up linux-5.1/lib/decompress_unzstd.c.19~ linux-5.1/lib/decompress_unzstd.c
+ return __unzstd(buf, len, fill, flush, out_buf, out_len, pos, error);
+}
+#endif
-diff -up linux-5.1/lib/Kconfig.19~ linux-5.1/lib/Kconfig
---- linux-5.1/lib/Kconfig.19~ 2019-05-06 02:42:58.000000000 +0200
-+++ linux-5.1/lib/Kconfig 2019-05-07 17:19:25.015249595 +0200
-@@ -312,6 +312,10 @@ config DECOMPRESS_LZ4
+diff -Naur linux-5.6.1/lib/Kconfig linux-5.6.1-p/lib/Kconfig
+--- linux-5.6.1/lib/Kconfig 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/lib/Kconfig 2020-04-01 13:28:53.998558979 +0200
+@@ -336,6 +336,10 @@
select LZ4_DECOMPRESS
tristate
@@ -453,10 +557,10 @@ diff -up linux-5.1/lib/Kconfig.19~ linux-5.1/lib/Kconfig
#
# Generic allocator support is selected if needed
#
-diff -up linux-5.1/lib/Makefile.19~ linux-5.1/lib/Makefile
---- linux-5.1/lib/Makefile.19~ 2019-05-07 17:19:25.005249746 +0200
-+++ linux-5.1/lib/Makefile 2019-05-07 17:19:25.016249580 +0200
-@@ -156,6 +156,7 @@ lib-$(CONFIG_DECOMPRESS_LZMA) += decompr
+diff -Naur linux-5.6.1/lib/Makefile linux-5.6.1-p/lib/Makefile
+--- linux-5.6.1/lib/Makefile 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/lib/Makefile 2020-04-01 13:28:53.998558979 +0200
+@@ -160,6 +160,7 @@
lib-$(CONFIG_DECOMPRESS_XZ) += decompress_unxz.o
lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
lib-$(CONFIG_DECOMPRESS_LZ4) += decompress_unlz4.o
@@ -464,10 +568,10 @@ diff -up linux-5.1/lib/Makefile.19~ linux-5.1/lib/Makefile
obj-$(CONFIG_TEXTSEARCH) += textsearch.o
obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
-diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
---- linux-5.1/lib/xxhash.c.19~ 2019-05-06 02:42:58.000000000 +0200
-+++ linux-5.1/lib/xxhash.c 2019-05-07 17:19:25.016249580 +0200
-@@ -80,13 +80,11 @@ void xxh32_copy_state(struct xxh32_state
+diff -Naur linux-5.6.1/lib/xxhash.c linux-5.6.1-p/lib/xxhash.c
+--- linux-5.6.1/lib/xxhash.c 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/lib/xxhash.c 2020-04-01 13:28:51.078540121 +0200
+@@ -80,13 +80,11 @@
{
memcpy(dst, src, sizeof(*dst));
}
@@ -481,7 +585,7 @@ diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
/*-***************************
* Simple Hash Functions
-@@ -151,7 +149,6 @@ uint32_t xxh32(const void *input, const
+@@ -151,7 +149,6 @@
return h32;
}
@@ -489,7 +593,7 @@ diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
static uint64_t xxh64_round(uint64_t acc, const uint64_t input)
{
-@@ -234,7 +231,6 @@ uint64_t xxh64(const void *input, const
+@@ -234,7 +231,6 @@
return h64;
}
@@ -497,7 +601,7 @@ diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
/*-**************************************************
* Advanced Hash Functions
-@@ -251,7 +247,6 @@ void xxh32_reset(struct xxh32_state *sta
+@@ -251,7 +247,6 @@
state.v4 = seed - PRIME32_1;
memcpy(statePtr, &state, sizeof(state));
}
@@ -505,7 +609,7 @@ diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed)
{
-@@ -265,7 +260,6 @@ void xxh64_reset(struct xxh64_state *sta
+@@ -265,7 +260,6 @@
state.v4 = seed - PRIME64_1;
memcpy(statePtr, &state, sizeof(state));
}
@@ -513,7 +617,7 @@ diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
int xxh32_update(struct xxh32_state *state, const void *input, const size_t len)
{
-@@ -334,7 +328,6 @@ int xxh32_update(struct xxh32_state *sta
+@@ -334,7 +328,6 @@
return 0;
}
@@ -521,7 +625,7 @@ diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
uint32_t xxh32_digest(const struct xxh32_state *state)
{
-@@ -372,7 +365,6 @@ uint32_t xxh32_digest(const struct xxh32
+@@ -372,7 +365,6 @@
return h32;
}
@@ -529,7 +633,7 @@ diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
int xxh64_update(struct xxh64_state *state, const void *input, const size_t len)
{
-@@ -439,7 +431,6 @@ int xxh64_update(struct xxh64_state *sta
+@@ -439,7 +431,6 @@
return 0;
}
@@ -537,12 +641,12 @@ diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
uint64_t xxh64_digest(const struct xxh64_state *state)
{
-@@ -494,7 +485,19 @@ uint64_t xxh64_digest(const struct xxh64
+@@ -494,7 +485,19 @@
return h64;
}
+
-+#ifndef PREBOOT
++#ifndef XXH_PREBOOT
+EXPORT_SYMBOL(xxh32_copy_state);
+EXPORT_SYMBOL(xxh64_copy_state);
+EXPORT_SYMBOL(xxh32);
@@ -557,25 +661,25 @@ diff -up linux-5.1/lib/xxhash.c.19~ linux-5.1/lib/xxhash.c
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("xxHash");
+#endif
-diff -up linux-5.1/lib/zstd/decompress.c.19~ linux-5.1/lib/zstd/decompress.c
---- linux-5.1/lib/zstd/decompress.c.19~ 2019-05-06 02:42:58.000000000 +0200
-+++ linux-5.1/lib/zstd/decompress.c 2019-05-07 17:19:25.016249580 +0200
-@@ -2487,6 +2487,7 @@ size_t ZSTD_decompressStream(ZSTD_DStrea
+diff -Naur linux-5.6.1/lib/zstd/decompress.c linux-5.6.1-p/lib/zstd/decompress.c
+--- linux-5.6.1/lib/zstd/decompress.c 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/lib/zstd/decompress.c 2020-04-01 13:28:46.908513191 +0200
+@@ -2490,6 +2490,7 @@
}
}
-+#ifndef PREBOOT
++#ifndef ZSTD_PREBOOT
EXPORT_SYMBOL(ZSTD_DCtxWorkspaceBound);
EXPORT_SYMBOL(ZSTD_initDCtx);
EXPORT_SYMBOL(ZSTD_decompressDCtx);
-@@ -2526,3 +2527,4 @@ EXPORT_SYMBOL(ZSTD_insertBlock);
+@@ -2529,3 +2530,4 @@
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("Zstd Decompressor");
+#endif
-diff -up linux-5.1/lib/zstd/fse_decompress.c.19~ linux-5.1/lib/zstd/fse_decompress.c
---- linux-5.1/lib/zstd/fse_decompress.c.19~ 2019-05-06 02:42:58.000000000 +0200
-+++ linux-5.1/lib/zstd/fse_decompress.c 2019-05-07 17:19:25.016249580 +0200
+diff -Naur linux-5.6.1/lib/zstd/fse_decompress.c linux-5.6.1-p/lib/zstd/fse_decompress.c
+--- linux-5.6.1/lib/zstd/fse_decompress.c 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/lib/zstd/fse_decompress.c 2020-04-01 13:28:46.908513191 +0200
@@ -47,6 +47,7 @@
****************************************************************/
#include "bitstream.h"
@@ -599,15 +703,16 @@ diff -up linux-5.1/lib/zstd/fse_decompress.c.19~ linux-5.1/lib/zstd/fse_decompre
/* **************************************************************
* Templates
****************************************************************/
-diff -up linux-5.1/lib/zstd/zstd_internal.h.19~ linux-5.1/lib/zstd/zstd_internal.h
---- linux-5.1/lib/zstd/zstd_internal.h.19~ 2019-05-06 02:42:58.000000000 +0200
-+++ linux-5.1/lib/zstd/zstd_internal.h 2019-05-07 17:19:25.016249580 +0200
-@@ -127,7 +127,13 @@ static const U32 OF_defaultNormLog = OF_
+diff -Naur linux-5.6.1/lib/zstd/zstd_internal.h linux-5.6.1-p/lib/zstd/zstd_internal.h
+--- linux-5.6.1/lib/zstd/zstd_internal.h 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/lib/zstd/zstd_internal.h 2020-04-01 13:28:46.908513191 +0200
+@@ -127,7 +127,14 @@
* Shared functions to include for inlining
*********************************************/
ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) {
- memcpy(dst, src, 8);
-+ /* zstd relies heavily on gcc being able to analyze and inline this
++ /*
++ * zstd relies heavily on gcc being able to analyze and inline this
+ * memcpy() call, since it is called in a tight loop. Preboot mode
+ * is compiled in freestanding mode, which stops gcc from analyzing
+ * memcpy(). Use __builtin_memcpy() to tell gcc to analyze this as a
@@ -617,15 +722,17 @@ diff -up linux-5.1/lib/zstd/zstd_internal.h.19~ linux-5.1/lib/zstd/zstd_internal
}
/*! ZSTD_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
-@@ -137,6 +143,7 @@ ZSTD_STATIC void ZSTD_wildcopy(void *dst
+@@ -137,13 +144,16 @@
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
-+#if GCC_VERSION >= 70000 && GCC_VERSION < 70200
- /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
+- /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
++#if defined(GCC_VERSION) && GCC_VERSION >= 70000 && GCC_VERSION < 70200
++ /*
++ * Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
* Avoid the bad case where the loop only runs once by handling the
* special case separately. This doesn't trigger the bug because it
-@@ -144,6 +151,7 @@ ZSTD_STATIC void ZSTD_wildcopy(void *dst
+ * doesn't involve pointer/integer overflow.
*/
if (length <= 8)
return ZSTD_copy8(dst, src);
@@ -633,10 +740,10 @@ diff -up linux-5.1/lib/zstd/zstd_internal.h.19~ linux-5.1/lib/zstd/zstd_internal
do {
ZSTD_copy8(op, ip);
op += 8;
-diff -up linux-5.1/scripts/Makefile.lib.19~ linux-5.1/scripts/Makefile.lib
---- linux-5.1/scripts/Makefile.lib.19~ 2019-05-07 17:19:25.017249565 +0200
-+++ linux-5.1/scripts/Makefile.lib 2019-05-07 17:19:58.549750595 +0200
-@@ -402,6 +402,21 @@ quiet_cmd_xzkern = XZKERN $@
+diff -Naur linux-5.6.1/scripts/Makefile.lib linux-5.6.1-p/scripts/Makefile.lib
+--- linux-5.6.1/scripts/Makefile.lib 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/scripts/Makefile.lib 2020-04-01 13:28:58.038585064 +0200
+@@ -394,6 +394,21 @@
quiet_cmd_xzmisc = XZMISC $@
cmd_xzmisc = cat $(real-prereqs) | xz --check=crc32 --lzma2=dict=1MiB > $@
@@ -651,17 +758,17 @@ diff -up linux-5.1/scripts/Makefile.lib.19~ linux-5.1/scripts/Makefile.lib
+
+quiet_cmd_zstd = ZSTD $@
+cmd_zstd = (cat $(filter-out FORCE,$^) | \
-+ zstd -19 && \
++ zstd -19 && \
+ $(call size_append, $(filter-out FORCE,$^))) > $@ || \
-+ (rm -f $@ ; false)
++ (rm -f $@ ; false)
+
# ASM offsets
# ---------------------------------------------------------------------------
-diff -up linux-5.1/usr/Kconfig.19~ linux-5.1/usr/Kconfig
---- linux-5.1/usr/Kconfig.19~ 2019-05-06 02:42:58.000000000 +0200
-+++ linux-5.1/usr/Kconfig 2019-05-07 17:19:25.017249565 +0200
-@@ -106,6 +106,15 @@ config RD_LZ4
+diff -Naur linux-5.6.1/usr/Kconfig linux-5.6.1-p/usr/Kconfig
+--- linux-5.6.1/usr/Kconfig 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/usr/Kconfig 2020-04-01 13:29:00.978604046 +0200
+@@ -100,6 +100,15 @@
Support loading of a LZ4 encoded initial ramdisk or cpio buffer
If unsure, say N.
@@ -676,10 +783,10 @@ diff -up linux-5.1/usr/Kconfig.19~ linux-5.1/usr/Kconfig
+
choice
prompt "Built-in initramfs compression mode"
- depends on INITRAMFS_SOURCE!=""
-@@ -214,6 +223,17 @@ config INITRAMFS_COMPRESSION_LZ4
- If you choose this, keep in mind that most distros don't provide lz4
- by default which could cause a build failure.
+ depends on INITRAMFS_SOURCE != ""
+@@ -207,4 +216,15 @@
+ short time at boot, while both the cpio image and the unpacked
+ filesystem image will be present in memory simultaneously
+config INITRAMFS_COMPRESSION_ZSTD
+ bool "ZSTD"
@@ -693,18 +800,14 @@ diff -up linux-5.1/usr/Kconfig.19~ linux-5.1/usr/Kconfig
+ tool to be able to compress the initram.
+
endchoice
+diff -Naur linux-5.6.1/usr/Makefile linux-5.6.1-p/usr/Makefile
+--- linux-5.6.1/usr/Makefile 2020-04-01 10:58:19.000000000 +0200
++++ linux-5.6.1-p/usr/Makefile 2020-04-01 13:29:00.978604046 +0200
+@@ -15,6 +15,7 @@
+ compress-$(CONFIG_INITRAMFS_COMPRESSION_XZ) := xzmisc
+ compress-$(CONFIG_INITRAMFS_COMPRESSION_LZO) := lzo
+ compress-$(CONFIG_INITRAMFS_COMPRESSION_LZ4) := lz4
++compress-$(CONFIG_INITRAMFS_COMPRESSION_ZSTD) := zstd
+
+ obj-$(CONFIG_BLK_DEV_INITRD) := initramfs_data.o
- config INITRAMFS_COMPRESSION
-@@ -226,10 +246,12 @@ config INITRAMFS_COMPRESSION
- default ".xz" if INITRAMFS_COMPRESSION_XZ
- default ".lzo" if INITRAMFS_COMPRESSION_LZO
- default ".lz4" if INITRAMFS_COMPRESSION_LZ4
-+ default ".zst" if INITRAMFS_COMPRESSION_ZSTD
- default ".gz" if RD_GZIP
- default ".lz4" if RD_LZ4
- default ".lzo" if RD_LZO
- default ".xz" if RD_XZ
- default ".lzma" if RD_LZMA
- default ".bz2" if RD_BZIP2
-+ default ".zst" if RD_ZSTD
- default ""
diff --git a/uksm-5.4.patch b/uksm-5.6.patch
similarity index 98%
rename from uksm-5.4.patch
rename to uksm-5.6.patch
index 31fc641..0fbc88d 100644
--- a/uksm-5.4.patch
+++ b/uksm-5.6.patch
@@ -66,10 +66,10 @@ index 000000000000..be19a3127001
+2016-09-10 UKSM 0.1.2.5 Fix a bug in dedup ratio calculation.
+2017-02-26 UKSM 0.1.2.6 Fix a bug in hugetlbpage handling and a race bug with page migration.
diff --git a/fs/exec.c b/fs/exec.c
-index 555e93c7dec8..7e1cfd568e6c 100644
+index db17be51b112..0ce4c6303f53 100644
--- a/fs/exec.c
+++ b/fs/exec.c
-@@ -63,6 +63,7 @@
+@@ -62,6 +62,7 @@
#include <linux/oom.h>
#include <linux/compat.h>
#include <linux/vmalloc.h>
@@ -77,7 +77,7 @@ index 555e93c7dec8..7e1cfd568e6c 100644
#include <linux/uaccess.h>
#include <asm/mmu_context.h>
-@@ -1384,6 +1385,7 @@ void setup_new_exec(struct linux_binprm * bprm)
+@@ -1387,6 +1388,7 @@ void setup_new_exec(struct linux_binprm * bprm)
/* An exec changes our domain. We are no longer part of the thread
group */
current->self_exec_id++;
@@ -102,10 +102,10 @@ index 8c1f1bb1a5ce..62e28cf10bbf 100644
global_node_page_state(NR_UNSTABLE_NFS));
show_val_kb(m, "Bounce: ",
diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
-index 818691846c90..b2169f37c696 100644
+index e2e2bef07dd2..59675a348878 100644
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
-@@ -855,12 +855,25 @@ extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
+@@ -866,12 +866,25 @@ extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
extern void untrack_pfn_moved(struct vm_area_struct *vma);
#endif
@@ -132,7 +132,7 @@ index 818691846c90..b2169f37c696 100644
}
#define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
-@@ -869,7 +882,7 @@ static inline int is_zero_pfn(unsigned long pfn)
+@@ -880,7 +893,7 @@ static inline int is_zero_pfn(unsigned long pfn)
static inline int is_zero_pfn(unsigned long pfn)
{
extern unsigned long zero_pfn;
@@ -147,7 +147,7 @@ index e48b1e453ff5..095d59310ce0 100644
+++ b/include/linux/ksm.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-+/* SPDX-License-Identifier: GPL-3.0 */
++/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_KSM_H
#define __LINUX_KSM_H
/*
@@ -219,10 +219,10 @@ index e48b1e453ff5..095d59310ce0 100644
+
#endif /* __LINUX_KSM_H */
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
-index 270aa8fd2800..8e1eaf6a2de3 100644
+index c28911c3afa8..d14689c20e56 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
-@@ -353,6 +353,9 @@ struct vm_area_struct {
+@@ -358,6 +358,9 @@ struct vm_area_struct {
struct mempolicy *vm_policy; /* NUMA policy for the VMA */
#endif
struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
@@ -233,7 +233,7 @@ index 270aa8fd2800..8e1eaf6a2de3 100644
struct core_thread {
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
-index bda20282746b..dab82e7cc3df 100644
+index 462f6873905a..43edd793d4e7 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -206,6 +206,9 @@ enum zone_stat_item {
@@ -485,10 +485,10 @@ index 000000000000..bb8651f534f2
+#endif /* !CONFIG_UKSM */
+#endif /* __LINUX_UKSM_H */
diff --git a/kernel/fork.c b/kernel/fork.c
-index 13b38794efb5..acb562745f1c 100644
+index d90af13431c7..b9d771b1acfb 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
-@@ -595,7 +595,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
+@@ -598,7 +598,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
__vma_link_rb(mm, tmp, rb_link, rb_parent);
rb_link = &tmp->vm_rb.rb_right;
rb_parent = &tmp->vm_rb;
@@ -498,18 +498,18 @@ index 13b38794efb5..acb562745f1c 100644
if (!(tmp->vm_flags & VM_WIPEONFORK))
retval = copy_page_range(mm, oldmm, mpnt);
diff --git a/lib/Makefile b/lib/Makefile
-index c5892807e06f..4b0d37da9fa0 100644
+index 611872c06926..1e3d86def26c 100644
--- a/lib/Makefile
+++ b/lib/Makefile
-@@ -25,7 +25,7 @@ CFLAGS_string.o := $(call cc-option, -fno-stack-protector)
+@@ -26,7 +26,7 @@ CFLAGS_string.o := $(call cc-option, -fno-stack-protector)
endif
lib-y := ctype.o string.o vsprintf.o cmdline.o \
- rbtree.o radix-tree.o timerqueue.o xarray.o \
+ rbtree.o radix-tree.o sradix-tree.o timerqueue.o xarray.o \
- idr.o extable.o \
- sha1.o chacha.o irq_regs.o argv_split.o \
+ idr.o extable.o sha1.o irq_regs.o argv_split.o \
flex_proportions.o ratelimit.o show_mem.o \
+ is_single_threaded.o plist.o decompress.o kobject_uevent.o \
diff --git a/lib/sradix-tree.c b/lib/sradix-tree.c
new file mode 100644
index 000000000000..ab21e6309b93
@@ -993,10 +993,10 @@ index 000000000000..ab21e6309b93
+ return 0;
+}
diff --git a/mm/Kconfig b/mm/Kconfig
-index a5dae9a7eb51..eda3a78226b0 100644
+index ab80933be65f..04796660748f 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
-@@ -299,6 +299,32 @@ config KSM
+@@ -300,6 +300,32 @@ config KSM
See Documentation/vm/ksm.rst for more information: KSM is inactive
until a program has madvised that an area is MADV_MERGEABLE, and
root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set).
@@ -1028,12 +1028,12 @@ index a5dae9a7eb51..eda3a78226b0 100644
+endchoice
config DEFAULT_MMAP_MIN_ADDR
- int "Low address space to protect from user allocation"
+ int "Low address space to protect from user allocation"
diff --git a/mm/Makefile b/mm/Makefile
-index d996846697ef..786cb9fad4b9 100644
+index 272e66039e70..20ec701eda60 100644
--- a/mm/Makefile
+++ b/mm/Makefile
-@@ -66,7 +66,8 @@ obj-$(CONFIG_SPARSEMEM) += sparse.o
+@@ -67,7 +67,8 @@ obj-$(CONFIG_SPARSEMEM) += sparse.o
obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o
obj-$(CONFIG_SLOB) += slob.o
obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
@@ -1044,7 +1044,7 @@ index d996846697ef..786cb9fad4b9 100644
obj-$(CONFIG_SLAB) += slab.o
obj-$(CONFIG_SLUB) += slub.o
diff --git a/mm/ksm.c b/mm/ksm.c
-index 7905934cd3ad..3e9d0064eaf2 100644
+index d17c7d57d0d8..d9942db0a21e 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -857,17 +857,6 @@ static int unmerge_ksm_pages(struct vm_area_struct *vma,
@@ -1066,10 +1066,10 @@ index 7905934cd3ad..3e9d0064eaf2 100644
/*
* Only called through the sysfs control interface:
diff --git a/mm/memory.c b/mm/memory.c
-index b1ca51a079f2..98a57e508b9b 100644
+index e8bfdf0d9d1d..3507bfef1c5e 100644
--- a/mm/memory.c
+++ b/mm/memory.c
-@@ -130,6 +130,25 @@ EXPORT_SYMBOL(zero_pfn);
+@@ -144,6 +144,25 @@ EXPORT_SYMBOL(zero_pfn);
unsigned long highest_memmap_pfn __read_mostly;
@@ -1095,15 +1095,15 @@ index b1ca51a079f2..98a57e508b9b 100644
/*
* CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
*/
-@@ -141,6 +160,7 @@ static int __init init_zero_pfn(void)
- core_initcall(init_zero_pfn);
-
+@@ -159,6 +178,7 @@ void mm_trace_rss_stat(struct mm_struct *mm, int member, long count)
+ trace_rss_stat(mm, member, count);
+ }
+
#if defined(SPLIT_RSS_COUNTING)
void sync_mm_rss(struct mm_struct *mm)
-@@ -772,8 +792,13 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+@@ -790,8 +810,13 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
get_page(page);
page_dup_rmap(page, false);
rss[mm_counter(page)]++;
@@ -1117,7 +1117,7 @@ index b1ca51a079f2..98a57e508b9b 100644
}
out_set_pte:
-@@ -1046,8 +1071,10 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
+@@ -1064,8 +1089,10 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
ptent = ptep_get_and_clear_full(mm, addr, pte,
tlb->fullmm);
tlb_remove_tlb_entry(tlb, pte, addr);
@@ -1129,19 +1129,15 @@ index b1ca51a079f2..98a57e508b9b 100644
if (!PageAnon(page)) {
if (pte_dirty(ptent)) {
-@@ -2169,8 +2196,10 @@ static inline void cow_user_page(struct page *dst, struct page *src, unsigned lo
- clear_page(kaddr);
- kunmap_atomic(kaddr);
- flush_dcache_page(dst);
-- } else
-+ } else {
- copy_user_highpage(dst, src, va, vma);
+@@ -2266,6 +2293,7 @@ static inline bool cow_user_page(struct page *dst, struct page *src,
+
+ if (likely(src)) {
+ copy_user_highpage(dst, src, addr, vma);
+ uksm_cow_page(vma, src);
-+ }
- }
+ return true;
+ }
- static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
-@@ -2322,6 +2351,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
+@@ -2512,6 +2540,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
vmf->address);
if (!new_page)
goto oom;
@@ -1149,7 +1145,7 @@ index b1ca51a079f2..98a57e508b9b 100644
} else {
new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
vmf->address);
-@@ -2351,7 +2381,9 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
+@@ -2553,7 +2582,9 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
mm_counter_file(old_page));
inc_mm_counter_fast(mm, MM_ANONPAGES);
}
@@ -1160,7 +1156,7 @@ index b1ca51a079f2..98a57e508b9b 100644
}
flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
diff --git a/mm/mmap.c b/mm/mmap.c
-index a7d8c84d19b7..797582240047 100644
+index d681a20eb4ea..acf07b12677c 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -46,6 +46,7 @@
@@ -1171,7 +1167,7 @@ index a7d8c84d19b7..797582240047 100644
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
-@@ -184,6 +185,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
+@@ -178,6 +179,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
if (vma->vm_file)
fput(vma->vm_file);
mpol_put(vma_policy(vma));
@@ -1179,7 +1175,7 @@ index a7d8c84d19b7..797582240047 100644
vm_area_free(vma);
return next;
}
-@@ -736,9 +738,16 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+@@ -705,9 +707,16 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
long adjust_next = 0;
int remove_next = 0;
@@ -1196,7 +1192,7 @@ index a7d8c84d19b7..797582240047 100644
if (end >= next->vm_end) {
/*
* vma expands, overlapping all the next, and
-@@ -871,6 +880,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+@@ -838,6 +847,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
end_changed = true;
}
vma->vm_pgoff = pgoff;
@@ -1204,7 +1200,7 @@ index a7d8c84d19b7..797582240047 100644
if (adjust_next) {
next->vm_start += adjust_next << PAGE_SHIFT;
next->vm_pgoff += adjust_next;
-@@ -976,6 +986,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+@@ -943,6 +953,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
if (remove_next == 2) {
remove_next = 1;
end = next->vm_end;
@@ -1212,7 +1208,7 @@ index a7d8c84d19b7..797582240047 100644
goto again;
}
else if (next)
-@@ -1002,10 +1013,14 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+@@ -969,10 +980,14 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
*/
VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma));
}
@@ -1227,7 +1223,7 @@ index a7d8c84d19b7..797582240047 100644
validate_mm(mm);
return 0;
-@@ -1465,6 +1480,9 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
+@@ -1431,6 +1446,9 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
@@ -1237,7 +1233,7 @@ index a7d8c84d19b7..797582240047 100644
if (flags & MAP_LOCKED)
if (!can_do_mlock())
return -EPERM;
-@@ -1835,6 +1853,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
+@@ -1799,6 +1817,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
allow_write_access(file);
}
file = vma->vm_file;
@@ -1245,7 +1241,7 @@ index a7d8c84d19b7..797582240047 100644
out:
perf_event_mmap(vma);
-@@ -1877,6 +1896,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
+@@ -1841,6 +1860,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
if (vm_flags & VM_DENYWRITE)
allow_write_access(file);
free_vma:
@@ -1253,7 +1249,7 @@ index a7d8c84d19b7..797582240047 100644
vm_area_free(vma);
unacct_error:
if (charged)
-@@ -2706,6 +2726,8 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
+@@ -2670,6 +2690,8 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
else
err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
@@ -1262,15 +1258,15 @@ index a7d8c84d19b7..797582240047 100644
/* Success. */
if (!err)
return 0;
-@@ -3011,6 +3033,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla
+@@ -2976,6 +2998,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla
if ((flags & (~VM_EXEC)) != 0)
return -EINVAL;
flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
+ uksm_vm_flags_mod(&flags);
- error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
- if (offset_in_page(error))
-@@ -3061,6 +3084,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla
+ mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
+ if (IS_ERR_VALUE(mapped_addr))
+@@ -3026,6 +3049,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla
vma->vm_flags = flags;
vma->vm_page_prot = vm_get_page_prot(flags);
vma_link(mm, vma, prev, rb_link, rb_parent);
@@ -1278,7 +1274,7 @@ index a7d8c84d19b7..797582240047 100644
out:
perf_event_mmap(vma);
mm->total_vm += len >> PAGE_SHIFT;
-@@ -3138,6 +3162,12 @@ void exit_mmap(struct mm_struct *mm)
+@@ -3103,6 +3127,12 @@ void exit_mmap(struct mm_struct *mm)
up_write(&mm->mmap_sem);
}
@@ -1291,7 +1287,7 @@ index a7d8c84d19b7..797582240047 100644
if (mm->locked_vm) {
vma = mm->mmap;
while (vma) {
-@@ -3172,6 +3202,11 @@ void exit_mmap(struct mm_struct *mm)
+@@ -3137,6 +3167,11 @@ void exit_mmap(struct mm_struct *mm)
vma = remove_vma(vma);
}
vm_unacct_memory(nr_accounted);
@@ -1303,7 +1299,7 @@ index a7d8c84d19b7..797582240047 100644
}
/* Insert vm structure into process list sorted by address
-@@ -3279,6 +3314,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
+@@ -3244,6 +3279,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
new_vma->vm_ops->open(new_vma);
vma_link(mm, new_vma, prev, rb_link, rb_parent);
*need_rmap_locks = false;
@@ -1311,7 +1307,7 @@ index a7d8c84d19b7..797582240047 100644
}
return new_vma;
-@@ -3429,6 +3465,7 @@ static struct vm_area_struct *__install_special_mapping(
+@@ -3396,6 +3432,7 @@ static struct vm_area_struct *__install_special_mapping(
vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
perf_event_mmap(vma);
@@ -6939,10 +6935,10 @@ index 000000000000..ef068d5dc307
+#endif
+
diff --git a/mm/vmstat.c b/mm/vmstat.c
-index a8222041bd44..7058e8322cbd 100644
+index 78d53378db99..fbb16b46b41b 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
-@@ -1168,6 +1168,9 @@ const char * const vmstat_text[] = {
+@@ -1169,6 +1169,9 @@ const char * const vmstat_text[] = {
"nr_written",
"nr_kernel_misc_reclaimable",
diff --git a/uksm-5.7.patch b/uksm-5.7.patch
new file mode 100644
index 0000000..471f357
--- /dev/null
+++ b/uksm-5.7.patch
@@ -0,0 +1,6936 @@
+diff --git a/Documentation/vm/uksm.txt b/Documentation/vm/uksm.txt
+new file mode 100644
+index 000000000000..be19a3127001
+--- /dev/null
++++ b/Documentation/vm/uksm.txt
+@@ -0,0 +1,61 @@
++The Ultra Kernel Samepage Merging feature
++----------------------------------------------
++/*
++ * Ultra KSM. Copyright (C) 2011-2012 Nai Xia
++ *
++ * This is an improvement upon KSM. Some basic data structures and routines
++ * are borrowed from ksm.c .
++ *
++ * Its new features:
++ * 1. Full system scan:
++ * It automatically scans all user processes' anonymous VMAs. Kernel-user
++ * interaction to submit a memory area to KSM is no longer needed.
++ *
++ * 2. Rich area detection:
++ * It automatically detects rich areas containing abundant duplicated
++ * pages based. Rich areas are given a full scan speed. Poor areas are
++ * sampled at a reasonable speed with very low CPU consumption.
++ *
++ * 3. Ultra Per-page scan speed improvement:
++ * A new hash algorithm is proposed. As a result, on a machine with
++ * Core(TM)2 Quad Q9300 CPU in 32-bit mode and 800MHZ DDR2 main memory, it
++ * can scan memory areas that does not contain duplicated pages at speed of
++ * 627MB/sec ~ 2445MB/sec and can merge duplicated areas at speed of
++ * 477MB/sec ~ 923MB/sec.
++ *
++ * 4. Thrashing area avoidance:
++ * Thrashing area(an VMA that has frequent Ksm page break-out) can be
++ * filtered out. My benchmark shows it's more efficient than KSM's per-page
++ * hash value based volatile page detection.
++ *
++ *
++ * 5. Misc changes upon KSM:
++ * * It has a fully x86-opitmized memcmp dedicated for 4-byte-aligned page
++ * comparison. It's much faster than default C version on x86.
++ * * rmap_item now has an struct *page member to loosely cache a
++ * address-->page mapping, which reduces too much time-costly
++ * follow_page().
++ * * The VMA creation/exit procedures are hooked to let the Ultra KSM know.
++ * * try_to_merge_two_pages() now can revert a pte if it fails. No break_
++ * ksm is needed for this case.
++ *
++ * 6. Full Zero Page consideration(contributed by Figo Zhang)
++ * Now uksmd consider full zero pages as special pages and merge them to an
++ * special unswappable uksm zero page.
++ */
++
++ChangeLog:
++
++2012-05-05 The creation of this Doc
++2012-05-08 UKSM 0.1.1.1 libc crash bug fix, api clean up, doc clean up.
++2012-05-28 UKSM 0.1.1.2 bug fix release
++2012-06-26 UKSM 0.1.2-beta1 first beta release for 0.1.2
++2012-07-2 UKSM 0.1.2-beta2
++2012-07-10 UKSM 0.1.2-beta3
++2012-07-26 UKSM 0.1.2 Fine grained speed control, more scan optimization.
++2012-10-13 UKSM 0.1.2.1 Bug fixes.
++2012-12-31 UKSM 0.1.2.2 Minor bug fixes.
++2014-07-02 UKSM 0.1.2.3 Fix a " __this_cpu_read() in preemptible bug".
++2015-04-22 UKSM 0.1.2.4 Fix a race condition that can sometimes trigger anonying warnings.
++2016-09-10 UKSM 0.1.2.5 Fix a bug in dedup ratio calculation.
++2017-02-26 UKSM 0.1.2.6 Fix a bug in hugetlbpage handling and a race bug with page migration.
+diff --git a/fs/exec.c b/fs/exec.c
+index 2c465119affc..50890203108a 100644
+--- a/fs/exec.c
++++ b/fs/exec.c
+@@ -62,6 +62,7 @@
+ #include <linux/oom.h>
+ #include <linux/compat.h>
+ #include <linux/vmalloc.h>
++#include <linux/ksm.h>
+
+ #include <linux/uaccess.h>
+ #include <asm/mmu_context.h>
+diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c
+index 8c1f1bb1a5ce..62e28cf10bbf 100644
+--- a/fs/proc/meminfo.c
++++ b/fs/proc/meminfo.c
+@@ -105,7 +105,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
+ global_zone_page_state(NR_KERNEL_STACK_KB));
+ show_val_kb(m, "PageTables: ",
+ global_zone_page_state(NR_PAGETABLE));
+-
++#ifdef CONFIG_UKSM
++ show_val_kb(m, "KsmZeroPages: ",
++ global_zone_page_state(NR_UKSM_ZERO_PAGES));
++#endif
+ show_val_kb(m, "NFS_Unstable: ",
+ global_node_page_state(NR_UNSTABLE_NFS));
+ show_val_kb(m, "Bounce: ",
+diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
+index 329b8c8ca703..22fb5a592b98 100644
+--- a/include/asm-generic/pgtable.h
++++ b/include/asm-generic/pgtable.h
+@@ -867,12 +867,25 @@ extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
+ extern void untrack_pfn_moved(struct vm_area_struct *vma);
+ #endif
+
++#ifdef CONFIG_UKSM
++static inline int is_uksm_zero_pfn(unsigned long pfn)
++{
++ extern unsigned long uksm_zero_pfn;
++ return pfn == uksm_zero_pfn;
++}
++#else
++static inline int is_uksm_zero_pfn(unsigned long pfn)
++{
++ return 0;
++}
++#endif
++
+ #ifdef __HAVE_COLOR_ZERO_PAGE
+ static inline int is_zero_pfn(unsigned long pfn)
+ {
+ extern unsigned long zero_pfn;
+ unsigned long offset_from_zero_pfn = pfn - zero_pfn;
+- return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
++ return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT) || is_uksm_zero_pfn(pfn);
+ }
+
+ #define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
+@@ -881,7 +894,7 @@ static inline int is_zero_pfn(unsigned long pfn)
+ static inline int is_zero_pfn(unsigned long pfn)
+ {
+ extern unsigned long zero_pfn;
+- return pfn == zero_pfn;
++ return (pfn == zero_pfn) || (is_uksm_zero_pfn(pfn));
+ }
+
+ static inline unsigned long my_zero_pfn(unsigned long addr)
+diff --git a/include/linux/ksm.h b/include/linux/ksm.h
+index e48b1e453ff5..8cc8077cebf4 100644
+--- a/include/linux/ksm.h
++++ b/include/linux/ksm.h
+@@ -21,20 +21,16 @@ struct mem_cgroup;
+ #ifdef CONFIG_KSM
+ int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
+ unsigned long end, int advice, unsigned long *vm_flags);
+-int __ksm_enter(struct mm_struct *mm);
+-void __ksm_exit(struct mm_struct *mm);
+
+-static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
++static inline struct stable_node *page_stable_node(struct page *page)
+ {
+- if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags))
+- return __ksm_enter(mm);
+- return 0;
++ return PageKsm(page) ? page_rmapping(page) : NULL;
+ }
+
+-static inline void ksm_exit(struct mm_struct *mm)
++static inline void set_page_stable_node(struct page *page,
++ struct stable_node *stable_node)
+ {
+- if (test_bit(MMF_VM_MERGEABLE, &mm->flags))
+- __ksm_exit(mm);
++ page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM);
+ }
+
+ /*
+@@ -56,6 +52,33 @@ void ksm_migrate_page(struct page *newpage, struct page *oldpage);
+ bool reuse_ksm_page(struct page *page,
+ struct vm_area_struct *vma, unsigned long address);
+
++#ifdef CONFIG_KSM_LEGACY
++int __ksm_enter(struct mm_struct *mm);
++void __ksm_exit(struct mm_struct *mm);
++static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
++{
++ if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags))
++ return __ksm_enter(mm);
++ return 0;
++}
++
++static inline void ksm_exit(struct mm_struct *mm)
++{
++ if (test_bit(MMF_VM_MERGEABLE, &mm->flags))
++ __ksm_exit(mm);
++}
++
++#elif defined(CONFIG_UKSM)
++static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
++{
++ return 0;
++}
++
++static inline void ksm_exit(struct mm_struct *mm)
++{
++}
++#endif /* !CONFIG_UKSM */
++
+ #else /* !CONFIG_KSM */
+
+ static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
+@@ -96,4 +119,6 @@ static inline bool reuse_ksm_page(struct page *page,
+ #endif /* CONFIG_MMU */
+ #endif /* !CONFIG_KSM */
+
++#include <linux/uksm.h>
++
+ #endif /* __LINUX_KSM_H */
+diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
+index 4aba6c0c2ba8..d4a58fce7f42 100644
+--- a/include/linux/mm_types.h
++++ b/include/linux/mm_types.h
+@@ -363,6 +363,9 @@ struct vm_area_struct {
+ struct mempolicy *vm_policy; /* NUMA policy for the VMA */
+ #endif
+ struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
++#ifdef CONFIG_UKSM
++ struct vma_slot *uksm_vma_slot;
++#endif
+ } __randomize_layout;
+
+ struct core_thread {
+diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
+index 1b9de7d220fb..84519d66a253 100644
+--- a/include/linux/mmzone.h
++++ b/include/linux/mmzone.h
+@@ -162,6 +162,9 @@ enum zone_stat_item {
+ NR_ZSPAGES, /* allocated in zsmalloc */
+ #endif
+ NR_FREE_CMA_PAGES,
++#ifdef CONFIG_UKSM
++ NR_UKSM_ZERO_PAGES,
++#endif
+ NR_VM_ZONE_STAT_ITEMS };
+
+ enum node_stat_item {
+diff --git a/include/linux/sradix-tree.h b/include/linux/sradix-tree.h
+new file mode 100644
+index 000000000000..d71edba6b63f
+--- /dev/null
++++ b/include/linux/sradix-tree.h
+@@ -0,0 +1,77 @@
++#ifndef _LINUX_SRADIX_TREE_H
++#define _LINUX_SRADIX_TREE_H
++
++
++#define INIT_SRADIX_TREE(root, mask) \
++do { \
++ (root)->height = 0; \
++ (root)->gfp_mask = (mask); \
++ (root)->rnode = NULL; \
++} while (0)
++
++#define ULONG_BITS (sizeof(unsigned long) * 8)
++#define SRADIX_TREE_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(unsigned long))
++//#define SRADIX_TREE_MAP_SHIFT 6
++//#define SRADIX_TREE_MAP_SIZE (1UL << SRADIX_TREE_MAP_SHIFT)
++//#define SRADIX_TREE_MAP_MASK (SRADIX_TREE_MAP_SIZE-1)
++
++struct sradix_tree_node {
++ unsigned int height; /* Height from the bottom */
++ unsigned int count;
++ unsigned int fulls; /* Number of full sublevel trees */
++ struct sradix_tree_node *parent;
++ void *stores[0];
++};
++
++/* A simple radix tree implementation */
++struct sradix_tree_root {
++ unsigned int height;
++ struct sradix_tree_node *rnode;
++
++ /* Where found to have available empty stores in its sublevels */
++ struct sradix_tree_node *enter_node;
++ unsigned int shift;
++ unsigned int stores_size;
++ unsigned int mask;
++ unsigned long min; /* The first hole index */
++ unsigned long num;
++ //unsigned long *height_to_maxindex;
++
++ /* How the node is allocated and freed. */
++ struct sradix_tree_node *(*alloc)(void);
++ void (*free)(struct sradix_tree_node *node);
++
++ /* When a new node is added and removed */
++ void (*extend)(struct sradix_tree_node *parent, struct sradix_tree_node *child);
++ void (*assign)(struct sradix_tree_node *node, unsigned int index, void *item);
++ void (*rm)(struct sradix_tree_node *node, unsigned int offset);
++};
++
++struct sradix_tree_path {
++ struct sradix_tree_node *node;
++ int offset;
++};
++
++static inline
++void init_sradix_tree_root(struct sradix_tree_root *root, unsigned long shift)
++{
++ root->height = 0;
++ root->rnode = NULL;
++ root->shift = shift;
++ root->stores_size = 1UL << shift;
++ root->mask = root->stores_size - 1;
++}
++
++
++extern void *sradix_tree_next(struct sradix_tree_root *root,
++ struct sradix_tree_node *node, unsigned long index,
++ int (*iter)(void *, unsigned long));
++
++extern int sradix_tree_enter(struct sradix_tree_root *root, void **item, int num);
++
++extern void sradix_tree_delete_from_leaf(struct sradix_tree_root *root,
++ struct sradix_tree_node *node, unsigned long index);
++
++extern void *sradix_tree_lookup(struct sradix_tree_root *root, unsigned long index);
++
++#endif /* _LINUX_SRADIX_TREE_H */
+diff --git a/include/linux/uksm.h b/include/linux/uksm.h
+new file mode 100644
+index 000000000000..bb8651f534f2
+--- /dev/null
++++ b/include/linux/uksm.h
+@@ -0,0 +1,149 @@
++#ifndef __LINUX_UKSM_H
++#define __LINUX_UKSM_H
++/*
++ * Memory merging support.
++ *
++ * This code enables dynamic sharing of identical pages found in different
++ * memory areas, even if they are not shared by fork().
++ */
++
++/* if !CONFIG_UKSM this file should not be compiled at all. */
++#ifdef CONFIG_UKSM
++
++#include <linux/bitops.h>
++#include <linux/mm.h>
++#include <linux/pagemap.h>
++#include <linux/rmap.h>
++#include <linux/sched.h>
++
++extern unsigned long zero_pfn __read_mostly;
++extern unsigned long uksm_zero_pfn __read_mostly;
++extern struct page *empty_uksm_zero_page;
++
++/* must be done before linked to mm */
++extern void uksm_vma_add_new(struct vm_area_struct *vma);
++extern void uksm_remove_vma(struct vm_area_struct *vma);
++
++#define UKSM_SLOT_NEED_SORT (1 << 0)
++#define UKSM_SLOT_NEED_RERAND (1 << 1)
++#define UKSM_SLOT_SCANNED (1 << 2) /* It's scanned in this round */
++#define UKSM_SLOT_FUL_SCANNED (1 << 3)
++#define UKSM_SLOT_IN_UKSM (1 << 4)
++
++struct vma_slot {
++ struct sradix_tree_node *snode;
++ unsigned long sindex;
++
++ struct list_head slot_list;
++ unsigned long fully_scanned_round;
++ unsigned long dedup_num;
++ unsigned long pages_scanned;
++ unsigned long this_sampled;
++ unsigned long last_scanned;
++ unsigned long pages_to_scan;
++ struct scan_rung *rung;
++ struct page **rmap_list_pool;
++ unsigned int *pool_counts;
++ unsigned long pool_size;
++ struct vm_area_struct *vma;
++ struct mm_struct *mm;
++ unsigned long ctime_j;
++ unsigned long pages;
++ unsigned long flags;
++ unsigned long pages_cowed; /* pages cowed this round */
++ unsigned long pages_merged; /* pages merged this round */
++ unsigned long pages_bemerged;
++
++ /* when it has page merged in this eval round */
++ struct list_head dedup_list;
++};
++
++static inline void uksm_unmap_zero_page(pte_t pte)
++{
++ if (pte_pfn(pte) == uksm_zero_pfn)
++ __dec_zone_page_state(empty_uksm_zero_page, NR_UKSM_ZERO_PAGES);
++}
++
++static inline void uksm_map_zero_page(pte_t pte)
++{
++ if (pte_pfn(pte) == uksm_zero_pfn)
++ __inc_zone_page_state(empty_uksm_zero_page, NR_UKSM_ZERO_PAGES);
++}
++
++static inline void uksm_cow_page(struct vm_area_struct *vma, struct page *page)
++{
++ if (vma->uksm_vma_slot && PageKsm(page))
++ vma->uksm_vma_slot->pages_cowed++;
++}
++
++static inline void uksm_cow_pte(struct vm_area_struct *vma, pte_t pte)
++{
++ if (vma->uksm_vma_slot && pte_pfn(pte) == uksm_zero_pfn)
++ vma->uksm_vma_slot->pages_cowed++;
++}
++
++static inline int uksm_flags_can_scan(unsigned long vm_flags)
++{
++#ifdef VM_SAO
++ if (vm_flags & VM_SAO)
++ return 0;
++#endif
++
++ return !(vm_flags & (VM_PFNMAP | VM_IO | VM_DONTEXPAND |
++ VM_HUGETLB | VM_MIXEDMAP | VM_SHARED
++ | VM_MAYSHARE | VM_GROWSUP | VM_GROWSDOWN));
++}
++
++static inline void uksm_vm_flags_mod(unsigned long *vm_flags_p)
++{
++ if (uksm_flags_can_scan(*vm_flags_p))
++ *vm_flags_p |= VM_MERGEABLE;
++}
++
++/*
++ * Just a wrapper for BUG_ON for where ksm_zeropage must not be. TODO: it will
++ * be removed when uksm zero page patch is stable enough.
++ */
++static inline void uksm_bugon_zeropage(pte_t pte)
++{
++ BUG_ON(pte_pfn(pte) == uksm_zero_pfn);
++}
++#else
++static inline void uksm_vma_add_new(struct vm_area_struct *vma)
++{
++}
++
++static inline void uksm_remove_vma(struct vm_area_struct *vma)
++{
++}
++
++static inline void uksm_unmap_zero_page(pte_t pte)
++{
++}
++
++static inline void uksm_map_zero_page(pte_t pte)
++{
++}
++
++static inline void uksm_cow_page(struct vm_area_struct *vma, struct page *page)
++{
++}
++
++static inline void uksm_cow_pte(struct vm_area_struct *vma, pte_t pte)
++{
++}
++
++static inline int uksm_flags_can_scan(unsigned long vm_flags)
++{
++ return 0;
++}
++
++static inline void uksm_vm_flags_mod(unsigned long *vm_flags_p)
++{
++}
++
++static inline void uksm_bugon_zeropage(pte_t pte)
++{
++}
++#endif /* !CONFIG_UKSM */
++#endif /* __LINUX_UKSM_H */
+diff --git a/kernel/fork.c b/kernel/fork.c
+index 48ed22774efa..9b5b1ff9babf 100644
+--- a/kernel/fork.c
++++ b/kernel/fork.c
+@@ -601,7 +601,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
+ __vma_link_rb(mm, tmp, rb_link, rb_parent);
+ rb_link = &tmp->vm_rb.rb_right;
+ rb_parent = &tmp->vm_rb;
+-
++ uksm_vma_add_new(tmp);
+ mm->map_count++;
+ if (!(tmp->vm_flags & VM_WIPEONFORK))
+ retval = copy_page_range(mm, oldmm, mpnt);
+diff --git a/lib/Makefile b/lib/Makefile
+index 685aee60de1d..e9584575b51c 100644
+--- a/lib/Makefile
++++ b/lib/Makefile
+@@ -26,7 +26,7 @@ CFLAGS_string.o := $(call cc-option, -fno-stack-protector)
+ endif
+
+ lib-y := ctype.o string.o vsprintf.o cmdline.o \
+- rbtree.o radix-tree.o timerqueue.o xarray.o \
++ rbtree.o radix-tree.o sradix-tree.o timerqueue.o xarray.o \
+ idr.o extable.o sha1.o irq_regs.o argv_split.o \
+ flex_proportions.o ratelimit.o show_mem.o \
+ is_single_threaded.o plist.o decompress.o kobject_uevent.o \
+diff --git a/lib/sradix-tree.c b/lib/sradix-tree.c
+new file mode 100644
+index 000000000000..ab21e6309b93
+--- /dev/null
++++ b/lib/sradix-tree.c
+@@ -0,0 +1,476 @@
++#include <linux/errno.h>
++#include <linux/mm.h>
++#include <linux/mman.h>
++#include <linux/spinlock.h>
++#include <linux/slab.h>
++#include <linux/gcd.h>
++#include <linux/sradix-tree.h>
++
++static inline int sradix_node_full(struct sradix_tree_root *root, struct sradix_tree_node *node)
++{
++ return node->fulls == root->stores_size ||
++ (node->height == 1 && node->count == root->stores_size);
++}
++
++/*
++ * Extend a sradix tree so it can store key @index.
++ */
++static int sradix_tree_extend(struct sradix_tree_root *root, unsigned long index)
++{
++ struct sradix_tree_node *node;
++ unsigned int height;
++
++ if (unlikely(root->rnode == NULL)) {
++ if (!(node = root->alloc()))
++ return -ENOMEM;
++
++ node->height = 1;
++ root->rnode = node;
++ root->height = 1;
++ }
++
++ /* Figure out what the height should be. */
++ height = root->height;
++ index >>= root->shift * height;
++
++ while (index) {
++ index >>= root->shift;
++ height++;
++ }
++
++ while (height > root->height) {
++ unsigned int newheight;
++
++ if (!(node = root->alloc()))
++ return -ENOMEM;
++
++ /* Increase the height. */
++ node->stores[0] = root->rnode;
++ root->rnode->parent = node;
++ if (root->extend)
++ root->extend(node, root->rnode);
++
++ newheight = root->height + 1;
++ node->height = newheight;
++ node->count = 1;
++ if (sradix_node_full(root, root->rnode))
++ node->fulls = 1;
++
++ root->rnode = node;
++ root->height = newheight;
++ }
++
++ return 0;
++}
++
++/*
++ * Search the next item from the current node, that is not NULL
++ * and can satify root->iter().
++ */
++void *sradix_tree_next(struct sradix_tree_root *root,
++ struct sradix_tree_node *node, unsigned long index,
++ int (*iter)(void *item, unsigned long height))
++{
++ unsigned long offset;
++ void *item;
++
++ if (unlikely(node == NULL)) {
++ node = root->rnode;
++ for (offset = 0; offset < root->stores_size; offset++) {
++ item = node->stores[offset];
++ if (item && (!iter || iter(item, node->height)))
++ break;
++ }
++
++ if (unlikely(offset >= root->stores_size))
++ return NULL;
++
++ if (node->height == 1)
++ return item;
++ else
++ goto go_down;
++ }
++
++ while (node) {
++ offset = (index & root->mask) + 1;
++ for (; offset < root->stores_size; offset++) {
++ item = node->stores[offset];
++ if (item && (!iter || iter(item, node->height)))
++ break;
++ }
++
++ if (offset < root->stores_size)
++ break;
++
++ node = node->parent;
++ index >>= root->shift;
++ }
++
++ if (!node)
++ return NULL;
++
++ while (node->height > 1) {
++go_down:
++ node = item;
++ for (offset = 0; offset < root->stores_size; offset++) {
++ item = node->stores[offset];
++ if (item && (!iter || iter(item, node->height)))
++ break;
++ }
++
++ if (unlikely(offset >= root->stores_size))
++ return NULL;
++ }
++
++ BUG_ON(offset > root->stores_size);
++
++ return item;
++}
++
++/*
++ * Blindly insert the item to the tree. Typically, we reuse the
++ * first empty store item.
++ */
++int sradix_tree_enter(struct sradix_tree_root *root, void **item, int num)
++{
++ unsigned long index;
++ unsigned int height;
++ struct sradix_tree_node *node, *tmp = NULL;
++ int offset, offset_saved;
++ void **store = NULL;
++ int error, i, j, shift;
++
++go_on:
++ index = root->min;
++
++ if (root->enter_node && !sradix_node_full(root, root->enter_node)) {
++ node = root->enter_node;
++ BUG_ON((index >> (root->shift * root->height)));
++ } else {
++ node = root->rnode;
++ if (node == NULL || (index >> (root->shift * root->height))
++ || sradix_node_full(root, node)) {
++ error = sradix_tree_extend(root, index);
++ if (error)
++ return error;
++
++ node = root->rnode;
++ }
++ }
++
++
++ height = node->height;
++ shift = (height - 1) * root->shift;
++ offset = (index >> shift) & root->mask;
++ while (shift > 0) {
++ offset_saved = offset;
++ for (; offset < root->stores_size; offset++) {
++ store = &node->stores[offset];
++ tmp = *store;
++
++ if (!tmp || !sradix_node_full(root, tmp))
++ break;
++ }
++ BUG_ON(offset >= root->stores_size);
++
++ if (offset != offset_saved) {
++ index += (offset - offset_saved) << shift;
++ index &= ~((1UL << shift) - 1);
++ }
++
++ if (!tmp) {
++ if (!(tmp = root->alloc()))
++ return -ENOMEM;
++
++ tmp->height = shift / root->shift;
++ *store = tmp;
++ tmp->parent = node;
++ node->count++;
++// if (root->extend)
++// root->extend(node, tmp);
++ }
++
++ node = tmp;
++ shift -= root->shift;
++ offset = (index >> shift) & root->mask;
++ }
++
++ BUG_ON(node->height != 1);
++
++
++ store = &node->stores[offset];
++ for (i = 0, j = 0;
++ j < root->stores_size - node->count &&
++ i < root->stores_size - offset && j < num; i++) {
++ if (!store[i]) {
++ store[i] = item[j];
++ if (root->assign)
++ root->assign(node, index + i, item[j]);
++ j++;
++ }
++ }
++
++ node->count += j;
++ root->num += j;
++ num -= j;
++
++ while (sradix_node_full(root, node)) {
++ node = node->parent;
++ if (!node)
++ break;
++
++ node->fulls++;
++ }
++
++ if (unlikely(!node)) {
++ /* All nodes are full */
++ root->min = 1 << (root->height * root->shift);
++ root->enter_node = NULL;
++ } else {
++ root->min = index + i - 1;
++ root->min |= (1UL << (node->height - 1)) - 1;
++ root->min++;
++ root->enter_node = node;
++ }
++
++ if (num) {
++ item += j;
++ goto go_on;
++ }
++
++ return 0;
++}
++
++
++/**
++ * sradix_tree_shrink - shrink height of a sradix tree to minimal
++ * @root sradix tree root
++ *
++ */
++static inline void sradix_tree_shrink(struct sradix_tree_root *root)
++{
++ /* try to shrink tree height */
++ while (root->height > 1) {
++ struct sradix_tree_node *to_free = root->rnode;
++
++ /*
++ * The candidate node has more than one child, or its child
++ * is not at the leftmost store, we cannot shrink.
++ */
++ if (to_free->count != 1 || !to_free->stores[0])
++ break;
++
++ root->rnode = to_free->stores[0];
++ root->rnode->parent = NULL;
++ root->height--;
++ if (unlikely(root->enter_node == to_free))
++ root->enter_node = NULL;
++ root->free(to_free);
++ }
++}
++
++/*
++ * Del the item on the known leaf node and index
++ */
++void sradix_tree_delete_from_leaf(struct sradix_tree_root *root,
++ struct sradix_tree_node *node, unsigned long index)
++{
++ unsigned int offset;
++ struct sradix_tree_node *start, *end;
++
++ BUG_ON(node->height != 1);
++
++ start = node;
++ while (node && !(--node->count))
++ node = node->parent;
++
++ end = node;
++ if (!node) {
++ root->rnode = NULL;
++ root->height = 0;
++ root->min = 0;
++ root->num = 0;
++ root->enter_node = NULL;
++ } else {
++ offset = (index >> (root->shift * (node->height - 1))) & root->mask;
++ if (root->rm)
++ root->rm(node, offset);
++ node->stores[offset] = NULL;
++ root->num--;
++ if (root->min > index) {
++ root->min = index;
++ root->enter_node = node;
++ }
++ }
++
++ if (start != end) {
++ do {
++ node = start;
++ start = start->parent;
++ if (unlikely(root->enter_node == node))
++ root->enter_node = end;
++ root->free(node);
++ } while (start != end);
++
++ /*
++ * Note that shrink may free "end", so enter_node still need to
++ * be checked inside.
++ */
++ sradix_tree_shrink(root);
++ } else if (node->count == root->stores_size - 1) {
++ /* It WAS a full leaf node. Update the ancestors */
++ node = node->parent;
++ while (node) {
++ node->fulls--;
++ if (node->fulls != root->stores_size - 1)
++ break;
++
++ node = node->parent;
++ }
++ }
++}
++
++void *sradix_tree_lookup(struct sradix_tree_root *root, unsigned long index)
++{
++ unsigned int height, offset;
++ struct sradix_tree_node *node;
++ int shift;
++
++ node = root->rnode;
++ if (node == NULL || (index >> (root->shift * root->height)))
++ return NULL;
++
++ height = root->height;
++ shift = (height - 1) * root->shift;
++
++ do {
++ offset = (index >> shift) & root->mask;
++ node = node->stores[offset];
++ if (!node)
++ return NULL;
++
++ shift -= root->shift;
++ } while (shift >= 0);
++
++ return node;
++}
++
++/*
++ * Return the item if it exists, otherwise create it in place
++ * and return the created item.
++ */
++void *sradix_tree_lookup_create(struct sradix_tree_root *root,
++ unsigned long index, void *(*item_alloc)(void))
++{
++ unsigned int height, offset;
++ struct sradix_tree_node *node, *tmp;
++ void *item;
++ int shift, error;
++
++ if (root->rnode == NULL || (index >> (root->shift * root->height))) {
++ if (item_alloc) {
++ error = sradix_tree_extend(root, index);
++ if (error)
++ return NULL;
++ } else {
++ return NULL;
++ }
++ }
++
++ node = root->rnode;
++ height = root->height;
++ shift = (height - 1) * root->shift;
++
++ do {
++ offset = (index >> shift) & root->mask;
++ if (!node->stores[offset]) {
++ if (!(tmp = root->alloc()))
++ return NULL;
++
++ tmp->height = shift / root->shift;
++ node->stores[offset] = tmp;
++ tmp->parent = node;
++ node->count++;
++ node = tmp;
++ } else {
++ node = node->stores[offset];
++ }
++
++ shift -= root->shift;
++ } while (shift > 0);
++
++ BUG_ON(node->height != 1);
++ offset = index & root->mask;
++ if (node->stores[offset]) {
++ return node->stores[offset];
++ } else if (item_alloc) {
++ if (!(item = item_alloc()))
++ return NULL;
++
++ node->stores[offset] = item;
++
++ /*
++ * NOTE: we do NOT call root->assign here, since this item is
++ * newly created by us having no meaning. Caller can call this
++ * if it's necessary to do so.
++ */
++
++ node->count++;
++ root->num++;
++
++ while (sradix_node_full(root, node)) {
++ node = node->parent;
++ if (!node)
++ break;
++
++ node->fulls++;
++ }
++
++ if (unlikely(!node)) {
++ /* All nodes are full */
++ root->min = 1 << (root->height * root->shift);
++ } else {
++ if (root->min == index) {
++ root->min |= (1UL << (node->height - 1)) - 1;
++ root->min++;
++ root->enter_node = node;
++ }
++ }
++
++ return item;
++ } else {
++ return NULL;
++ }
++
++}
++
++int sradix_tree_delete(struct sradix_tree_root *root, unsigned long index)
++{
++ unsigned int height, offset;
++ struct sradix_tree_node *node;
++ int shift;
++
++ node = root->rnode;
++ if (node == NULL || (index >> (root->shift * root->height)))
++ return -ENOENT;
++
++ height = root->height;
++ shift = (height - 1) * root->shift;
++
++ do {
++ offset = (index >> shift) & root->mask;
++ node = node->stores[offset];
++ if (!node)
++ return -ENOENT;
++
++ shift -= root->shift;
++ } while (shift > 0);
++
++ offset = index & root->mask;
++ if (!node->stores[offset])
++ return -ENOENT;
++
++ sradix_tree_delete_from_leaf(root, node, index);
++
++ return 0;
++}
+diff --git a/mm/Kconfig b/mm/Kconfig
+index c1acc34c1c35..ecdcd88f5275 100644
+--- a/mm/Kconfig
++++ b/mm/Kconfig
+@@ -316,6 +316,32 @@ config KSM
+ See Documentation/vm/ksm.rst for more information: KSM is inactive
+ until a program has madvised that an area is MADV_MERGEABLE, and
+ root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set).
++choice
++ prompt "Choose UKSM/KSM strategy"
++ default UKSM
++ depends on KSM
++ help
++ This option allows to select a UKSM/KSM stragety.
++
++config UKSM
++ bool "Ultra-KSM for page merging"
++ depends on KSM
++ help
++ UKSM is inspired by the Linux kernel project \u2014 KSM(Kernel Same
++ page Merging), but with a fundamentally rewritten core algorithm. With
++ an advanced algorithm, UKSM now can transparently scans all anonymously
++ mapped user space applications with an significantly improved scan speed
++ and CPU efficiency. Since KVM is friendly to KSM, KVM can also benefit from
++ UKSM. Now UKSM has its first stable release and first real world enterprise user.
++ For more information, please goto its project page.
++ (github.com/dolohow/uksm)
++
++config KSM_LEGACY
++ bool "Legacy KSM implementation"
++ depends on KSM
++ help
++ The legacy KSM implementation from Red Hat.
++endchoice
+
+ config DEFAULT_MMAP_MIN_ADDR
+ int "Low address space to protect from user allocation"
+diff --git a/mm/Makefile b/mm/Makefile
+index fccd3756b25f..c157e08bd87a 100644
+--- a/mm/Makefile
++++ b/mm/Makefile
+@@ -68,7 +68,8 @@ obj-$(CONFIG_SPARSEMEM) += sparse.o
+ obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o
+ obj-$(CONFIG_SLOB) += slob.o
+ obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
+-obj-$(CONFIG_KSM) += ksm.o
++obj-$(CONFIG_KSM_LEGACY) += ksm.o
++obj-$(CONFIG_UKSM) += uksm.o
+ obj-$(CONFIG_PAGE_POISONING) += page_poison.o
+ obj-$(CONFIG_SLAB) += slab.o
+ obj-$(CONFIG_SLUB) += slub.o
+diff --git a/mm/ksm.c b/mm/ksm.c
+index 281c00129a2e..48d9ac624335 100644
+--- a/mm/ksm.c
++++ b/mm/ksm.c
+@@ -857,17 +857,6 @@ static int unmerge_ksm_pages(struct vm_area_struct *vma,
+ return err;
+ }
+
+-static inline struct stable_node *page_stable_node(struct page *page)
+-{
+- return PageKsm(page) ? page_rmapping(page) : NULL;
+-}
+-
+-static inline void set_page_stable_node(struct page *page,
+- struct stable_node *stable_node)
+-{
+- page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM);
+-}
+-
+ #ifdef CONFIG_SYSFS
+ /*
+ * Only called through the sysfs control interface:
+diff --git a/mm/memory.c b/mm/memory.c
+index f703fe8c8346..0bedec37098f 100644
+--- a/mm/memory.c
++++ b/mm/memory.c
+@@ -144,6 +144,25 @@ EXPORT_SYMBOL(zero_pfn);
+
+ unsigned long highest_memmap_pfn __read_mostly;
+
++#ifdef CONFIG_UKSM
++unsigned long uksm_zero_pfn __read_mostly;
++EXPORT_SYMBOL_GPL(uksm_zero_pfn);
++struct page *empty_uksm_zero_page;
++
++static int __init setup_uksm_zero_page(void)
++{
++ empty_uksm_zero_page = alloc_pages(__GFP_ZERO & ~__GFP_MOVABLE, 0);
++ if (!empty_uksm_zero_page)
++ panic("Oh boy, that early out of memory?");
++
++ SetPageReserved(empty_uksm_zero_page);
++ uksm_zero_pfn = page_to_pfn(empty_uksm_zero_page);
++
++ return 0;
++}
++core_initcall(setup_uksm_zero_page);
++#endif
++
+ /*
+ * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
+ */
+@@ -159,6 +178,7 @@ void mm_trace_rss_stat(struct mm_struct *mm, int member, long count)
+ trace_rss_stat(mm, member, count);
+ }
+
++
+ #if defined(SPLIT_RSS_COUNTING)
+
+ void sync_mm_rss(struct mm_struct *mm)
+@@ -802,8 +822,13 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+ get_page(page);
+ page_dup_rmap(page, false);
+ rss[mm_counter(page)]++;
++
++ /* Should return NULL in vm_normal_page() */
++ uksm_bugon_zeropage(pte);
+ } else if (pte_devmap(pte)) {
+ page = pte_page(pte);
++ } else {
++ uksm_map_zero_page(pte);
+ }
+
+ out_set_pte:
+@@ -1076,8 +1101,10 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
+ ptent = ptep_get_and_clear_full(mm, addr, pte,
+ tlb->fullmm);
+ tlb_remove_tlb_entry(tlb, pte, addr);
+- if (unlikely(!page))
++ if (unlikely(!page)) {
++ uksm_unmap_zero_page(ptent);
+ continue;
++ }
+
+ if (!PageAnon(page)) {
+ if (pte_dirty(ptent)) {
+@@ -2412,6 +2439,7 @@ static inline bool cow_user_page(struct page *dst, struct page *src,
+
+ if (likely(src)) {
+ copy_user_highpage(dst, src, addr, vma);
++ uksm_cow_page(vma, src);
+ return true;
+ }
+
+@@ -2658,6 +2686,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
+ vmf->address);
+ if (!new_page)
+ goto oom;
++ uksm_cow_pte(vma, vmf->orig_pte);
+ } else {
+ new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
+ vmf->address);
+@@ -2699,7 +2728,9 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
+ mm_counter_file(old_page));
+ inc_mm_counter_fast(mm, MM_ANONPAGES);
+ }
++ uksm_bugon_zeropage(vmf->orig_pte);
+ } else {
++ uksm_unmap_zero_page(vmf->orig_pte);
+ inc_mm_counter_fast(mm, MM_ANONPAGES);
+ }
+ flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
+diff --git a/mm/mmap.c b/mm/mmap.c
+index f609e9ec4a25..464058e2f637 100644
+--- a/mm/mmap.c
++++ b/mm/mmap.c
+@@ -46,6 +46,7 @@
+ #include <linux/moduleparam.h>
+ #include <linux/pkeys.h>
+ #include <linux/oom.h>
++#include <linux/ksm.h>
+ #include <linux/sched/mm.h>
+
+ #include <linux/uaccess.h>
+@@ -181,6 +182,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
+ if (vma->vm_file)
+ fput(vma->vm_file);
+ mpol_put(vma_policy(vma));
++ uksm_remove_vma(vma);
+ vm_area_free(vma);
+ return next;
+ }
+@@ -708,9 +710,16 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ long adjust_next = 0;
+ int remove_next = 0;
+
++/*
++ * to avoid deadlock, ksm_remove_vma must be done before any spin_lock is
++ * acquired
++ */
++ uksm_remove_vma(vma);
++
+ if (next && !insert) {
+ struct vm_area_struct *exporter = NULL, *importer = NULL;
+
++ uksm_remove_vma(next);
+ if (end >= next->vm_end) {
+ /*
+ * vma expands, overlapping all the next, and
+@@ -841,6 +850,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ end_changed = true;
+ }
+ vma->vm_pgoff = pgoff;
++
+ if (adjust_next) {
+ next->vm_start += adjust_next << PAGE_SHIFT;
+ next->vm_pgoff += adjust_next;
+@@ -946,6 +956,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ if (remove_next == 2) {
+ remove_next = 1;
+ end = next->vm_end;
++ uksm_remove_vma(next);
+ goto again;
+ }
+ else if (next)
+@@ -972,10 +983,14 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
+ */
+ VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma));
+ }
++ } else {
++ if (next && !insert)
++ uksm_vma_add_new(next);
+ }
+ if (insert && file)
+ uprobe_mmap(insert);
+
++ uksm_vma_add_new(vma);
+ validate_mm(mm);
+
+ return 0;
+@@ -1434,6 +1449,9 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
+ vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
+ mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
+
++ /* If uksm is enabled, we add VM_MERGEABLE to new VMAs. */
++ uksm_vm_flags_mod(&vm_flags);
++
+ if (flags & MAP_LOCKED)
+ if (!can_do_mlock())
+ return -EPERM;
+@@ -1801,6 +1819,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
+ allow_write_access(file);
+ }
+ file = vma->vm_file;
++ uksm_vma_add_new(vma);
+ out:
+ perf_event_mmap(vma);
+
+@@ -1843,6 +1862,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
+ if (vm_flags & VM_DENYWRITE)
+ allow_write_access(file);
+ free_vma:
++ uksm_remove_vma(vma);
+ vm_area_free(vma);
+ unacct_error:
+ if (charged)
+@@ -2694,6 +2714,8 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
+ else
+ err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
+
++ uksm_vma_add_new(new);
++
+ /* Success. */
+ if (!err)
+ return 0;
+@@ -3000,6 +3022,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla
+ if ((flags & (~VM_EXEC)) != 0)
+ return -EINVAL;
+ flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
++ uksm_vm_flags_mod(&flags);
+
+ mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
+ if (IS_ERR_VALUE(mapped_addr))
+@@ -3050,6 +3073,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla
+ vma->vm_flags = flags;
+ vma->vm_page_prot = vm_get_page_prot(flags);
+ vma_link(mm, vma, prev, rb_link, rb_parent);
++ uksm_vma_add_new(vma);
+ out:
+ perf_event_mmap(vma);
+ mm->total_vm += len >> PAGE_SHIFT;
+@@ -3127,6 +3151,12 @@ void exit_mmap(struct mm_struct *mm)
+ up_write(&mm->mmap_sem);
+ }
+
++ /*
++ * Taking write lock on mmap_sem does not harm others,
++ * but it's crucial for uksm to avoid races.
++ */
++ down_write(&mm->mmap_sem);
++
+ if (mm->locked_vm) {
+ vma = mm->mmap;
+ while (vma) {
+@@ -3161,6 +3191,11 @@ void exit_mmap(struct mm_struct *mm)
+ vma = remove_vma(vma);
+ }
+ vm_unacct_memory(nr_accounted);
++
++ mm->mmap = NULL;
++ mm->mm_rb = RB_ROOT;
++ vmacache_invalidate(mm);
++ up_write(&mm->mmap_sem);
+ }
+
+ /* Insert vm structure into process list sorted by address
+@@ -3268,6 +3303,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
+ new_vma->vm_ops->open(new_vma);
+ vma_link(mm, new_vma, prev, rb_link, rb_parent);
+ *need_rmap_locks = false;
++ uksm_vma_add_new(new_vma);
+ }
+ return new_vma;
+
+@@ -3420,6 +3456,7 @@ static struct vm_area_struct *__install_special_mapping(
+ vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
+
+ perf_event_mmap(vma);
++ uksm_vma_add_new(vma);
+
+ return vma;
+
+diff --git a/mm/uksm.c b/mm/uksm.c
+new file mode 100644
+index 000000000000..ef068d5dc307
+--- /dev/null
++++ b/mm/uksm.c
+@@ -0,0 +1,5613 @@
++/*
++ * Ultra KSM. Copyright (C) 2011-2012 Nai Xia
++ *
++ * This is an improvement upon KSM. Some basic data structures and routines
++ * are borrowed from ksm.c .
++ *
++ * Its new features:
++ * 1. Full system scan:
++ * It automatically scans all user processes' anonymous VMAs. Kernel-user
++ * interaction to submit a memory area to KSM is no longer needed.
++ *
++ * 2. Rich area detection:
++ * It automatically detects rich areas containing abundant duplicated
++ * pages based. Rich areas are given a full scan speed. Poor areas are
++ * sampled at a reasonable speed with very low CPU consumption.
++ *
++ * 3. Ultra Per-page scan speed improvement:
++ * A new hash algorithm is proposed. As a result, on a machine with
++ * Core(TM)2 Quad Q9300 CPU in 32-bit mode and 800MHZ DDR2 main memory, it
++ * can scan memory areas that does not contain duplicated pages at speed of
++ * 627MB/sec ~ 2445MB/sec and can merge duplicated areas at speed of
++ * 477MB/sec ~ 923MB/sec.
++ *
++ * 4. Thrashing area avoidance:
++ * Thrashing area(an VMA that has frequent Ksm page break-out) can be
++ * filtered out. My benchmark shows it's more efficient than KSM's per-page
++ * hash value based volatile page detection.
++ *
++ *
++ * 5. Misc changes upon KSM:
++ * * It has a fully x86-opitmized memcmp dedicated for 4-byte-aligned page
++ * comparison. It's much faster than default C version on x86.
++ * * rmap_item now has an struct *page member to loosely cache a
++ * address-->page mapping, which reduces too much time-costly
++ * follow_page().
++ * * The VMA creation/exit procedures are hooked to let the Ultra KSM know.
++ * * try_to_merge_two_pages() now can revert a pte if it fails. No break_
++ * ksm is needed for this case.
++ *
++ * 6. Full Zero Page consideration(contributed by Figo Zhang)
++ * Now uksmd consider full zero pages as special pages and merge them to an
++ * special unswappable uksm zero page.
++ */
++
++#include <linux/errno.h>
++#include <linux/mm.h>
++#include <linux/fs.h>
++#include <linux/mman.h>
++#include <linux/sched.h>
++#include <linux/sched/mm.h>
++#include <linux/sched/coredump.h>
++#include <linux/sched/cputime.h>
++#include <linux/rwsem.h>
++#include <linux/pagemap.h>
++#include <linux/rmap.h>
++#include <linux/spinlock.h>
++#include <linux/jhash.h>
++#include <linux/delay.h>
++#include <linux/kthread.h>
++#include <linux/wait.h>
++#include <linux/slab.h>
++#include <linux/rbtree.h>
++#include <linux/memory.h>
++#include <linux/mmu_notifier.h>
++#include <linux/swap.h>
++#include <linux/ksm.h>
++#include <linux/crypto.h>
++#include <linux/scatterlist.h>
++#include <crypto/hash.h>
++#include <linux/random.h>
++#include <linux/math64.h>
++#include <linux/gcd.h>
++#include <linux/freezer.h>
++#include <linux/oom.h>
++#include <linux/numa.h>
++#include <linux/sradix-tree.h>
++
++#include <asm/tlbflush.h>
++#include "internal.h"
++
++#ifdef CONFIG_X86
++#undef memcmp
++
++#ifdef CONFIG_X86_32
++#define memcmp memcmpx86_32
++/*
++ * Compare 4-byte-aligned address s1 and s2, with length n
++ */
++int memcmpx86_32(void *s1, void *s2, size_t n)
++{
++ size_t num = n / 4;
++ register int res;
++
++ __asm__ __volatile__
++ (
++ "testl %3,%3\n\t"
++ "repe; cmpsd\n\t"
++ "je 1f\n\t"
++ "sbbl %0,%0\n\t"
++ "orl $1,%0\n"
++ "1:"
++ : "=&a" (res), "+&S" (s1), "+&D" (s2), "+&c" (num)
++ : "0" (0)
++ : "cc");
++
++ return res;
++}
++
++/*
++ * Check the page is all zero ?
++ */
++static int is_full_zero(const void *s1, size_t len)
++{
++ unsigned char same;
++
++ len /= 4;
++
++ __asm__ __volatile__
++ ("repe; scasl;"
++ "sete %0"
++ : "=qm" (same), "+D" (s1), "+c" (len)
++ : "a" (0)
++ : "cc");
++
++ return same;
++}
++
++
++#elif defined(CONFIG_X86_64)
++#define memcmp memcmpx86_64
++/*
++ * Compare 8-byte-aligned address s1 and s2, with length n
++ */
++int memcmpx86_64(void *s1, void *s2, size_t n)
++{
++ size_t num = n / 8;
++ register int res;
++
++ __asm__ __volatile__
++ (
++ "testq %q3,%q3\n\t"
++ "repe; cmpsq\n\t"
++ "je 1f\n\t"
++ "sbbq %q0,%q0\n\t"
++ "orq $1,%q0\n"
++ "1:"
++ : "=&a" (res), "+&S" (s1), "+&D" (s2), "+&c" (num)
++ : "0" (0)
++ : "cc");
++
++ return res;
++}
++
++static int is_full_zero(const void *s1, size_t len)
++{
++ unsigned char same;
++
++ len /= 8;
++
++ __asm__ __volatile__
++ ("repe; scasq;"
++ "sete %0"
++ : "=qm" (same), "+D" (s1), "+c" (len)
++ : "a" (0)
++ : "cc");
++
++ return same;
++}
++
++#endif
++#else
++static int is_full_zero(const void *s1, size_t len)
++{
++ unsigned long *src = s1;
++ int i;
++
++ len /= sizeof(*src);
++
++ for (i = 0; i < len; i++) {
++ if (src[i])
++ return 0;
++ }
++
++ return 1;
++}
++#endif
++
++#define UKSM_RUNG_ROUND_FINISHED (1 << 0)
++#define TIME_RATIO_SCALE 10000
++
++#define SLOT_TREE_NODE_SHIFT 8
++#define SLOT_TREE_NODE_STORE_SIZE (1UL << SLOT_TREE_NODE_SHIFT)
++struct slot_tree_node {
++ unsigned long size;
++ struct sradix_tree_node snode;
++ void *stores[SLOT_TREE_NODE_STORE_SIZE];
++};
++
++static struct kmem_cache *slot_tree_node_cachep;
++
++static struct sradix_tree_node *slot_tree_node_alloc(void)
++{
++ struct slot_tree_node *p;
++
++ p = kmem_cache_zalloc(slot_tree_node_cachep, GFP_KERNEL |
++ __GFP_NORETRY | __GFP_NOWARN);
++ if (!p)
++ return NULL;
++
++ return &p->snode;
++}
++
++static void slot_tree_node_free(struct sradix_tree_node *node)
++{
++ struct slot_tree_node *p;
++
++ p = container_of(node, struct slot_tree_node, snode);
++ kmem_cache_free(slot_tree_node_cachep, p);
++}
++
++static void slot_tree_node_extend(struct sradix_tree_node *parent,
++ struct sradix_tree_node *child)
++{
++ struct slot_tree_node *p, *c;
++
++ p = container_of(parent, struct slot_tree_node, snode);
++ c = container_of(child, struct slot_tree_node, snode);
++
++ p->size += c->size;
++}
++
++void slot_tree_node_assign(struct sradix_tree_node *node,
++ unsigned int index, void *item)
++{
++ struct vma_slot *slot = item;
++ struct slot_tree_node *cur;
++
++ slot->snode = node;
++ slot->sindex = index;
++
++ while (node) {
++ cur = container_of(node, struct slot_tree_node, snode);
++ cur->size += slot->pages;
++ node = node->parent;
++ }
++}
++
++void slot_tree_node_rm(struct sradix_tree_node *node, unsigned int offset)
++{
++ struct vma_slot *slot;
++ struct slot_tree_node *cur;
++ unsigned long pages;
++
++ if (node->height == 1) {
++ slot = node->stores[offset];
++ pages = slot->pages;
++ } else {
++ cur = container_of(node->stores[offset],
++ struct slot_tree_node, snode);
++ pages = cur->size;
++ }
++
++ while (node) {
++ cur = container_of(node, struct slot_tree_node, snode);
++ cur->size -= pages;
++ node = node->parent;
++ }
++}
++
++unsigned long slot_iter_index;
++int slot_iter(void *item, unsigned long height)
++{
++ struct slot_tree_node *node;
++ struct vma_slot *slot;
++
++ if (height == 1) {
++ slot = item;
++ if (slot_iter_index < slot->pages) {
++ /*in this one*/
++ return 1;
++ } else {
++ slot_iter_index -= slot->pages;
++ return 0;
++ }
++
++ } else {
++ node = container_of(item, struct slot_tree_node, snode);
++ if (slot_iter_index < node->size) {
++ /*in this one*/
++ return 1;
++ } else {
++ slot_iter_index -= node->size;
++ return 0;
++ }
++ }
++}
++
++
++static inline void slot_tree_init_root(struct sradix_tree_root *root)
++{
++ init_sradix_tree_root(root, SLOT_TREE_NODE_SHIFT);
++ root->alloc = slot_tree_node_alloc;
++ root->free = slot_tree_node_free;
++ root->extend = slot_tree_node_extend;
++ root->assign = slot_tree_node_assign;
++ root->rm = slot_tree_node_rm;
++}
++
++void slot_tree_init(void)
++{
++ slot_tree_node_cachep = kmem_cache_create("slot_tree_node",
++ sizeof(struct slot_tree_node), 0,
++ SLAB_PANIC | SLAB_RECLAIM_ACCOUNT,
++ NULL);
++}
++
++
++/* Each rung of this ladder is a list of VMAs having a same scan ratio */
++struct scan_rung {
++ //struct list_head scanned_list;
++ struct sradix_tree_root vma_root;
++ struct sradix_tree_root vma_root2;
++
++ struct vma_slot *current_scan;
++ unsigned long current_offset;
++
++ /*
++ * The initial value for current_offset, it should loop over
++ * [0~ step - 1] to let all slot have its chance to be scanned.
++ */
++ unsigned long offset_init;
++ unsigned long step; /* dynamic step for current_offset */
++ unsigned int flags;
++ unsigned long pages_to_scan;
++ //unsigned long fully_scanned_slots;
++ /*
++ * a little bit tricky - if cpu_time_ratio > 0, then the value is the
++ * the cpu time ratio it can spend in rung_i for every scan
++ * period. if < 0, then it is the cpu time ratio relative to the
++ * max cpu percentage user specified. Both in unit of
++ * 1/TIME_RATIO_SCALE
++ */
++ int cpu_ratio;
++
++ /*
++ * How long it will take for all slots in this rung to be fully
++ * scanned? If it's zero, we don't care about the cover time:
++ * it's fully scanned.
++ */
++ unsigned int cover_msecs;
++ //unsigned long vma_num;
++ //unsigned long pages; /* Sum of all slot's pages in rung */
++};
++
++/**
++ * node of either the stable or unstale rbtree
++ *
++ */
++struct tree_node {
++ struct rb_node node; /* link in the main (un)stable rbtree */
++ struct rb_root sub_root; /* rb_root for sublevel collision rbtree */
++ u32 hash;
++ unsigned long count; /* TODO: merged with sub_root */
++ struct list_head all_list; /* all tree nodes in stable/unstable tree */
++};
++
++/**
++ * struct stable_node - node of the stable rbtree
++ * @node: rb node of this ksm page in the stable tree
++ * @hlist: hlist head of rmap_items using this ksm page
++ * @kpfn: page frame number of this ksm page
++ */
++struct stable_node {
++ struct rb_node node; /* link in sub-rbtree */
++ struct tree_node *tree_node; /* it's tree node root in stable tree, NULL if it's in hell list */
++ struct hlist_head hlist;
++ unsigned long kpfn;
++ u32 hash_max; /* if ==0 then it's not been calculated yet */
++ struct list_head all_list; /* in a list for all stable nodes */
++};
++
++/**
++ * struct node_vma - group rmap_items linked in a same stable
++ * node together.
++ */
++struct node_vma {
++ union {
++ struct vma_slot *slot;
++ unsigned long key; /* slot is used as key sorted on hlist */
++ };
++ struct hlist_node hlist;
++ struct hlist_head rmap_hlist;
++ struct stable_node *head;
++};
++
++/**
++ * struct rmap_item - reverse mapping item for virtual addresses
++ * @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
++ * @anon_vma: pointer to anon_vma for this mm,address, when in stable tree
++ * @mm: the memory structure this rmap_item is pointing into
++ * @address: the virtual address this rmap_item tracks (+ flags in low bits)
++ * @node: rb node of this rmap_item in the unstable tree
++ * @head: pointer to stable_node heading this list in the stable tree
++ * @hlist: link into hlist of rmap_items hanging off that stable_node
++ */
++struct rmap_item {
++ struct vma_slot *slot;
++ struct page *page;
++ unsigned long address; /* + low bits used for flags below */
++ unsigned long hash_round;
++ unsigned long entry_index;
++ union {
++ struct {/* when in unstable tree */
++ struct rb_node node;
++ struct tree_node *tree_node;
++ u32 hash_max;
++ };
++ struct { /* when in stable tree */
++ struct node_vma *head;
++ struct hlist_node hlist;
++ struct anon_vma *anon_vma;
++ };
++ };
++} __aligned(4);
++
++struct rmap_list_entry {
++ union {
++ struct rmap_item *item;
++ unsigned long addr;
++ };
++ /* lowest bit is used for is_addr tag */
++} __aligned(4); /* 4 aligned to fit in to pages*/
++
++
++/* Basic data structure definition ends */
++
++
++/*
++ * Flags for rmap_item to judge if it's listed in the stable/unstable tree.
++ * The flags use the low bits of rmap_item.address
++ */
++#define UNSTABLE_FLAG 0x1
++#define STABLE_FLAG 0x2
++#define get_rmap_addr(x) ((x)->address & PAGE_MASK)
++
++/*
++ * rmap_list_entry helpers
++ */
++#define IS_ADDR_FLAG 1
++#define is_addr(ptr) ((unsigned long)(ptr) & IS_ADDR_FLAG)
++#define set_is_addr(ptr) ((ptr) |= IS_ADDR_FLAG)
++#define get_clean_addr(ptr) (((ptr) & ~(__typeof__(ptr))IS_ADDR_FLAG))
++
++
++/*
++ * High speed caches for frequently allocated and freed structs
++ */
++static struct kmem_cache *rmap_item_cache;
++static struct kmem_cache *stable_node_cache;
++static struct kmem_cache *node_vma_cache;
++static struct kmem_cache *vma_slot_cache;
++static struct kmem_cache *tree_node_cache;
++#define UKSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("uksm_"#__struct,\
++ sizeof(struct __struct), __alignof__(struct __struct),\
++ (__flags), NULL)
++
++/* Array of all scan_rung, uksm_scan_ladder[0] having the minimum scan ratio */
++#define SCAN_LADDER_SIZE 4
++static struct scan_rung uksm_scan_ladder[SCAN_LADDER_SIZE];
++
++/* The evaluation rounds uksmd has finished */
++static unsigned long long uksm_eval_round = 1;
++
++/*
++ * we add 1 to this var when we consider we should rebuild the whole
++ * unstable tree.
++ */
++static unsigned long uksm_hash_round = 1;
++
++/*
++ * How many times the whole memory is scanned.
++ */
++static unsigned long long fully_scanned_round = 1;
++
++/* The total number of virtual pages of all vma slots */
++static u64 uksm_pages_total;
++
++/* The number of pages has been scanned since the start up */
++static u64 uksm_pages_scanned;
++
++static u64 scanned_virtual_pages;
++
++/* The number of pages has been scanned since last encode_benefit call */
++static u64 uksm_pages_scanned_last;
++
++/* If the scanned number is tooo large, we encode it here */
++static u64 pages_scanned_stored;
++
++static unsigned long pages_scanned_base;
++
++/* The number of nodes in the stable tree */
++static unsigned long uksm_pages_shared;
++
++/* The number of page slots additionally sharing those nodes */
++static unsigned long uksm_pages_sharing;
++
++/* The number of nodes in the unstable tree */
++static unsigned long uksm_pages_unshared;
++
++/*
++ * Milliseconds ksmd should sleep between scans,
++ * >= 100ms to be consistent with
++ * scan_time_to_sleep_msec()
++ */
++static unsigned int uksm_sleep_jiffies;
++
++/* The real value for the uksmd next sleep */
++static unsigned int uksm_sleep_real;
++
++/* Saved value for user input uksm_sleep_jiffies when it's enlarged */
++static unsigned int uksm_sleep_saved;
++
++/* Max percentage of cpu utilization ksmd can take to scan in one batch */
++static unsigned int uksm_max_cpu_percentage;
++
++static int uksm_cpu_governor;
++
++static char *uksm_cpu_governor_str[4] = { "full", "medium", "low", "quiet" };
++
++struct uksm_cpu_preset_s {
++ int cpu_ratio[SCAN_LADDER_SIZE];
++ unsigned int cover_msecs[SCAN_LADDER_SIZE];
++ unsigned int max_cpu; /* percentage */
++};
++
++struct uksm_cpu_preset_s uksm_cpu_preset[4] = {
++ { {20, 40, -2500, -10000}, {1000, 500, 200, 50}, 95},
++ { {20, 30, -2500, -10000}, {1000, 500, 400, 100}, 50},
++ { {10, 20, -5000, -10000}, {1500, 1000, 1000, 250}, 20},
++ { {10, 20, 40, 75}, {2000, 1000, 1000, 1000}, 1},
++};
++
++/* The default value for uksm_ema_page_time if it's not initialized */
++#define UKSM_PAGE_TIME_DEFAULT 500
++
++/*cost to scan one page by expotional moving average in nsecs */
++static unsigned long uksm_ema_page_time = UKSM_PAGE_TIME_DEFAULT;
++
++/* The expotional moving average alpha weight, in percentage. */
++#define EMA_ALPHA 20
++
++/*
++ * The threshold used to filter out thrashing areas,
++ * If it == 0, filtering is disabled, otherwise it's the percentage up-bound
++ * of the thrashing ratio of all areas. Any area with a bigger thrashing ratio
++ * will be considered as having a zero duplication ratio.
++ */
++static unsigned int uksm_thrash_threshold = 50;
++
++/* How much dedup ratio is considered to be abundant*/
++static unsigned int uksm_abundant_threshold = 10;
++
++/* All slots having merged pages in this eval round. */
++struct list_head vma_slot_dedup = LIST_HEAD_INIT(vma_slot_dedup);
++
++/* How many times the ksmd has slept since startup */
++static unsigned long long uksm_sleep_times;
++
++#define UKSM_RUN_STOP 0
++#define UKSM_RUN_MERGE 1
++static unsigned int uksm_run = 1;
++
++static DECLARE_WAIT_QUEUE_HEAD(uksm_thread_wait);
++static DEFINE_MUTEX(uksm_thread_mutex);
++
++/*
++ * List vma_slot_new is for newly created vma_slot waiting to be added by
++ * ksmd. If one cannot be added(e.g. due to it's too small), it's moved to
++ * vma_slot_noadd. vma_slot_del is the list for vma_slot whose corresponding
++ * VMA has been removed/freed.
++ */
++struct list_head vma_slot_new = LIST_HEAD_INIT(vma_slot_new);
++struct list_head vma_slot_noadd = LIST_HEAD_INIT(vma_slot_noadd);
++struct list_head vma_slot_del = LIST_HEAD_INIT(vma_slot_del);
++static DEFINE_SPINLOCK(vma_slot_list_lock);
++
++/* The unstable tree heads */
++static struct rb_root root_unstable_tree = RB_ROOT;
++
++/*
++ * All tree_nodes are in a list to be freed at once when unstable tree is
++ * freed after each scan round.
++ */
++static struct list_head unstable_tree_node_list =
++ LIST_HEAD_INIT(unstable_tree_node_list);
++
++/* List contains all stable nodes */
++static struct list_head stable_node_list = LIST_HEAD_INIT(stable_node_list);
++
++/*
++ * When the hash strength is changed, the stable tree must be delta_hashed and
++ * re-structured. We use two set of below structs to speed up the
++ * re-structuring of stable tree.
++ */
++static struct list_head
++stable_tree_node_list[2] = {LIST_HEAD_INIT(stable_tree_node_list[0]),
++ LIST_HEAD_INIT(stable_tree_node_list[1])};
++
++static struct list_head *stable_tree_node_listp = &stable_tree_node_list[0];
++static struct rb_root root_stable_tree[2] = {RB_ROOT, RB_ROOT};
++static struct rb_root *root_stable_treep = &root_stable_tree[0];
++static unsigned long stable_tree_index;
++
++/* The hash strength needed to hash a full page */
++#define HASH_STRENGTH_FULL (PAGE_SIZE / sizeof(u32))
++
++/* The hash strength needed for loop-back hashing */
++#define HASH_STRENGTH_MAX (HASH_STRENGTH_FULL + 10)
++
++/* The random offsets in a page */
++static u32 *random_nums;
++
++/* The hash strength */
++static unsigned long hash_strength = HASH_STRENGTH_FULL >> 4;
++
++/* The delta value each time the hash strength increases or decreases */
++static unsigned long hash_strength_delta;
++#define HASH_STRENGTH_DELTA_MAX 5
++
++/* The time we have saved due to random_sample_hash */
++static u64 rshash_pos;
++
++/* The time we have wasted due to hash collision */
++static u64 rshash_neg;
++
++struct uksm_benefit {
++ u64 pos;
++ u64 neg;
++ u64 scanned;
++ unsigned long base;
++} benefit;
++
++/*
++ * The relative cost of memcmp, compared to 1 time unit of random sample
++ * hash, this value is tested when ksm module is initialized
++ */
++static unsigned long memcmp_cost;
++
++static unsigned long rshash_neg_cont_zero;
++static unsigned long rshash_cont_obscure;
++
++/* The possible states of hash strength adjustment heuristic */
++enum rshash_states {
++ RSHASH_STILL,
++ RSHASH_TRYUP,
++ RSHASH_TRYDOWN,
++ RSHASH_NEW,
++ RSHASH_PRE_STILL,
++};
++
++/* The possible direction we are about to adjust hash strength */
++enum rshash_direct {
++ GO_UP,
++ GO_DOWN,
++ OBSCURE,
++ STILL,
++};
++
++/* random sampling hash state machine */
++static struct {
++ enum rshash_states state;
++ enum rshash_direct pre_direct;
++ u8 below_count;
++ /* Keep a lookup window of size 5, iff above_count/below_count > 3
++ * in this window we stop trying.
++ */
++ u8 lookup_window_index;
++ u64 stable_benefit;
++ unsigned long turn_point_down;
++ unsigned long turn_benefit_down;
++ unsigned long turn_point_up;
++ unsigned long turn_benefit_up;
++ unsigned long stable_point;
++} rshash_state;
++
++/*zero page hash table, hash_strength [0 ~ HASH_STRENGTH_MAX]*/
++static u32 *zero_hash_table;
++
++static inline struct node_vma *alloc_node_vma(void)
++{
++ struct node_vma *node_vma;
++
++ node_vma = kmem_cache_zalloc(node_vma_cache, GFP_KERNEL |
++ __GFP_NORETRY | __GFP_NOWARN);
++ if (node_vma) {
++ INIT_HLIST_HEAD(&node_vma->rmap_hlist);
++ INIT_HLIST_NODE(&node_vma->hlist);
++ }
++ return node_vma;
++}
++
++static inline void free_node_vma(struct node_vma *node_vma)
++{
++ kmem_cache_free(node_vma_cache, node_vma);
++}
++
++
++static inline struct vma_slot *alloc_vma_slot(void)
++{
++ struct vma_slot *slot;
++
++ /*
++ * In case ksm is not initialized by now.
++ * Oops, we need to consider the call site of uksm_init() in the future.
++ */
++ if (!vma_slot_cache)
++ return NULL;
++
++ slot = kmem_cache_zalloc(vma_slot_cache, GFP_KERNEL |
++ __GFP_NORETRY | __GFP_NOWARN);
++ if (slot) {
++ INIT_LIST_HEAD(&slot->slot_list);
++ INIT_LIST_HEAD(&slot->dedup_list);
++ slot->flags |= UKSM_SLOT_NEED_RERAND;
++ }
++ return slot;
++}
++
++static inline void free_vma_slot(struct vma_slot *vma_slot)
++{
++ kmem_cache_free(vma_slot_cache, vma_slot);
++}
++
++
++
++static inline struct rmap_item *alloc_rmap_item(void)
++{
++ struct rmap_item *rmap_item;
++
++ rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |
++ __GFP_NORETRY | __GFP_NOWARN);
++ if (rmap_item) {
++ /* bug on lowest bit is not clear for flag use */
++ BUG_ON(is_addr(rmap_item));
++ }
++ return rmap_item;
++}
++
++static inline void free_rmap_item(struct rmap_item *rmap_item)
++{
++ rmap_item->slot = NULL; /* debug safety */
++ kmem_cache_free(rmap_item_cache, rmap_item);
++}
++
++static inline struct stable_node *alloc_stable_node(void)
++{
++ struct stable_node *node;
++
++ node = kmem_cache_alloc(stable_node_cache, GFP_KERNEL |
++ __GFP_NORETRY | __GFP_NOWARN);
++ if (!node)
++ return NULL;
++
++ INIT_HLIST_HEAD(&node->hlist);
++ list_add(&node->all_list, &stable_node_list);
++ return node;
++}
++
++static inline void free_stable_node(struct stable_node *stable_node)
++{
++ list_del(&stable_node->all_list);
++ kmem_cache_free(stable_node_cache, stable_node);
++}
++
++static inline struct tree_node *alloc_tree_node(struct list_head *list)
++{
++ struct tree_node *node;
++
++ node = kmem_cache_zalloc(tree_node_cache, GFP_KERNEL |
++ __GFP_NORETRY | __GFP_NOWARN);
++ if (!node)
++ return NULL;
++
++ list_add(&node->all_list, list);
++ return node;
++}
++
++static inline void free_tree_node(struct tree_node *node)
++{
++ list_del(&node->all_list);
++ kmem_cache_free(tree_node_cache, node);
++}
++
++static void uksm_drop_anon_vma(struct rmap_item *rmap_item)
++{
++ struct anon_vma *anon_vma = rmap_item->anon_vma;
++
++ put_anon_vma(anon_vma);
++}
++
++
++/**
++ * Remove a stable node from stable_tree, may unlink from its tree_node and
++ * may remove its parent tree_node if no other stable node is pending.
++ *
++ * @stable_node The node need to be removed
++ * @unlink_rb Will this node be unlinked from the rbtree?
++ * @remove_tree_ node Will its tree_node be removed if empty?
++ */
++static void remove_node_from_stable_tree(struct stable_node *stable_node,
++ int unlink_rb, int remove_tree_node)
++{
++ struct node_vma *node_vma;
++ struct rmap_item *rmap_item;
++ struct hlist_node *n;
++
++ if (!hlist_empty(&stable_node->hlist)) {
++ hlist_for_each_entry_safe(node_vma, n,
++ &stable_node->hlist, hlist) {
++ hlist_for_each_entry(rmap_item, &node_vma->rmap_hlist, hlist) {
++ uksm_pages_sharing--;
++
++ uksm_drop_anon_vma(rmap_item);
++ rmap_item->address &= PAGE_MASK;
++ }
++ free_node_vma(node_vma);
++ cond_resched();
++ }
++
++ /* the last one is counted as shared */
++ uksm_pages_shared--;
++ uksm_pages_sharing++;
++ }
++
++ if (stable_node->tree_node && unlink_rb) {
++ rb_erase(&stable_node->node,
++ &stable_node->tree_node->sub_root);
++
++ if (RB_EMPTY_ROOT(&stable_node->tree_node->sub_root) &&
++ remove_tree_node) {
++ rb_erase(&stable_node->tree_node->node,
++ root_stable_treep);
++ free_tree_node(stable_node->tree_node);
++ } else {
++ stable_node->tree_node->count--;
++ }
++ }
++
++ free_stable_node(stable_node);
++}
++
++
++/*
++ * get_uksm_page: checks if the page indicated by the stable node
++ * is still its ksm page, despite having held no reference to it.
++ * In which case we can trust the content of the page, and it
++ * returns the gotten page; but if the page has now been zapped,
++ * remove the stale node from the stable tree and return NULL.
++ *
++ * You would expect the stable_node to hold a reference to the ksm page.
++ * But if it increments the page's count, swapping out has to wait for
++ * ksmd to come around again before it can free the page, which may take
++ * seconds or even minutes: much too unresponsive. So instead we use a
++ * "keyhole reference": access to the ksm page from the stable node peeps
++ * out through its keyhole to see if that page still holds the right key,
++ * pointing back to this stable node. This relies on freeing a PageAnon
++ * page to reset its page->mapping to NULL, and relies on no other use of
++ * a page to put something that might look like our key in page->mapping.
++ *
++ * include/linux/pagemap.h page_cache_get_speculative() is a good reference,
++ * but this is different - made simpler by uksm_thread_mutex being held, but
++ * interesting for assuming that no other use of the struct page could ever
++ * put our expected_mapping into page->mapping (or a field of the union which
++ * coincides with page->mapping). The RCU calls are not for KSM at all, but
++ * to keep the page_count protocol described with page_cache_get_speculative.
++ *
++ * Note: it is possible that get_uksm_page() will return NULL one moment,
++ * then page the next, if the page is in between page_freeze_refs() and
++ * page_unfreeze_refs(): this shouldn't be a problem anywhere, the page
++ * is on its way to being freed; but it is an anomaly to bear in mind.
++ *
++ * @unlink_rb: if the removal of this node will firstly unlink from
++ * its rbtree. stable_node_reinsert will prevent this when restructuring the
++ * node from its old tree.
++ *
++ * @remove_tree_node: if this is the last one of its tree_node, will the
++ * tree_node be freed ? If we are inserting stable node, this tree_node may
++ * be reused, so don't free it.
++ */
++static struct page *get_uksm_page(struct stable_node *stable_node,
++ int unlink_rb, int remove_tree_node)
++{
++ struct page *page;
++ void *expected_mapping;
++ unsigned long kpfn;
++
++ expected_mapping = (void *)((unsigned long)stable_node |
++ PAGE_MAPPING_KSM);
++again:
++ kpfn = READ_ONCE(stable_node->kpfn);
++ page = pfn_to_page(kpfn);
++
++ /*
++ * page is computed from kpfn, so on most architectures reading
++ * page->mapping is naturally ordered after reading node->kpfn,
++ * but on Alpha we need to be more careful.
++ */
++ smp_read_barrier_depends();
++
++ if (READ_ONCE(page->mapping) != expected_mapping)
++ goto stale;
++
++ /*
++ * We cannot do anything with the page while its refcount is 0.
++ * Usually 0 means free, or tail of a higher-order page: in which
++ * case this node is no longer referenced, and should be freed;
++ * however, it might mean that the page is under page_freeze_refs().
++ * The __remove_mapping() case is easy, again the node is now stale;
++ * but if page is swapcache in migrate_page_move_mapping(), it might
++ * still be our page, in which case it's essential to keep the node.
++ */
++ while (!get_page_unless_zero(page)) {
++ /*
++ * Another check for page->mapping != expected_mapping would
++ * work here too. We have chosen the !PageSwapCache test to
++ * optimize the common case, when the page is or is about to
++ * be freed: PageSwapCache is cleared (under spin_lock_irq)
++ * in the freeze_refs section of __remove_mapping(); but Anon
++ * page->mapping reset to NULL later, in free_pages_prepare().
++ */
++ if (!PageSwapCache(page))
++ goto stale;
++ cpu_relax();
++ }
++
++ if (READ_ONCE(page->mapping) != expected_mapping) {
++ put_page(page);
++ goto stale;
++ }
++
++ lock_page(page);
++ if (READ_ONCE(page->mapping) != expected_mapping) {
++ unlock_page(page);
++ put_page(page);
++ goto stale;
++ }
++ unlock_page(page);
++ return page;
++stale:
++ /*
++ * We come here from above when page->mapping or !PageSwapCache
++ * suggests that the node is stale; but it might be under migration.
++ * We need smp_rmb(), matching the smp_wmb() in ksm_migrate_page(),
++ * before checking whether node->kpfn has been changed.
++ */
++ smp_rmb();
++ if (stable_node->kpfn != kpfn)
++ goto again;
++
++ remove_node_from_stable_tree(stable_node, unlink_rb, remove_tree_node);
++
++ return NULL;
++}
++
++/*
++ * Removing rmap_item from stable or unstable tree.
++ * This function will clean the information from the stable/unstable tree.
++ */
++static inline void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
++{
++ if (rmap_item->address & STABLE_FLAG) {
++ struct stable_node *stable_node;
++ struct node_vma *node_vma;
++ struct page *page;
++
++ node_vma = rmap_item->head;
++ stable_node = node_vma->head;
++ page = get_uksm_page(stable_node, 1, 1);
++ if (!page)
++ goto out;
++
++ /*
++ * page lock is needed because it's racing with
++ * try_to_unmap_ksm(), etc.
++ */
++ lock_page(page);
++ hlist_del(&rmap_item->hlist);
++
++ if (hlist_empty(&node_vma->rmap_hlist)) {
++ hlist_del(&node_vma->hlist);
++ free_node_vma(node_vma);
++ }
++ unlock_page(page);
++
++ put_page(page);
++ if (hlist_empty(&stable_node->hlist)) {
++ /* do NOT call remove_node_from_stable_tree() here,
++ * it's possible for a forked rmap_item not in
++ * stable tree while the in-tree rmap_items were
++ * deleted.
++ */
++ uksm_pages_shared--;
++ } else
++ uksm_pages_sharing--;
++
++
++ uksm_drop_anon_vma(rmap_item);
++ } else if (rmap_item->address & UNSTABLE_FLAG) {
++ if (rmap_item->hash_round == uksm_hash_round) {
++
++ rb_erase(&rmap_item->node,
++ &rmap_item->tree_node->sub_root);
++ if (RB_EMPTY_ROOT(&rmap_item->tree_node->sub_root)) {
++ rb_erase(&rmap_item->tree_node->node,
++ &root_unstable_tree);
++
++ free_tree_node(rmap_item->tree_node);
++ } else
++ rmap_item->tree_node->count--;
++ }
++ uksm_pages_unshared--;
++ }
++
++ rmap_item->address &= PAGE_MASK;
++ rmap_item->hash_max = 0;
++
++out:
++ cond_resched(); /* we're called from many long loops */
++}
++
++static inline int slot_in_uksm(struct vma_slot *slot)
++{
++ return list_empty(&slot->slot_list);
++}
++
++/*
++ * Test if the mm is exiting
++ */
++static inline bool uksm_test_exit(struct mm_struct *mm)
++{
++ return atomic_read(&mm->mm_users) == 0;
++}
++
++static inline unsigned long vma_pool_size(struct vma_slot *slot)
++{
++ return round_up(sizeof(struct rmap_list_entry) * slot->pages,
++ PAGE_SIZE) >> PAGE_SHIFT;
++}
++
++#define CAN_OVERFLOW_U64(x, delta) (U64_MAX - (x) < (delta))
++
++/* must be done with sem locked */
++static int slot_pool_alloc(struct vma_slot *slot)
++{
++ unsigned long pool_size;
++
++ if (slot->rmap_list_pool)
++ return 0;
++
++ pool_size = vma_pool_size(slot);
++ slot->rmap_list_pool = kcalloc(pool_size, sizeof(struct page *),
++ GFP_KERNEL);
++ if (!slot->rmap_list_pool)
++ return -ENOMEM;
++
++ slot->pool_counts = kcalloc(pool_size, sizeof(unsigned int),
++ GFP_KERNEL);
++ if (!slot->pool_counts) {
++ kfree(slot->rmap_list_pool);
++ return -ENOMEM;
++ }
++
++ slot->pool_size = pool_size;
++ BUG_ON(CAN_OVERFLOW_U64(uksm_pages_total, slot->pages));
++ slot->flags |= UKSM_SLOT_IN_UKSM;
++ uksm_pages_total += slot->pages;
++
++ return 0;
++}
++
++/*
++ * Called after vma is unlinked from its mm
++ */
++void uksm_remove_vma(struct vm_area_struct *vma)
++{
++ struct vma_slot *slot;
++
++ if (!vma->uksm_vma_slot)
++ return;
++
++ spin_lock(&vma_slot_list_lock);
++ slot = vma->uksm_vma_slot;
++ if (!slot)
++ goto out;
++
++ if (slot_in_uksm(slot)) {
++ /**
++ * This slot has been added by ksmd, so move to the del list
++ * waiting ksmd to free it.
++ */
++ list_add_tail(&slot->slot_list, &vma_slot_del);
++ } else {
++ /**
++ * It's still on new list. It's ok to free slot directly.
++ */
++ list_del(&slot->slot_list);
++ free_vma_slot(slot);
++ }
++out:
++ vma->uksm_vma_slot = NULL;
++ spin_unlock(&vma_slot_list_lock);
++}
++
++/**
++ * Need to do two things:
++ * 1. check if slot was moved to del list
++ * 2. make sure the mmap_sem is manipulated under valid vma.
++ *
++ * My concern here is that in some cases, this may make
++ * vma_slot_list_lock() waiters to serialized further by some
++ * sem->wait_lock, can this really be expensive?
++ *
++ *
++ * @return
++ * 0: if successfully locked mmap_sem
++ * -ENOENT: this slot was moved to del list
++ * -EBUSY: vma lock failed
++ */
++static int try_down_read_slot_mmap_sem(struct vma_slot *slot)
++{
++ struct vm_area_struct *vma;
++ struct mm_struct *mm;
++ struct rw_semaphore *sem;
++
++ spin_lock(&vma_slot_list_lock);
++
++ /* the slot_list was removed and inited from new list, when it enters
++ * uksm_list. If now it's not empty, then it must be moved to del list
++ */
++ if (!slot_in_uksm(slot)) {
++ spin_unlock(&vma_slot_list_lock);
++ return -ENOENT;
++ }
++
++ BUG_ON(slot->pages != vma_pages(slot->vma));
++ /* Ok, vma still valid */
++ vma = slot->vma;
++ mm = vma->vm_mm;
++ sem = &mm->mmap_sem;
++
++ if (uksm_test_exit(mm)) {
++ spin_unlock(&vma_slot_list_lock);
++ return -ENOENT;
++ }
++
++ if (down_read_trylock(sem)) {
++ spin_unlock(&vma_slot_list_lock);
++ if (slot_pool_alloc(slot)) {
++ uksm_remove_vma(vma);
++ up_read(sem);
++ return -ENOENT;
++ }
++ return 0;
++ }
++
++ spin_unlock(&vma_slot_list_lock);
++ return -EBUSY;
++}
++
++static inline unsigned long
++vma_page_address(struct page *page, struct vm_area_struct *vma)
++{
++ pgoff_t pgoff = page->index;
++ unsigned long address;
++
++ address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
++ if (unlikely(address < vma->vm_start || address >= vma->vm_end)) {
++ /* page should be within @vma mapping range */
++ return -EFAULT;
++ }
++ return address;
++}
++
++
++/* return 0 on success with the item's mmap_sem locked */
++static inline int get_mergeable_page_lock_mmap(struct rmap_item *item)
++{
++ struct mm_struct *mm;
++ struct vma_slot *slot = item->slot;
++ int err = -EINVAL;
++
++ struct page *page;
++
++ /*
++ * try_down_read_slot_mmap_sem() returns non-zero if the slot
++ * has been removed by uksm_remove_vma().
++ */
++ if (try_down_read_slot_mmap_sem(slot))
++ return -EBUSY;
++
++ mm = slot->vma->vm_mm;
++
++ if (uksm_test_exit(mm))
++ goto failout_up;
++
++ page = item->page;
++ rcu_read_lock();
++ if (!get_page_unless_zero(page)) {
++ rcu_read_unlock();
++ goto failout_up;
++ }
++
++ /* No need to consider huge page here. */
++ if (item->slot->vma->anon_vma != page_anon_vma(page) ||
++ vma_page_address(page, item->slot->vma) != get_rmap_addr(item)) {
++ /*
++ * TODO:
++ * should we release this item becase of its stale page
++ * mapping?
++ */
++ put_page(page);
++ rcu_read_unlock();
++ goto failout_up;
++ }
++ rcu_read_unlock();
++ return 0;
++
++failout_up:
++ up_read(&mm->mmap_sem);
++ return err;
++}
++
++/*
++ * What kind of VMA is considered ?
++ */
++static inline int vma_can_enter(struct vm_area_struct *vma)
++{
++ return uksm_flags_can_scan(vma->vm_flags);
++}
++
++/*
++ * Called whenever a fresh new vma is created A new vma_slot.
++ * is created and inserted into a global list Must be called.
++ * after vma is inserted to its mm.
++ */
++void uksm_vma_add_new(struct vm_area_struct *vma)
++{
++ struct vma_slot *slot;
++
++ if (!vma_can_enter(vma)) {
++ vma->uksm_vma_slot = NULL;
++ return;
++ }
++
++ slot = alloc_vma_slot();
++ if (!slot) {
++ vma->uksm_vma_slot = NULL;
++ return;
++ }
++
++ vma->uksm_vma_slot = slot;
++ vma->vm_flags |= VM_MERGEABLE;
++ slot->vma = vma;
++ slot->mm = vma->vm_mm;
++ slot->ctime_j = jiffies;
++ slot->pages = vma_pages(vma);
++ spin_lock(&vma_slot_list_lock);
++ list_add_tail(&slot->slot_list, &vma_slot_new);
++ spin_unlock(&vma_slot_list_lock);
++}
++
++/* 32/3 < they < 32/2 */
++#define shiftl 8
++#define shiftr 12
++
++#define HASH_FROM_TO(from, to) \
++for (index = from; index < to; index++) { \
++ pos = random_nums[index]; \
++ hash += key[pos]; \
++ hash += (hash << shiftl); \
++ hash ^= (hash >> shiftr); \
++}
++
++
++#define HASH_FROM_DOWN_TO(from, to) \
++for (index = from - 1; index >= to; index--) { \
++ hash ^= (hash >> shiftr); \
++ hash ^= (hash >> (shiftr*2)); \
++ hash -= (hash << shiftl); \
++ hash += (hash << (shiftl*2)); \
++ pos = random_nums[index]; \
++ hash -= key[pos]; \
++}
++
++/*
++ * The main random sample hash function.
++ */
++static u32 random_sample_hash(void *addr, u32 hash_strength)
++{
++ u32 hash = 0xdeadbeef;
++ int index, pos, loop = hash_strength;
++ u32 *key = (u32 *)addr;
++
++ if (loop > HASH_STRENGTH_FULL)
++ loop = HASH_STRENGTH_FULL;
++
++ HASH_FROM_TO(0, loop);
++
++ if (hash_strength > HASH_STRENGTH_FULL) {
++ loop = hash_strength - HASH_STRENGTH_FULL;
++ HASH_FROM_TO(0, loop);
++ }
++
++ return hash;
++}
++
++
++/**
++ * It's used when hash strength is adjusted
++ *
++ * @addr The page's virtual address
++ * @from The original hash strength
++ * @to The hash strength changed to
++ * @hash The hash value generated with "from" hash value
++ *
++ * return the hash value
++ */
++static u32 delta_hash(void *addr, int from, int to, u32 hash)
++{
++ u32 *key = (u32 *)addr;
++ int index, pos; /* make sure they are int type */
++
++ if (to > from) {
++ if (from >= HASH_STRENGTH_FULL) {
++ from -= HASH_STRENGTH_FULL;
++ to -= HASH_STRENGTH_FULL;
++ HASH_FROM_TO(from, to);
++ } else if (to <= HASH_STRENGTH_FULL) {
++ HASH_FROM_TO(from, to);
++ } else {
++ HASH_FROM_TO(from, HASH_STRENGTH_FULL);
++ HASH_FROM_TO(0, to - HASH_STRENGTH_FULL);
++ }
++ } else {
++ if (from <= HASH_STRENGTH_FULL) {
++ HASH_FROM_DOWN_TO(from, to);
++ } else if (to >= HASH_STRENGTH_FULL) {
++ from -= HASH_STRENGTH_FULL;
++ to -= HASH_STRENGTH_FULL;
++ HASH_FROM_DOWN_TO(from, to);
++ } else {
++ HASH_FROM_DOWN_TO(from - HASH_STRENGTH_FULL, 0);
++ HASH_FROM_DOWN_TO(HASH_STRENGTH_FULL, to);
++ }
++ }
++
++ return hash;
++}
++
++/**
++ *
++ * Called when: rshash_pos or rshash_neg is about to overflow or a scan round
++ * has finished.
++ *
++ * return 0 if no page has been scanned since last call, 1 otherwise.
++ */
++static inline int encode_benefit(void)
++{
++ u64 scanned_delta, pos_delta, neg_delta;
++ unsigned long base = benefit.base;
++
++ scanned_delta = uksm_pages_scanned - uksm_pages_scanned_last;
++
++ if (!scanned_delta)
++ return 0;
++
++ scanned_delta >>= base;
++ pos_delta = rshash_pos >> base;
++ neg_delta = rshash_neg >> base;
++
++ if (CAN_OVERFLOW_U64(benefit.pos, pos_delta) ||
++ CAN_OVERFLOW_U64(benefit.neg, neg_delta) ||
++ CAN_OVERFLOW_U64(benefit.scanned, scanned_delta)) {
++ benefit.scanned >>= 1;
++ benefit.neg >>= 1;
++ benefit.pos >>= 1;
++ benefit.base++;
++ scanned_delta >>= 1;
++ pos_delta >>= 1;
++ neg_delta >>= 1;
++ }
++
++ benefit.pos += pos_delta;
++ benefit.neg += neg_delta;
++ benefit.scanned += scanned_delta;
++
++ BUG_ON(!benefit.scanned);
++
++ rshash_pos = rshash_neg = 0;
++ uksm_pages_scanned_last = uksm_pages_scanned;
++
++ return 1;
++}
++
++static inline void reset_benefit(void)
++{
++ benefit.pos = 0;
++ benefit.neg = 0;
++ benefit.base = 0;
++ benefit.scanned = 0;
++}
++
++static inline void inc_rshash_pos(unsigned long delta)
++{
++ if (CAN_OVERFLOW_U64(rshash_pos, delta))
++ encode_benefit();
++
++ rshash_pos += delta;
++}
++
++static inline void inc_rshash_neg(unsigned long delta)
++{
++ if (CAN_OVERFLOW_U64(rshash_neg, delta))
++ encode_benefit();
++
++ rshash_neg += delta;
++}
++
++
++static inline u32 page_hash(struct page *page, unsigned long hash_strength,
++ int cost_accounting)
++{
++ u32 val;
++ unsigned long delta;
++
++ void *addr = kmap_atomic(page);
++
++ val = random_sample_hash(addr, hash_strength);
++ kunmap_atomic(addr);
++
++ if (cost_accounting) {
++ if (hash_strength < HASH_STRENGTH_FULL)
++ delta = HASH_STRENGTH_FULL - hash_strength;
++ else
++ delta = 0;
++
++ inc_rshash_pos(delta);
++ }
++
++ return val;
++}
++
++static int memcmp_pages_with_cost(struct page *page1, struct page *page2,
++ int cost_accounting)
++{
++ char *addr1, *addr2;
++ int ret;
++
++ addr1 = kmap_atomic(page1);
++ addr2 = kmap_atomic(page2);
++ ret = memcmp(addr1, addr2, PAGE_SIZE);
++ kunmap_atomic(addr2);
++ kunmap_atomic(addr1);
++
++ if (cost_accounting)
++ inc_rshash_neg(memcmp_cost);
++
++ return ret;
++}
++
++static inline int pages_identical_with_cost(struct page *page1, struct page *page2)
++{
++ return !memcmp_pages_with_cost(page1, page2, 0);
++}
++
++static inline int is_page_full_zero(struct page *page)
++{
++ char *addr;
++ int ret;
++
++ addr = kmap_atomic(page);
++ ret = is_full_zero(addr, PAGE_SIZE);
++ kunmap_atomic(addr);
++
++ return ret;
++}
++
++static int write_protect_page(struct vm_area_struct *vma, struct page *page,
++ pte_t *orig_pte, pte_t *old_pte)
++{
++ struct mm_struct *mm = vma->vm_mm;
++ struct page_vma_mapped_walk pvmw = {
++ .page = page,
++ .vma = vma,
++ };
++ struct mmu_notifier_range range;
++ int swapped;
++ int err = -EFAULT;
++
++ pvmw.address = page_address_in_vma(page, vma);
++ if (pvmw.address == -EFAULT)
++ goto out;
++
++ BUG_ON(PageTransCompound(page));
++
++ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, pvmw.address,
++ pvmw.address + PAGE_SIZE);
++ mmu_notifier_invalidate_range_start(&range);
++
++ if (!page_vma_mapped_walk(&pvmw))
++ goto out_mn;
++ if (WARN_ONCE(!pvmw.pte, "Unexpected PMD mapping?"))
++ goto out_unlock;
++
++ if (old_pte)
++ *old_pte = *pvmw.pte;
++
++ if (pte_write(*pvmw.pte) || pte_dirty(*pvmw.pte) ||
++ (pte_protnone(*pvmw.pte) && pte_savedwrite(*pvmw.pte)) || mm_tlb_flush_pending(mm)) {
++ pte_t entry;
++
++ swapped = PageSwapCache(page);
++ flush_cache_page(vma, pvmw.address, page_to_pfn(page));
++ /*
++ * Ok this is tricky, when get_user_pages_fast() run it doesn't
++ * take any lock, therefore the check that we are going to make
++ * with the pagecount against the mapcount is racey and
++ * O_DIRECT can happen right after the check.
++ * So we clear the pte and flush the tlb before the check
++ * this assure us that no O_DIRECT can happen after the check
++ * or in the middle of the check.
++ */
++ entry = ptep_clear_flush_notify(vma, pvmw.address, pvmw.pte);
++ /*
++ * Check that no O_DIRECT or similar I/O is in progress on the
++ * page
++ */
++ if (page_mapcount(page) + 1 + swapped != page_count(page)) {
++ set_pte_at(mm, pvmw.address, pvmw.pte, entry);
++ goto out_unlock;
++ }
++ if (pte_dirty(entry))
++ set_page_dirty(page);
++
++ if (pte_protnone(entry))
++ entry = pte_mkclean(pte_clear_savedwrite(entry));
++ else
++ entry = pte_mkclean(pte_wrprotect(entry));
++
++ set_pte_at_notify(mm, pvmw.address, pvmw.pte, entry);
++ }
++ *orig_pte = *pvmw.pte;
++ err = 0;
++
++out_unlock:
++ page_vma_mapped_walk_done(&pvmw);
++out_mn:
++ mmu_notifier_invalidate_range_end(&range);
++out:
++ return err;
++}
++
++#define MERGE_ERR_PGERR 1 /* the page is invalid cannot continue */
++#define MERGE_ERR_COLLI 2 /* there is a collision */
++#define MERGE_ERR_COLLI_MAX 3 /* collision at the max hash strength */
++#define MERGE_ERR_CHANGED 4 /* the page has changed since last hash */
++
++
++/**
++ * replace_page - replace page in vma by new ksm page
++ * @vma: vma that holds the pte pointing to page
++ * @page: the page we are replacing by kpage
++ * @kpage: the ksm page we replace page by
++ * @orig_pte: the original value of the pte
++ *
++ * Returns 0 on success, MERGE_ERR_PGERR on failure.
++ */
++static int replace_page(struct vm_area_struct *vma, struct page *page,
++ struct page *kpage, pte_t orig_pte)
++{
++ struct mm_struct *mm = vma->vm_mm;
++ struct mmu_notifier_range range;
++ pgd_t *pgd;
++ p4d_t *p4d;
++ pud_t *pud;
++ pmd_t *pmd;
++ pte_t *ptep;
++ spinlock_t *ptl;
++ pte_t entry;
++
++ unsigned long addr;
++ int err = MERGE_ERR_PGERR;
++
++ addr = page_address_in_vma(page, vma);
++ if (addr == -EFAULT)
++ goto out;
++
++ pgd = pgd_offset(mm, addr);
++ if (!pgd_present(*pgd))
++ goto out;
++
++ p4d = p4d_offset(pgd, addr);
++ pud = pud_offset(p4d, addr);
++ if (!pud_present(*pud))
++ goto out;
++
++ pmd = pmd_offset(pud, addr);
++ BUG_ON(pmd_trans_huge(*pmd));
++ if (!pmd_present(*pmd))
++ goto out;
++
++ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,
++ addr + PAGE_SIZE);
++ mmu_notifier_invalidate_range_start(&range);
++
++ ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
++ if (!pte_same(*ptep, orig_pte)) {
++ pte_unmap_unlock(ptep, ptl);
++ goto out_mn;
++ }
++
++ flush_cache_page(vma, addr, pte_pfn(*ptep));
++ ptep_clear_flush_notify(vma, addr, ptep);
++ entry = mk_pte(kpage, vma->vm_page_prot);
++
++ /* special treatment is needed for zero_page */
++ if ((page_to_pfn(kpage) == uksm_zero_pfn) ||
++ (page_to_pfn(kpage) == zero_pfn)) {
++ entry = pte_mkspecial(entry);
++ dec_mm_counter(mm, MM_ANONPAGES);
++ inc_zone_page_state(page, NR_UKSM_ZERO_PAGES);
++ } else {
++ get_page(kpage);
++ page_add_anon_rmap(kpage, vma, addr, false);
++ }
++
++ set_pte_at_notify(mm, addr, ptep, entry);
++
++ page_remove_rmap(page, false);
++ if (!page_mapped(page))
++ try_to_free_swap(page);
++ put_page(page);
++
++ pte_unmap_unlock(ptep, ptl);
++ err = 0;
++out_mn:
++ mmu_notifier_invalidate_range_end(&range);
++out:
++ return err;
++}
++
++
++/**
++ * Fully hash a page with HASH_STRENGTH_MAX return a non-zero hash value. The
++ * zero hash value at HASH_STRENGTH_MAX is used to indicated that its
++ * hash_max member has not been calculated.
++ *
++ * @page The page needs to be hashed
++ * @hash_old The hash value calculated with current hash strength
++ *
++ * return the new hash value calculated at HASH_STRENGTH_MAX
++ */
++static inline u32 page_hash_max(struct page *page, u32 hash_old)
++{
++ u32 hash_max = 0;
++ void *addr;
++
++ addr = kmap_atomic(page);
++ hash_max = delta_hash(addr, hash_strength,
++ HASH_STRENGTH_MAX, hash_old);
++
++ kunmap_atomic(addr);
++
++ if (!hash_max)
++ hash_max = 1;
++
++ inc_rshash_neg(HASH_STRENGTH_MAX - hash_strength);
++ return hash_max;
++}
++
++/*
++ * We compare the hash again, to ensure that it is really a hash collision
++ * instead of being caused by page write.
++ */
++static inline int check_collision(struct rmap_item *rmap_item,
++ u32 hash)
++{
++ int err;
++ struct page *page = rmap_item->page;
++
++ /* if this rmap_item has already been hash_maxed, then the collision
++ * must appears in the second-level rbtree search. In this case we check
++ * if its hash_max value has been changed. Otherwise, the collision
++ * happens in the first-level rbtree search, so we check against it's
++ * current hash value.
++ */
++ if (rmap_item->hash_max) {
++ inc_rshash_neg(memcmp_cost);
++ inc_rshash_neg(HASH_STRENGTH_MAX - hash_strength);
++
++ if (rmap_item->hash_max == page_hash_max(page, hash))
++ err = MERGE_ERR_COLLI;
++ else
++ err = MERGE_ERR_CHANGED;
++ } else {
++ inc_rshash_neg(memcmp_cost + hash_strength);
++
++ if (page_hash(page, hash_strength, 0) == hash)
++ err = MERGE_ERR_COLLI;
++ else
++ err = MERGE_ERR_CHANGED;
++ }
++
++ return err;
++}
++
++/**
++ * Try to merge a rmap_item.page with a kpage in stable node. kpage must
++ * already be a ksm page.
++ *
++ * @return 0 if the pages were merged, -EFAULT otherwise.
++ */
++static int try_to_merge_with_uksm_page(struct rmap_item *rmap_item,
++ struct page *kpage, u32 hash)
++{
++ struct vm_area_struct *vma = rmap_item->slot->vma;
++ struct mm_struct *mm = vma->vm_mm;
++ pte_t orig_pte = __pte(0);
++ int err = MERGE_ERR_PGERR;
++ struct page *page;
++
++ if (uksm_test_exit(mm))
++ goto out;
++
++ page = rmap_item->page;
++
++ if (page == kpage) { /* ksm page forked */
++ err = 0;
++ goto out;
++ }
++
++ /*
++ * We need the page lock to read a stable PageSwapCache in
++ * write_protect_page(). We use trylock_page() instead of
++ * lock_page() because we don't want to wait here - we
++ * prefer to continue scanning and merging different pages,
++ * then come back to this page when it is unlocked.
++ */
++ if (!trylock_page(page))
++ goto out;
++
++ if (!PageAnon(page) || !PageKsm(kpage))
++ goto out_unlock;
++
++ if (PageTransCompound(page)) {
++ err = split_huge_page(page);
++ if (err)
++ goto out_unlock;
++ }
++
++ /*
++ * If this anonymous page is mapped only here, its pte may need
++ * to be write-protected. If it's mapped elsewhere, all of its
++ * ptes are necessarily already write-protected. But in either
++ * case, we need to lock and check page_count is not raised.
++ */
++ if (write_protect_page(vma, page, &orig_pte, NULL) == 0) {
++ if (pages_identical_with_cost(page, kpage))
++ err = replace_page(vma, page, kpage, orig_pte);
++ else
++ err = check_collision(rmap_item, hash);
++ }
++
++ if ((vma->vm_flags & VM_LOCKED) && kpage && !err) {
++ munlock_vma_page(page);
++ if (!PageMlocked(kpage)) {
++ unlock_page(page);
++ lock_page(kpage);
++ mlock_vma_page(kpage);
++ page = kpage; /* for final unlock */
++ }
++ }
++
++out_unlock:
++ unlock_page(page);
++out:
++ return err;
++}
++
++
++
++/**
++ * If two pages fail to merge in try_to_merge_two_pages, then we have a chance
++ * to restore a page mapping that has been changed in try_to_merge_two_pages.
++ *
++ * @return 0 on success.
++ */
++static int restore_uksm_page_pte(struct vm_area_struct *vma, unsigned long addr,
++ pte_t orig_pte, pte_t wprt_pte)
++{
++ struct mm_struct *mm = vma->vm_mm;
++ pgd_t *pgd;
++ p4d_t *p4d;
++ pud_t *pud;
++ pmd_t *pmd;
++ pte_t *ptep;
++ spinlock_t *ptl;
++
++ int err = -EFAULT;
++
++ pgd = pgd_offset(mm, addr);
++ if (!pgd_present(*pgd))
++ goto out;
++
++ p4d = p4d_offset(pgd, addr);
++ pud = pud_offset(p4d, addr);
++ if (!pud_present(*pud))
++ goto out;
++
++ pmd = pmd_offset(pud, addr);
++ if (!pmd_present(*pmd))
++ goto out;
++
++ ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
++ if (!pte_same(*ptep, wprt_pte)) {
++ /* already copied, let it be */
++ pte_unmap_unlock(ptep, ptl);
++ goto out;
++ }
++
++ /*
++ * Good boy, still here. When we still get the ksm page, it does not
++ * return to the free page pool, there is no way that a pte was changed
++ * to other page and gets back to this page. And remind that ksm page
++ * do not reuse in do_wp_page(). So it's safe to restore the original
++ * pte.
++ */
++ flush_cache_page(vma, addr, pte_pfn(*ptep));
++ ptep_clear_flush_notify(vma, addr, ptep);
++ set_pte_at_notify(mm, addr, ptep, orig_pte);
++
++ pte_unmap_unlock(ptep, ptl);
++ err = 0;
++out:
++ return err;
++}
++
++/**
++ * try_to_merge_two_pages() - take two identical pages and prepare
++ * them to be merged into one page(rmap_item->page)
++ *
++ * @return 0 if we successfully merged two identical pages into
++ * one ksm page. MERGE_ERR_COLLI if it's only a hash collision
++ * search in rbtree. MERGE_ERR_CHANGED if rmap_item has been
++ * changed since it's hashed. MERGE_ERR_PGERR otherwise.
++ *
++ */
++static int try_to_merge_two_pages(struct rmap_item *rmap_item,
++ struct rmap_item *tree_rmap_item,
++ u32 hash)
++{
++ pte_t orig_pte1 = __pte(0), orig_pte2 = __pte(0);
++ pte_t wprt_pte1 = __pte(0), wprt_pte2 = __pte(0);
++ struct vm_area_struct *vma1 = rmap_item->slot->vma;
++ struct vm_area_struct *vma2 = tree_rmap_item->slot->vma;
++ struct page *page = rmap_item->page;
++ struct page *tree_page = tree_rmap_item->page;
++ int err = MERGE_ERR_PGERR;
++ struct address_space *saved_mapping;
++
++
++ if (rmap_item->page == tree_rmap_item->page)
++ goto out;
++
++ if (!trylock_page(page))
++ goto out;
++
++ if (!PageAnon(page))
++ goto out_unlock;
++
++ if (PageTransCompound(page)) {
++ err = split_huge_page(page);
++ if (err)
++ goto out_unlock;
++ }
++
++ if (write_protect_page(vma1, page, &wprt_pte1, &orig_pte1) != 0) {
++ unlock_page(page);
++ goto out;
++ }
++
++ /*
++ * While we hold page lock, upgrade page from
++ * PageAnon+anon_vma to PageKsm+NULL stable_node:
++ * stable_tree_insert() will update stable_node.
++ */
++ saved_mapping = page->mapping;
++ set_page_stable_node(page, NULL);
++ mark_page_accessed(page);
++ if (!PageDirty(page))
++ SetPageDirty(page);
++
++ unlock_page(page);
++
++ if (!trylock_page(tree_page))
++ goto restore_out;
++
++ if (!PageAnon(tree_page)) {
++ unlock_page(tree_page);
++ goto restore_out;
++ }
++
++ if (PageTransCompound(tree_page)) {
++ err = split_huge_page(tree_page);
++ if (err) {
++ unlock_page(tree_page);
++ goto restore_out;
++ }
++ }
++
++ if (write_protect_page(vma2, tree_page, &wprt_pte2, &orig_pte2) != 0) {
++ unlock_page(tree_page);
++ goto restore_out;
++ }
++
++ if (pages_identical_with_cost(page, tree_page)) {
++ err = replace_page(vma2, tree_page, page, wprt_pte2);
++ if (err) {
++ unlock_page(tree_page);
++ goto restore_out;
++ }
++
++ if ((vma2->vm_flags & VM_LOCKED)) {
++ munlock_vma_page(tree_page);
++ if (!PageMlocked(page)) {
++ unlock_page(tree_page);
++ lock_page(page);
++ mlock_vma_page(page);
++ tree_page = page; /* for final unlock */
++ }
++ }
++
++ unlock_page(tree_page);
++
++ goto out; /* success */
++
++ } else {
++ if (tree_rmap_item->hash_max &&
++ tree_rmap_item->hash_max == rmap_item->hash_max) {
++ err = MERGE_ERR_COLLI_MAX;
++ } else if (page_hash(page, hash_strength, 0) ==
++ page_hash(tree_page, hash_strength, 0)) {
++ inc_rshash_neg(memcmp_cost + hash_strength * 2);
++ err = MERGE_ERR_COLLI;
++ } else {
++ err = MERGE_ERR_CHANGED;
++ }
++
++ unlock_page(tree_page);
++ }
++
++restore_out:
++ lock_page(page);
++ if (!restore_uksm_page_pte(vma1, get_rmap_addr(rmap_item),
++ orig_pte1, wprt_pte1))
++ page->mapping = saved_mapping;
++
++out_unlock:
++ unlock_page(page);
++out:
++ return err;
++}
++
++static inline int hash_cmp(u32 new_val, u32 node_val)
++{
++ if (new_val > node_val)
++ return 1;
++ else if (new_val < node_val)
++ return -1;
++ else
++ return 0;
++}
++
++static inline u32 rmap_item_hash_max(struct rmap_item *item, u32 hash)
++{
++ u32 hash_max = item->hash_max;
++
++ if (!hash_max) {
++ hash_max = page_hash_max(item->page, hash);
++
++ item->hash_max = hash_max;
++ }
++
++ return hash_max;
++}
++
++
++
++/**
++ * stable_tree_search() - search the stable tree for a page
++ *
++ * @item: the rmap_item we are comparing with
++ * @hash: the hash value of this item->page already calculated
++ *
++ * @return the page we have found, NULL otherwise. The page returned has
++ * been gotten.
++ */
++static struct page *stable_tree_search(struct rmap_item *item, u32 hash)
++{
++ struct rb_node *node = root_stable_treep->rb_node;
++ struct tree_node *tree_node;
++ unsigned long hash_max;
++ struct page *page = item->page;
++ struct stable_node *stable_node;
++
++ stable_node = page_stable_node(page);
++ if (stable_node) {
++ /* ksm page forked, that is
++ * if (PageKsm(page) && !in_stable_tree(rmap_item))
++ * it's actually gotten once outside.
++ */
++ get_page(page);
++ return page;
++ }
++
++ while (node) {
++ int cmp;
++
++ tree_node = rb_entry(node, struct tree_node, node);
++
++ cmp = hash_cmp(hash, tree_node->hash);
++
++ if (cmp < 0)
++ node = node->rb_left;
++ else if (cmp > 0)
++ node = node->rb_right;
++ else
++ break;
++ }
++
++ if (!node)
++ return NULL;
++
++ if (tree_node->count == 1) {
++ stable_node = rb_entry(tree_node->sub_root.rb_node,
++ struct stable_node, node);
++ BUG_ON(!stable_node);
++
++ goto get_page_out;
++ }
++
++ /*
++ * ok, we have to search the second
++ * level subtree, hash the page to a
++ * full strength.
++ */
++ node = tree_node->sub_root.rb_node;
++ BUG_ON(!node);
++ hash_max = rmap_item_hash_max(item, hash);
++
++ while (node) {
++ int cmp;
++
++ stable_node = rb_entry(node, struct stable_node, node);
++
++ cmp = hash_cmp(hash_max, stable_node->hash_max);
++
++ if (cmp < 0)
++ node = node->rb_left;
++ else if (cmp > 0)
++ node = node->rb_right;
++ else
++ goto get_page_out;
++ }
++
++ return NULL;
++
++get_page_out:
++ page = get_uksm_page(stable_node, 1, 1);
++ return page;
++}
++
++static int try_merge_rmap_item(struct rmap_item *item,
++ struct page *kpage,
++ struct page *tree_page)
++{
++ struct vm_area_struct *vma = item->slot->vma;
++ struct page_vma_mapped_walk pvmw = {
++ .page = kpage,
++ .vma = vma,
++ };
++
++ pvmw.address = get_rmap_addr(item);
++ if (!page_vma_mapped_walk(&pvmw))
++ return 0;
++
++ if (pte_write(*pvmw.pte)) {
++ /* has changed, abort! */
++ page_vma_mapped_walk_done(&pvmw);
++ return 0;
++ }
++
++ get_page(tree_page);
++ page_add_anon_rmap(tree_page, vma, pvmw.address, false);
++
++ flush_cache_page(vma, pvmw.address, page_to_pfn(kpage));
++ ptep_clear_flush_notify(vma, pvmw.address, pvmw.pte);
++ set_pte_at_notify(vma->vm_mm, pvmw.address, pvmw.pte,
++ mk_pte(tree_page, vma->vm_page_prot));
++
++ page_remove_rmap(kpage, false);
++ put_page(kpage);
++
++ page_vma_mapped_walk_done(&pvmw);
++
++ return 1;
++}
++
++/**
++ * try_to_merge_with_stable_page() - when two rmap_items need to be inserted
++ * into stable tree, the page was found to be identical to a stable ksm page,
++ * this is the last chance we can merge them into one.
++ *
++ * @item1: the rmap_item holding the page which we wanted to insert
++ * into stable tree.
++ * @item2: the other rmap_item we found when unstable tree search
++ * @oldpage: the page currently mapped by the two rmap_items
++ * @tree_page: the page we found identical in stable tree node
++ * @success1: return if item1 is successfully merged
++ * @success2: return if item2 is successfully merged
++ */
++static void try_merge_with_stable(struct rmap_item *item1,
++ struct rmap_item *item2,
++ struct page **kpage,
++ struct page *tree_page,
++ int *success1, int *success2)
++{
++ struct vm_area_struct *vma1 = item1->slot->vma;
++ struct vm_area_struct *vma2 = item2->slot->vma;
++ *success1 = 0;
++ *success2 = 0;
++
++ if (unlikely(*kpage == tree_page)) {
++ /* I don't think this can really happen */
++ pr_warn("UKSM: unexpected condition detected in "
++ "%s -- *kpage == tree_page !\n", __func__);
++ *success1 = 1;
++ *success2 = 1;
++ return;
++ }
++
++ if (!PageAnon(*kpage) || !PageKsm(*kpage))
++ goto failed;
++
++ if (!trylock_page(tree_page))
++ goto failed;
++
++ /* If the oldpage is still ksm and still pointed
++ * to in the right place, and still write protected,
++ * we are confident it's not changed, no need to
++ * memcmp anymore.
++ * be ware, we cannot take nested pte locks,
++ * deadlock risk.
++ */
++ if (!try_merge_rmap_item(item1, *kpage, tree_page))
++ goto unlock_failed;
++
++ /* ok, then vma2, remind that pte1 already set */
++ if (!try_merge_rmap_item(item2, *kpage, tree_page))
++ goto success_1;
++
++ *success2 = 1;
++success_1:
++ *success1 = 1;
++
++
++ if ((*success1 && vma1->vm_flags & VM_LOCKED) ||
++ (*success2 && vma2->vm_flags & VM_LOCKED)) {
++ munlock_vma_page(*kpage);
++ if (!PageMlocked(tree_page))
++ mlock_vma_page(tree_page);
++ }
++
++ /*
++ * We do not need oldpage any more in the caller, so can break the lock
++ * now.
++ */
++ unlock_page(*kpage);
++ *kpage = tree_page; /* Get unlocked outside. */
++ return;
++
++unlock_failed:
++ unlock_page(tree_page);
++failed:
++ return;
++}
++
++static inline void stable_node_hash_max(struct stable_node *node,
++ struct page *page, u32 hash)
++{
++ u32 hash_max = node->hash_max;
++
++ if (!hash_max) {
++ hash_max = page_hash_max(page, hash);
++ node->hash_max = hash_max;
++ }
++}
++
++static inline
++struct stable_node *new_stable_node(struct tree_node *tree_node,
++ struct page *kpage, u32 hash_max)
++{
++ struct stable_node *new_stable_node;
++
++ new_stable_node = alloc_stable_node();
++ if (!new_stable_node)
++ return NULL;
++
++ new_stable_node->kpfn = page_to_pfn(kpage);
++ new_stable_node->hash_max = hash_max;
++ new_stable_node->tree_node = tree_node;
++ set_page_stable_node(kpage, new_stable_node);
++
++ return new_stable_node;
++}
++
++static inline
++struct stable_node *first_level_insert(struct tree_node *tree_node,
++ struct rmap_item *rmap_item,
++ struct rmap_item *tree_rmap_item,
++ struct page **kpage, u32 hash,
++ int *success1, int *success2)
++{
++ int cmp;
++ struct page *tree_page;
++ u32 hash_max = 0;
++ struct stable_node *stable_node, *new_snode;
++ struct rb_node *parent = NULL, **new;
++
++ /* this tree node contains no sub-tree yet */
++ stable_node = rb_entry(tree_node->sub_root.rb_node,
++ struct stable_node, node);
++
++ tree_page = get_uksm_page(stable_node, 1, 0);
++ if (tree_page) {
++ cmp = memcmp_pages_with_cost(*kpage, tree_page, 1);
++ if (!cmp) {
++ try_merge_with_stable(rmap_item, tree_rmap_item, kpage,
++ tree_page, success1, success2);
++ put_page(tree_page);
++ if (!*success1 && !*success2)
++ goto failed;
++
++ return stable_node;
++
++ } else {
++ /*
++ * collision in first level try to create a subtree.
++ * A new node need to be created.
++ */
++ put_page(tree_page);
++
++ stable_node_hash_max(stable_node, tree_page,
++ tree_node->hash);
++ hash_max = rmap_item_hash_max(rmap_item, hash);
++ cmp = hash_cmp(hash_max, stable_node->hash_max);
++
++ parent = &stable_node->node;
++ if (cmp < 0)
++ new = &parent->rb_left;
++ else if (cmp > 0)
++ new = &parent->rb_right;
++ else
++ goto failed;
++ }
++
++ } else {
++ /* the only stable_node deleted, we reuse its tree_node.
++ */
++ parent = NULL;
++ new = &tree_node->sub_root.rb_node;
++ }
++
++ new_snode = new_stable_node(tree_node, *kpage, hash_max);
++ if (!new_snode)
++ goto failed;
++
++ rb_link_node(&new_snode->node, parent, new);
++ rb_insert_color(&new_snode->node, &tree_node->sub_root);
++ tree_node->count++;
++ *success1 = *success2 = 1;
++
++ return new_snode;
++
++failed:
++ return NULL;
++}
++
++static inline
++struct stable_node *stable_subtree_insert(struct tree_node *tree_node,
++ struct rmap_item *rmap_item,
++ struct rmap_item *tree_rmap_item,
++ struct page **kpage, u32 hash,
++ int *success1, int *success2)
++{
++ struct page *tree_page;
++ u32 hash_max;
++ struct stable_node *stable_node, *new_snode;
++ struct rb_node *parent, **new;
++
++research:
++ parent = NULL;
++ new = &tree_node->sub_root.rb_node;
++ BUG_ON(!*new);
++ hash_max = rmap_item_hash_max(rmap_item, hash);
++ while (*new) {
++ int cmp;
++
++ stable_node = rb_entry(*new, struct stable_node, node);
++
++ cmp = hash_cmp(hash_max, stable_node->hash_max);
++
++ if (cmp < 0) {
++ parent = *new;
++ new = &parent->rb_left;
++ } else if (cmp > 0) {
++ parent = *new;
++ new = &parent->rb_right;
++ } else {
++ tree_page = get_uksm_page(stable_node, 1, 0);
++ if (tree_page) {
++ cmp = memcmp_pages_with_cost(*kpage, tree_page, 1);
++ if (!cmp) {
++ try_merge_with_stable(rmap_item,
++ tree_rmap_item, kpage,
++ tree_page, success1, success2);
++
++ put_page(tree_page);
++ if (!*success1 && !*success2)
++ goto failed;
++ /*
++ * successfully merged with a stable
++ * node
++ */
++ return stable_node;
++ } else {
++ put_page(tree_page);
++ goto failed;
++ }
++ } else {
++ /*
++ * stable node may be deleted,
++ * and subtree maybe
++ * restructed, cannot
++ * continue, research it.
++ */
++ if (tree_node->count) {
++ goto research;
++ } else {
++ /* reuse the tree node*/
++ parent = NULL;
++ new = &tree_node->sub_root.rb_node;
++ }
++ }
++ }
++ }
++
++ new_snode = new_stable_node(tree_node, *kpage, hash_max);
++ if (!new_snode)
++ goto failed;
++
++ rb_link_node(&new_snode->node, parent, new);
++ rb_insert_color(&new_snode->node, &tree_node->sub_root);
++ tree_node->count++;
++ *success1 = *success2 = 1;
++
++ return new_snode;
++
++failed:
++ return NULL;
++}
++
++
++/**
++ * stable_tree_insert() - try to insert a merged page in unstable tree to
++ * the stable tree
++ *
++ * @kpage: the page need to be inserted
++ * @hash: the current hash of this page
++ * @rmap_item: the rmap_item being scanned
++ * @tree_rmap_item: the rmap_item found on unstable tree
++ * @success1: return if rmap_item is merged
++ * @success2: return if tree_rmap_item is merged
++ *
++ * @return the stable_node on stable tree if at least one
++ * rmap_item is inserted into stable tree, NULL
++ * otherwise.
++ */
++static struct stable_node *
++stable_tree_insert(struct page **kpage, u32 hash,
++ struct rmap_item *rmap_item,
++ struct rmap_item *tree_rmap_item,
++ int *success1, int *success2)
++{
++ struct rb_node **new = &root_stable_treep->rb_node;
++ struct rb_node *parent = NULL;
++ struct stable_node *stable_node;
++ struct tree_node *tree_node;
++ u32 hash_max = 0;
++
++ *success1 = *success2 = 0;
++
++ while (*new) {
++ int cmp;
++
++ tree_node = rb_entry(*new, struct tree_node, node);
++
++ cmp = hash_cmp(hash, tree_node->hash);
++
++ if (cmp < 0) {
++ parent = *new;
++ new = &parent->rb_left;
++ } else if (cmp > 0) {
++ parent = *new;
++ new = &parent->rb_right;
++ } else
++ break;
++ }
++
++ if (*new) {
++ if (tree_node->count == 1) {
++ stable_node = first_level_insert(tree_node, rmap_item,
++ tree_rmap_item, kpage,
++ hash, success1, success2);
++ } else {
++ stable_node = stable_subtree_insert(tree_node,
++ rmap_item, tree_rmap_item, kpage,
++ hash, success1, success2);
++ }
++ } else {
++
++ /* no tree node found */
++ tree_node = alloc_tree_node(stable_tree_node_listp);
++ if (!tree_node) {
++ stable_node = NULL;
++ goto out;
++ }
++
++ stable_node = new_stable_node(tree_node, *kpage, hash_max);
++ if (!stable_node) {
++ free_tree_node(tree_node);
++ goto out;
++ }
++
++ tree_node->hash = hash;
++ rb_link_node(&tree_node->node, parent, new);
++ rb_insert_color(&tree_node->node, root_stable_treep);
++ parent = NULL;
++ new = &tree_node->sub_root.rb_node;
++
++ rb_link_node(&stable_node->node, parent, new);
++ rb_insert_color(&stable_node->node, &tree_node->sub_root);
++ tree_node->count++;
++ *success1 = *success2 = 1;
++ }
++
++out:
++ return stable_node;
++}
++
++
++/**
++ * get_tree_rmap_item_page() - try to get the page and lock the mmap_sem
++ *
++ * @return 0 on success, -EBUSY if unable to lock the mmap_sem,
++ * -EINVAL if the page mapping has been changed.
++ */
++static inline int get_tree_rmap_item_page(struct rmap_item *tree_rmap_item)
++{
++ int err;
++
++ err = get_mergeable_page_lock_mmap(tree_rmap_item);
++
++ if (err == -EINVAL) {
++ /* its page map has been changed, remove it */
++ remove_rmap_item_from_tree(tree_rmap_item);
++ }
++
++ /* The page is gotten and mmap_sem is locked now. */
++ return err;
++}
++
++
++/**
++ * unstable_tree_search_insert() - search an unstable tree rmap_item with the
++ * same hash value. Get its page and trylock the mmap_sem
++ */
++static inline
++struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,
++ u32 hash)
++
++{
++ struct rb_node **new = &root_unstable_tree.rb_node;
++ struct rb_node *parent = NULL;
++ struct tree_node *tree_node;
++ u32 hash_max;
++ struct rmap_item *tree_rmap_item;
++
++ while (*new) {
++ int cmp;
++
++ tree_node = rb_entry(*new, struct tree_node, node);
++
++ cmp = hash_cmp(hash, tree_node->hash);
++
++ if (cmp < 0) {
++ parent = *new;
++ new = &parent->rb_left;
++ } else if (cmp > 0) {
++ parent = *new;
++ new = &parent->rb_right;
++ } else
++ break;
++ }
++
++ if (*new) {
++ /* got the tree_node */
++ if (tree_node->count == 1) {
++ tree_rmap_item = rb_entry(tree_node->sub_root.rb_node,
++ struct rmap_item, node);
++ BUG_ON(!tree_rmap_item);
++
++ goto get_page_out;
++ }
++
++ /* well, search the collision subtree */
++ new = &tree_node->sub_root.rb_node;
++ BUG_ON(!*new);
++ hash_max = rmap_item_hash_max(rmap_item, hash);
++
++ while (*new) {
++ int cmp;
++
++ tree_rmap_item = rb_entry(*new, struct rmap_item,
++ node);
++
++ cmp = hash_cmp(hash_max, tree_rmap_item->hash_max);
++ parent = *new;
++ if (cmp < 0)
++ new = &parent->rb_left;
++ else if (cmp > 0)
++ new = &parent->rb_right;
++ else
++ goto get_page_out;
++ }
++ } else {
++ /* alloc a new tree_node */
++ tree_node = alloc_tree_node(&unstable_tree_node_list);
++ if (!tree_node)
++ return NULL;
++
++ tree_node->hash = hash;
++ rb_link_node(&tree_node->node, parent, new);
++ rb_insert_color(&tree_node->node, &root_unstable_tree);
++ parent = NULL;
++ new = &tree_node->sub_root.rb_node;
++ }
++
++ /* did not found even in sub-tree */
++ rmap_item->tree_node = tree_node;
++ rmap_item->address |= UNSTABLE_FLAG;
++ rmap_item->hash_round = uksm_hash_round;
++ rb_link_node(&rmap_item->node, parent, new);
++ rb_insert_color(&rmap_item->node, &tree_node->sub_root);
++
++ uksm_pages_unshared++;
++ return NULL;
++
++get_page_out:
++ if (tree_rmap_item->page == rmap_item->page)
++ return NULL;
++
++ if (get_tree_rmap_item_page(tree_rmap_item))
++ return NULL;
++
++ return tree_rmap_item;
++}
++
++static void hold_anon_vma(struct rmap_item *rmap_item,
++ struct anon_vma *anon_vma)
++{
++ rmap_item->anon_vma = anon_vma;
++ get_anon_vma(anon_vma);
++}
++
++
++/**
++ * stable_tree_append() - append a rmap_item to a stable node. Deduplication
++ * ratio statistics is done in this function.
++ *
++ */
++static void stable_tree_append(struct rmap_item *rmap_item,
++ struct stable_node *stable_node, int logdedup)
++{
++ struct node_vma *node_vma = NULL, *new_node_vma, *node_vma_cont = NULL;
++ unsigned long key = (unsigned long)rmap_item->slot;
++ unsigned long factor = rmap_item->slot->rung->step;
++
++ BUG_ON(!stable_node);
++ rmap_item->address |= STABLE_FLAG;
++
++ if (hlist_empty(&stable_node->hlist)) {
++ uksm_pages_shared++;
++ goto node_vma_new;
++ } else {
++ uksm_pages_sharing++;
++ }
++
++ hlist_for_each_entry(node_vma, &stable_node->hlist, hlist) {
++ if (node_vma->key >= key)
++ break;
++
++ if (logdedup) {
++ node_vma->slot->pages_bemerged += factor;
++ if (list_empty(&node_vma->slot->dedup_list))
++ list_add(&node_vma->slot->dedup_list,
++ &vma_slot_dedup);
++ }
++ }
++
++ if (node_vma) {
++ if (node_vma->key == key) {
++ node_vma_cont = hlist_entry_safe(node_vma->hlist.next, struct node_vma, hlist);
++ goto node_vma_ok;
++ } else if (node_vma->key > key) {
++ node_vma_cont = node_vma;
++ }
++ }
++
++node_vma_new:
++ /* no same vma already in node, alloc a new node_vma */
++ new_node_vma = alloc_node_vma();
++ BUG_ON(!new_node_vma);
++ new_node_vma->head = stable_node;
++ new_node_vma->slot = rmap_item->slot;
++
++ if (!node_vma) {
++ hlist_add_head(&new_node_vma->hlist, &stable_node->hlist);
++ } else if (node_vma->key != key) {
++ if (node_vma->key < key)
++ hlist_add_behind(&new_node_vma->hlist, &node_vma->hlist);
++ else {
++ hlist_add_before(&new_node_vma->hlist,
++ &node_vma->hlist);
++ }
++
++ }
++ node_vma = new_node_vma;
++
++node_vma_ok: /* ok, ready to add to the list */
++ rmap_item->head = node_vma;
++ hlist_add_head(&rmap_item->hlist, &node_vma->rmap_hlist);
++ hold_anon_vma(rmap_item, rmap_item->slot->vma->anon_vma);
++ if (logdedup) {
++ rmap_item->slot->pages_merged++;
++ if (node_vma_cont) {
++ node_vma = node_vma_cont;
++ hlist_for_each_entry_continue(node_vma, hlist) {
++ node_vma->slot->pages_bemerged += factor;
++ if (list_empty(&node_vma->slot->dedup_list))
++ list_add(&node_vma->slot->dedup_list,
++ &vma_slot_dedup);
++ }
++ }
++ }
++}
++
++/*
++ * We use break_ksm to break COW on a ksm page: it's a stripped down
++ *
++ * if (get_user_pages(addr, 1, 1, 1, &page, NULL) == 1)
++ * put_page(page);
++ *
++ * but taking great care only to touch a ksm page, in a VM_MERGEABLE vma,
++ * in case the application has unmapped and remapped mm,addr meanwhile.
++ * Could a ksm page appear anywhere else? Actually yes, in a VM_PFNMAP
++ * mmap of /dev/mem or /dev/kmem, where we would not want to touch it.
++ */
++static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
++{
++ struct page *page;
++ int ret = 0;
++
++ do {
++ cond_resched();
++ page = follow_page(vma, addr, FOLL_GET | FOLL_MIGRATION | FOLL_REMOTE);
++ if (IS_ERR_OR_NULL(page))
++ break;
++ if (PageKsm(page)) {
++ ret = handle_mm_fault(vma, addr,
++ FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE);
++ } else
++ ret = VM_FAULT_WRITE;
++ put_page(page);
++ } while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | VM_FAULT_OOM)));
++ /*
++ * We must loop because handle_mm_fault() may back out if there's
++ * any difficulty e.g. if pte accessed bit gets updated concurrently.
++ *
++ * VM_FAULT_WRITE is what we have been hoping for: it indicates that
++ * COW has been broken, even if the vma does not permit VM_WRITE;
++ * but note that a concurrent fault might break PageKsm for us.
++ *
++ * VM_FAULT_SIGBUS could occur if we race with truncation of the
++ * backing file, which also invalidates anonymous pages: that's
++ * okay, that truncation will have unmapped the PageKsm for us.
++ *
++ * VM_FAULT_OOM: at the time of writing (late July 2009), setting
++ * aside mem_cgroup limits, VM_FAULT_OOM would only be set if the
++ * current task has TIF_MEMDIE set, and will be OOM killed on return
++ * to user; and ksmd, having no mm, would never be chosen for that.
++ *
++ * But if the mm is in a limited mem_cgroup, then the fault may fail
++ * with VM_FAULT_OOM even if the current task is not TIF_MEMDIE; and
++ * even ksmd can fail in this way - though it's usually breaking ksm
++ * just to undo a merge it made a moment before, so unlikely to oom.
++ *
++ * That's a pity: we might therefore have more kernel pages allocated
++ * than we're counting as nodes in the stable tree; but uksm_do_scan
++ * will retry to break_cow on each pass, so should recover the page
++ * in due course. The important thing is to not let VM_MERGEABLE
++ * be cleared while any such pages might remain in the area.
++ */
++ return (ret & VM_FAULT_OOM) ? -ENOMEM : 0;
++}
++
++static void break_cow(struct rmap_item *rmap_item)
++{
++ struct vm_area_struct *vma = rmap_item->slot->vma;
++ struct mm_struct *mm = vma->vm_mm;
++ unsigned long addr = get_rmap_addr(rmap_item);
++
++ if (uksm_test_exit(mm))
++ goto out;
++
++ break_ksm(vma, addr);
++out:
++ return;
++}
++
++/*
++ * Though it's very tempting to unmerge in_stable_tree(rmap_item)s rather
++ * than check every pte of a given vma, the locking doesn't quite work for
++ * that - an rmap_item is assigned to the stable tree after inserting ksm
++ * page and upping mmap_sem. Nor does it fit with the way we skip dup'ing
++ * rmap_items from parent to child at fork time (so as not to waste time
++ * if exit comes before the next scan reaches it).
++ *
++ * Similarly, although we'd like to remove rmap_items (so updating counts
++ * and freeing memory) when unmerging an area, it's easier to leave that
++ * to the next pass of ksmd - consider, for example, how ksmd might be
++ * in cmp_and_merge_page on one of the rmap_items we would be removing.
++ */
++inline int unmerge_uksm_pages(struct vm_area_struct *vma,
++ unsigned long start, unsigned long end)
++{
++ unsigned long addr;
++ int err = 0;
++
++ for (addr = start; addr < end && !err; addr += PAGE_SIZE) {
++ if (uksm_test_exit(vma->vm_mm))
++ break;
++ if (signal_pending(current))
++ err = -ERESTARTSYS;
++ else
++ err = break_ksm(vma, addr);
++ }
++ return err;
++}
++
++static inline void inc_uksm_pages_scanned(void)
++{
++ u64 delta;
++
++
++ if (uksm_pages_scanned == U64_MAX) {
++ encode_benefit();
++
++ delta = uksm_pages_scanned >> pages_scanned_base;
++
++ if (CAN_OVERFLOW_U64(pages_scanned_stored, delta)) {
++ pages_scanned_stored >>= 1;
++ delta >>= 1;
++ pages_scanned_base++;
++ }
++
++ pages_scanned_stored += delta;
++
++ uksm_pages_scanned = uksm_pages_scanned_last = 0;
++ }
++
++ uksm_pages_scanned++;
++}
++
++static inline int find_zero_page_hash(int strength, u32 hash)
++{
++ return (zero_hash_table[strength] == hash);
++}
++
++static
++int cmp_and_merge_zero_page(struct vm_area_struct *vma, struct page *page)
++{
++ struct page *zero_page = empty_uksm_zero_page;
++ struct mm_struct *mm = vma->vm_mm;
++ pte_t orig_pte = __pte(0);
++ int err = -EFAULT;
++
++ if (uksm_test_exit(mm))
++ goto out;
++
++ if (!trylock_page(page))
++ goto out;
++
++ if (!PageAnon(page))
++ goto out_unlock;
++
++ if (PageTransCompound(page)) {
++ err = split_huge_page(page);
++ if (err)
++ goto out_unlock;
++ }
++
++ if (write_protect_page(vma, page, &orig_pte, 0) == 0) {
++ if (is_page_full_zero(page))
++ err = replace_page(vma, page, zero_page, orig_pte);
++ }
++
++out_unlock:
++ unlock_page(page);
++out:
++ return err;
++}
++
++/*
++ * cmp_and_merge_page() - first see if page can be merged into the stable
++ * tree; if not, compare hash to previous and if it's the same, see if page
++ * can be inserted into the unstable tree, or merged with a page already there
++ * and both transferred to the stable tree.
++ *
++ * @page: the page that we are searching identical page to.
++ * @rmap_item: the reverse mapping into the virtual address of this page
++ */
++static void cmp_and_merge_page(struct rmap_item *rmap_item, u32 hash)
++{
++ struct rmap_item *tree_rmap_item;
++ struct page *page;
++ struct page *kpage = NULL;
++ u32 hash_max;
++ int err;
++ unsigned int success1, success2;
++ struct stable_node *snode;
++ int cmp;
++ struct rb_node *parent = NULL, **new;
++
++ remove_rmap_item_from_tree(rmap_item);
++ page = rmap_item->page;
++
++ /* We first start with searching the page inside the stable tree */
++ kpage = stable_tree_search(rmap_item, hash);
++ if (kpage) {
++ err = try_to_merge_with_uksm_page(rmap_item, kpage,
++ hash);
++ if (!err) {
++ /*
++ * The page was successfully merged, add
++ * its rmap_item to the stable tree.
++ * page lock is needed because it's
++ * racing with try_to_unmap_ksm(), etc.
++ */
++ lock_page(kpage);
++ snode = page_stable_node(kpage);
++ stable_tree_append(rmap_item, snode, 1);
++ unlock_page(kpage);
++ put_page(kpage);
++ return; /* success */
++ }
++ put_page(kpage);
++
++ /*
++ * if it's a collision and it has been search in sub-rbtree
++ * (hash_max != 0), we want to abort, because if it is
++ * successfully merged in unstable tree, the collision trends to
++ * happen again.
++ */
++ if (err == MERGE_ERR_COLLI && rmap_item->hash_max)
++ return;
++ }
++
++ tree_rmap_item =
++ unstable_tree_search_insert(rmap_item, hash);
++ if (tree_rmap_item) {
++ err = try_to_merge_two_pages(rmap_item, tree_rmap_item, hash);
++ /*
++ * As soon as we merge this page, we want to remove the
++ * rmap_item of the page we have merged with from the unstable
++ * tree, and insert it instead as new node in the stable tree.
++ */
++ if (!err) {
++ kpage = page;
++ remove_rmap_item_from_tree(tree_rmap_item);
++ lock_page(kpage);
++ snode = stable_tree_insert(&kpage, hash,
++ rmap_item, tree_rmap_item,
++ &success1, &success2);
++
++ /*
++ * Do not log dedup for tree item, it's not counted as
++ * scanned in this round.
++ */
++ if (success2)
++ stable_tree_append(tree_rmap_item, snode, 0);
++
++ /*
++ * The order of these two stable append is important:
++ * we are scanning rmap_item.
++ */
++ if (success1)
++ stable_tree_append(rmap_item, snode, 1);
++
++ /*
++ * The original kpage may be unlocked inside
++ * stable_tree_insert() already. This page
++ * should be unlocked before doing
++ * break_cow().
++ */
++ unlock_page(kpage);
++
++ if (!success1)
++ break_cow(rmap_item);
++
++ if (!success2)
++ break_cow(tree_rmap_item);
++
++ } else if (err == MERGE_ERR_COLLI) {
++ BUG_ON(tree_rmap_item->tree_node->count > 1);
++
++ rmap_item_hash_max(tree_rmap_item,
++ tree_rmap_item->tree_node->hash);
++
++ hash_max = rmap_item_hash_max(rmap_item, hash);
++ cmp = hash_cmp(hash_max, tree_rmap_item->hash_max);
++ parent = &tree_rmap_item->node;
++ if (cmp < 0)
++ new = &parent->rb_left;
++ else if (cmp > 0)
++ new = &parent->rb_right;
++ else
++ goto put_up_out;
++
++ rmap_item->tree_node = tree_rmap_item->tree_node;
++ rmap_item->address |= UNSTABLE_FLAG;
++ rmap_item->hash_round = uksm_hash_round;
++ rb_link_node(&rmap_item->node, parent, new);
++ rb_insert_color(&rmap_item->node,
++ &tree_rmap_item->tree_node->sub_root);
++ rmap_item->tree_node->count++;
++ } else {
++ /*
++ * either one of the page has changed or they collide
++ * at the max hash, we consider them as ill items.
++ */
++ remove_rmap_item_from_tree(tree_rmap_item);
++ }
++put_up_out:
++ put_page(tree_rmap_item->page);
++ up_read(&tree_rmap_item->slot->vma->vm_mm->mmap_sem);
++ }
++}
++
++
++
++
++static inline unsigned long get_pool_index(struct vma_slot *slot,
++ unsigned long index)
++{
++ unsigned long pool_index;
++
++ pool_index = (sizeof(struct rmap_list_entry *) * index) >> PAGE_SHIFT;
++ if (pool_index >= slot->pool_size)
++ BUG();
++ return pool_index;
++}
++
++static inline unsigned long index_page_offset(unsigned long index)
++{
++ return offset_in_page(sizeof(struct rmap_list_entry *) * index);
++}
++
++static inline
++struct rmap_list_entry *get_rmap_list_entry(struct vma_slot *slot,
++ unsigned long index, int need_alloc)
++{
++ unsigned long pool_index;
++ struct page *page;
++ void *addr;
++
++
++ pool_index = get_pool_index(slot, index);
++ if (!slot->rmap_list_pool[pool_index]) {
++ if (!need_alloc)
++ return NULL;
++
++ page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN);
++ if (!page)
++ return NULL;
++
++ slot->rmap_list_pool[pool_index] = page;
++ }
++
++ addr = kmap(slot->rmap_list_pool[pool_index]);
++ addr += index_page_offset(index);
++
++ return addr;
++}
++
++static inline void put_rmap_list_entry(struct vma_slot *slot,
++ unsigned long index)
++{
++ unsigned long pool_index;
++
++ pool_index = get_pool_index(slot, index);
++ BUG_ON(!slot->rmap_list_pool[pool_index]);
++ kunmap(slot->rmap_list_pool[pool_index]);
++}
++
++static inline int entry_is_new(struct rmap_list_entry *entry)
++{
++ return !entry->item;
++}
++
++static inline unsigned long get_index_orig_addr(struct vma_slot *slot,
++ unsigned long index)
++{
++ return slot->vma->vm_start + (index << PAGE_SHIFT);
++}
++
++static inline unsigned long get_entry_address(struct rmap_list_entry *entry)
++{
++ unsigned long addr;
++
++ if (is_addr(entry->addr))
++ addr = get_clean_addr(entry->addr);
++ else if (entry->item)
++ addr = get_rmap_addr(entry->item);
++ else
++ BUG();
++
++ return addr;
++}
++
++static inline struct rmap_item *get_entry_item(struct rmap_list_entry *entry)
++{
++ if (is_addr(entry->addr))
++ return NULL;
++
++ return entry->item;
++}
++
++static inline void inc_rmap_list_pool_count(struct vma_slot *slot,
++ unsigned long index)
++{
++ unsigned long pool_index;
++
++ pool_index = get_pool_index(slot, index);
++ BUG_ON(!slot->rmap_list_pool[pool_index]);
++ slot->pool_counts[pool_index]++;
++}
++
++static inline void dec_rmap_list_pool_count(struct vma_slot *slot,
++ unsigned long index)
++{
++ unsigned long pool_index;
++
++ pool_index = get_pool_index(slot, index);
++ BUG_ON(!slot->rmap_list_pool[pool_index]);
++ BUG_ON(!slot->pool_counts[pool_index]);
++ slot->pool_counts[pool_index]--;
++}
++
++static inline int entry_has_rmap(struct rmap_list_entry *entry)
++{
++ return !is_addr(entry->addr) && entry->item;
++}
++
++static inline void swap_entries(struct rmap_list_entry *entry1,
++ unsigned long index1,
++ struct rmap_list_entry *entry2,
++ unsigned long index2)
++{
++ struct rmap_list_entry tmp;
++
++ /* swapping two new entries is meaningless */
++ BUG_ON(entry_is_new(entry1) && entry_is_new(entry2));
++
++ tmp = *entry1;
++ *entry1 = *entry2;
++ *entry2 = tmp;
++
++ if (entry_has_rmap(entry1))
++ entry1->item->entry_index = index1;
++
++ if (entry_has_rmap(entry2))
++ entry2->item->entry_index = index2;
++
++ if (entry_has_rmap(entry1) && !entry_has_rmap(entry2)) {
++ inc_rmap_list_pool_count(entry1->item->slot, index1);
++ dec_rmap_list_pool_count(entry1->item->slot, index2);
++ } else if (!entry_has_rmap(entry1) && entry_has_rmap(entry2)) {
++ inc_rmap_list_pool_count(entry2->item->slot, index2);
++ dec_rmap_list_pool_count(entry2->item->slot, index1);
++ }
++}
++
++static inline void free_entry_item(struct rmap_list_entry *entry)
++{
++ unsigned long index;
++ struct rmap_item *item;
++
++ if (!is_addr(entry->addr)) {
++ BUG_ON(!entry->item);
++ item = entry->item;
++ entry->addr = get_rmap_addr(item);
++ set_is_addr(entry->addr);
++ index = item->entry_index;
++ remove_rmap_item_from_tree(item);
++ dec_rmap_list_pool_count(item->slot, index);
++ free_rmap_item(item);
++ }
++}
++
++static inline int pool_entry_boundary(unsigned long index)
++{
++ unsigned long linear_addr;
++
++ linear_addr = sizeof(struct rmap_list_entry *) * index;
++ return index && !offset_in_page(linear_addr);
++}
++
++static inline void try_free_last_pool(struct vma_slot *slot,
++ unsigned long index)
++{
++ unsigned long pool_index;
++
++ pool_index = get_pool_index(slot, index);
++ if (slot->rmap_list_pool[pool_index] &&
++ !slot->pool_counts[pool_index]) {
++ __free_page(slot->rmap_list_pool[pool_index]);
++ slot->rmap_list_pool[pool_index] = NULL;
++ slot->flags |= UKSM_SLOT_NEED_SORT;
++ }
++
++}
++
++static inline unsigned long vma_item_index(struct vm_area_struct *vma,
++ struct rmap_item *item)
++{
++ return (get_rmap_addr(item) - vma->vm_start) >> PAGE_SHIFT;
++}
++
++static int within_same_pool(struct vma_slot *slot,
++ unsigned long i, unsigned long j)
++{
++ unsigned long pool_i, pool_j;
++
++ pool_i = get_pool_index(slot, i);
++ pool_j = get_pool_index(slot, j);
++
++ return (pool_i == pool_j);
++}
++
++static void sort_rmap_entry_list(struct vma_slot *slot)
++{
++ unsigned long i, j;
++ struct rmap_list_entry *entry, *swap_entry;
++
++ entry = get_rmap_list_entry(slot, 0, 0);
++ for (i = 0; i < slot->pages; ) {
++
++ if (!entry)
++ goto skip_whole_pool;
++
++ if (entry_is_new(entry))
++ goto next_entry;
++
++ if (is_addr(entry->addr)) {
++ entry->addr = 0;
++ goto next_entry;
++ }
++
++ j = vma_item_index(slot->vma, entry->item);
++ if (j == i)
++ goto next_entry;
++
++ if (within_same_pool(slot, i, j))
++ swap_entry = entry + j - i;
++ else
++ swap_entry = get_rmap_list_entry(slot, j, 1);
++
++ swap_entries(entry, i, swap_entry, j);
++ if (!within_same_pool(slot, i, j))
++ put_rmap_list_entry(slot, j);
++ continue;
++
++skip_whole_pool:
++ i += PAGE_SIZE / sizeof(*entry);
++ if (i < slot->pages)
++ entry = get_rmap_list_entry(slot, i, 0);
++ continue;
++
++next_entry:
++ if (i >= slot->pages - 1 ||
++ !within_same_pool(slot, i, i + 1)) {
++ put_rmap_list_entry(slot, i);
++ if (i + 1 < slot->pages)
++ entry = get_rmap_list_entry(slot, i + 1, 0);
++ } else
++ entry++;
++ i++;
++ continue;
++ }
++
++ /* free empty pool entries which contain no rmap_item */
++ /* CAN be simplied to based on only pool_counts when bug freed !!!!! */
++ for (i = 0; i < slot->pool_size; i++) {
++ unsigned char has_rmap;
++ void *addr;
++
++ if (!slot->rmap_list_pool[i])
++ continue;
++
++ has_rmap = 0;
++ addr = kmap(slot->rmap_list_pool[i]);
++ BUG_ON(!addr);
++ for (j = 0; j < PAGE_SIZE / sizeof(*entry); j++) {
++ entry = (struct rmap_list_entry *)addr + j;
++ if (is_addr(entry->addr))
++ continue;
++ if (!entry->item)
++ continue;
++ has_rmap = 1;
++ }
++ kunmap(slot->rmap_list_pool[i]);
++ if (!has_rmap) {
++ BUG_ON(slot->pool_counts[i]);
++ __free_page(slot->rmap_list_pool[i]);
++ slot->rmap_list_pool[i] = NULL;
++ }
++ }
++
++ slot->flags &= ~UKSM_SLOT_NEED_SORT;
++}
++
++/*
++ * vma_fully_scanned() - if all the pages in this slot have been scanned.
++ */
++static inline int vma_fully_scanned(struct vma_slot *slot)
++{
++ return slot->pages_scanned == slot->pages;
++}
++
++/**
++ * get_next_rmap_item() - Get the next rmap_item in a vma_slot according to
++ * its random permutation. This function is embedded with the random
++ * permutation index management code.
++ */
++static struct rmap_item *get_next_rmap_item(struct vma_slot *slot, u32 *hash)
++{
++ unsigned long rand_range, addr, swap_index, scan_index;
++ struct rmap_item *item = NULL;
++ struct rmap_list_entry *scan_entry, *swap_entry = NULL;
++ struct page *page;
++
++ scan_index = swap_index = slot->pages_scanned % slot->pages;
++
++ if (pool_entry_boundary(scan_index))
++ try_free_last_pool(slot, scan_index - 1);
++
++ if (vma_fully_scanned(slot)) {
++ if (slot->flags & UKSM_SLOT_NEED_SORT)
++ slot->flags |= UKSM_SLOT_NEED_RERAND;
++ else
++ slot->flags &= ~UKSM_SLOT_NEED_RERAND;
++ if (slot->flags & UKSM_SLOT_NEED_SORT)
++ sort_rmap_entry_list(slot);
++ }
++
++ scan_entry = get_rmap_list_entry(slot, scan_index, 1);
++ if (!scan_entry)
++ return NULL;
++
++ if (entry_is_new(scan_entry)) {
++ scan_entry->addr = get_index_orig_addr(slot, scan_index);
++ set_is_addr(scan_entry->addr);
++ }
++
++ if (slot->flags & UKSM_SLOT_NEED_RERAND) {
++ rand_range = slot->pages - scan_index;
++ BUG_ON(!rand_range);
++ swap_index = scan_index + (prandom_u32() % rand_range);
++ }
++
++ if (swap_index != scan_index) {
++ swap_entry = get_rmap_list_entry(slot, swap_index, 1);
++
++ if (!swap_entry)
++ return NULL;
++
++ if (entry_is_new(swap_entry)) {
++ swap_entry->addr = get_index_orig_addr(slot,
++ swap_index);
++ set_is_addr(swap_entry->addr);
++ }
++ swap_entries(scan_entry, scan_index, swap_entry, swap_index);
++ }
++
++ addr = get_entry_address(scan_entry);
++ item = get_entry_item(scan_entry);
++ BUG_ON(addr > slot->vma->vm_end || addr < slot->vma->vm_start);
++
++ page = follow_page(slot->vma, addr, FOLL_GET);
++ if (IS_ERR_OR_NULL(page))
++ goto nopage;
++
++ if (!PageAnon(page))
++ goto putpage;
++
++ /*check is zero_page pfn or uksm_zero_page*/
++ if ((page_to_pfn(page) == zero_pfn)
++ || (page_to_pfn(page) == uksm_zero_pfn))
++ goto putpage;
++
++ flush_anon_page(slot->vma, page, addr);
++ flush_dcache_page(page);
++
++
++ *hash = page_hash(page, hash_strength, 1);
++ inc_uksm_pages_scanned();
++ /*if the page content all zero, re-map to zero-page*/
++ if (find_zero_page_hash(hash_strength, *hash)) {
++ if (!cmp_and_merge_zero_page(slot->vma, page)) {
++ slot->pages_merged++;
++
++ /* For full-zero pages, no need to create rmap item */
++ goto putpage;
++ } else {
++ inc_rshash_neg(memcmp_cost / 2);
++ }
++ }
++
++ if (!item) {
++ item = alloc_rmap_item();
++ if (item) {
++ /* It has already been zeroed */
++ item->slot = slot;
++ item->address = addr;
++ item->entry_index = scan_index;
++ scan_entry->item = item;
++ inc_rmap_list_pool_count(slot, scan_index);
++ } else
++ goto putpage;
++ }
++
++ BUG_ON(item->slot != slot);
++ /* the page may have changed */
++ item->page = page;
++ put_rmap_list_entry(slot, scan_index);
++ if (swap_entry)
++ put_rmap_list_entry(slot, swap_index);
++ return item;
++
++putpage:
++ put_page(page);
++ page = NULL;
++nopage:
++ /* no page, store addr back and free rmap_item if possible */
++ free_entry_item(scan_entry);
++ put_rmap_list_entry(slot, scan_index);
++ if (swap_entry)
++ put_rmap_list_entry(slot, swap_index);
++ return NULL;
++}
++
++static inline int in_stable_tree(struct rmap_item *rmap_item)
++{
++ return rmap_item->address & STABLE_FLAG;
++}
++
++/**
++ * scan_vma_one_page() - scan the next page in a vma_slot. Called with
++ * mmap_sem locked.
++ */
++static noinline void scan_vma_one_page(struct vma_slot *slot)
++{
++ u32 hash;
++ struct mm_struct *mm;
++ struct rmap_item *rmap_item = NULL;
++ struct vm_area_struct *vma = slot->vma;
++
++ mm = vma->vm_mm;
++ BUG_ON(!mm);
++ BUG_ON(!slot);
++
++ rmap_item = get_next_rmap_item(slot, &hash);
++ if (!rmap_item)
++ goto out1;
++
++ if (PageKsm(rmap_item->page) && in_stable_tree(rmap_item))
++ goto out2;
++
++ cmp_and_merge_page(rmap_item, hash);
++out2:
++ put_page(rmap_item->page);
++out1:
++ slot->pages_scanned++;
++ slot->this_sampled++;
++ if (slot->fully_scanned_round != fully_scanned_round)
++ scanned_virtual_pages++;
++
++ if (vma_fully_scanned(slot))
++ slot->fully_scanned_round = fully_scanned_round;
++}
++
++static inline unsigned long rung_get_pages(struct scan_rung *rung)
++{
++ struct slot_tree_node *node;
++
++ if (!rung->vma_root.rnode)
++ return 0;
++
++ node = container_of(rung->vma_root.rnode, struct slot_tree_node, snode);
++
++ return node->size;
++}
++
++#define RUNG_SAMPLED_MIN 3
++
++static inline
++void uksm_calc_rung_step(struct scan_rung *rung,
++ unsigned long page_time, unsigned long ratio)
++{
++ unsigned long sampled, pages;
++
++ /* will be fully scanned ? */
++ if (!rung->cover_msecs) {
++ rung->step = 1;
++ return;
++ }
++
++ sampled = rung->cover_msecs * (NSEC_PER_MSEC / TIME_RATIO_SCALE)
++ * ratio / page_time;
++
++ /*
++ * Before we finsish a scan round and expensive per-round jobs,
++ * we need to have a chance to estimate the per page time. So
++ * the sampled number can not be too small.
++ */
++ if (sampled < RUNG_SAMPLED_MIN)
++ sampled = RUNG_SAMPLED_MIN;
++
++ pages = rung_get_pages(rung);
++ if (likely(pages > sampled))
++ rung->step = pages / sampled;
++ else
++ rung->step = 1;
++}
++
++static inline int step_need_recalc(struct scan_rung *rung)
++{
++ unsigned long pages, stepmax;
++
++ pages = rung_get_pages(rung);
++ stepmax = pages / RUNG_SAMPLED_MIN;
++
++ return pages && (rung->step > pages ||
++ (stepmax && rung->step > stepmax));
++}
++
++static inline
++void reset_current_scan(struct scan_rung *rung, int finished, int step_recalc)
++{
++ struct vma_slot *slot;
++
++ if (finished)
++ rung->flags |= UKSM_RUNG_ROUND_FINISHED;
++
++ if (step_recalc || step_need_recalc(rung)) {
++ uksm_calc_rung_step(rung, uksm_ema_page_time, rung->cpu_ratio);
++ BUG_ON(step_need_recalc(rung));
++ }
++
++ slot_iter_index = prandom_u32() % rung->step;
++ BUG_ON(!rung->vma_root.rnode);
++ slot = sradix_tree_next(&rung->vma_root, NULL, 0, slot_iter);
++ BUG_ON(!slot);
++
++ rung->current_scan = slot;
++ rung->current_offset = slot_iter_index;
++}
++
++static inline struct sradix_tree_root *slot_get_root(struct vma_slot *slot)
++{
++ return &slot->rung->vma_root;
++}
++
++/*
++ * return if resetted.
++ */
++static int advance_current_scan(struct scan_rung *rung)
++{
++ unsigned short n;
++ struct vma_slot *slot, *next = NULL;
++
++ BUG_ON(!rung->vma_root.num);
++
++ slot = rung->current_scan;
++ n = (slot->pages - rung->current_offset) % rung->step;
++ slot_iter_index = rung->step - n;
++ next = sradix_tree_next(&rung->vma_root, slot->snode,
++ slot->sindex, slot_iter);
++
++ if (next) {
++ rung->current_offset = slot_iter_index;
++ rung->current_scan = next;
++ return 0;
++ } else {
++ reset_current_scan(rung, 1, 0);
++ return 1;
++ }
++}
++
++static inline void rung_rm_slot(struct vma_slot *slot)
++{
++ struct scan_rung *rung = slot->rung;
++ struct sradix_tree_root *root;
++
++ if (rung->current_scan == slot)
++ advance_current_scan(rung);
++
++ root = slot_get_root(slot);
++ sradix_tree_delete_from_leaf(root, slot->snode, slot->sindex);
++ slot->snode = NULL;
++ if (step_need_recalc(rung)) {
++ uksm_calc_rung_step(rung, uksm_ema_page_time, rung->cpu_ratio);
++ BUG_ON(step_need_recalc(rung));
++ }
++
++ /* In case advance_current_scan loop back to this slot again */
++ if (rung->vma_root.num && rung->current_scan == slot)
++ reset_current_scan(slot->rung, 1, 0);
++}
++
++static inline void rung_add_new_slots(struct scan_rung *rung,
++ struct vma_slot **slots, unsigned long num)
++{
++ int err;
++ struct vma_slot *slot;
++ unsigned long i;
++ struct sradix_tree_root *root = &rung->vma_root;
++
++ err = sradix_tree_enter(root, (void **)slots, num);
++ BUG_ON(err);
++
++ for (i = 0; i < num; i++) {
++ slot = slots[i];
++ slot->rung = rung;
++ BUG_ON(vma_fully_scanned(slot));
++ }
++
++ if (rung->vma_root.num == num)
++ reset_current_scan(rung, 0, 1);
++}
++
++static inline int rung_add_one_slot(struct scan_rung *rung,
++ struct vma_slot *slot)
++{
++ int err;
++
++ err = sradix_tree_enter(&rung->vma_root, (void **)&slot, 1);
++ if (err)
++ return err;
++
++ slot->rung = rung;
++ if (rung->vma_root.num == 1)
++ reset_current_scan(rung, 0, 1);
++
++ return 0;
++}
++
++/*
++ * Return true if the slot is deleted from its rung.
++ */
++static inline int vma_rung_enter(struct vma_slot *slot, struct scan_rung *rung)
++{
++ struct scan_rung *old_rung = slot->rung;
++ int err;
++
++ if (old_rung == rung)
++ return 0;
++
++ rung_rm_slot(slot);
++ err = rung_add_one_slot(rung, slot);
++ if (err) {
++ err = rung_add_one_slot(old_rung, slot);
++ WARN_ON(err); /* OOPS, badly OOM, we lost this slot */
++ }
++
++ return 1;
++}
++
++static inline int vma_rung_up(struct vma_slot *slot)
++{
++ struct scan_rung *rung;
++
++ rung = slot->rung;
++ if (slot->rung != &uksm_scan_ladder[SCAN_LADDER_SIZE-1])
++ rung++;
++
++ return vma_rung_enter(slot, rung);
++}
++
++static inline int vma_rung_down(struct vma_slot *slot)
++{
++ struct scan_rung *rung;
++
++ rung = slot->rung;
++ if (slot->rung != &uksm_scan_ladder[0])
++ rung--;
++
++ return vma_rung_enter(slot, rung);
++}
++
++/**
++ * cal_dedup_ratio() - Calculate the deduplication ratio for this slot.
++ */
++static unsigned long cal_dedup_ratio(struct vma_slot *slot)
++{
++ unsigned long ret;
++ unsigned long pages;
++
++ pages = slot->this_sampled;
++ if (!pages)
++ return 0;
++
++ BUG_ON(slot->pages_scanned == slot->last_scanned);
++
++ ret = slot->pages_merged;
++
++ /* Thrashing area filtering */
++ if (ret && uksm_thrash_threshold) {
++ if (slot->pages_cowed * 100 / slot->pages_merged
++ > uksm_thrash_threshold) {
++ ret = 0;
++ } else {
++ ret = slot->pages_merged - slot->pages_cowed;
++ }
++ }
++
++ return ret * 100 / pages;
++}
++
++/**
++ * cal_dedup_ratio() - Calculate the deduplication ratio for this slot.
++ */
++static unsigned long cal_dedup_ratio_old(struct vma_slot *slot)
++{
++ unsigned long ret;
++ unsigned long pages;
++
++ pages = slot->pages;
++ if (!pages)
++ return 0;
++
++ ret = slot->pages_bemerged;
++
++ /* Thrashing area filtering */
++ if (ret && uksm_thrash_threshold) {
++ if (slot->pages_cowed * 100 / slot->pages_bemerged
++ > uksm_thrash_threshold) {
++ ret = 0;
++ } else {
++ ret = slot->pages_bemerged - slot->pages_cowed;
++ }
++ }
++
++ return ret * 100 / pages;
++}
++
++/**
++ * stable_node_reinsert() - When the hash_strength has been adjusted, the
++ * stable tree need to be restructured, this is the function re-inserting the
++ * stable node.
++ */
++static inline void stable_node_reinsert(struct stable_node *new_node,
++ struct page *page,
++ struct rb_root *root_treep,
++ struct list_head *tree_node_listp,
++ u32 hash)
++{
++ struct rb_node **new = &root_treep->rb_node;
++ struct rb_node *parent = NULL;
++ struct stable_node *stable_node;
++ struct tree_node *tree_node;
++ struct page *tree_page;
++ int cmp;
++
++ while (*new) {
++ int cmp;
++
++ tree_node = rb_entry(*new, struct tree_node, node);
++
++ cmp = hash_cmp(hash, tree_node->hash);
++
++ if (cmp < 0) {
++ parent = *new;
++ new = &parent->rb_left;
++ } else if (cmp > 0) {
++ parent = *new;
++ new = &parent->rb_right;
++ } else
++ break;
++ }
++
++ if (*new) {
++ /* find a stable tree node with same first level hash value */
++ stable_node_hash_max(new_node, page, hash);
++ if (tree_node->count == 1) {
++ stable_node = rb_entry(tree_node->sub_root.rb_node,
++ struct stable_node, node);
++ tree_page = get_uksm_page(stable_node, 1, 0);
++ if (tree_page) {
++ stable_node_hash_max(stable_node,
++ tree_page, hash);
++ put_page(tree_page);
++
++ /* prepare for stable node insertion */
++
++ cmp = hash_cmp(new_node->hash_max,
++ stable_node->hash_max);
++ parent = &stable_node->node;
++ if (cmp < 0)
++ new = &parent->rb_left;
++ else if (cmp > 0)
++ new = &parent->rb_right;
++ else
++ goto failed;
++
++ goto add_node;
++ } else {
++ /* the only stable_node deleted, the tree node
++ * was not deleted.
++ */
++ goto tree_node_reuse;
++ }
++ }
++
++ /* well, search the collision subtree */
++ new = &tree_node->sub_root.rb_node;
++ parent = NULL;
++ BUG_ON(!*new);
++ while (*new) {
++ int cmp;
++
++ stable_node = rb_entry(*new, struct stable_node, node);
++
++ cmp = hash_cmp(new_node->hash_max,
++ stable_node->hash_max);
++
++ if (cmp < 0) {
++ parent = *new;
++ new = &parent->rb_left;
++ } else if (cmp > 0) {
++ parent = *new;
++ new = &parent->rb_right;
++ } else {
++ /* oh, no, still a collision */
++ goto failed;
++ }
++ }
++
++ goto add_node;
++ }
++
++ /* no tree node found */
++ tree_node = alloc_tree_node(tree_node_listp);
++ if (!tree_node) {
++ pr_err("UKSM: memory allocation error!\n");
++ goto failed;
++ } else {
++ tree_node->hash = hash;
++ rb_link_node(&tree_node->node, parent, new);
++ rb_insert_color(&tree_node->node, root_treep);
++
++tree_node_reuse:
++ /* prepare for stable node insertion */
++ parent = NULL;
++ new = &tree_node->sub_root.rb_node;
++ }
++
++add_node:
++ rb_link_node(&new_node->node, parent, new);
++ rb_insert_color(&new_node->node, &tree_node->sub_root);
++ new_node->tree_node = tree_node;
++ tree_node->count++;
++ return;
++
++failed:
++ /* This can only happen when two nodes have collided
++ * in two levels.
++ */
++ new_node->tree_node = NULL;
++ return;
++}
++
++static inline void free_all_tree_nodes(struct list_head *list)
++{
++ struct tree_node *node, *tmp;
++
++ list_for_each_entry_safe(node, tmp, list, all_list) {
++ free_tree_node(node);
++ }
++}
++
++/**
++ * stable_tree_delta_hash() - Delta hash the stable tree from previous hash
++ * strength to the current hash_strength. It re-structures the hole tree.
++ */
++static inline void stable_tree_delta_hash(u32 prev_hash_strength)
++{
++ struct stable_node *node, *tmp;
++ struct rb_root *root_new_treep;
++ struct list_head *new_tree_node_listp;
++
++ stable_tree_index = (stable_tree_index + 1) % 2;
++ root_new_treep = &root_stable_tree[stable_tree_index];
++ new_tree_node_listp = &stable_tree_node_list[stable_tree_index];
++ *root_new_treep = RB_ROOT;
++ BUG_ON(!list_empty(new_tree_node_listp));
++
++ /*
++ * we need to be safe, the node could be removed by get_uksm_page()
++ */
++ list_for_each_entry_safe(node, tmp, &stable_node_list, all_list) {
++ void *addr;
++ struct page *node_page;
++ u32 hash;
++
++ /*
++ * We are completely re-structuring the stable nodes to a new
++ * stable tree. We don't want to touch the old tree unlinks and
++ * old tree_nodes. The old tree_nodes will be freed at once.
++ */
++ node_page = get_uksm_page(node, 0, 0);
++ if (!node_page)
++ continue;
++
++ if (node->tree_node) {
++ hash = node->tree_node->hash;
++
++ addr = kmap_atomic(node_page);
++
++ hash = delta_hash(addr, prev_hash_strength,
++ hash_strength, hash);
++ kunmap_atomic(addr);
++ } else {
++ /*
++ *it was not inserted to rbtree due to collision in last
++ *round scan.
++ */
++ hash = page_hash(node_page, hash_strength, 0);
++ }
++
++ stable_node_reinsert(node, node_page, root_new_treep,
++ new_tree_node_listp, hash);
++ put_page(node_page);
++ }
++
++ root_stable_treep = root_new_treep;
++ free_all_tree_nodes(stable_tree_node_listp);
++ BUG_ON(!list_empty(stable_tree_node_listp));
++ stable_tree_node_listp = new_tree_node_listp;
++}
++
++static inline void inc_hash_strength(unsigned long delta)
++{
++ hash_strength += 1 << delta;
++ if (hash_strength > HASH_STRENGTH_MAX)
++ hash_strength = HASH_STRENGTH_MAX;
++}
++
++static inline void dec_hash_strength(unsigned long delta)
++{
++ unsigned long change = 1 << delta;
++
++ if (hash_strength <= change + 1)
++ hash_strength = 1;
++ else
++ hash_strength -= change;
++}
++
++static inline void inc_hash_strength_delta(void)
++{
++ hash_strength_delta++;
++ if (hash_strength_delta > HASH_STRENGTH_DELTA_MAX)
++ hash_strength_delta = HASH_STRENGTH_DELTA_MAX;
++}
++
++static inline unsigned long get_current_neg_ratio(void)
++{
++ u64 pos = benefit.pos;
++ u64 neg = benefit.neg;
++
++ if (!neg)
++ return 0;
++
++ if (!pos || neg > pos)
++ return 100;
++
++ if (neg > div64_u64(U64_MAX, 100))
++ pos = div64_u64(pos, 100);
++ else
++ neg *= 100;
++
++ return div64_u64(neg, pos);
++}
++
++static inline unsigned long get_current_benefit(void)
++{
++ u64 pos = benefit.pos;
++ u64 neg = benefit.neg;
++ u64 scanned = benefit.scanned;
++
++ if (neg > pos)
++ return 0;
++
++ return div64_u64((pos - neg), scanned);
++}
++
++static inline int judge_rshash_direction(void)
++{
++ u64 current_neg_ratio, stable_benefit;
++ u64 current_benefit, delta = 0;
++ int ret = STILL;
++
++ /*
++ * Try to probe a value after the boot, and in case the system
++ * are still for a long time.
++ */
++ if ((fully_scanned_round & 0xFFULL) == 10) {
++ ret = OBSCURE;
++ goto out;
++ }
++
++ current_neg_ratio = get_current_neg_ratio();
++
++ if (current_neg_ratio == 0) {
++ rshash_neg_cont_zero++;
++ if (rshash_neg_cont_zero > 2)
++ return GO_DOWN;
++ else
++ return STILL;
++ }
++ rshash_neg_cont_zero = 0;
++
++ if (current_neg_ratio > 90) {
++ ret = GO_UP;
++ goto out;
++ }
++
++ current_benefit = get_current_benefit();
++ stable_benefit = rshash_state.stable_benefit;
++
++ if (!stable_benefit) {
++ ret = OBSCURE;
++ goto out;
++ }
++
++ if (current_benefit > stable_benefit)
++ delta = current_benefit - stable_benefit;
++ else if (current_benefit < stable_benefit)
++ delta = stable_benefit - current_benefit;
++
++ delta = div64_u64(100 * delta, stable_benefit);
++
++ if (delta > 50) {
++ rshash_cont_obscure++;
++ if (rshash_cont_obscure > 2)
++ return OBSCURE;
++ else
++ return STILL;
++ }
++
++out:
++ rshash_cont_obscure = 0;
++ return ret;
++}
++
++/**
++ * rshash_adjust() - The main function to control the random sampling state
++ * machine for hash strength adapting.
++ *
++ * return true if hash_strength has changed.
++ */
++static inline int rshash_adjust(void)
++{
++ unsigned long prev_hash_strength = hash_strength;
++
++ if (!encode_benefit())
++ return 0;
++
++ switch (rshash_state.state) {
++ case RSHASH_STILL:
++ switch (judge_rshash_direction()) {
++ case GO_UP:
++ if (rshash_state.pre_direct == GO_DOWN)
++ hash_strength_delta = 0;
++
++ inc_hash_strength(hash_strength_delta);
++ inc_hash_strength_delta();
++ rshash_state.stable_benefit = get_current_benefit();
++ rshash_state.pre_direct = GO_UP;
++ break;
++
++ case GO_DOWN:
++ if (rshash_state.pre_direct == GO_UP)
++ hash_strength_delta = 0;
++
++ dec_hash_strength(hash_strength_delta);
++ inc_hash_strength_delta();
++ rshash_state.stable_benefit = get_current_benefit();
++ rshash_state.pre_direct = GO_DOWN;
++ break;
++
++ case OBSCURE:
++ rshash_state.stable_point = hash_strength;
++ rshash_state.turn_point_down = hash_strength;
++ rshash_state.turn_point_up = hash_strength;
++ rshash_state.turn_benefit_down = get_current_benefit();
++ rshash_state.turn_benefit_up = get_current_benefit();
++ rshash_state.lookup_window_index = 0;
++ rshash_state.state = RSHASH_TRYDOWN;
++ dec_hash_strength(hash_strength_delta);
++ inc_hash_strength_delta();
++ break;
++
++ case STILL:
++ break;
++ default:
++ BUG();
++ }
++ break;
++
++ case RSHASH_TRYDOWN:
++ if (rshash_state.lookup_window_index++ % 5 == 0)
++ rshash_state.below_count = 0;
++
++ if (get_current_benefit() < rshash_state.stable_benefit)
++ rshash_state.below_count++;
++ else if (get_current_benefit() >
++ rshash_state.turn_benefit_down) {
++ rshash_state.turn_point_down = hash_strength;
++ rshash_state.turn_benefit_down = get_current_benefit();
++ }
++
++ if (rshash_state.below_count >= 3 ||
++ judge_rshash_direction() == GO_UP ||
++ hash_strength == 1) {
++ hash_strength = rshash_state.stable_point;
++ hash_strength_delta = 0;
++ inc_hash_strength(hash_strength_delta);
++ inc_hash_strength_delta();
++ rshash_state.lookup_window_index = 0;
++ rshash_state.state = RSHASH_TRYUP;
++ hash_strength_delta = 0;
++ } else {
++ dec_hash_strength(hash_strength_delta);
++ inc_hash_strength_delta();
++ }
++ break;
++
++ case RSHASH_TRYUP:
++ if (rshash_state.lookup_window_index++ % 5 == 0)
++ rshash_state.below_count = 0;
++
++ if (get_current_benefit() < rshash_state.turn_benefit_down)
++ rshash_state.below_count++;
++ else if (get_current_benefit() > rshash_state.turn_benefit_up) {
++ rshash_state.turn_point_up = hash_strength;
++ rshash_state.turn_benefit_up = get_current_benefit();
++ }
++
++ if (rshash_state.below_count >= 3 ||
++ judge_rshash_direction() == GO_DOWN ||
++ hash_strength == HASH_STRENGTH_MAX) {
++ hash_strength = rshash_state.turn_benefit_up >
++ rshash_state.turn_benefit_down ?
++ rshash_state.turn_point_up :
++ rshash_state.turn_point_down;
++
++ rshash_state.state = RSHASH_PRE_STILL;
++ } else {
++ inc_hash_strength(hash_strength_delta);
++ inc_hash_strength_delta();
++ }
++
++ break;
++
++ case RSHASH_NEW:
++ case RSHASH_PRE_STILL:
++ rshash_state.stable_benefit = get_current_benefit();
++ rshash_state.state = RSHASH_STILL;
++ hash_strength_delta = 0;
++ break;
++ default:
++ BUG();
++ }
++
++ /* rshash_neg = rshash_pos = 0; */
++ reset_benefit();
++
++ if (prev_hash_strength != hash_strength)
++ stable_tree_delta_hash(prev_hash_strength);
++
++ return prev_hash_strength != hash_strength;
++}
++
++/**
++ * round_update_ladder() - The main function to do update of all the
++ * adjustments whenever a scan round is finished.
++ */
++static noinline void round_update_ladder(void)
++{
++ int i;
++ unsigned long dedup;
++ struct vma_slot *slot, *tmp_slot;
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++)
++ uksm_scan_ladder[i].flags &= ~UKSM_RUNG_ROUND_FINISHED;
++
++ list_for_each_entry_safe(slot, tmp_slot, &vma_slot_dedup, dedup_list) {
++
++ /* slot may be rung_rm_slot() when mm exits */
++ if (slot->snode) {
++ dedup = cal_dedup_ratio_old(slot);
++ if (dedup && dedup >= uksm_abundant_threshold)
++ vma_rung_up(slot);
++ }
++
++ slot->pages_bemerged = 0;
++ slot->pages_cowed = 0;
++
++ list_del_init(&slot->dedup_list);
++ }
++}
++
++static void uksm_del_vma_slot(struct vma_slot *slot)
++{
++ int i, j;
++ struct rmap_list_entry *entry;
++
++ if (slot->snode) {
++ /*
++ * In case it just failed when entering the rung, it's not
++ * necessary.
++ */
++ rung_rm_slot(slot);
++ }
++
++ if (!list_empty(&slot->dedup_list))
++ list_del(&slot->dedup_list);
++
++ if (!slot->rmap_list_pool || !slot->pool_counts) {
++ /* In case it OOMed in uksm_vma_enter() */
++ goto out;
++ }
++
++ for (i = 0; i < slot->pool_size; i++) {
++ void *addr;
++
++ if (!slot->rmap_list_pool[i])
++ continue;
++
++ addr = kmap(slot->rmap_list_pool[i]);
++ for (j = 0; j < PAGE_SIZE / sizeof(*entry); j++) {
++ entry = (struct rmap_list_entry *)addr + j;
++ if (is_addr(entry->addr))
++ continue;
++ if (!entry->item)
++ continue;
++
++ remove_rmap_item_from_tree(entry->item);
++ free_rmap_item(entry->item);
++ slot->pool_counts[i]--;
++ }
++ BUG_ON(slot->pool_counts[i]);
++ kunmap(slot->rmap_list_pool[i]);
++ __free_page(slot->rmap_list_pool[i]);
++ }
++ kfree(slot->rmap_list_pool);
++ kfree(slot->pool_counts);
++
++out:
++ slot->rung = NULL;
++ if (slot->flags & UKSM_SLOT_IN_UKSM) {
++ BUG_ON(uksm_pages_total < slot->pages);
++ uksm_pages_total -= slot->pages;
++ }
++
++ if (slot->fully_scanned_round == fully_scanned_round)
++ scanned_virtual_pages -= slot->pages;
++ else
++ scanned_virtual_pages -= slot->pages_scanned;
++ free_vma_slot(slot);
++}
++
++
++#define SPIN_LOCK_PERIOD 32
++static struct vma_slot *cleanup_slots[SPIN_LOCK_PERIOD];
++static inline void cleanup_vma_slots(void)
++{
++ struct vma_slot *slot;
++ int i;
++
++ i = 0;
++ spin_lock(&vma_slot_list_lock);
++ while (!list_empty(&vma_slot_del)) {
++ slot = list_entry(vma_slot_del.next,
++ struct vma_slot, slot_list);
++ list_del(&slot->slot_list);
++ cleanup_slots[i++] = slot;
++ if (i == SPIN_LOCK_PERIOD) {
++ spin_unlock(&vma_slot_list_lock);
++ while (--i >= 0)
++ uksm_del_vma_slot(cleanup_slots[i]);
++ i = 0;
++ spin_lock(&vma_slot_list_lock);
++ }
++ }
++ spin_unlock(&vma_slot_list_lock);
++
++ while (--i >= 0)
++ uksm_del_vma_slot(cleanup_slots[i]);
++}
++
++/*
++ * Expotional moving average formula
++ */
++static inline unsigned long ema(unsigned long curr, unsigned long last_ema)
++{
++ /*
++ * For a very high burst, even the ema cannot work well, a false very
++ * high per-page time estimation can result in feedback in very high
++ * overhead of context switch and rung update -- this will then lead
++ * to higher per-paper time, this may not converge.
++ *
++ * Instead, we try to approach this value in a binary manner.
++ */
++ if (curr > last_ema * 10)
++ return last_ema * 2;
++
++ return (EMA_ALPHA * curr + (100 - EMA_ALPHA) * last_ema) / 100;
++}
++
++/*
++ * convert cpu ratio in 1/TIME_RATIO_SCALE configured by user to
++ * nanoseconds based on current uksm_sleep_jiffies.
++ */
++static inline unsigned long cpu_ratio_to_nsec(unsigned int ratio)
++{
++ return NSEC_PER_USEC * jiffies_to_usecs(uksm_sleep_jiffies) /
++ (TIME_RATIO_SCALE - ratio) * ratio;
++}
++
++
++static inline unsigned long rung_real_ratio(int cpu_time_ratio)
++{
++ unsigned long ret;
++
++ BUG_ON(!cpu_time_ratio);
++
++ if (cpu_time_ratio > 0)
++ ret = cpu_time_ratio;
++ else
++ ret = (unsigned long)(-cpu_time_ratio) *
++ uksm_max_cpu_percentage / 100UL;
++
++ return ret ? ret : 1;
++}
++
++static noinline void uksm_calc_scan_pages(void)
++{
++ struct scan_rung *ladder = uksm_scan_ladder;
++ unsigned long sleep_usecs, nsecs;
++ unsigned long ratio;
++ int i;
++ unsigned long per_page;
++
++ if (uksm_ema_page_time > 100000 ||
++ (((unsigned long) uksm_eval_round & (256UL - 1)) == 0UL))
++ uksm_ema_page_time = UKSM_PAGE_TIME_DEFAULT;
++
++ per_page = uksm_ema_page_time;
++ BUG_ON(!per_page);
++
++ /*
++ * For every 8 eval round, we try to probe a uksm_sleep_jiffies value
++ * based on saved user input.
++ */
++ if (((unsigned long) uksm_eval_round & (8UL - 1)) == 0UL)
++ uksm_sleep_jiffies = uksm_sleep_saved;
++
++ /* We require a rung scan at least 1 page in a period. */
++ nsecs = per_page;
++ ratio = rung_real_ratio(ladder[0].cpu_ratio);
++ if (cpu_ratio_to_nsec(ratio) < nsecs) {
++ sleep_usecs = nsecs * (TIME_RATIO_SCALE - ratio) / ratio
++ / NSEC_PER_USEC;
++ uksm_sleep_jiffies = usecs_to_jiffies(sleep_usecs) + 1;
++ }
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ ratio = rung_real_ratio(ladder[i].cpu_ratio);
++ ladder[i].pages_to_scan = cpu_ratio_to_nsec(ratio) /
++ per_page;
++ BUG_ON(!ladder[i].pages_to_scan);
++ uksm_calc_rung_step(&ladder[i], per_page, ratio);
++ }
++}
++
++/*
++ * From the scan time of this round (ns) to next expected min sleep time
++ * (ms), be careful of the possible overflows. ratio is taken from
++ * rung_real_ratio()
++ */
++static inline
++unsigned int scan_time_to_sleep(unsigned long long scan_time, unsigned long ratio)
++{
++ scan_time >>= 20; /* to msec level now */
++ BUG_ON(scan_time > (ULONG_MAX / TIME_RATIO_SCALE));
++
++ return (unsigned int) ((unsigned long) scan_time *
++ (TIME_RATIO_SCALE - ratio) / ratio);
++}
++
++#define __round_mask(x, y) ((__typeof__(x))((y)-1))
++#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
++
++static void uksm_vma_enter(struct vma_slot **slots, unsigned long num)
++{
++ struct scan_rung *rung;
++
++ rung = &uksm_scan_ladder[0];
++ rung_add_new_slots(rung, slots, num);
++}
++
++static struct vma_slot *batch_slots[SLOT_TREE_NODE_STORE_SIZE];
++
++static void uksm_enter_all_slots(void)
++{
++ struct vma_slot *slot;
++ unsigned long index;
++ struct list_head empty_vma_list;
++ int i;
++
++ i = 0;
++ index = 0;
++ INIT_LIST_HEAD(&empty_vma_list);
++
++ spin_lock(&vma_slot_list_lock);
++ while (!list_empty(&vma_slot_new)) {
++ slot = list_entry(vma_slot_new.next,
++ struct vma_slot, slot_list);
++
++ if (!slot->vma->anon_vma) {
++ list_move(&slot->slot_list, &empty_vma_list);
++ } else if (vma_can_enter(slot->vma)) {
++ batch_slots[index++] = slot;
++ list_del_init(&slot->slot_list);
++ } else {
++ list_move(&slot->slot_list, &vma_slot_noadd);
++ }
++
++ if (++i == SPIN_LOCK_PERIOD ||
++ (index && !(index % SLOT_TREE_NODE_STORE_SIZE))) {
++ spin_unlock(&vma_slot_list_lock);
++
++ if (index && !(index % SLOT_TREE_NODE_STORE_SIZE)) {
++ uksm_vma_enter(batch_slots, index);
++ index = 0;
++ }
++ i = 0;
++ cond_resched();
++ spin_lock(&vma_slot_list_lock);
++ }
++ }
++
++ list_splice(&empty_vma_list, &vma_slot_new);
++
++ spin_unlock(&vma_slot_list_lock);
++
++ if (index)
++ uksm_vma_enter(batch_slots, index);
++
++}
++
++static inline int rung_round_finished(struct scan_rung *rung)
++{
++ return rung->flags & UKSM_RUNG_ROUND_FINISHED;
++}
++
++static inline void judge_slot(struct vma_slot *slot)
++{
++ struct scan_rung *rung = slot->rung;
++ unsigned long dedup;
++ int deleted;
++
++ dedup = cal_dedup_ratio(slot);
++ if (vma_fully_scanned(slot) && uksm_thrash_threshold)
++ deleted = vma_rung_enter(slot, &uksm_scan_ladder[0]);
++ else if (dedup && dedup >= uksm_abundant_threshold)
++ deleted = vma_rung_up(slot);
++ else
++ deleted = vma_rung_down(slot);
++
++ slot->pages_merged = 0;
++ slot->pages_cowed = 0;
++ slot->this_sampled = 0;
++
++ if (vma_fully_scanned(slot))
++ slot->pages_scanned = 0;
++
++ slot->last_scanned = slot->pages_scanned;
++
++ /* If its deleted in above, then rung was already advanced. */
++ if (!deleted)
++ advance_current_scan(rung);
++}
++
++
++static inline int hash_round_finished(void)
++{
++ if (scanned_virtual_pages > (uksm_pages_total >> 2)) {
++ scanned_virtual_pages = 0;
++ if (uksm_pages_scanned)
++ fully_scanned_round++;
++
++ return 1;
++ } else {
++ return 0;
++ }
++}
++
++#define UKSM_MMSEM_BATCH 5
++#define BUSY_RETRY 100
++
++/**
++ * uksm_do_scan() - the main worker function.
++ */
++static noinline void uksm_do_scan(void)
++{
++ struct vma_slot *slot, *iter;
++ struct mm_struct *busy_mm;
++ unsigned char round_finished, all_rungs_emtpy;
++ int i, err, mmsem_batch;
++ unsigned long pcost;
++ long long delta_exec;
++ unsigned long vpages, max_cpu_ratio;
++ unsigned long long start_time, end_time, scan_time;
++ unsigned int expected_jiffies;
++
++ might_sleep();
++
++ vpages = 0;
++
++ start_time = task_sched_runtime(current);
++ max_cpu_ratio = 0;
++ mmsem_batch = 0;
++
++ for (i = 0; i < SCAN_LADDER_SIZE;) {
++ struct scan_rung *rung = &uksm_scan_ladder[i];
++ unsigned long ratio;
++ int busy_retry;
++
++ if (!rung->pages_to_scan) {
++ i++;
++ continue;
++ }
++
++ if (!rung->vma_root.num) {
++ rung->pages_to_scan = 0;
++ i++;
++ continue;
++ }
++
++ ratio = rung_real_ratio(rung->cpu_ratio);
++ if (ratio > max_cpu_ratio)
++ max_cpu_ratio = ratio;
++
++ busy_retry = BUSY_RETRY;
++ /*
++ * Do not consider rung_round_finished() here, just used up the
++ * rung->pages_to_scan quota.
++ */
++ while (rung->pages_to_scan && rung->vma_root.num &&
++ likely(!freezing(current))) {
++ int reset = 0;
++
++ slot = rung->current_scan;
++
++ BUG_ON(vma_fully_scanned(slot));
++
++ if (mmsem_batch)
++ err = 0;
++ else
++ err = try_down_read_slot_mmap_sem(slot);
++
++ if (err == -ENOENT) {
++rm_slot:
++ rung_rm_slot(slot);
++ continue;
++ }
++
++ busy_mm = slot->mm;
++
++ if (err == -EBUSY) {
++ /* skip other vmas on the same mm */
++ do {
++ reset = advance_current_scan(rung);
++ iter = rung->current_scan;
++ busy_retry--;
++ if (iter->vma->vm_mm != busy_mm ||
++ !busy_retry || reset)
++ break;
++ } while (1);
++
++ if (iter->vma->vm_mm != busy_mm) {
++ continue;
++ } else {
++ /* scan round finsished */
++ break;
++ }
++ }
++
++ BUG_ON(!vma_can_enter(slot->vma));
++ if (uksm_test_exit(slot->vma->vm_mm)) {
++ mmsem_batch = 0;
++ up_read(&slot->vma->vm_mm->mmap_sem);
++ goto rm_slot;
++ }
++
++ if (mmsem_batch)
++ mmsem_batch--;
++ else
++ mmsem_batch = UKSM_MMSEM_BATCH;
++
++ /* Ok, we have take the mmap_sem, ready to scan */
++ scan_vma_one_page(slot);
++ rung->pages_to_scan--;
++ vpages++;
++
++ if (rung->current_offset + rung->step > slot->pages - 1
++ || vma_fully_scanned(slot)) {
++ up_read(&slot->vma->vm_mm->mmap_sem);
++ judge_slot(slot);
++ mmsem_batch = 0;
++ } else {
++ rung->current_offset += rung->step;
++ if (!mmsem_batch)
++ up_read(&slot->vma->vm_mm->mmap_sem);
++ }
++
++ busy_retry = BUSY_RETRY;
++ cond_resched();
++ }
++
++ if (mmsem_batch) {
++ up_read(&slot->vma->vm_mm->mmap_sem);
++ mmsem_batch = 0;
++ }
++
++ if (freezing(current))
++ break;
++
++ cond_resched();
++ }
++ end_time = task_sched_runtime(current);
++ delta_exec = end_time - start_time;
++
++ if (freezing(current))
++ return;
++
++ cleanup_vma_slots();
++ uksm_enter_all_slots();
++
++ round_finished = 1;
++ all_rungs_emtpy = 1;
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ struct scan_rung *rung = &uksm_scan_ladder[i];
++
++ if (rung->vma_root.num) {
++ all_rungs_emtpy = 0;
++ if (!rung_round_finished(rung))
++ round_finished = 0;
++ }
++ }
++
++ if (all_rungs_emtpy)
++ round_finished = 0;
++
++ if (round_finished) {
++ round_update_ladder();
++ uksm_eval_round++;
++
++ if (hash_round_finished() && rshash_adjust()) {
++ /* Reset the unstable root iff hash strength changed */
++ uksm_hash_round++;
++ root_unstable_tree = RB_ROOT;
++ free_all_tree_nodes(&unstable_tree_node_list);
++ }
++
++ /*
++ * A number of pages can hang around indefinitely on per-cpu
++ * pagevecs, raised page count preventing write_protect_page
++ * from merging them. Though it doesn't really matter much,
++ * it is puzzling to see some stuck in pages_volatile until
++ * other activity jostles them out, and they also prevented
++ * LTP's KSM test from succeeding deterministically; so drain
++ * them here (here rather than on entry to uksm_do_scan(),
++ * so we don't IPI too often when pages_to_scan is set low).
++ */
++ lru_add_drain_all();
++ }
++
++
++ if (vpages && delta_exec > 0) {
++ pcost = (unsigned long) delta_exec / vpages;
++ if (likely(uksm_ema_page_time))
++ uksm_ema_page_time = ema(pcost, uksm_ema_page_time);
++ else
++ uksm_ema_page_time = pcost;
++ }
++
++ uksm_calc_scan_pages();
++ uksm_sleep_real = uksm_sleep_jiffies;
++ /* in case of radical cpu bursts, apply the upper bound */
++ end_time = task_sched_runtime(current);
++ if (max_cpu_ratio && end_time > start_time) {
++ scan_time = end_time - start_time;
++ expected_jiffies = msecs_to_jiffies(
++ scan_time_to_sleep(scan_time, max_cpu_ratio));
++
++ if (expected_jiffies > uksm_sleep_real)
++ uksm_sleep_real = expected_jiffies;
++
++ /* We have a 1 second up bound for responsiveness. */
++ if (jiffies_to_msecs(uksm_sleep_real) > MSEC_PER_SEC)
++ uksm_sleep_real = msecs_to_jiffies(1000);
++ }
++
++ return;
++}
++
++static int ksmd_should_run(void)
++{
++ return uksm_run & UKSM_RUN_MERGE;
++}
++
++static int uksm_scan_thread(void *nothing)
++{
++ set_freezable();
++ set_user_nice(current, 5);
++
++ while (!kthread_should_stop()) {
++ mutex_lock(&uksm_thread_mutex);
++ if (ksmd_should_run())
++ uksm_do_scan();
++ mutex_unlock(&uksm_thread_mutex);
++
++ try_to_freeze();
++
++ if (ksmd_should_run()) {
++ schedule_timeout_interruptible(uksm_sleep_real);
++ uksm_sleep_times++;
++ } else {
++ wait_event_freezable(uksm_thread_wait,
++ ksmd_should_run() || kthread_should_stop());
++ }
++ }
++ return 0;
++}
++
++void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)
++{
++ struct stable_node *stable_node;
++ struct node_vma *node_vma;
++ struct rmap_item *rmap_item;
++ int search_new_forks = 0;
++ unsigned long address;
++
++ VM_BUG_ON_PAGE(!PageKsm(page), page);
++ VM_BUG_ON_PAGE(!PageLocked(page), page);
++
++ stable_node = page_stable_node(page);
++ if (!stable_node)
++ return;
++again:
++ hlist_for_each_entry(node_vma, &stable_node->hlist, hlist) {
++ hlist_for_each_entry(rmap_item, &node_vma->rmap_hlist, hlist) {
++ struct anon_vma *anon_vma = rmap_item->anon_vma;
++ struct anon_vma_chain *vmac;
++ struct vm_area_struct *vma;
++
++ cond_resched();
++ anon_vma_lock_read(anon_vma);
++ anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
++ 0, ULONG_MAX) {
++ cond_resched();
++ vma = vmac->vma;
++ address = get_rmap_addr(rmap_item);
++
++ if (address < vma->vm_start ||
++ address >= vma->vm_end)
++ continue;
++
++ if ((rmap_item->slot->vma == vma) ==
++ search_new_forks)
++ continue;
++
++ if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
++ continue;
++
++ if (!rwc->rmap_one(page, vma, address, rwc->arg)) {
++ anon_vma_unlock_read(anon_vma);
++ return;
++ }
++
++ if (rwc->done && rwc->done(page)) {
++ anon_vma_unlock_read(anon_vma);
++ return;
++ }
++ }
++ anon_vma_unlock_read(anon_vma);
++ }
++ }
++ if (!search_new_forks++)
++ goto again;
++}
++
++#ifdef CONFIG_MIGRATION
++/* Common ksm interface but may be specific to uksm */
++void ksm_migrate_page(struct page *newpage, struct page *oldpage)
++{
++ struct stable_node *stable_node;
++
++ VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
++ VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
++ VM_BUG_ON(newpage->mapping != oldpage->mapping);
++
++ stable_node = page_stable_node(newpage);
++ if (stable_node) {
++ VM_BUG_ON(stable_node->kpfn != page_to_pfn(oldpage));
++ stable_node->kpfn = page_to_pfn(newpage);
++ /*
++ * newpage->mapping was set in advance; now we need smp_wmb()
++ * to make sure that the new stable_node->kpfn is visible
++ * to get_ksm_page() before it can see that oldpage->mapping
++ * has gone stale (or that PageSwapCache has been cleared).
++ */
++ smp_wmb();
++ set_page_stable_node(oldpage, NULL);
++ }
++}
++#endif /* CONFIG_MIGRATION */
++
++#ifdef CONFIG_MEMORY_HOTREMOVE
++static struct stable_node *uksm_check_stable_tree(unsigned long start_pfn,
++ unsigned long end_pfn)
++{
++ struct rb_node *node;
++
++ for (node = rb_first(root_stable_treep); node; node = rb_next(node)) {
++ struct stable_node *stable_node;
++
++ stable_node = rb_entry(node, struct stable_node, node);
++ if (stable_node->kpfn >= start_pfn &&
++ stable_node->kpfn < end_pfn)
++ return stable_node;
++ }
++ return NULL;
++}
++
++static int uksm_memory_callback(struct notifier_block *self,
++ unsigned long action, void *arg)
++{
++ struct memory_notify *mn = arg;
++ struct stable_node *stable_node;
++
++ switch (action) {
++ case MEM_GOING_OFFLINE:
++ /*
++ * Keep it very simple for now: just lock out ksmd and
++ * MADV_UNMERGEABLE while any memory is going offline.
++ * mutex_lock_nested() is necessary because lockdep was alarmed
++ * that here we take uksm_thread_mutex inside notifier chain
++ * mutex, and later take notifier chain mutex inside
++ * uksm_thread_mutex to unlock it. But that's safe because both
++ * are inside mem_hotplug_mutex.
++ */
++ mutex_lock_nested(&uksm_thread_mutex, SINGLE_DEPTH_NESTING);
++ break;
++
++ case MEM_OFFLINE:
++ /*
++ * Most of the work is done by page migration; but there might
++ * be a few stable_nodes left over, still pointing to struct
++ * pages which have been offlined: prune those from the tree.
++ */
++ while ((stable_node = uksm_check_stable_tree(mn->start_pfn,
++ mn->start_pfn + mn->nr_pages)) != NULL)
++ remove_node_from_stable_tree(stable_node, 1, 1);
++ /* fallthrough */
++
++ case MEM_CANCEL_OFFLINE:
++ mutex_unlock(&uksm_thread_mutex);
++ break;
++ }
++ return NOTIFY_OK;
++}
++#endif /* CONFIG_MEMORY_HOTREMOVE */
++
++#ifdef CONFIG_SYSFS
++/*
++ * This all compiles without CONFIG_SYSFS, but is a waste of space.
++ */
++
++#define UKSM_ATTR_RO(_name) \
++ static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
++#define UKSM_ATTR(_name) \
++ static struct kobj_attribute _name##_attr = \
++ __ATTR(_name, 0644, _name##_show, _name##_store)
++
++static ssize_t max_cpu_percentage_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%u\n", uksm_max_cpu_percentage);
++}
++
++static ssize_t max_cpu_percentage_store(struct kobject *kobj,
++ struct kobj_attribute *attr,
++ const char *buf, size_t count)
++{
++ unsigned long max_cpu_percentage;
++ int err;
++
++ err = kstrtoul(buf, 10, &max_cpu_percentage);
++ if (err || max_cpu_percentage > 100)
++ return -EINVAL;
++
++ if (max_cpu_percentage == 100)
++ max_cpu_percentage = 99;
++ else if (max_cpu_percentage < 10)
++ max_cpu_percentage = 10;
++
++ uksm_max_cpu_percentage = max_cpu_percentage;
++
++ return count;
++}
++UKSM_ATTR(max_cpu_percentage);
++
++static ssize_t sleep_millisecs_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%u\n", jiffies_to_msecs(uksm_sleep_jiffies));
++}
++
++static ssize_t sleep_millisecs_store(struct kobject *kobj,
++ struct kobj_attribute *attr,
++ const char *buf, size_t count)
++{
++ unsigned long msecs;
++ int err;
++
++ err = kstrtoul(buf, 10, &msecs);
++ if (err || msecs > MSEC_PER_SEC)
++ return -EINVAL;
++
++ uksm_sleep_jiffies = msecs_to_jiffies(msecs);
++ uksm_sleep_saved = uksm_sleep_jiffies;
++
++ return count;
++}
++UKSM_ATTR(sleep_millisecs);
++
++
++static ssize_t cpu_governor_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ int n = sizeof(uksm_cpu_governor_str) / sizeof(char *);
++ int i;
++
++ buf[0] = '\0';
++ for (i = 0; i < n ; i++) {
++ if (uksm_cpu_governor == i)
++ strcat(buf, "[");
++
++ strcat(buf, uksm_cpu_governor_str[i]);
++
++ if (uksm_cpu_governor == i)
++ strcat(buf, "]");
++
++ strcat(buf, " ");
++ }
++ strcat(buf, "\n");
++
++ return strlen(buf);
++}
++
++static inline void init_performance_values(void)
++{
++ int i;
++ struct scan_rung *rung;
++ struct uksm_cpu_preset_s *preset = uksm_cpu_preset + uksm_cpu_governor;
++
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ rung = uksm_scan_ladder + i;
++ rung->cpu_ratio = preset->cpu_ratio[i];
++ rung->cover_msecs = preset->cover_msecs[i];
++ }
++
++ uksm_max_cpu_percentage = preset->max_cpu;
++}
++
++static ssize_t cpu_governor_store(struct kobject *kobj,
++ struct kobj_attribute *attr,
++ const char *buf, size_t count)
++{
++ int n = sizeof(uksm_cpu_governor_str) / sizeof(char *);
++
++ for (n--; n >= 0 ; n--) {
++ if (!strncmp(buf, uksm_cpu_governor_str[n],
++ strlen(uksm_cpu_governor_str[n])))
++ break;
++ }
++
++ if (n < 0)
++ return -EINVAL;
++ else
++ uksm_cpu_governor = n;
++
++ init_performance_values();
++
++ return count;
++}
++UKSM_ATTR(cpu_governor);
++
++static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr,
++ char *buf)
++{
++ return sprintf(buf, "%u\n", uksm_run);
++}
++
++static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr,
++ const char *buf, size_t count)
++{
++ int err;
++ unsigned long flags;
++
++ err = kstrtoul(buf, 10, &flags);
++ if (err || flags > UINT_MAX)
++ return -EINVAL;
++ if (flags > UKSM_RUN_MERGE)
++ return -EINVAL;
++
++ mutex_lock(&uksm_thread_mutex);
++ if (uksm_run != flags)
++ uksm_run = flags;
++ mutex_unlock(&uksm_thread_mutex);
++
++ if (flags & UKSM_RUN_MERGE)
++ wake_up_interruptible(&uksm_thread_wait);
++
++ return count;
++}
++UKSM_ATTR(run);
++
++static ssize_t abundant_threshold_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%u\n", uksm_abundant_threshold);
++}
++
++static ssize_t abundant_threshold_store(struct kobject *kobj,
++ struct kobj_attribute *attr,
++ const char *buf, size_t count)
++{
++ int err;
++ unsigned long flags;
++
++ err = kstrtoul(buf, 10, &flags);
++ if (err || flags > 99)
++ return -EINVAL;
++
++ uksm_abundant_threshold = flags;
++
++ return count;
++}
++UKSM_ATTR(abundant_threshold);
++
++static ssize_t thrash_threshold_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%u\n", uksm_thrash_threshold);
++}
++
++static ssize_t thrash_threshold_store(struct kobject *kobj,
++ struct kobj_attribute *attr,
++ const char *buf, size_t count)
++{
++ int err;
++ unsigned long flags;
++
++ err = kstrtoul(buf, 10, &flags);
++ if (err || flags > 99)
++ return -EINVAL;
++
++ uksm_thrash_threshold = flags;
++
++ return count;
++}
++UKSM_ATTR(thrash_threshold);
++
++static ssize_t cpu_ratios_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ int i, size;
++ struct scan_rung *rung;
++ char *p = buf;
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ rung = &uksm_scan_ladder[i];
++
++ if (rung->cpu_ratio > 0)
++ size = sprintf(p, "%d ", rung->cpu_ratio);
++ else
++ size = sprintf(p, "MAX/%d ",
++ TIME_RATIO_SCALE / -rung->cpu_ratio);
++
++ p += size;
++ }
++
++ *p++ = '\n';
++ *p = '\0';
++
++ return p - buf;
++}
++
++static ssize_t cpu_ratios_store(struct kobject *kobj,
++ struct kobj_attribute *attr,
++ const char *buf, size_t count)
++{
++ int i, cpuratios[SCAN_LADDER_SIZE], err;
++ unsigned long value;
++ struct scan_rung *rung;
++ char *p, *end = NULL;
++
++ p = kzalloc(count, GFP_KERNEL);
++ if (!p)
++ return -ENOMEM;
++
++ memcpy(p, buf, count);
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ if (i != SCAN_LADDER_SIZE - 1) {
++ end = strchr(p, ' ');
++ if (!end)
++ return -EINVAL;
++
++ *end = '\0';
++ }
++
++ if (strstr(p, "MAX/")) {
++ p = strchr(p, '/') + 1;
++ err = kstrtoul(p, 10, &value);
++ if (err || value > TIME_RATIO_SCALE || !value)
++ return -EINVAL;
++
++ cpuratios[i] = -(int) (TIME_RATIO_SCALE / value);
++ } else {
++ err = kstrtoul(p, 10, &value);
++ if (err || value > TIME_RATIO_SCALE || !value)
++ return -EINVAL;
++
++ cpuratios[i] = value;
++ }
++
++ p = end + 1;
++ }
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ rung = &uksm_scan_ladder[i];
++
++ rung->cpu_ratio = cpuratios[i];
++ }
++
++ return count;
++}
++UKSM_ATTR(cpu_ratios);
++
++static ssize_t eval_intervals_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ int i, size;
++ struct scan_rung *rung;
++ char *p = buf;
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ rung = &uksm_scan_ladder[i];
++ size = sprintf(p, "%u ", rung->cover_msecs);
++ p += size;
++ }
++
++ *p++ = '\n';
++ *p = '\0';
++
++ return p - buf;
++}
++
++static ssize_t eval_intervals_store(struct kobject *kobj,
++ struct kobj_attribute *attr,
++ const char *buf, size_t count)
++{
++ int i, err;
++ unsigned long values[SCAN_LADDER_SIZE];
++ struct scan_rung *rung;
++ char *p, *end = NULL;
++ ssize_t ret = count;
++
++ p = kzalloc(count + 2, GFP_KERNEL);
++ if (!p)
++ return -ENOMEM;
++
++ memcpy(p, buf, count);
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ if (i != SCAN_LADDER_SIZE - 1) {
++ end = strchr(p, ' ');
++ if (!end) {
++ ret = -EINVAL;
++ goto out;
++ }
++
++ *end = '\0';
++ }
++
++ err = kstrtoul(p, 10, &values[i]);
++ if (err) {
++ ret = -EINVAL;
++ goto out;
++ }
++
++ p = end + 1;
++ }
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ rung = &uksm_scan_ladder[i];
++
++ rung->cover_msecs = values[i];
++ }
++
++out:
++ kfree(p);
++ return ret;
++}
++UKSM_ATTR(eval_intervals);
++
++static ssize_t ema_per_page_time_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%lu\n", uksm_ema_page_time);
++}
++UKSM_ATTR_RO(ema_per_page_time);
++
++static ssize_t pages_shared_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%lu\n", uksm_pages_shared);
++}
++UKSM_ATTR_RO(pages_shared);
++
++static ssize_t pages_sharing_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%lu\n", uksm_pages_sharing);
++}
++UKSM_ATTR_RO(pages_sharing);
++
++static ssize_t pages_unshared_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%lu\n", uksm_pages_unshared);
++}
++UKSM_ATTR_RO(pages_unshared);
++
++static ssize_t full_scans_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%llu\n", fully_scanned_round);
++}
++UKSM_ATTR_RO(full_scans);
++
++static ssize_t pages_scanned_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ unsigned long base = 0;
++ u64 delta, ret;
++
++ if (pages_scanned_stored) {
++ base = pages_scanned_base;
++ ret = pages_scanned_stored;
++ delta = uksm_pages_scanned >> base;
++ if (CAN_OVERFLOW_U64(ret, delta)) {
++ ret >>= 1;
++ delta >>= 1;
++ base++;
++ ret += delta;
++ }
++ } else {
++ ret = uksm_pages_scanned;
++ }
++
++ while (ret > ULONG_MAX) {
++ ret >>= 1;
++ base++;
++ }
++
++ if (base)
++ return sprintf(buf, "%lu * 2^%lu\n", (unsigned long)ret, base);
++ else
++ return sprintf(buf, "%lu\n", (unsigned long)ret);
++}
++UKSM_ATTR_RO(pages_scanned);
++
++static ssize_t hash_strength_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%lu\n", hash_strength);
++}
++UKSM_ATTR_RO(hash_strength);
++
++static ssize_t sleep_times_show(struct kobject *kobj,
++ struct kobj_attribute *attr, char *buf)
++{
++ return sprintf(buf, "%llu\n", uksm_sleep_times);
++}
++UKSM_ATTR_RO(sleep_times);
++
++
++static struct attribute *uksm_attrs[] = {
++ &max_cpu_percentage_attr.attr,
++ &sleep_millisecs_attr.attr,
++ &cpu_governor_attr.attr,
++ &run_attr.attr,
++ &ema_per_page_time_attr.attr,
++ &pages_shared_attr.attr,
++ &pages_sharing_attr.attr,
++ &pages_unshared_attr.attr,
++ &full_scans_attr.attr,
++ &pages_scanned_attr.attr,
++ &hash_strength_attr.attr,
++ &sleep_times_attr.attr,
++ &thrash_threshold_attr.attr,
++ &abundant_threshold_attr.attr,
++ &cpu_ratios_attr.attr,
++ &eval_intervals_attr.attr,
++ NULL,
++};
++
++static struct attribute_group uksm_attr_group = {
++ .attrs = uksm_attrs,
++ .name = "uksm",
++};
++#endif /* CONFIG_SYSFS */
++
++static inline void init_scan_ladder(void)
++{
++ int i;
++ struct scan_rung *rung;
++
++ for (i = 0; i < SCAN_LADDER_SIZE; i++) {
++ rung = uksm_scan_ladder + i;
++ slot_tree_init_root(&rung->vma_root);
++ }
++
++ init_performance_values();
++ uksm_calc_scan_pages();
++}
++
++static inline int cal_positive_negative_costs(void)
++{
++ struct page *p1, *p2;
++ unsigned char *addr1, *addr2;
++ unsigned long i, time_start, hash_cost;
++ unsigned long loopnum = 0;
++
++ /*IMPORTANT: volatile is needed to prevent over-optimization by gcc. */
++ volatile u32 hash;
++ volatile int ret;
++
++ p1 = alloc_page(GFP_KERNEL);
++ if (!p1)
++ return -ENOMEM;
++
++ p2 = alloc_page(GFP_KERNEL);
++ if (!p2)
++ return -ENOMEM;
++
++ addr1 = kmap_atomic(p1);
++ addr2 = kmap_atomic(p2);
++ memset(addr1, prandom_u32(), PAGE_SIZE);
++ memcpy(addr2, addr1, PAGE_SIZE);
++
++ /* make sure that the two pages differ in last byte */
++ addr2[PAGE_SIZE-1] = ~addr2[PAGE_SIZE-1];
++ kunmap_atomic(addr2);
++ kunmap_atomic(addr1);
++
++ time_start = jiffies;
++ while (jiffies - time_start < 100) {
++ for (i = 0; i < 100; i++)
++ hash = page_hash(p1, HASH_STRENGTH_FULL, 0);
++ loopnum += 100;
++ }
++ hash_cost = (jiffies - time_start);
++
++ time_start = jiffies;
++ for (i = 0; i < loopnum; i++)
++ ret = pages_identical_with_cost(p1, p2);
++ memcmp_cost = HASH_STRENGTH_FULL * (jiffies - time_start);
++ memcmp_cost /= hash_cost;
++ pr_info("UKSM: relative memcmp_cost = %lu "
++ "hash=%u cmp_ret=%d.\n",
++ memcmp_cost, hash, ret);
++
++ __free_page(p1);
++ __free_page(p2);
++ return 0;
++}
++
++static int init_zeropage_hash_table(void)
++{
++ struct page *page;
++ char *addr;
++ int i;
++
++ page = alloc_page(GFP_KERNEL);
++ if (!page)
++ return -ENOMEM;
++
++ addr = kmap_atomic(page);
++ memset(addr, 0, PAGE_SIZE);
++ kunmap_atomic(addr);
++
++ zero_hash_table = kmalloc_array(HASH_STRENGTH_MAX, sizeof(u32),
++ GFP_KERNEL);
++ if (!zero_hash_table)
++ return -ENOMEM;
++
++ for (i = 0; i < HASH_STRENGTH_MAX; i++)
++ zero_hash_table[i] = page_hash(page, i, 0);
++
++ __free_page(page);
++
++ return 0;
++}
++
++static inline int init_random_sampling(void)
++{
++ unsigned long i;
++
++ random_nums = kmalloc(PAGE_SIZE, GFP_KERNEL);
++ if (!random_nums)
++ return -ENOMEM;
++
++ for (i = 0; i < HASH_STRENGTH_FULL; i++)
++ random_nums[i] = i;
++
++ for (i = 0; i < HASH_STRENGTH_FULL; i++) {
++ unsigned long rand_range, swap_index, tmp;
++
++ rand_range = HASH_STRENGTH_FULL - i;
++ swap_index = i + prandom_u32() % rand_range;
++ tmp = random_nums[i];
++ random_nums[i] = random_nums[swap_index];
++ random_nums[swap_index] = tmp;
++ }
++
++ rshash_state.state = RSHASH_NEW;
++ rshash_state.below_count = 0;
++ rshash_state.lookup_window_index = 0;
++
++ return cal_positive_negative_costs();
++}
++
++static int __init uksm_slab_init(void)
++{
++ rmap_item_cache = UKSM_KMEM_CACHE(rmap_item, 0);
++ if (!rmap_item_cache)
++ goto out;
++
++ stable_node_cache = UKSM_KMEM_CACHE(stable_node, 0);
++ if (!stable_node_cache)
++ goto out_free1;
++
++ node_vma_cache = UKSM_KMEM_CACHE(node_vma, 0);
++ if (!node_vma_cache)
++ goto out_free2;
++
++ vma_slot_cache = UKSM_KMEM_CACHE(vma_slot, 0);
++ if (!vma_slot_cache)
++ goto out_free3;
++
++ tree_node_cache = UKSM_KMEM_CACHE(tree_node, 0);
++ if (!tree_node_cache)
++ goto out_free4;
++
++ return 0;
++
++out_free4:
++ kmem_cache_destroy(vma_slot_cache);
++out_free3:
++ kmem_cache_destroy(node_vma_cache);
++out_free2:
++ kmem_cache_destroy(stable_node_cache);
++out_free1:
++ kmem_cache_destroy(rmap_item_cache);
++out:
++ return -ENOMEM;
++}
++
++static void __init uksm_slab_free(void)
++{
++ kmem_cache_destroy(stable_node_cache);
++ kmem_cache_destroy(rmap_item_cache);
++ kmem_cache_destroy(node_vma_cache);
++ kmem_cache_destroy(vma_slot_cache);
++ kmem_cache_destroy(tree_node_cache);
++}
++
++/* Common interface to ksm, different to it. */
++int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
++ unsigned long end, int advice, unsigned long *vm_flags)
++{
++ int err;
++
++ switch (advice) {
++ case MADV_MERGEABLE:
++ return 0; /* just ignore the advice */
++
++ case MADV_UNMERGEABLE:
++ if (!(*vm_flags & VM_MERGEABLE) || !uksm_flags_can_scan(*vm_flags))
++ return 0; /* just ignore the advice */
++
++ if (vma->anon_vma) {
++ err = unmerge_uksm_pages(vma, start, end);
++ if (err)
++ return err;
++ }
++
++ uksm_remove_vma(vma);
++ *vm_flags &= ~VM_MERGEABLE;
++ break;
++ }
++
++ return 0;
++}
++
++/* Common interface to ksm, actually the same. */
++struct page *ksm_might_need_to_copy(struct page *page,
++ struct vm_area_struct *vma, unsigned long address)
++{
++ struct anon_vma *anon_vma = page_anon_vma(page);
++ struct page *new_page;
++
++ if (PageKsm(page)) {
++ if (page_stable_node(page))
++ return page; /* no need to copy it */
++ } else if (!anon_vma) {
++ return page; /* no need to copy it */
++ } else if (anon_vma->root == vma->anon_vma->root &&
++ page->index == linear_page_index(vma, address)) {
++ return page; /* still no need to copy it */
++ }
++ if (!PageUptodate(page))
++ return page; /* let do_swap_page report the error */
++
++ new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
++ if (new_page) {
++ copy_user_highpage(new_page, page, address, vma);
++
++ SetPageDirty(new_page);
++ __SetPageUptodate(new_page);
++ __SetPageLocked(new_page);
++ }
++
++ return new_page;
++}
++
++/* Copied from mm/ksm.c and required from 5.1 */
++bool reuse_ksm_page(struct page *page,
++ struct vm_area_struct *vma,
++ unsigned long address)
++{
++#ifdef CONFIG_DEBUG_VM
++ if (WARN_ON(is_zero_pfn(page_to_pfn(page))) ||
++ WARN_ON(!page_mapped(page)) ||
++ WARN_ON(!PageLocked(page))) {
++ dump_page(page, "reuse_ksm_page");
++ return false;
++ }
++#endif
++
++ if (PageSwapCache(page) || !page_stable_node(page))
++ return false;
++ /* Prohibit parallel get_ksm_page() */
++ if (!page_ref_freeze(page, 1))
++ return false;
++
++ page_move_anon_rmap(page, vma);
++ page->index = linear_page_index(vma, address);
++ page_ref_unfreeze(page, 1);
++
++ return true;
++}
++
++static int __init uksm_init(void)
++{
++ struct task_struct *uksm_thread;
++ int err;
++
++ uksm_sleep_jiffies = msecs_to_jiffies(100);
++ uksm_sleep_saved = uksm_sleep_jiffies;
++
++ slot_tree_init();
++ init_scan_ladder();
++
++
++ err = init_random_sampling();
++ if (err)
++ goto out_free2;
++
++ err = uksm_slab_init();
++ if (err)
++ goto out_free1;
++
++ err = init_zeropage_hash_table();
++ if (err)
++ goto out_free0;
++
++ uksm_thread = kthread_run(uksm_scan_thread, NULL, "uksmd");
++ if (IS_ERR(uksm_thread)) {
++ pr_err("uksm: creating kthread failed\n");
++ err = PTR_ERR(uksm_thread);
++ goto out_free;
++ }
++
++#ifdef CONFIG_SYSFS
++ err = sysfs_create_group(mm_kobj, &uksm_attr_group);
++ if (err) {
++ pr_err("uksm: register sysfs failed\n");
++ kthread_stop(uksm_thread);
++ goto out_free;
++ }
++#else
++ uksm_run = UKSM_RUN_MERGE; /* no way for user to start it */
++
++#endif /* CONFIG_SYSFS */
++
++#ifdef CONFIG_MEMORY_HOTREMOVE
++ /*
++ * Choose a high priority since the callback takes uksm_thread_mutex:
++ * later callbacks could only be taking locks which nest within that.
++ */
++ hotplug_memory_notifier(uksm_memory_callback, 100);
++#endif
++ return 0;
++
++out_free:
++ kfree(zero_hash_table);
++out_free0:
++ uksm_slab_free();
++out_free1:
++ kfree(random_nums);
++out_free2:
++ kfree(uksm_scan_ladder);
++ return err;
++}
++
++#ifdef MODULE
++subsys_initcall(ksm_init);
++#else
++late_initcall(uksm_init);
++#endif
++
+diff --git a/mm/vmstat.c b/mm/vmstat.c
+index 96d21a792b57..f1f6bb6f6901 100644
+--- a/mm/vmstat.c
++++ b/mm/vmstat.c
+@@ -1171,6 +1171,9 @@ const char * const vmstat_text[] = {
+ "nr_foll_pin_acquired",
+ "nr_foll_pin_released",
+
++#ifdef CONFIG_UKSM
++ "nr_uksm_zero_pages",
++#endif
+ /* enum writeback_stat_item counters */
+ "nr_dirty_threshold",
+ "nr_dirty_background_threshold",
diff --git a/v19-fs-Add-VirtualBox-guest-shared-folder-vboxsf-support.diff b/v19-fs-Add-VirtualBox-guest-shared-folder-vboxsf-support.diff
deleted file mode 100644
index 56008db..0000000
--- a/v19-fs-Add-VirtualBox-guest-shared-folder-vboxsf-support.diff
+++ /dev/null
@@ -1,3370 +0,0 @@
-diff --git a/MAINTAINERS b/MAINTAINERS
-index 02d5278a4c9a..90302540b12d 100644
---- a/MAINTAINERS
-+++ b/MAINTAINERS
-@@ -17591,6 +17591,12 @@ F: include/linux/vbox_utils.h
- F: include/uapi/linux/vbox*.h
- F: drivers/virt/vboxguest/
-
-+VIRTUAL BOX SHARED FOLDER VFS DRIVER:
-+M: Hans de Goede <hdegoede@redhat.com>
-+L: linux-fsdevel@vger.kernel.org
-+S: Maintained
-+F: fs/vboxsf/*
-+
- VIRTUAL SERIO DEVICE DRIVER
- M: Stephen Chandler Paul <thatslyude@gmail.com>
- S: Maintained
-diff --git a/fs/Kconfig b/fs/Kconfig
-index 7b623e9fc1b0..8493a3f0c4b1 100644
---- a/fs/Kconfig
-+++ b/fs/Kconfig
-@@ -264,6 +264,7 @@ source "fs/pstore/Kconfig"
- source "fs/sysv/Kconfig"
- source "fs/ufs/Kconfig"
- source "fs/erofs/Kconfig"
-+source "fs/vboxsf/Kconfig"
-
- endif # MISC_FILESYSTEMS
-
-diff --git a/fs/Makefile b/fs/Makefile
-index 1148c555c4d3..b807cbff3790 100644
---- a/fs/Makefile
-+++ b/fs/Makefile
-@@ -133,3 +133,4 @@ obj-$(CONFIG_CEPH_FS) += ceph/
- obj-$(CONFIG_PSTORE) += pstore/
- obj-$(CONFIG_EFIVAR_FS) += efivarfs/
- obj-$(CONFIG_EROFS_FS) += erofs/
-+obj-$(CONFIG_VBOXSF_FS) += vboxsf/
-diff --git a/fs/vboxsf/Kconfig b/fs/vboxsf/Kconfig
-new file mode 100644
-index 000000000000..b84586ae08b3
---- /dev/null
-+++ b/fs/vboxsf/Kconfig
-@@ -0,0 +1,10 @@
-+config VBOXSF_FS
-+ tristate "VirtualBox guest shared folder (vboxsf) support"
-+ depends on X86 && VBOXGUEST
-+ select NLS
-+ help
-+ VirtualBox hosts can share folders with guests, this driver
-+ implements the Linux-guest side of this allowing folders exported
-+ by the host to be mounted under Linux.
-+
-+ If you want to use shared folders in VirtualBox guests, answer Y or M.
-diff --git a/fs/vboxsf/Makefile b/fs/vboxsf/Makefile
-new file mode 100644
-index 000000000000..9e4328e79623
---- /dev/null
-+++ b/fs/vboxsf/Makefile
-@@ -0,0 +1,5 @@
-+# SPDX-License-Identifier: MIT
-+
-+obj-$(CONFIG_VBOXSF_FS) += vboxsf.o
-+
-+vboxsf-y := dir.o file.o utils.o vboxsf_wrappers.o super.o
-diff --git a/fs/vboxsf/dir.c b/fs/vboxsf/dir.c
-new file mode 100644
-index 000000000000..dd147b490982
---- /dev/null
-+++ b/fs/vboxsf/dir.c
-@@ -0,0 +1,427 @@
-+// SPDX-License-Identifier: MIT
-+/*
-+ * VirtualBox Guest Shared Folders support: Directory inode and file operations
-+ *
-+ * Copyright (C) 2006-2018 Oracle Corporation
-+ */
-+
-+#include <linux/namei.h>
-+#include <linux/vbox_utils.h>
-+#include "vfsmod.h"
-+
-+static int vboxsf_dir_open(struct inode *inode, struct file *file)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(inode->i_sb);
-+ struct shfl_createparms params = {};
-+ struct vboxsf_dir_info *sf_d;
-+ int err;
-+
-+ sf_d = vboxsf_dir_info_alloc();
-+ if (!sf_d)
-+ return -ENOMEM;
-+
-+ params.handle = SHFL_HANDLE_NIL;
-+ params.create_flags = SHFL_CF_DIRECTORY | SHFL_CF_ACT_OPEN_IF_EXISTS |
-+ SHFL_CF_ACT_FAIL_IF_NEW | SHFL_CF_ACCESS_READ;
-+
-+ err = vboxsf_create_at_dentry(file_dentry(file), ¶ms);
-+ if (err)
-+ goto err_free_dir_info;
-+
-+ if (params.result != SHFL_FILE_EXISTS) {
-+ err = -ENOENT;
-+ goto err_close;
-+ }
-+
-+ err = vboxsf_dir_read_all(sbi, sf_d, params.handle);
-+ if (err)
-+ goto err_close;
-+
-+ vboxsf_close(sbi->root, params.handle);
-+ file->private_data = sf_d;
-+ return 0;
-+
-+err_close:
-+ vboxsf_close(sbi->root, params.handle);
-+err_free_dir_info:
-+ vboxsf_dir_info_free(sf_d);
-+ return err;
-+}
-+
-+static int vboxsf_dir_release(struct inode *inode, struct file *file)
-+{
-+ if (file->private_data)
-+ vboxsf_dir_info_free(file->private_data);
-+
-+ return 0;
-+}
-+
-+static unsigned int vboxsf_get_d_type(u32 mode)
-+{
-+ unsigned int d_type;
-+
-+ switch (mode & SHFL_TYPE_MASK) {
-+ case SHFL_TYPE_FIFO:
-+ d_type = DT_FIFO;
-+ break;
-+ case SHFL_TYPE_DEV_CHAR:
-+ d_type = DT_CHR;
-+ break;
-+ case SHFL_TYPE_DIRECTORY:
-+ d_type = DT_DIR;
-+ break;
-+ case SHFL_TYPE_DEV_BLOCK:
-+ d_type = DT_BLK;
-+ break;
-+ case SHFL_TYPE_FILE:
-+ d_type = DT_REG;
-+ break;
-+ case SHFL_TYPE_SYMLINK:
-+ d_type = DT_LNK;
-+ break;
-+ case SHFL_TYPE_SOCKET:
-+ d_type = DT_SOCK;
-+ break;
-+ case SHFL_TYPE_WHITEOUT:
-+ d_type = DT_WHT;
-+ break;
-+ default:
-+ d_type = DT_UNKNOWN;
-+ break;
-+ }
-+ return d_type;
-+}
-+
-+static bool vboxsf_dir_emit(struct file *dir, struct dir_context *ctx)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(file_inode(dir)->i_sb);
-+ struct vboxsf_dir_info *sf_d = dir->private_data;
-+ struct shfl_dirinfo *info;
-+ struct vboxsf_dir_buf *b;
-+ unsigned int d_type;
-+ loff_t i, cur = 0;
-+ ino_t fake_ino;
-+ void *end;
-+ int err;
-+
-+ list_for_each_entry(b, &sf_d->info_list, head) {
-+try_next_entry:
-+ if (ctx->pos >= cur + b->entries) {
-+ cur += b->entries;
-+ continue;
-+ }
-+
-+ /*
-+ * Note the vboxsf_dir_info objects we are iterating over here
-+ * are variable sized, so the info pointer may end up being
-+ * unaligned. This is how we get the data from the host.
-+ * Since vboxsf is only supported on x86 machines this is not
-+ * a problem.
-+ */
-+ for (i = 0, info = b->buf; i < ctx->pos - cur; i++) {
-+ end = &info->name.string.utf8[info->name.size];
-+ /* Only happens if the host gives us corrupt data */
-+ if (WARN_ON(end > (b->buf + b->used)))
-+ return false;
-+ info = end;
-+ }
-+
-+ end = &info->name.string.utf8[info->name.size];
-+ if (WARN_ON(end > (b->buf + b->used)))
-+ return false;
-+
-+ /* Info now points to the right entry, emit it. */
-+ d_type = vboxsf_get_d_type(info->info.attr.mode);
-+
-+ /*
-+ * On 32 bit systems pos is 64 signed, while ino is 32 bit
-+ * unsigned so fake_ino may overflow, check for this.
-+ */
-+ if ((ino_t)(ctx->pos + 1) != (u64)(ctx->pos + 1)) {
-+ vbg_err("vboxsf: fake ino overflow, truncating dir\n");
-+ return false;
-+ }
-+ fake_ino = ctx->pos + 1;
-+
-+ if (sbi->nls) {
-+ char d_name[NAME_MAX];
-+
-+ err = vboxsf_nlscpy(sbi, d_name, NAME_MAX,
-+ info->name.string.utf8,
-+ info->name.length);
-+ if (err) {
-+ /* skip erroneous entry and proceed */
-+ ctx->pos += 1;
-+ goto try_next_entry;
-+ }
-+
-+ return dir_emit(ctx, d_name, strlen(d_name),
-+ fake_ino, d_type);
-+ }
-+
-+ return dir_emit(ctx, info->name.string.utf8, info->name.length,
-+ fake_ino, d_type);
-+ }
-+
-+ return false;
-+}
-+
-+static int vboxsf_dir_iterate(struct file *dir, struct dir_context *ctx)
-+{
-+ bool emitted;
-+
-+ do {
-+ emitted = vboxsf_dir_emit(dir, ctx);
-+ if (emitted)
-+ ctx->pos += 1;
-+ } while (emitted);
-+
-+ return 0;
-+}
-+
-+const struct file_operations vboxsf_dir_fops = {
-+ .open = vboxsf_dir_open,
-+ .iterate = vboxsf_dir_iterate,
-+ .release = vboxsf_dir_release,
-+ .read = generic_read_dir,
-+ .llseek = generic_file_llseek,
-+};
-+
-+/*
-+ * This is called during name resolution/lookup to check if the @dentry in
-+ * the cache is still valid. the job is handled by vboxsf_inode_revalidate.
-+ */
-+static int vboxsf_dentry_revalidate(struct dentry *dentry, unsigned int flags)
-+{
-+ if (flags & LOOKUP_RCU)
-+ return -ECHILD;
-+
-+ if (d_really_is_positive(dentry))
-+ return vboxsf_inode_revalidate(dentry) == 0;
-+ else
-+ return vboxsf_stat_dentry(dentry, NULL) == -ENOENT;
-+}
-+
-+const struct dentry_operations vboxsf_dentry_ops = {
-+ .d_revalidate = vboxsf_dentry_revalidate
-+};
-+
-+/* iops */
-+
-+static struct dentry *vboxsf_dir_lookup(struct inode *parent,
-+ struct dentry *dentry,
-+ unsigned int flags)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb);
-+ struct shfl_fsobjinfo fsinfo;
-+ struct inode *inode;
-+ int err;
-+
-+ dentry->d_time = jiffies;
-+
-+ err = vboxsf_stat_dentry(dentry, &fsinfo);
-+ if (err) {
-+ inode = (err == -ENOENT) ? NULL : ERR_PTR(err);
-+ } else {
-+ inode = vboxsf_new_inode(parent->i_sb);
-+ if (!IS_ERR(inode))
-+ vboxsf_init_inode(sbi, inode, &fsinfo);
-+ }
-+
-+ return d_splice_alias(inode, dentry);
-+}
-+
-+static int vboxsf_dir_instantiate(struct inode *parent, struct dentry *dentry,
-+ struct shfl_fsobjinfo *info)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb);
-+ struct vboxsf_inode *sf_i;
-+ struct inode *inode;
-+
-+ inode = vboxsf_new_inode(parent->i_sb);
-+ if (IS_ERR(inode))
-+ return PTR_ERR(inode);
-+
-+ sf_i = VBOXSF_I(inode);
-+ /* The host may have given us different attr then requested */
-+ sf_i->force_restat = 1;
-+ vboxsf_init_inode(sbi, inode, info);
-+
-+ d_instantiate(dentry, inode);
-+
-+ return 0;
-+}
-+
-+static int vboxsf_dir_create(struct inode *parent, struct dentry *dentry,
-+ umode_t mode, int is_dir)
-+{
-+ struct vboxsf_inode *sf_parent_i = VBOXSF_I(parent);
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb);
-+ struct shfl_createparms params = {};
-+ int err;
-+
-+ params.handle = SHFL_HANDLE_NIL;
-+ params.create_flags = SHFL_CF_ACT_CREATE_IF_NEW |
-+ SHFL_CF_ACT_FAIL_IF_EXISTS |
-+ SHFL_CF_ACCESS_READWRITE |
-+ (is_dir ? SHFL_CF_DIRECTORY : 0);
-+ params.info.attr.mode = (mode & 0777) |
-+ (is_dir ? SHFL_TYPE_DIRECTORY : SHFL_TYPE_FILE);
-+ params.info.attr.additional = SHFLFSOBJATTRADD_NOTHING;
-+
-+ err = vboxsf_create_at_dentry(dentry, ¶ms);
-+ if (err)
-+ return err;
-+
-+ if (params.result != SHFL_FILE_CREATED)
-+ return -EPERM;
-+
-+ vboxsf_close(sbi->root, params.handle);
-+
-+ err = vboxsf_dir_instantiate(parent, dentry, ¶ms.info);
-+ if (err)
-+ return err;
-+
-+ /* parent directory access/change time changed */
-+ sf_parent_i->force_restat = 1;
-+
-+ return 0;
-+}
-+
-+static int vboxsf_dir_mkfile(struct inode *parent, struct dentry *dentry,
-+ umode_t mode, bool excl)
-+{
-+ return vboxsf_dir_create(parent, dentry, mode, 0);
-+}
-+
-+static int vboxsf_dir_mkdir(struct inode *parent, struct dentry *dentry,
-+ umode_t mode)
-+{
-+ return vboxsf_dir_create(parent, dentry, mode, 1);
-+}
-+
-+static int vboxsf_dir_unlink(struct inode *parent, struct dentry *dentry)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb);
-+ struct vboxsf_inode *sf_parent_i = VBOXSF_I(parent);
-+ struct inode *inode = d_inode(dentry);
-+ struct shfl_string *path;
-+ u32 flags;
-+ int err;
-+
-+ if (S_ISDIR(inode->i_mode))
-+ flags = SHFL_REMOVE_DIR;
-+ else
-+ flags = SHFL_REMOVE_FILE;
-+
-+ if (S_ISLNK(inode->i_mode))
-+ flags |= SHFL_REMOVE_SYMLINK;
-+
-+ path = vboxsf_path_from_dentry(sbi, dentry);
-+ if (IS_ERR(path))
-+ return PTR_ERR(path);
-+
-+ err = vboxsf_remove(sbi->root, path, flags);
-+ __putname(path);
-+ if (err)
-+ return err;
-+
-+ /* parent directory access/change time changed */
-+ sf_parent_i->force_restat = 1;
-+
-+ return 0;
-+}
-+
-+static int vboxsf_dir_rename(struct inode *old_parent,
-+ struct dentry *old_dentry,
-+ struct inode *new_parent,
-+ struct dentry *new_dentry,
-+ unsigned int flags)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(old_parent->i_sb);
-+ struct vboxsf_inode *sf_old_parent_i = VBOXSF_I(old_parent);
-+ struct vboxsf_inode *sf_new_parent_i = VBOXSF_I(new_parent);
-+ u32 shfl_flags = SHFL_RENAME_FILE | SHFL_RENAME_REPLACE_IF_EXISTS;
-+ struct shfl_string *old_path, *new_path;
-+ int err;
-+
-+ if (flags)
-+ return -EINVAL;
-+
-+ old_path = vboxsf_path_from_dentry(sbi, old_dentry);
-+ if (IS_ERR(old_path))
-+ return PTR_ERR(old_path);
-+
-+ new_path = vboxsf_path_from_dentry(sbi, new_dentry);
-+ if (IS_ERR(new_path)) {
-+ err = PTR_ERR(new_path);
-+ goto err_put_old_path;
-+ }
-+
-+ if (d_inode(old_dentry)->i_mode & S_IFDIR)
-+ shfl_flags = 0;
-+
-+ err = vboxsf_rename(sbi->root, old_path, new_path, shfl_flags);
-+ if (err == 0) {
-+ /* parent directories access/change time changed */
-+ sf_new_parent_i->force_restat = 1;
-+ sf_old_parent_i->force_restat = 1;
-+ }
-+
-+ __putname(new_path);
-+err_put_old_path:
-+ __putname(old_path);
-+ return err;
-+}
-+
-+static int vboxsf_dir_symlink(struct inode *parent, struct dentry *dentry,
-+ const char *symname)
-+{
-+ struct vboxsf_inode *sf_parent_i = VBOXSF_I(parent);
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb);
-+ int symname_size = strlen(symname) + 1;
-+ struct shfl_string *path, *ssymname;
-+ struct shfl_fsobjinfo info;
-+ int err;
-+
-+ path = vboxsf_path_from_dentry(sbi, dentry);
-+ if (IS_ERR(path))
-+ return PTR_ERR(path);
-+
-+ ssymname = kmalloc(SHFLSTRING_HEADER_SIZE + symname_size, GFP_KERNEL);
-+ if (!ssymname) {
-+ __putname(path);
-+ return -ENOMEM;
-+ }
-+ ssymname->length = symname_size - 1;
-+ ssymname->size = symname_size;
-+ memcpy(ssymname->string.utf8, symname, symname_size);
-+
-+ err = vboxsf_symlink(sbi->root, path, ssymname, &info);
-+ kfree(ssymname);
-+ __putname(path);
-+ if (err) {
-+ /* -EROFS means symlinks are note support -> -EPERM */
-+ return (err == -EROFS) ? -EPERM : err;
-+ }
-+
-+ err = vboxsf_dir_instantiate(parent, dentry, &info);
-+ if (err)
-+ return err;
-+
-+ /* parent directory access/change time changed */
-+ sf_parent_i->force_restat = 1;
-+ return 0;
-+}
-+
-+const struct inode_operations vboxsf_dir_iops = {
-+ .lookup = vboxsf_dir_lookup,
-+ .create = vboxsf_dir_mkfile,
-+ .mkdir = vboxsf_dir_mkdir,
-+ .rmdir = vboxsf_dir_unlink,
-+ .unlink = vboxsf_dir_unlink,
-+ .rename = vboxsf_dir_rename,
-+ .symlink = vboxsf_dir_symlink,
-+ .getattr = vboxsf_getattr,
-+ .setattr = vboxsf_setattr,
-+};
-diff --git a/fs/vboxsf/file.c b/fs/vboxsf/file.c
-new file mode 100644
-index 000000000000..c4ab5996d97a
---- /dev/null
-+++ b/fs/vboxsf/file.c
-@@ -0,0 +1,379 @@
-+// SPDX-License-Identifier: MIT
-+/*
-+ * VirtualBox Guest Shared Folders support: Regular file inode and file ops.
-+ *
-+ * Copyright (C) 2006-2018 Oracle Corporation
-+ */
-+
-+#include <linux/mm.h>
-+#include <linux/page-flags.h>
-+#include <linux/pagemap.h>
-+#include <linux/highmem.h>
-+#include <linux/sizes.h>
-+#include "vfsmod.h"
-+
-+struct vboxsf_handle {
-+ u64 handle;
-+ u32 root;
-+ u32 access_flags;
-+ struct kref refcount;
-+ struct list_head head;
-+};
-+
-+static int vboxsf_file_open(struct inode *inode, struct file *file)
-+{
-+ struct vboxsf_inode *sf_i = VBOXSF_I(inode);
-+ struct shfl_createparms params = {};
-+ struct vboxsf_handle *sf_handle;
-+ u32 access_flags = 0;
-+ int err;
-+
-+ sf_handle = kmalloc(sizeof(*sf_handle), GFP_KERNEL);
-+ if (!sf_handle)
-+ return -ENOMEM;
-+
-+ /*
-+ * We check the value of params.handle afterwards to find out if
-+ * the call succeeded or failed, as the API does not seem to cleanly
-+ * distinguish error and informational messages.
-+ *
-+ * Furthermore, we must set params.handle to SHFL_HANDLE_NIL to
-+ * make the shared folders host service use our mode parameter.
-+ */
-+ params.handle = SHFL_HANDLE_NIL;
-+ if (file->f_flags & O_CREAT) {
-+ params.create_flags |= SHFL_CF_ACT_CREATE_IF_NEW;
-+ /*
-+ * We ignore O_EXCL, as the Linux kernel seems to call create
-+ * beforehand itself, so O_EXCL should always fail.
-+ */
-+ if (file->f_flags & O_TRUNC)
-+ params.create_flags |= SHFL_CF_ACT_OVERWRITE_IF_EXISTS;
-+ else
-+ params.create_flags |= SHFL_CF_ACT_OPEN_IF_EXISTS;
-+ } else {
-+ params.create_flags |= SHFL_CF_ACT_FAIL_IF_NEW;
-+ if (file->f_flags & O_TRUNC)
-+ params.create_flags |= SHFL_CF_ACT_OVERWRITE_IF_EXISTS;
-+ }
-+
-+ switch (file->f_flags & O_ACCMODE) {
-+ case O_RDONLY:
-+ access_flags |= SHFL_CF_ACCESS_READ;
-+ break;
-+
-+ case O_WRONLY:
-+ access_flags |= SHFL_CF_ACCESS_WRITE;
-+ break;
-+
-+ case O_RDWR:
-+ access_flags |= SHFL_CF_ACCESS_READWRITE;
-+ break;
-+
-+ default:
-+ WARN_ON(1);
-+ }
-+
-+ if (file->f_flags & O_APPEND)
-+ access_flags |= SHFL_CF_ACCESS_APPEND;
-+
-+ params.create_flags |= access_flags;
-+ params.info.attr.mode = inode->i_mode;
-+
-+ err = vboxsf_create_at_dentry(file_dentry(file), ¶ms);
-+ if (err == 0 && params.handle == SHFL_HANDLE_NIL)
-+ err = (params.result == SHFL_FILE_EXISTS) ? -EEXIST : -ENOENT;
-+ if (err) {
-+ kfree(sf_handle);
-+ return err;
-+ }
-+
-+ /* the host may have given us different attr then requested */
-+ sf_i->force_restat = 1;
-+
-+ /* init our handle struct and add it to the inode's handles list */
-+ sf_handle->handle = params.handle;
-+ sf_handle->root = VBOXSF_SBI(inode->i_sb)->root;
-+ sf_handle->access_flags = access_flags;
-+ kref_init(&sf_handle->refcount);
-+
-+ mutex_lock(&sf_i->handle_list_mutex);
-+ list_add(&sf_handle->head, &sf_i->handle_list);
-+ mutex_unlock(&sf_i->handle_list_mutex);
-+
-+ file->private_data = sf_handle;
-+ return 0;
-+}
-+
-+static void vboxsf_handle_release(struct kref *refcount)
-+{
-+ struct vboxsf_handle *sf_handle =
-+ container_of(refcount, struct vboxsf_handle, refcount);
-+
-+ vboxsf_close(sf_handle->root, sf_handle->handle);
-+ kfree(sf_handle);
-+}
-+
-+static int vboxsf_file_release(struct inode *inode, struct file *file)
-+{
-+ struct vboxsf_inode *sf_i = VBOXSF_I(inode);
-+ struct vboxsf_handle *sf_handle = file->private_data;
-+
-+ /*
-+ * When a file is closed on our (the guest) side, we want any subsequent
-+ * accesses done on the host side to see all changes done from our side.
-+ */
-+ filemap_write_and_wait(inode->i_mapping);
-+
-+ mutex_lock(&sf_i->handle_list_mutex);
-+ list_del(&sf_handle->head);
-+ mutex_unlock(&sf_i->handle_list_mutex);
-+
-+ kref_put(&sf_handle->refcount, vboxsf_handle_release);
-+ return 0;
-+}
-+
-+/*
-+ * Write back dirty pages now, because there may not be any suitable
-+ * open files later
-+ */
-+static void vboxsf_vma_close(struct vm_area_struct *vma)
-+{
-+ filemap_write_and_wait(vma->vm_file->f_mapping);
-+}
-+
-+static const struct vm_operations_struct vboxsf_file_vm_ops = {
-+ .close = vboxsf_vma_close,
-+ .fault = filemap_fault,
-+ .map_pages = filemap_map_pages,
-+};
-+
-+static int vboxsf_file_mmap(struct file *file, struct vm_area_struct *vma)
-+{
-+ int err;
-+
-+ err = generic_file_mmap(file, vma);
-+ if (!err)
-+ vma->vm_ops = &vboxsf_file_vm_ops;
-+
-+ return err;
-+}
-+
-+/*
-+ * Note that since we are accessing files on the host's filesystem, files
-+ * may always be changed underneath us by the host!
-+ *
-+ * The vboxsf API between the guest and the host does not offer any functions
-+ * to deal with this. There is no inode-generation to check for changes, no
-+ * events / callback on changes and no way to lock files.
-+ *
-+ * To avoid returning stale data when a file gets *opened* on our (the guest)
-+ * side, we do a "stat" on the host side, then compare the mtime with the
-+ * last known mtime and invalidate the page-cache if they differ.
-+ * This is done from vboxsf_inode_revalidate().
-+ *
-+ * When reads are done through the read_iter fop, it is possible to do
-+ * further cache revalidation then, there are 3 options to deal with this:
-+ *
-+ * 1) Rely solely on the revalidation done at open time
-+ * 2) Do another "stat" and compare mtime again. Unfortunately the vboxsf
-+ * host API does not allow stat on handles, so we would need to use
-+ * file->f_path.dentry and the stat will then fail if the file was unlinked
-+ * or renamed (and there is no thing like NFS' silly-rename). So we get:
-+ * 2a) "stat" and compare mtime, on stat failure invalidate the cache
-+ * 2b) "stat" and compare mtime, on stat failure do nothing
-+ * 3) Simply always call invalidate_inode_pages2_range on the range of the read
-+ *
-+ * Currently we are keeping things KISS and using option 1. this allows
-+ * directly using generic_file_read_iter without wrapping it.
-+ *
-+ * This means that only data written on the host side before open() on
-+ * the guest side is guaranteed to be seen by the guest. If necessary
-+ * we may provide other read-cache strategies in the future and make this
-+ * configurable through a mount option.
-+ */
-+const struct file_operations vboxsf_reg_fops = {
-+ .llseek = generic_file_llseek,
-+ .read_iter = generic_file_read_iter,
-+ .write_iter = generic_file_write_iter,
-+ .mmap = vboxsf_file_mmap,
-+ .open = vboxsf_file_open,
-+ .release = vboxsf_file_release,
-+ .fsync = noop_fsync,
-+ .splice_read = generic_file_splice_read,
-+};
-+
-+const struct inode_operations vboxsf_reg_iops = {
-+ .getattr = vboxsf_getattr,
-+ .setattr = vboxsf_setattr
-+};
-+
-+static int vboxsf_readpage(struct file *file, struct page *page)
-+{
-+ struct vboxsf_handle *sf_handle = file->private_data;
-+ loff_t off = page_offset(page);
-+ u32 nread = PAGE_SIZE;
-+ u8 *buf;
-+ int err;
-+
-+ buf = kmap(page);
-+
-+ err = vboxsf_read(sf_handle->root, sf_handle->handle, off, &nread, buf);
-+ if (err == 0) {
-+ memset(&buf[nread], 0, PAGE_SIZE - nread);
-+ flush_dcache_page(page);
-+ SetPageUptodate(page);
-+ } else {
-+ SetPageError(page);
-+ }
-+
-+ kunmap(page);
-+ unlock_page(page);
-+ return err;
-+}
-+
-+static struct vboxsf_handle *vboxsf_get_write_handle(struct vboxsf_inode *sf_i)
-+{
-+ struct vboxsf_handle *h, *sf_handle = NULL;
-+
-+ mutex_lock(&sf_i->handle_list_mutex);
-+ list_for_each_entry(h, &sf_i->handle_list, head) {
-+ if (h->access_flags == SHFL_CF_ACCESS_WRITE ||
-+ h->access_flags == SHFL_CF_ACCESS_READWRITE) {
-+ kref_get(&h->refcount);
-+ sf_handle = h;
-+ break;
-+ }
-+ }
-+ mutex_unlock(&sf_i->handle_list_mutex);
-+
-+ return sf_handle;
-+}
-+
-+static int vboxsf_writepage(struct page *page, struct writeback_control *wbc)
-+{
-+ struct inode *inode = page->mapping->host;
-+ struct vboxsf_inode *sf_i = VBOXSF_I(inode);
-+ struct vboxsf_handle *sf_handle;
-+ loff_t off = page_offset(page);
-+ loff_t size = i_size_read(inode);
-+ u32 nwrite = PAGE_SIZE;
-+ u8 *buf;
-+ int err;
-+
-+ if (off + PAGE_SIZE > size)
-+ nwrite = size & ~PAGE_MASK;
-+
-+ sf_handle = vboxsf_get_write_handle(sf_i);
-+ if (!sf_handle)
-+ return -EBADF;
-+
-+ buf = kmap(page);
-+ err = vboxsf_write(sf_handle->root, sf_handle->handle,
-+ off, &nwrite, buf);
-+ kunmap(page);
-+
-+ kref_put(&sf_handle->refcount, vboxsf_handle_release);
-+
-+ if (err == 0) {
-+ ClearPageError(page);
-+ /* mtime changed */
-+ sf_i->force_restat = 1;
-+ } else {
-+ ClearPageUptodate(page);
-+ }
-+
-+ unlock_page(page);
-+ return err;
-+}
-+
-+static int vboxsf_write_end(struct file *file, struct address_space *mapping,
-+ loff_t pos, unsigned int len, unsigned int copied,
-+ struct page *page, void *fsdata)
-+{
-+ struct inode *inode = mapping->host;
-+ struct vboxsf_handle *sf_handle = file->private_data;
-+ unsigned int from = pos & ~PAGE_MASK;
-+ u32 nwritten = len;
-+ u8 *buf;
-+ int err;
-+
-+ /* zero the stale part of the page if we did a short copy */
-+ if (!PageUptodate(page) && copied < len)
-+ zero_user(page, from + copied, len - copied);
-+
-+ buf = kmap(page);
-+ err = vboxsf_write(sf_handle->root, sf_handle->handle,
-+ pos, &nwritten, buf + from);
-+ kunmap(page);
-+
-+ if (err) {
-+ nwritten = 0;
-+ goto out;
-+ }
-+
-+ /* mtime changed */
-+ VBOXSF_I(inode)->force_restat = 1;
-+
-+ if (!PageUptodate(page) && nwritten == PAGE_SIZE)
-+ SetPageUptodate(page);
-+
-+ pos += nwritten;
-+ if (pos > inode->i_size)
-+ i_size_write(inode, pos);
-+
-+out:
-+ unlock_page(page);
-+ put_page(page);
-+
-+ return nwritten;
-+}
-+
-+/*
-+ * Note simple_write_begin does not read the page from disk on partial writes
-+ * this is ok since vboxsf_write_end only writes the written parts of the
-+ * page and it does not call SetPageUptodate for partial writes.
-+ */
-+const struct address_space_operations vboxsf_reg_aops = {
-+ .readpage = vboxsf_readpage,
-+ .writepage = vboxsf_writepage,
-+ .set_page_dirty = __set_page_dirty_nobuffers,
-+ .write_begin = simple_write_begin,
-+ .write_end = vboxsf_write_end,
-+};
-+
-+static const char *vboxsf_get_link(struct dentry *dentry, struct inode *inode,
-+ struct delayed_call *done)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(inode->i_sb);
-+ struct shfl_string *path;
-+ char *link;
-+ int err;
-+
-+ if (!dentry)
-+ return ERR_PTR(-ECHILD);
-+
-+ path = vboxsf_path_from_dentry(sbi, dentry);
-+ if (IS_ERR(path))
-+ return ERR_CAST(path);
-+
-+ link = kzalloc(PATH_MAX, GFP_KERNEL);
-+ if (!link) {
-+ __putname(path);
-+ return ERR_PTR(-ENOMEM);
-+ }
-+
-+ err = vboxsf_readlink(sbi->root, path, PATH_MAX, link);
-+ __putname(path);
-+ if (err) {
-+ kfree(link);
-+ return ERR_PTR(err);
-+ }
-+
-+ set_delayed_call(done, kfree_link, link);
-+ return link;
-+}
-+
-+const struct inode_operations vboxsf_lnk_iops = {
-+ .get_link = vboxsf_get_link
-+};
-diff --git a/fs/vboxsf/shfl_hostintf.h b/fs/vboxsf/shfl_hostintf.h
-new file mode 100644
-index 000000000000..aca829062c12
---- /dev/null
-+++ b/fs/vboxsf/shfl_hostintf.h
-@@ -0,0 +1,901 @@
-+/* SPDX-License-Identifier: MIT */
-+/*
-+ * VirtualBox Shared Folders: host interface definition.
-+ *
-+ * Copyright (C) 2006-2018 Oracle Corporation
-+ */
-+
-+#ifndef SHFL_HOSTINTF_H
-+#define SHFL_HOSTINTF_H
-+
-+#include <linux/vbox_vmmdev_types.h>
-+
-+/* The max in/out buffer size for a FN_READ or FN_WRITE call */
-+#define SHFL_MAX_RW_COUNT (16 * SZ_1M)
-+
-+/*
-+ * Structures shared between guest and the service
-+ * can be relocated and use offsets to point to variable
-+ * length parts.
-+ *
-+ * Shared folders protocol works with handles.
-+ * Before doing any action on a file system object,
-+ * one have to obtain the object handle via a SHFL_FN_CREATE
-+ * request. A handle must be closed with SHFL_FN_CLOSE.
-+ */
-+
-+enum {
-+ SHFL_FN_QUERY_MAPPINGS = 1, /* Query mappings changes. */
-+ SHFL_FN_QUERY_MAP_NAME = 2, /* Query map name. */
-+ SHFL_FN_CREATE = 3, /* Open/create object. */
-+ SHFL_FN_CLOSE = 4, /* Close object handle. */
-+ SHFL_FN_READ = 5, /* Read object content. */
-+ SHFL_FN_WRITE = 6, /* Write new object content. */
-+ SHFL_FN_LOCK = 7, /* Lock/unlock a range in the object. */
-+ SHFL_FN_LIST = 8, /* List object content. */
-+ SHFL_FN_INFORMATION = 9, /* Query/set object information. */
-+ /* Note function number 10 is not used! */
-+ SHFL_FN_REMOVE = 11, /* Remove object */
-+ SHFL_FN_MAP_FOLDER_OLD = 12, /* Map folder (legacy) */
-+ SHFL_FN_UNMAP_FOLDER = 13, /* Unmap folder */
-+ SHFL_FN_RENAME = 14, /* Rename object */
-+ SHFL_FN_FLUSH = 15, /* Flush file */
-+ SHFL_FN_SET_UTF8 = 16, /* Select UTF8 filename encoding */
-+ SHFL_FN_MAP_FOLDER = 17, /* Map folder */
-+ SHFL_FN_READLINK = 18, /* Read symlink dest (as of VBox 4.0) */
-+ SHFL_FN_SYMLINK = 19, /* Create symlink (as of VBox 4.0) */
-+ SHFL_FN_SET_SYMLINKS = 20, /* Ask host to show symlinks (4.0+) */
-+};
-+
-+/* Root handles for a mapping are of type u32, Root handles are unique. */
-+#define SHFL_ROOT_NIL UINT_MAX
-+
-+/* Shared folders handle for an opened object are of type u64. */
-+#define SHFL_HANDLE_NIL ULLONG_MAX
-+
-+/* Hardcoded maximum length (in chars) of a shared folder name. */
-+#define SHFL_MAX_LEN (256)
-+/* Hardcoded maximum number of shared folder mapping available to the guest. */
-+#define SHFL_MAX_MAPPINGS (64)
-+
-+/** Shared folder string buffer structure. */
-+struct shfl_string {
-+ /** Allocated size of the string member in bytes. */
-+ u16 size;
-+
-+ /** Length of string without trailing nul in bytes. */
-+ u16 length;
-+
-+ /** UTF-8 or UTF-16 string. Nul terminated. */
-+ union {
-+ u8 utf8[2];
-+ u16 utf16[1];
-+ u16 ucs2[1]; /* misnomer, use utf16. */
-+ } string;
-+};
-+VMMDEV_ASSERT_SIZE(shfl_string, 6);
-+
-+/* The size of shfl_string w/o the string part. */
-+#define SHFLSTRING_HEADER_SIZE 4
-+
-+/* Calculate size of the string. */
-+static inline u32 shfl_string_buf_size(const struct shfl_string *string)
-+{
-+ return string ? SHFLSTRING_HEADER_SIZE + string->size : 0;
-+}
-+
-+/* Set user id on execution (S_ISUID). */
-+#define SHFL_UNIX_ISUID 0004000U
-+/* Set group id on execution (S_ISGID). */
-+#define SHFL_UNIX_ISGID 0002000U
-+/* Sticky bit (S_ISVTX / S_ISTXT). */
-+#define SHFL_UNIX_ISTXT 0001000U
-+
-+/* Owner readable (S_IRUSR). */
-+#define SHFL_UNIX_IRUSR 0000400U
-+/* Owner writable (S_IWUSR). */
-+#define SHFL_UNIX_IWUSR 0000200U
-+/* Owner executable (S_IXUSR). */
-+#define SHFL_UNIX_IXUSR 0000100U
-+
-+/* Group readable (S_IRGRP). */
-+#define SHFL_UNIX_IRGRP 0000040U
-+/* Group writable (S_IWGRP). */
-+#define SHFL_UNIX_IWGRP 0000020U
-+/* Group executable (S_IXGRP). */
-+#define SHFL_UNIX_IXGRP 0000010U
-+
-+/* Other readable (S_IROTH). */
-+#define SHFL_UNIX_IROTH 0000004U
-+/* Other writable (S_IWOTH). */
-+#define SHFL_UNIX_IWOTH 0000002U
-+/* Other executable (S_IXOTH). */
-+#define SHFL_UNIX_IXOTH 0000001U
-+
-+/* Named pipe (fifo) (S_IFIFO). */
-+#define SHFL_TYPE_FIFO 0010000U
-+/* Character device (S_IFCHR). */
-+#define SHFL_TYPE_DEV_CHAR 0020000U
-+/* Directory (S_IFDIR). */
-+#define SHFL_TYPE_DIRECTORY 0040000U
-+/* Block device (S_IFBLK). */
-+#define SHFL_TYPE_DEV_BLOCK 0060000U
-+/* Regular file (S_IFREG). */
-+#define SHFL_TYPE_FILE 0100000U
-+/* Symbolic link (S_IFLNK). */
-+#define SHFL_TYPE_SYMLINK 0120000U
-+/* Socket (S_IFSOCK). */
-+#define SHFL_TYPE_SOCKET 0140000U
-+/* Whiteout (S_IFWHT). */
-+#define SHFL_TYPE_WHITEOUT 0160000U
-+/* Type mask (S_IFMT). */
-+#define SHFL_TYPE_MASK 0170000U
-+
-+/* Checks the mode flags indicate a directory (S_ISDIR). */
-+#define SHFL_IS_DIRECTORY(m) (((m) & SHFL_TYPE_MASK) == SHFL_TYPE_DIRECTORY)
-+/* Checks the mode flags indicate a symbolic link (S_ISLNK). */
-+#define SHFL_IS_SYMLINK(m) (((m) & SHFL_TYPE_MASK) == SHFL_TYPE_SYMLINK)
-+
-+/** The available additional information in a shfl_fsobjattr object. */
-+enum shfl_fsobjattr_add {
-+ /** No additional information is available / requested. */
-+ SHFLFSOBJATTRADD_NOTHING = 1,
-+ /**
-+ * The additional unix attributes (shfl_fsobjattr::u::unix_attr) are
-+ * available / requested.
-+ */
-+ SHFLFSOBJATTRADD_UNIX,
-+ /**
-+ * The additional extended attribute size (shfl_fsobjattr::u::size) is
-+ * available / requested.
-+ */
-+ SHFLFSOBJATTRADD_EASIZE,
-+ /**
-+ * The last valid item (inclusive).
-+ * The valid range is SHFLFSOBJATTRADD_NOTHING thru
-+ * SHFLFSOBJATTRADD_LAST.
-+ */
-+ SHFLFSOBJATTRADD_LAST = SHFLFSOBJATTRADD_EASIZE,
-+
-+ /** The usual 32-bit hack. */
-+ SHFLFSOBJATTRADD_32BIT_SIZE_HACK = 0x7fffffff
-+};
-+
-+/**
-+ * Additional unix Attributes, these are available when
-+ * shfl_fsobjattr.additional == SHFLFSOBJATTRADD_UNIX.
-+ */
-+struct shfl_fsobjattr_unix {
-+ /**
-+ * The user owning the filesystem object (st_uid).
-+ * This field is ~0U if not supported.
-+ */
-+ u32 uid;
-+
-+ /**
-+ * The group the filesystem object is assigned (st_gid).
-+ * This field is ~0U if not supported.
-+ */
-+ u32 gid;
-+
-+ /**
-+ * Number of hard links to this filesystem object (st_nlink).
-+ * This field is 1 if the filesystem doesn't support hardlinking or
-+ * the information isn't available.
-+ */
-+ u32 hardlinks;
-+
-+ /**
-+ * The device number of the device which this filesystem object resides
-+ * on (st_dev). This field is 0 if this information is not available.
-+ */
-+ u32 inode_id_device;
-+
-+ /**
-+ * The unique identifier (within the filesystem) of this filesystem
-+ * object (st_ino). Together with inode_id_device, this field can be
-+ * used as a OS wide unique id, when both their values are not 0.
-+ * This field is 0 if the information is not available.
-+ */
-+ u64 inode_id;
-+
-+ /**
-+ * User flags (st_flags).
-+ * This field is 0 if this information is not available.
-+ */
-+ u32 flags;
-+
-+ /**
-+ * The current generation number (st_gen).
-+ * This field is 0 if this information is not available.
-+ */
-+ u32 generation_id;
-+
-+ /**
-+ * The device number of a char. or block device type object (st_rdev).
-+ * This field is 0 if the file isn't a char. or block device or when
-+ * the OS doesn't use the major+minor device idenfication scheme.
-+ */
-+ u32 device;
-+} __packed;
-+
-+/** Extended attribute size. */
-+struct shfl_fsobjattr_easize {
-+ /** Size of EAs. */
-+ s64 cb;
-+} __packed;
-+
-+/** Shared folder filesystem object attributes. */
-+struct shfl_fsobjattr {
-+ /** Mode flags (st_mode). SHFL_UNIX_*, SHFL_TYPE_*, and SHFL_DOS_*. */
-+ u32 mode;
-+
-+ /** The additional attributes available. */
-+ enum shfl_fsobjattr_add additional;
-+
-+ /**
-+ * Additional attributes.
-+ *
-+ * Unless explicitly specified to an API, the API can provide additional
-+ * data as it is provided by the underlying OS.
-+ */
-+ union {
-+ struct shfl_fsobjattr_unix unix_attr;
-+ struct shfl_fsobjattr_easize size;
-+ } __packed u;
-+} __packed;
-+VMMDEV_ASSERT_SIZE(shfl_fsobjattr, 44);
-+
-+struct shfl_timespec {
-+ s64 ns_relative_to_unix_epoch;
-+};
-+
-+/** Filesystem object information structure. */
-+struct shfl_fsobjinfo {
-+ /**
-+ * Logical size (st_size).
-+ * For normal files this is the size of the file.
-+ * For symbolic links, this is the length of the path name contained
-+ * in the symbolic link.
-+ * For other objects this fields needs to be specified.
-+ */
-+ s64 size;
-+
-+ /** Disk allocation size (st_blocks * DEV_BSIZE). */
-+ s64 allocated;
-+
-+ /** Time of last access (st_atime). */
-+ struct shfl_timespec access_time;
-+
-+ /** Time of last data modification (st_mtime). */
-+ struct shfl_timespec modification_time;
-+
-+ /**
-+ * Time of last status change (st_ctime).
-+ * If not available this is set to modification_time.
-+ */
-+ struct shfl_timespec change_time;
-+
-+ /**
-+ * Time of file birth (st_birthtime).
-+ * If not available this is set to change_time.
-+ */
-+ struct shfl_timespec birth_time;
-+
-+ /** Attributes. */
-+ struct shfl_fsobjattr attr;
-+
-+} __packed;
-+VMMDEV_ASSERT_SIZE(shfl_fsobjinfo, 92);
-+
-+/**
-+ * result of an open/create request.
-+ * Along with handle value the result code
-+ * identifies what has happened while
-+ * trying to open the object.
-+ */
-+enum shfl_create_result {
-+ SHFL_NO_RESULT,
-+ /** Specified path does not exist. */
-+ SHFL_PATH_NOT_FOUND,
-+ /** Path to file exists, but the last component does not. */
-+ SHFL_FILE_NOT_FOUND,
-+ /** File already exists and either has been opened or not. */
-+ SHFL_FILE_EXISTS,
-+ /** New file was created. */
-+ SHFL_FILE_CREATED,
-+ /** Existing file was replaced or overwritten. */
-+ SHFL_FILE_REPLACED
-+};
-+
-+/* No flags. Initialization value. */
-+#define SHFL_CF_NONE (0x00000000)
-+
-+/*
-+ * Only lookup the object, do not return a handle. When this is set all other
-+ * flags are ignored.
-+ */
-+#define SHFL_CF_LOOKUP (0x00000001)
-+
-+/*
-+ * Open parent directory of specified object.
-+ * Useful for the corresponding Windows FSD flag
-+ * and for opening paths like \\dir\\*.* to search the 'dir'.
-+ */
-+#define SHFL_CF_OPEN_TARGET_DIRECTORY (0x00000002)
-+
-+/* Create/open a directory. */
-+#define SHFL_CF_DIRECTORY (0x00000004)
-+
-+/*
-+ * Open/create action to do if object exists
-+ * and if the object does not exists.
-+ * REPLACE file means atomically DELETE and CREATE.
-+ * OVERWRITE file means truncating the file to 0 and
-+ * setting new size.
-+ * When opening an existing directory REPLACE and OVERWRITE
-+ * actions are considered invalid, and cause returning
-+ * FILE_EXISTS with NIL handle.
-+ */
-+#define SHFL_CF_ACT_MASK_IF_EXISTS (0x000000f0)
-+#define SHFL_CF_ACT_MASK_IF_NEW (0x00000f00)
-+
-+/* What to do if object exists. */
-+#define SHFL_CF_ACT_OPEN_IF_EXISTS (0x00000000)
-+#define SHFL_CF_ACT_FAIL_IF_EXISTS (0x00000010)
-+#define SHFL_CF_ACT_REPLACE_IF_EXISTS (0x00000020)
-+#define SHFL_CF_ACT_OVERWRITE_IF_EXISTS (0x00000030)
-+
-+/* What to do if object does not exist. */
-+#define SHFL_CF_ACT_CREATE_IF_NEW (0x00000000)
-+#define SHFL_CF_ACT_FAIL_IF_NEW (0x00000100)
-+
-+/* Read/write requested access for the object. */
-+#define SHFL_CF_ACCESS_MASK_RW (0x00003000)
-+
-+/* No access requested. */
-+#define SHFL_CF_ACCESS_NONE (0x00000000)
-+/* Read access requested. */
-+#define SHFL_CF_ACCESS_READ (0x00001000)
-+/* Write access requested. */
-+#define SHFL_CF_ACCESS_WRITE (0x00002000)
-+/* Read/Write access requested. */
-+#define SHFL_CF_ACCESS_READWRITE (0x00003000)
-+
-+/* Requested share access for the object. */
-+#define SHFL_CF_ACCESS_MASK_DENY (0x0000c000)
-+
-+/* Allow any access. */
-+#define SHFL_CF_ACCESS_DENYNONE (0x00000000)
-+/* Do not allow read. */
-+#define SHFL_CF_ACCESS_DENYREAD (0x00004000)
-+/* Do not allow write. */
-+#define SHFL_CF_ACCESS_DENYWRITE (0x00008000)
-+/* Do not allow access. */
-+#define SHFL_CF_ACCESS_DENYALL (0x0000c000)
-+
-+/* Requested access to attributes of the object. */
-+#define SHFL_CF_ACCESS_MASK_ATTR (0x00030000)
-+
-+/* No access requested. */
-+#define SHFL_CF_ACCESS_ATTR_NONE (0x00000000)
-+/* Read access requested. */
-+#define SHFL_CF_ACCESS_ATTR_READ (0x00010000)
-+/* Write access requested. */
-+#define SHFL_CF_ACCESS_ATTR_WRITE (0x00020000)
-+/* Read/Write access requested. */
-+#define SHFL_CF_ACCESS_ATTR_READWRITE (0x00030000)
-+
-+/*
-+ * The file is opened in append mode.
-+ * Ignored if SHFL_CF_ACCESS_WRITE is not set.
-+ */
-+#define SHFL_CF_ACCESS_APPEND (0x00040000)
-+
-+/** Create parameters buffer struct for SHFL_FN_CREATE call */
-+struct shfl_createparms {
-+ /** Returned handle of opened object. */
-+ u64 handle;
-+
-+ /** Returned result of the operation */
-+ enum shfl_create_result result;
-+
-+ /** SHFL_CF_* */
-+ u32 create_flags;
-+
-+ /**
-+ * Attributes of object to create and
-+ * returned actual attributes of opened/created object.
-+ */
-+ struct shfl_fsobjinfo info;
-+} __packed;
-+
-+/** Shared Folder directory information */
-+struct shfl_dirinfo {
-+ /** Full information about the object. */
-+ struct shfl_fsobjinfo info;
-+ /**
-+ * The length of the short field (number of UTF16 chars).
-+ * It is 16-bit for reasons of alignment.
-+ */
-+ u16 short_name_len;
-+ /**
-+ * The short name for 8.3 compatibility.
-+ * Empty string if not available.
-+ */
-+ u16 short_name[14];
-+ struct shfl_string name;
-+};
-+
-+/** Shared folder filesystem properties. */
-+struct shfl_fsproperties {
-+ /**
-+ * The maximum size of a filesystem object name.
-+ * This does not include the '\\0'.
-+ */
-+ u32 max_component_len;
-+
-+ /**
-+ * True if the filesystem is remote.
-+ * False if the filesystem is local.
-+ */
-+ bool remote;
-+
-+ /**
-+ * True if the filesystem is case sensitive.
-+ * False if the filesystem is case insensitive.
-+ */
-+ bool case_sensitive;
-+
-+ /**
-+ * True if the filesystem is mounted read only.
-+ * False if the filesystem is mounted read write.
-+ */
-+ bool read_only;
-+
-+ /**
-+ * True if the filesystem can encode unicode object names.
-+ * False if it can't.
-+ */
-+ bool supports_unicode;
-+
-+ /**
-+ * True if the filesystem is compresses.
-+ * False if it isn't or we don't know.
-+ */
-+ bool compressed;
-+
-+ /**
-+ * True if the filesystem compresses of individual files.
-+ * False if it doesn't or we don't know.
-+ */
-+ bool file_compression;
-+};
-+VMMDEV_ASSERT_SIZE(shfl_fsproperties, 12);
-+
-+struct shfl_volinfo {
-+ s64 total_allocation_bytes;
-+ s64 available_allocation_bytes;
-+ u32 bytes_per_allocation_unit;
-+ u32 bytes_per_sector;
-+ u32 serial;
-+ struct shfl_fsproperties properties;
-+};
-+
-+
-+/** SHFL_FN_MAP_FOLDER Parameters structure. */
-+struct shfl_map_folder {
-+ /**
-+ * pointer, in:
-+ * Points to struct shfl_string buffer.
-+ */
-+ struct vmmdev_hgcm_function_parameter path;
-+
-+ /**
-+ * pointer, out: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * pointer, in: UTF16
-+ * Path delimiter
-+ */
-+ struct vmmdev_hgcm_function_parameter delimiter;
-+
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Case senstive flag
-+ */
-+ struct vmmdev_hgcm_function_parameter case_sensitive;
-+
-+};
-+
-+/* Number of parameters */
-+#define SHFL_CPARMS_MAP_FOLDER (4)
-+
-+
-+/** SHFL_FN_UNMAP_FOLDER Parameters structure. */
-+struct shfl_unmap_folder {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+};
-+
-+/* Number of parameters */
-+#define SHFL_CPARMS_UNMAP_FOLDER (1)
-+
-+
-+/** SHFL_FN_CREATE Parameters structure. */
-+struct shfl_create {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * pointer, in:
-+ * Points to struct shfl_string buffer.
-+ */
-+ struct vmmdev_hgcm_function_parameter path;
-+
-+ /**
-+ * pointer, in/out:
-+ * Points to struct shfl_createparms buffer.
-+ */
-+ struct vmmdev_hgcm_function_parameter parms;
-+
-+};
-+
-+/* Number of parameters */
-+#define SHFL_CPARMS_CREATE (3)
-+
-+
-+/** SHFL_FN_CLOSE Parameters structure. */
-+struct shfl_close {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * value64, in:
-+ * SHFLHANDLE (u64) of object to close.
-+ */
-+ struct vmmdev_hgcm_function_parameter handle;
-+
-+};
-+
-+/* Number of parameters */
-+#define SHFL_CPARMS_CLOSE (2)
-+
-+
-+/** SHFL_FN_READ Parameters structure. */
-+struct shfl_read {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * value64, in:
-+ * SHFLHANDLE (u64) of object to read from.
-+ */
-+ struct vmmdev_hgcm_function_parameter handle;
-+
-+ /**
-+ * value64, in:
-+ * Offset to read from.
-+ */
-+ struct vmmdev_hgcm_function_parameter offset;
-+
-+ /**
-+ * value64, in/out:
-+ * Bytes to read/How many were read.
-+ */
-+ struct vmmdev_hgcm_function_parameter cb;
-+
-+ /**
-+ * pointer, out:
-+ * Buffer to place data to.
-+ */
-+ struct vmmdev_hgcm_function_parameter buffer;
-+
-+};
-+
-+/* Number of parameters */
-+#define SHFL_CPARMS_READ (5)
-+
-+
-+/** SHFL_FN_WRITE Parameters structure. */
-+struct shfl_write {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * value64, in:
-+ * SHFLHANDLE (u64) of object to write to.
-+ */
-+ struct vmmdev_hgcm_function_parameter handle;
-+
-+ /**
-+ * value64, in:
-+ * Offset to write to.
-+ */
-+ struct vmmdev_hgcm_function_parameter offset;
-+
-+ /**
-+ * value64, in/out:
-+ * Bytes to write/How many were written.
-+ */
-+ struct vmmdev_hgcm_function_parameter cb;
-+
-+ /**
-+ * pointer, in:
-+ * Data to write.
-+ */
-+ struct vmmdev_hgcm_function_parameter buffer;
-+
-+};
-+
-+/* Number of parameters */
-+#define SHFL_CPARMS_WRITE (5)
-+
-+
-+/*
-+ * SHFL_FN_LIST
-+ * Listing information includes variable length RTDIRENTRY[EX] structures.
-+ */
-+
-+#define SHFL_LIST_NONE 0
-+#define SHFL_LIST_RETURN_ONE 1
-+
-+/** SHFL_FN_LIST Parameters structure. */
-+struct shfl_list {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * value64, in:
-+ * SHFLHANDLE (u64) of object to be listed.
-+ */
-+ struct vmmdev_hgcm_function_parameter handle;
-+
-+ /**
-+ * value32, in:
-+ * List flags SHFL_LIST_*.
-+ */
-+ struct vmmdev_hgcm_function_parameter flags;
-+
-+ /**
-+ * value32, in/out:
-+ * Bytes to be used for listing information/How many bytes were used.
-+ */
-+ struct vmmdev_hgcm_function_parameter cb;
-+
-+ /**
-+ * pointer, in/optional
-+ * Points to struct shfl_string buffer that specifies a search path.
-+ */
-+ struct vmmdev_hgcm_function_parameter path;
-+
-+ /**
-+ * pointer, out:
-+ * Buffer to place listing information to. (struct shfl_dirinfo)
-+ */
-+ struct vmmdev_hgcm_function_parameter buffer;
-+
-+ /**
-+ * value32, in/out:
-+ * Indicates a key where the listing must be resumed.
-+ * in: 0 means start from begin of object.
-+ * out: 0 means listing completed.
-+ */
-+ struct vmmdev_hgcm_function_parameter resume_point;
-+
-+ /**
-+ * pointer, out:
-+ * Number of files returned
-+ */
-+ struct vmmdev_hgcm_function_parameter file_count;
-+};
-+
-+/* Number of parameters */
-+#define SHFL_CPARMS_LIST (8)
-+
-+
-+/** SHFL_FN_READLINK Parameters structure. */
-+struct shfl_readLink {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * pointer, in:
-+ * Points to struct shfl_string buffer.
-+ */
-+ struct vmmdev_hgcm_function_parameter path;
-+
-+ /**
-+ * pointer, out:
-+ * Buffer to place data to.
-+ */
-+ struct vmmdev_hgcm_function_parameter buffer;
-+
-+};
-+
-+/* Number of parameters */
-+#define SHFL_CPARMS_READLINK (3)
-+
-+
-+/* SHFL_FN_INFORMATION */
-+
-+/* Mask of Set/Get bit. */
-+#define SHFL_INFO_MODE_MASK (0x1)
-+/* Get information */
-+#define SHFL_INFO_GET (0x0)
-+/* Set information */
-+#define SHFL_INFO_SET (0x1)
-+
-+/* Get name of the object. */
-+#define SHFL_INFO_NAME (0x2)
-+/* Set size of object (extend/trucate); only applies to file objects */
-+#define SHFL_INFO_SIZE (0x4)
-+/* Get/Set file object info. */
-+#define SHFL_INFO_FILE (0x8)
-+/* Get volume information. */
-+#define SHFL_INFO_VOLUME (0x10)
-+
-+/** SHFL_FN_INFORMATION Parameters structure. */
-+struct shfl_information {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * value64, in:
-+ * SHFLHANDLE (u64) of object to be listed.
-+ */
-+ struct vmmdev_hgcm_function_parameter handle;
-+
-+ /**
-+ * value32, in:
-+ * SHFL_INFO_*
-+ */
-+ struct vmmdev_hgcm_function_parameter flags;
-+
-+ /**
-+ * value32, in/out:
-+ * Bytes to be used for information/How many bytes were used.
-+ */
-+ struct vmmdev_hgcm_function_parameter cb;
-+
-+ /**
-+ * pointer, in/out:
-+ * Information to be set/get (shfl_fsobjinfo or shfl_string). Do not
-+ * forget to set the shfl_fsobjinfo::attr::additional for a get
-+ * operation as well.
-+ */
-+ struct vmmdev_hgcm_function_parameter info;
-+
-+};
-+
-+/* Number of parameters */
-+#define SHFL_CPARMS_INFORMATION (5)
-+
-+
-+/* SHFL_FN_REMOVE */
-+
-+#define SHFL_REMOVE_FILE (0x1)
-+#define SHFL_REMOVE_DIR (0x2)
-+#define SHFL_REMOVE_SYMLINK (0x4)
-+
-+/** SHFL_FN_REMOVE Parameters structure. */
-+struct shfl_remove {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * pointer, in:
-+ * Points to struct shfl_string buffer.
-+ */
-+ struct vmmdev_hgcm_function_parameter path;
-+
-+ /**
-+ * value32, in:
-+ * remove flags (file/directory)
-+ */
-+ struct vmmdev_hgcm_function_parameter flags;
-+
-+};
-+
-+#define SHFL_CPARMS_REMOVE (3)
-+
-+
-+/* SHFL_FN_RENAME */
-+
-+#define SHFL_RENAME_FILE (0x1)
-+#define SHFL_RENAME_DIR (0x2)
-+#define SHFL_RENAME_REPLACE_IF_EXISTS (0x4)
-+
-+/** SHFL_FN_RENAME Parameters structure. */
-+struct shfl_rename {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * pointer, in:
-+ * Points to struct shfl_string src.
-+ */
-+ struct vmmdev_hgcm_function_parameter src;
-+
-+ /**
-+ * pointer, in:
-+ * Points to struct shfl_string dest.
-+ */
-+ struct vmmdev_hgcm_function_parameter dest;
-+
-+ /**
-+ * value32, in:
-+ * rename flags (file/directory)
-+ */
-+ struct vmmdev_hgcm_function_parameter flags;
-+
-+};
-+
-+#define SHFL_CPARMS_RENAME (4)
-+
-+
-+/** SHFL_FN_SYMLINK Parameters structure. */
-+struct shfl_symlink {
-+ /**
-+ * pointer, in: SHFLROOT (u32)
-+ * Root handle of the mapping which name is queried.
-+ */
-+ struct vmmdev_hgcm_function_parameter root;
-+
-+ /**
-+ * pointer, in:
-+ * Points to struct shfl_string of path for the new symlink.
-+ */
-+ struct vmmdev_hgcm_function_parameter new_path;
-+
-+ /**
-+ * pointer, in:
-+ * Points to struct shfl_string of destination for symlink.
-+ */
-+ struct vmmdev_hgcm_function_parameter old_path;
-+
-+ /**
-+ * pointer, out:
-+ * Information about created symlink.
-+ */
-+ struct vmmdev_hgcm_function_parameter info;
-+
-+};
-+
-+#define SHFL_CPARMS_SYMLINK (4)
-+
-+#endif
-diff --git a/fs/vboxsf/super.c b/fs/vboxsf/super.c
-new file mode 100644
-index 000000000000..b14ecd2948f4
---- /dev/null
-+++ b/fs/vboxsf/super.c
-@@ -0,0 +1,497 @@
-+// SPDX-License-Identifier: MIT
-+/*
-+ * VirtualBox Guest Shared Folders support: Virtual File System.
-+ *
-+ * Module initialization/finalization
-+ * File system registration/deregistration
-+ * Superblock reading
-+ * Few utility functions
-+ *
-+ * Copyright (C) 2006-2018 Oracle Corporation
-+ */
-+
-+#include <linux/idr.h>
-+#include <linux/fs_parser.h>
-+#include <linux/magic.h>
-+#include <linux/module.h>
-+#include <linux/nls.h>
-+#include <linux/statfs.h>
-+#include <linux/vbox_utils.h>
-+#include "vfsmod.h"
-+
-+#define VBOXSF_SUPER_MAGIC 0x786f4256 /* 'VBox' little endian */
-+
-+#define VBSF_MOUNT_SIGNATURE_BYTE_0 ('\000')
-+#define VBSF_MOUNT_SIGNATURE_BYTE_1 ('\377')
-+#define VBSF_MOUNT_SIGNATURE_BYTE_2 ('\376')
-+#define VBSF_MOUNT_SIGNATURE_BYTE_3 ('\375')
-+
-+static int follow_symlinks;
-+module_param(follow_symlinks, int, 0444);
-+MODULE_PARM_DESC(follow_symlinks,
-+ "Let host resolve symlinks rather than showing them");
-+
-+static DEFINE_IDA(vboxsf_bdi_ida);
-+static DEFINE_MUTEX(vboxsf_setup_mutex);
-+static bool vboxsf_setup_done;
-+static struct super_operations vboxsf_super_ops; /* forward declaration */
-+static struct kmem_cache *vboxsf_inode_cachep;
-+
-+static char * const vboxsf_default_nls = CONFIG_NLS_DEFAULT;
-+
-+enum { opt_nls, opt_uid, opt_gid, opt_ttl, opt_dmode, opt_fmode,
-+ opt_dmask, opt_fmask };
-+
-+static const struct fs_parameter_spec vboxsf_param_specs[] = {
-+ fsparam_string ("nls", opt_nls),
-+ fsparam_u32 ("uid", opt_uid),
-+ fsparam_u32 ("gid", opt_gid),
-+ fsparam_u32 ("ttl", opt_ttl),
-+ fsparam_u32oct ("dmode", opt_dmode),
-+ fsparam_u32oct ("fmode", opt_fmode),
-+ fsparam_u32oct ("dmask", opt_dmask),
-+ fsparam_u32oct ("fmask", opt_fmask),
-+ {}
-+};
-+
-+static const struct fs_parameter_description vboxsf_fs_parameters = {
-+ .name = "vboxsf",
-+ .specs = vboxsf_param_specs,
-+};
-+
-+static int vboxsf_parse_param(struct fs_context *fc, struct fs_parameter *param)
-+{
-+ struct vboxsf_fs_context *ctx = fc->fs_private;
-+ struct fs_parse_result result;
-+ kuid_t uid;
-+ kgid_t gid;
-+ int opt;
-+
-+ opt = fs_parse(fc, &vboxsf_fs_parameters, param, &result);
-+ if (opt < 0)
-+ return opt;
-+
-+ switch (opt) {
-+ case opt_nls:
-+ if (ctx->nls_name || fc->purpose != FS_CONTEXT_FOR_MOUNT) {
-+ vbg_err("vboxsf: Cannot reconfigure nls option\n");
-+ return -EINVAL;
-+ }
-+ ctx->nls_name = param->string;
-+ param->string = NULL;
-+ break;
-+ case opt_uid:
-+ uid = make_kuid(current_user_ns(), result.uint_32);
-+ if (!uid_valid(uid))
-+ return -EINVAL;
-+ ctx->o.uid = uid;
-+ break;
-+ case opt_gid:
-+ gid = make_kgid(current_user_ns(), result.uint_32);
-+ if (!gid_valid(gid))
-+ return -EINVAL;
-+ ctx->o.gid = gid;
-+ break;
-+ case opt_ttl:
-+ ctx->o.ttl = msecs_to_jiffies(result.uint_32);
-+ break;
-+ case opt_dmode:
-+ if (result.uint_32 & ~0777)
-+ return -EINVAL;
-+ ctx->o.dmode = result.uint_32;
-+ ctx->o.dmode_set = true;
-+ break;
-+ case opt_fmode:
-+ if (result.uint_32 & ~0777)
-+ return -EINVAL;
-+ ctx->o.fmode = result.uint_32;
-+ ctx->o.fmode_set = true;
-+ break;
-+ case opt_dmask:
-+ if (result.uint_32 & ~07777)
-+ return -EINVAL;
-+ ctx->o.dmask = result.uint_32;
-+ break;
-+ case opt_fmask:
-+ if (result.uint_32 & ~07777)
-+ return -EINVAL;
-+ ctx->o.fmask = result.uint_32;
-+ break;
-+ default:
-+ return -EINVAL;
-+ }
-+
-+ return 0;
-+}
-+
-+static int vboxsf_fill_super(struct super_block *sb, struct fs_context *fc)
-+{
-+ struct vboxsf_fs_context *ctx = fc->fs_private;
-+ struct shfl_string *folder_name, root_path;
-+ struct vboxsf_sbi *sbi;
-+ struct dentry *droot;
-+ struct inode *iroot;
-+ char *nls_name;
-+ size_t size;
-+ int err;
-+
-+ if (!fc->source)
-+ return -EINVAL;
-+
-+ sbi = kzalloc(sizeof(*sbi), GFP_KERNEL);
-+ if (!sbi)
-+ return -ENOMEM;
-+
-+ sbi->o = ctx->o;
-+ idr_init(&sbi->ino_idr);
-+ spin_lock_init(&sbi->ino_idr_lock);
-+ sbi->next_generation = 1;
-+ sbi->bdi_id = -1;
-+
-+ /* Load nls if not utf8 */
-+ nls_name = ctx->nls_name ? ctx->nls_name : vboxsf_default_nls;
-+ if (strcmp(nls_name, "utf8") != 0) {
-+ if (nls_name == vboxsf_default_nls)
-+ sbi->nls = load_nls_default();
-+ else
-+ sbi->nls = load_nls(nls_name);
-+
-+ if (!sbi->nls) {
-+ vbg_err("vboxsf: Count not load '%s' nls\n", nls_name);
-+ err = -EINVAL;
-+ goto fail_free;
-+ }
-+ }
-+
-+ sbi->bdi_id = ida_simple_get(&vboxsf_bdi_ida, 0, 0, GFP_KERNEL);
-+ if (sbi->bdi_id < 0) {
-+ err = sbi->bdi_id;
-+ goto fail_free;
-+ }
-+
-+ err = super_setup_bdi_name(sb, "vboxsf-%s.%d", fc->source, sbi->bdi_id);
-+ if (err)
-+ goto fail_free;
-+
-+ /* Turn source into a shfl_string and map the folder */
-+ size = strlen(fc->source) + 1;
-+ folder_name = kmalloc(SHFLSTRING_HEADER_SIZE + size, GFP_KERNEL);
-+ if (!folder_name) {
-+ err = -ENOMEM;
-+ goto fail_free;
-+ }
-+ folder_name->size = size;
-+ folder_name->length = size - 1;
-+ strlcpy(folder_name->string.utf8, fc->source, size);
-+ err = vboxsf_map_folder(folder_name, &sbi->root);
-+ kfree(folder_name);
-+ if (err) {
-+ vbg_err("vboxsf: Host rejected mount of '%s' with error %d\n",
-+ fc->source, err);
-+ goto fail_free;
-+ }
-+
-+ root_path.length = 1;
-+ root_path.size = 2;
-+ root_path.string.utf8[0] = '/';
-+ root_path.string.utf8[1] = 0;
-+ err = vboxsf_stat(sbi, &root_path, &sbi->root_info);
-+ if (err)
-+ goto fail_unmap;
-+
-+ sb->s_magic = VBOXSF_SUPER_MAGIC;
-+ sb->s_blocksize = 1024;
-+ sb->s_maxbytes = MAX_LFS_FILESIZE;
-+ sb->s_op = &vboxsf_super_ops;
-+ sb->s_d_op = &vboxsf_dentry_ops;
-+
-+ iroot = iget_locked(sb, 0);
-+ if (!iroot) {
-+ err = -ENOMEM;
-+ goto fail_unmap;
-+ }
-+ vboxsf_init_inode(sbi, iroot, &sbi->root_info);
-+ unlock_new_inode(iroot);
-+
-+ droot = d_make_root(iroot);
-+ if (!droot) {
-+ err = -ENOMEM;
-+ goto fail_unmap;
-+ }
-+
-+ sb->s_root = droot;
-+ sb->s_fs_info = sbi;
-+ return 0;
-+
-+fail_unmap:
-+ vboxsf_unmap_folder(sbi->root);
-+fail_free:
-+ if (sbi->bdi_id >= 0)
-+ ida_simple_remove(&vboxsf_bdi_ida, sbi->bdi_id);
-+ if (sbi->nls)
-+ unload_nls(sbi->nls);
-+ idr_destroy(&sbi->ino_idr);
-+ kfree(sbi);
-+ return err;
-+}
-+
-+static void vboxsf_inode_init_once(void *data)
-+{
-+ struct vboxsf_inode *sf_i = data;
-+
-+ mutex_init(&sf_i->handle_list_mutex);
-+ inode_init_once(&sf_i->vfs_inode);
-+}
-+
-+static struct inode *vboxsf_alloc_inode(struct super_block *sb)
-+{
-+ struct vboxsf_inode *sf_i;
-+
-+ sf_i = kmem_cache_alloc(vboxsf_inode_cachep, GFP_NOFS);
-+ if (!sf_i)
-+ return NULL;
-+
-+ sf_i->force_restat = 0;
-+ INIT_LIST_HEAD(&sf_i->handle_list);
-+
-+ return &sf_i->vfs_inode;
-+}
-+
-+static void vboxsf_free_inode(struct inode *inode)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(inode->i_sb);
-+ unsigned long flags;
-+
-+ spin_lock_irqsave(&sbi->ino_idr_lock, flags);
-+ idr_remove(&sbi->ino_idr, inode->i_ino);
-+ spin_unlock_irqrestore(&sbi->ino_idr_lock, flags);
-+ kmem_cache_free(vboxsf_inode_cachep, VBOXSF_I(inode));
-+}
-+
-+static void vboxsf_put_super(struct super_block *sb)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(sb);
-+
-+ vboxsf_unmap_folder(sbi->root);
-+ if (sbi->bdi_id >= 0)
-+ ida_simple_remove(&vboxsf_bdi_ida, sbi->bdi_id);
-+ if (sbi->nls)
-+ unload_nls(sbi->nls);
-+
-+ /*
-+ * vboxsf_free_inode uses the idr, make sure all delayed rcu free
-+ * inodes are flushed.
-+ */
-+ rcu_barrier();
-+ idr_destroy(&sbi->ino_idr);
-+ kfree(sbi);
-+}
-+
-+static int vboxsf_statfs(struct dentry *dentry, struct kstatfs *stat)
-+{
-+ struct super_block *sb = dentry->d_sb;
-+ struct shfl_volinfo shfl_volinfo;
-+ struct vboxsf_sbi *sbi;
-+ u32 buf_len;
-+ int err;
-+
-+ sbi = VBOXSF_SBI(sb);
-+ buf_len = sizeof(shfl_volinfo);
-+ err = vboxsf_fsinfo(sbi->root, 0, SHFL_INFO_GET | SHFL_INFO_VOLUME,
-+ &buf_len, &shfl_volinfo);
-+ if (err)
-+ return err;
-+
-+ stat->f_type = VBOXSF_SUPER_MAGIC;
-+ stat->f_bsize = shfl_volinfo.bytes_per_allocation_unit;
-+
-+ do_div(shfl_volinfo.total_allocation_bytes,
-+ shfl_volinfo.bytes_per_allocation_unit);
-+ stat->f_blocks = shfl_volinfo.total_allocation_bytes;
-+
-+ do_div(shfl_volinfo.available_allocation_bytes,
-+ shfl_volinfo.bytes_per_allocation_unit);
-+ stat->f_bfree = shfl_volinfo.available_allocation_bytes;
-+ stat->f_bavail = shfl_volinfo.available_allocation_bytes;
-+
-+ stat->f_files = 1000;
-+ /*
-+ * Don't return 0 here since the guest may then think that it is not
-+ * possible to create any more files.
-+ */
-+ stat->f_ffree = 1000000;
-+ stat->f_fsid.val[0] = 0;
-+ stat->f_fsid.val[1] = 0;
-+ stat->f_namelen = 255;
-+ return 0;
-+}
-+
-+static struct super_operations vboxsf_super_ops = {
-+ .alloc_inode = vboxsf_alloc_inode,
-+ .free_inode = vboxsf_free_inode,
-+ .put_super = vboxsf_put_super,
-+ .statfs = vboxsf_statfs,
-+};
-+
-+static int vboxsf_setup(void)
-+{
-+ int err;
-+
-+ mutex_lock(&vboxsf_setup_mutex);
-+
-+ if (vboxsf_setup_done)
-+ goto success;
-+
-+ vboxsf_inode_cachep =
-+ kmem_cache_create("vboxsf_inode_cache",
-+ sizeof(struct vboxsf_inode), 0,
-+ (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD |
-+ SLAB_ACCOUNT),
-+ vboxsf_inode_init_once);
-+ if (!vboxsf_inode_cachep) {
-+ err = -ENOMEM;
-+ goto fail_nomem;
-+ }
-+
-+ err = vboxsf_connect();
-+ if (err) {
-+ vbg_err("vboxsf: err %d connecting to guest PCI-device\n", err);
-+ vbg_err("vboxsf: make sure you are inside a VirtualBox VM\n");
-+ vbg_err("vboxsf: and check dmesg for vboxguest errors\n");
-+ goto fail_free_cache;
-+ }
-+
-+ err = vboxsf_set_utf8();
-+ if (err) {
-+ vbg_err("vboxsf_setutf8 error %d\n", err);
-+ goto fail_disconnect;
-+ }
-+
-+ if (!follow_symlinks) {
-+ err = vboxsf_set_symlinks();
-+ if (err)
-+ vbg_warn("vboxsf: Unable to show symlinks: %d\n", err);
-+ }
-+
-+ vboxsf_setup_done = true;
-+success:
-+ mutex_unlock(&vboxsf_setup_mutex);
-+ return 0;
-+
-+fail_disconnect:
-+ vboxsf_disconnect();
-+fail_free_cache:
-+ kmem_cache_destroy(vboxsf_inode_cachep);
-+fail_nomem:
-+ mutex_unlock(&vboxsf_setup_mutex);
-+ return err;
-+}
-+
-+static int vboxsf_parse_monolithic(struct fs_context *fc, void *data)
-+{
-+ char *options = data;
-+
-+ if (options && options[0] == VBSF_MOUNT_SIGNATURE_BYTE_0 &&
-+ options[1] == VBSF_MOUNT_SIGNATURE_BYTE_1 &&
-+ options[2] == VBSF_MOUNT_SIGNATURE_BYTE_2 &&
-+ options[3] == VBSF_MOUNT_SIGNATURE_BYTE_3) {
-+ vbg_err("vboxsf: Old binary mount data not supported, remove obsolete mount.vboxsf and/or update your VBoxService.\n");
-+ return -EINVAL;
-+ }
-+
-+ return generic_parse_monolithic(fc, data);
-+}
-+
-+static int vboxsf_get_tree(struct fs_context *fc)
-+{
-+ int err;
-+
-+ err = vboxsf_setup();
-+ if (err)
-+ return err;
-+
-+ return get_tree_nodev(fc, vboxsf_fill_super);
-+}
-+
-+static int vboxsf_reconfigure(struct fs_context *fc)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(fc->root->d_sb);
-+ struct vboxsf_fs_context *ctx = fc->fs_private;
-+ struct inode *iroot = fc->root->d_sb->s_root->d_inode;
-+
-+ /* Apply changed options to the root inode */
-+ sbi->o = ctx->o;
-+ vboxsf_init_inode(sbi, iroot, &sbi->root_info);
-+
-+ return 0;
-+}
-+
-+static void vboxsf_free_fc(struct fs_context *fc)
-+{
-+ struct vboxsf_fs_context *ctx = fc->fs_private;
-+
-+ kfree(ctx->nls_name);
-+ kfree(ctx);
-+}
-+
-+static const struct fs_context_operations vboxsf_context_ops = {
-+ .free = vboxsf_free_fc,
-+ .parse_param = vboxsf_parse_param,
-+ .parse_monolithic = vboxsf_parse_monolithic,
-+ .get_tree = vboxsf_get_tree,
-+ .reconfigure = vboxsf_reconfigure,
-+};
-+
-+static int vboxsf_init_fs_context(struct fs_context *fc)
-+{
-+ struct vboxsf_fs_context *ctx;
-+
-+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
-+ if (!ctx)
-+ return -ENOMEM;
-+
-+ current_uid_gid(&ctx->o.uid, &ctx->o.gid);
-+
-+ fc->fs_private = ctx;
-+ fc->ops = &vboxsf_context_ops;
-+ return 0;
-+}
-+
-+static struct file_system_type vboxsf_fs_type = {
-+ .owner = THIS_MODULE,
-+ .name = "vboxsf",
-+ .init_fs_context = vboxsf_init_fs_context,
-+ .parameters = &vboxsf_fs_parameters,
-+ .kill_sb = kill_anon_super
-+};
-+
-+/* Module initialization/finalization handlers */
-+static int __init vboxsf_init(void)
-+{
-+ return register_filesystem(&vboxsf_fs_type);
-+}
-+
-+static void __exit vboxsf_fini(void)
-+{
-+ unregister_filesystem(&vboxsf_fs_type);
-+
-+ mutex_lock(&vboxsf_setup_mutex);
-+ if (vboxsf_setup_done) {
-+ vboxsf_disconnect();
-+ /*
-+ * Make sure all delayed rcu free inodes are flushed
-+ * before we destroy the cache.
-+ */
-+ rcu_barrier();
-+ kmem_cache_destroy(vboxsf_inode_cachep);
-+ }
-+ mutex_unlock(&vboxsf_setup_mutex);
-+}
-+
-+module_init(vboxsf_init);
-+module_exit(vboxsf_fini);
-+
-+MODULE_DESCRIPTION("Oracle VM VirtualBox Module for Host File System Access");
-+MODULE_AUTHOR("Oracle Corporation");
-+MODULE_LICENSE("GPL v2");
-+MODULE_ALIAS_FS("vboxsf");
-diff --git a/fs/vboxsf/utils.c b/fs/vboxsf/utils.c
-new file mode 100644
-index 000000000000..96bd160da48b
---- /dev/null
-+++ b/fs/vboxsf/utils.c
-@@ -0,0 +1,551 @@
-+// SPDX-License-Identifier: MIT
-+/*
-+ * VirtualBox Guest Shared Folders support: Utility functions.
-+ * Mainly conversion from/to VirtualBox/Linux data structures.
-+ *
-+ * Copyright (C) 2006-2018 Oracle Corporation
-+ */
-+
-+#include <linux/namei.h>
-+#include <linux/nls.h>
-+#include <linux/sizes.h>
-+#include <linux/vfs.h>
-+#include "vfsmod.h"
-+
-+struct inode *vboxsf_new_inode(struct super_block *sb)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(sb);
-+ struct inode *inode;
-+ unsigned long flags;
-+ int cursor, ret;
-+ u32 gen;
-+
-+ inode = new_inode(sb);
-+ if (!inode)
-+ return ERR_PTR(-ENOMEM);
-+
-+ idr_preload(GFP_KERNEL);
-+ spin_lock_irqsave(&sbi->ino_idr_lock, flags);
-+ cursor = idr_get_cursor(&sbi->ino_idr);
-+ ret = idr_alloc_cyclic(&sbi->ino_idr, inode, 1, 0, GFP_ATOMIC);
-+ if (ret >= 0 && ret < cursor)
-+ sbi->next_generation++;
-+ gen = sbi->next_generation;
-+ spin_unlock_irqrestore(&sbi->ino_idr_lock, flags);
-+ idr_preload_end();
-+
-+ if (ret < 0) {
-+ iput(inode);
-+ return ERR_PTR(ret);
-+ }
-+
-+ inode->i_ino = ret;
-+ inode->i_generation = gen;
-+ return inode;
-+}
-+
-+/* set [inode] attributes based on [info], uid/gid based on [sbi] */
-+void vboxsf_init_inode(struct vboxsf_sbi *sbi, struct inode *inode,
-+ const struct shfl_fsobjinfo *info)
-+{
-+ const struct shfl_fsobjattr *attr;
-+ s64 allocated;
-+ int mode;
-+
-+ attr = &info->attr;
-+
-+#define mode_set(r) ((attr->mode & (SHFL_UNIX_##r)) ? (S_##r) : 0)
-+
-+ mode = mode_set(IRUSR);
-+ mode |= mode_set(IWUSR);
-+ mode |= mode_set(IXUSR);
-+
-+ mode |= mode_set(IRGRP);
-+ mode |= mode_set(IWGRP);
-+ mode |= mode_set(IXGRP);
-+
-+ mode |= mode_set(IROTH);
-+ mode |= mode_set(IWOTH);
-+ mode |= mode_set(IXOTH);
-+
-+#undef mode_set
-+
-+ /* We use the host-side values for these */
-+ inode->i_flags |= S_NOATIME | S_NOCMTIME;
-+ inode->i_mapping->a_ops = &vboxsf_reg_aops;
-+
-+ if (SHFL_IS_DIRECTORY(attr->mode)) {
-+ inode->i_mode = sbi->o.dmode_set ? sbi->o.dmode : mode;
-+ inode->i_mode &= ~sbi->o.dmask;
-+ inode->i_mode |= S_IFDIR;
-+ inode->i_op = &vboxsf_dir_iops;
-+ inode->i_fop = &vboxsf_dir_fops;
-+ /*
-+ * XXX: this probably should be set to the number of entries
-+ * in the directory plus two (. ..)
-+ */
-+ set_nlink(inode, 1);
-+ } else if (SHFL_IS_SYMLINK(attr->mode)) {
-+ inode->i_mode = sbi->o.fmode_set ? sbi->o.fmode : mode;
-+ inode->i_mode &= ~sbi->o.fmask;
-+ inode->i_mode |= S_IFLNK;
-+ inode->i_op = &vboxsf_lnk_iops;
-+ set_nlink(inode, 1);
-+ } else {
-+ inode->i_mode = sbi->o.fmode_set ? sbi->o.fmode : mode;
-+ inode->i_mode &= ~sbi->o.fmask;
-+ inode->i_mode |= S_IFREG;
-+ inode->i_op = &vboxsf_reg_iops;
-+ inode->i_fop = &vboxsf_reg_fops;
-+ set_nlink(inode, 1);
-+ }
-+
-+ inode->i_uid = sbi->o.uid;
-+ inode->i_gid = sbi->o.gid;
-+
-+ inode->i_size = info->size;
-+ inode->i_blkbits = 12;
-+ /* i_blocks always in units of 512 bytes! */
-+ allocated = info->allocated + 511;
-+ do_div(allocated, 512);
-+ inode->i_blocks = allocated;
-+
-+ inode->i_atime = ns_to_timespec64(
-+ info->access_time.ns_relative_to_unix_epoch);
-+ inode->i_ctime = ns_to_timespec64(
-+ info->change_time.ns_relative_to_unix_epoch);
-+ inode->i_mtime = ns_to_timespec64(
-+ info->modification_time.ns_relative_to_unix_epoch);
-+}
-+
-+int vboxsf_create_at_dentry(struct dentry *dentry,
-+ struct shfl_createparms *params)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb);
-+ struct shfl_string *path;
-+ int err;
-+
-+ path = vboxsf_path_from_dentry(sbi, dentry);
-+ if (IS_ERR(path))
-+ return PTR_ERR(path);
-+
-+ err = vboxsf_create(sbi->root, path, params);
-+ __putname(path);
-+
-+ return err;
-+}
-+
-+int vboxsf_stat(struct vboxsf_sbi *sbi, struct shfl_string *path,
-+ struct shfl_fsobjinfo *info)
-+{
-+ struct shfl_createparms params = {};
-+ int err;
-+
-+ params.handle = SHFL_HANDLE_NIL;
-+ params.create_flags = SHFL_CF_LOOKUP | SHFL_CF_ACT_FAIL_IF_NEW;
-+
-+ err = vboxsf_create(sbi->root, path, ¶ms);
-+ if (err)
-+ return err;
-+
-+ if (params.result != SHFL_FILE_EXISTS)
-+ return -ENOENT;
-+
-+ if (info)
-+ *info = params.info;
-+
-+ return 0;
-+}
-+
-+int vboxsf_stat_dentry(struct dentry *dentry, struct shfl_fsobjinfo *info)
-+{
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb);
-+ struct shfl_string *path;
-+ int err;
-+
-+ path = vboxsf_path_from_dentry(sbi, dentry);
-+ if (IS_ERR(path))
-+ return PTR_ERR(path);
-+
-+ err = vboxsf_stat(sbi, path, info);
-+ __putname(path);
-+ return err;
-+}
-+
-+int vboxsf_inode_revalidate(struct dentry *dentry)
-+{
-+ struct vboxsf_sbi *sbi;
-+ struct vboxsf_inode *sf_i;
-+ struct shfl_fsobjinfo info;
-+ struct timespec64 prev_mtime;
-+ struct inode *inode;
-+ int err;
-+
-+ if (!dentry || !d_really_is_positive(dentry))
-+ return -EINVAL;
-+
-+ inode = d_inode(dentry);
-+ prev_mtime = inode->i_mtime;
-+ sf_i = VBOXSF_I(inode);
-+ sbi = VBOXSF_SBI(dentry->d_sb);
-+ if (!sf_i->force_restat) {
-+ if (time_before(jiffies, dentry->d_time + sbi->o.ttl))
-+ return 0;
-+ }
-+
-+ err = vboxsf_stat_dentry(dentry, &info);
-+ if (err)
-+ return err;
-+
-+ dentry->d_time = jiffies;
-+ sf_i->force_restat = 0;
-+ vboxsf_init_inode(sbi, inode, &info);
-+
-+ /*
-+ * If the file was changed on the host side we need to invalidate the
-+ * page-cache for it. Note this also gets triggered by our own writes,
-+ * this is unavoidable.
-+ */
-+ if (timespec64_compare(&inode->i_mtime, &prev_mtime) > 0)
-+ invalidate_inode_pages2(inode->i_mapping);
-+
-+ return 0;
-+}
-+
-+int vboxsf_getattr(const struct path *path, struct kstat *kstat,
-+ u32 request_mask, unsigned int flags)
-+{
-+ int err;
-+ struct dentry *dentry = path->dentry;
-+ struct inode *inode = d_inode(dentry);
-+ struct vboxsf_inode *sf_i = VBOXSF_I(inode);
-+
-+ switch (flags & AT_STATX_SYNC_TYPE) {
-+ case AT_STATX_DONT_SYNC:
-+ err = 0;
-+ break;
-+ case AT_STATX_FORCE_SYNC:
-+ sf_i->force_restat = 1;
-+ /* fall-through */
-+ default:
-+ err = vboxsf_inode_revalidate(dentry);
-+ }
-+ if (err)
-+ return err;
-+
-+ generic_fillattr(d_inode(dentry), kstat);
-+ return 0;
-+}
-+
-+int vboxsf_setattr(struct dentry *dentry, struct iattr *iattr)
-+{
-+ struct vboxsf_inode *sf_i = VBOXSF_I(d_inode(dentry));
-+ struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb);
-+ struct shfl_createparms params = {};
-+ struct shfl_fsobjinfo info = {};
-+ u32 buf_len;
-+ int err;
-+
-+ params.handle = SHFL_HANDLE_NIL;
-+ params.create_flags = SHFL_CF_ACT_OPEN_IF_EXISTS |
-+ SHFL_CF_ACT_FAIL_IF_NEW |
-+ SHFL_CF_ACCESS_ATTR_WRITE;
-+
-+ /* this is at least required for Posix hosts */
-+ if (iattr->ia_valid & ATTR_SIZE)
-+ params.create_flags |= SHFL_CF_ACCESS_WRITE;
-+
-+ err = vboxsf_create_at_dentry(dentry, ¶ms);
-+ if (err || params.result != SHFL_FILE_EXISTS)
-+ return err ? err : -ENOENT;
-+
-+#define mode_set(r) ((iattr->ia_mode & (S_##r)) ? SHFL_UNIX_##r : 0)
-+
-+ /*
-+ * Setting the file size and setting the other attributes has to
-+ * be handled separately.
-+ */
-+ if (iattr->ia_valid & (ATTR_MODE | ATTR_ATIME | ATTR_MTIME)) {
-+ if (iattr->ia_valid & ATTR_MODE) {
-+ info.attr.mode = mode_set(IRUSR);
-+ info.attr.mode |= mode_set(IWUSR);
-+ info.attr.mode |= mode_set(IXUSR);
-+ info.attr.mode |= mode_set(IRGRP);
-+ info.attr.mode |= mode_set(IWGRP);
-+ info.attr.mode |= mode_set(IXGRP);
-+ info.attr.mode |= mode_set(IROTH);
-+ info.attr.mode |= mode_set(IWOTH);
-+ info.attr.mode |= mode_set(IXOTH);
-+
-+ if (iattr->ia_mode & S_IFDIR)
-+ info.attr.mode |= SHFL_TYPE_DIRECTORY;
-+ else
-+ info.attr.mode |= SHFL_TYPE_FILE;
-+ }
-+
-+ if (iattr->ia_valid & ATTR_ATIME)
-+ info.access_time.ns_relative_to_unix_epoch =
-+ timespec64_to_ns(&iattr->ia_atime);
-+
-+ if (iattr->ia_valid & ATTR_MTIME)
-+ info.modification_time.ns_relative_to_unix_epoch =
-+ timespec64_to_ns(&iattr->ia_mtime);
-+
-+ /*
-+ * Ignore ctime (inode change time) as it can't be set
-+ * from userland anyway.
-+ */
-+
-+ buf_len = sizeof(info);
-+ err = vboxsf_fsinfo(sbi->root, params.handle,
-+ SHFL_INFO_SET | SHFL_INFO_FILE, &buf_len,
-+ &info);
-+ if (err) {
-+ vboxsf_close(sbi->root, params.handle);
-+ return err;
-+ }
-+
-+ /* the host may have given us different attr then requested */
-+ sf_i->force_restat = 1;
-+ }
-+
-+#undef mode_set
-+
-+ if (iattr->ia_valid & ATTR_SIZE) {
-+ memset(&info, 0, sizeof(info));
-+ info.size = iattr->ia_size;
-+ buf_len = sizeof(info);
-+ err = vboxsf_fsinfo(sbi->root, params.handle,
-+ SHFL_INFO_SET | SHFL_INFO_SIZE, &buf_len,
-+ &info);
-+ if (err) {
-+ vboxsf_close(sbi->root, params.handle);
-+ return err;
-+ }
-+
-+ /* the host may have given us different attr then requested */
-+ sf_i->force_restat = 1;
-+ }
-+
-+ vboxsf_close(sbi->root, params.handle);
-+
-+ /* Update the inode with what the host has actually given us. */
-+ if (sf_i->force_restat)
-+ vboxsf_inode_revalidate(dentry);
-+
-+ return 0;
-+}
-+
-+/*
-+ * [dentry] contains string encoded in coding system that corresponds
-+ * to [sbi]->nls, we must convert it to UTF8 here.
-+ * Returns a shfl_string allocated through __getname (must be freed using
-+ * __putname), or an ERR_PTR on error.
-+ */
-+struct shfl_string *vboxsf_path_from_dentry(struct vboxsf_sbi *sbi,
-+ struct dentry *dentry)
-+{
-+ struct shfl_string *shfl_path;
-+ int path_len, out_len, nb;
-+ char *buf, *path;
-+ wchar_t uni;
-+ u8 *out;
-+
-+ buf = __getname();
-+ if (!buf)
-+ return ERR_PTR(-ENOMEM);
-+
-+ path = dentry_path_raw(dentry, buf, PATH_MAX);
-+ if (IS_ERR(path)) {
-+ __putname(buf);
-+ return ERR_CAST(path);
-+ }
-+ path_len = strlen(path);
-+
-+ if (sbi->nls) {
-+ shfl_path = __getname();
-+ if (!shfl_path) {
-+ __putname(buf);
-+ return ERR_PTR(-ENOMEM);
-+ }
-+
-+ out = shfl_path->string.utf8;
-+ out_len = PATH_MAX - SHFLSTRING_HEADER_SIZE - 1;
-+
-+ while (path_len) {
-+ nb = sbi->nls->char2uni(path, path_len, &uni);
-+ if (nb < 0) {
-+ __putname(shfl_path);
-+ __putname(buf);
-+ return ERR_PTR(-EINVAL);
-+ }
-+ path += nb;
-+ path_len -= nb;
-+
-+ nb = utf32_to_utf8(uni, out, out_len);
-+ if (nb < 0) {
-+ __putname(shfl_path);
-+ __putname(buf);
-+ return ERR_PTR(-ENAMETOOLONG);
-+ }
-+ out += nb;
-+ out_len -= nb;
-+ }
-+ *out = 0;
-+ shfl_path->length = out - shfl_path->string.utf8;
-+ shfl_path->size = shfl_path->length + 1;
-+ __putname(buf);
-+ } else {
-+ if ((SHFLSTRING_HEADER_SIZE + path_len + 1) > PATH_MAX) {
-+ __putname(buf);
-+ return ERR_PTR(-ENAMETOOLONG);
-+ }
-+ /*
-+ * dentry_path stores the name at the end of buf, but the
-+ * shfl_string string we return must be properly aligned.
-+ */
-+ shfl_path = (struct shfl_string *)buf;
-+ memmove(shfl_path->string.utf8, path, path_len);
-+ shfl_path->string.utf8[path_len] = 0;
-+ shfl_path->length = path_len;
-+ shfl_path->size = path_len + 1;
-+ }
-+
-+ return shfl_path;
-+}
-+
-+int vboxsf_nlscpy(struct vboxsf_sbi *sbi, char *name, size_t name_bound_len,
-+ const unsigned char *utf8_name, size_t utf8_len)
-+{
-+ const char *in;
-+ char *out;
-+ size_t out_len;
-+ size_t out_bound_len;
-+ size_t in_bound_len;
-+
-+ in = utf8_name;
-+ in_bound_len = utf8_len;
-+
-+ out = name;
-+ out_len = 0;
-+ /* Reserve space for terminating 0 */
-+ out_bound_len = name_bound_len - 1;
-+
-+ while (in_bound_len) {
-+ int nb;
-+ unicode_t uni;
-+
-+ nb = utf8_to_utf32(in, in_bound_len, &uni);
-+ if (nb < 0)
-+ return -EINVAL;
-+
-+ in += nb;
-+ in_bound_len -= nb;
-+
-+ nb = sbi->nls->uni2char(uni, out, out_bound_len);
-+ if (nb < 0)
-+ return nb;
-+
-+ out += nb;
-+ out_bound_len -= nb;
-+ out_len += nb;
-+ }
-+
-+ *out = 0;
-+
-+ return 0;
-+}
-+
-+static struct vboxsf_dir_buf *vboxsf_dir_buf_alloc(struct list_head *list)
-+{
-+ struct vboxsf_dir_buf *b;
-+
-+ b = kmalloc(sizeof(*b), GFP_KERNEL);
-+ if (!b)
-+ return NULL;
-+
-+ b->buf = kmalloc(DIR_BUFFER_SIZE, GFP_KERNEL);
-+ if (!b->buf) {
-+ kfree(b);
-+ return NULL;
-+ }
-+
-+ b->entries = 0;
-+ b->used = 0;
-+ b->free = DIR_BUFFER_SIZE;
-+ list_add(&b->head, list);
-+
-+ return b;
-+}
-+
-+static void vboxsf_dir_buf_free(struct vboxsf_dir_buf *b)
-+{
-+ list_del(&b->head);
-+ kfree(b->buf);
-+ kfree(b);
-+}
-+
-+struct vboxsf_dir_info *vboxsf_dir_info_alloc(void)
-+{
-+ struct vboxsf_dir_info *p;
-+
-+ p = kmalloc(sizeof(*p), GFP_KERNEL);
-+ if (!p)
-+ return NULL;
-+
-+ INIT_LIST_HEAD(&p->info_list);
-+ return p;
-+}
-+
-+void vboxsf_dir_info_free(struct vboxsf_dir_info *p)
-+{
-+ struct list_head *list, *pos, *tmp;
-+
-+ list = &p->info_list;
-+ list_for_each_safe(pos, tmp, list) {
-+ struct vboxsf_dir_buf *b;
-+
-+ b = list_entry(pos, struct vboxsf_dir_buf, head);
-+ vboxsf_dir_buf_free(b);
-+ }
-+ kfree(p);
-+}
-+
-+int vboxsf_dir_read_all(struct vboxsf_sbi *sbi, struct vboxsf_dir_info *sf_d,
-+ u64 handle)
-+{
-+ struct vboxsf_dir_buf *b;
-+ u32 entries, size;
-+ int err = 0;
-+ void *buf;
-+
-+ /* vboxsf_dirinfo returns 1 on end of dir */
-+ while (err == 0) {
-+ b = vboxsf_dir_buf_alloc(&sf_d->info_list);
-+ if (!b) {
-+ err = -ENOMEM;
-+ break;
-+ }
-+
-+ buf = b->buf;
-+ size = b->free;
-+
-+ err = vboxsf_dirinfo(sbi->root, handle, NULL, 0, 0,
-+ &size, buf, &entries);
-+ if (err < 0)
-+ break;
-+
-+ b->entries += entries;
-+ b->free -= size;
-+ b->used += size;
-+ }
-+
-+ if (b && b->used == 0)
-+ vboxsf_dir_buf_free(b);
-+
-+ /* -EILSEQ means the host could not translate a filename, ignore */
-+ if (err > 0 || err == -EILSEQ)
-+ err = 0;
-+
-+ return err;
-+}
-diff --git a/fs/vboxsf/vboxsf_wrappers.c b/fs/vboxsf/vboxsf_wrappers.c
-new file mode 100644
-index 000000000000..bfc78a097dae
---- /dev/null
-+++ b/fs/vboxsf/vboxsf_wrappers.c
-@@ -0,0 +1,371 @@
-+// SPDX-License-Identifier: MIT
-+/*
-+ * Wrapper functions for the shfl host calls.
-+ *
-+ * Copyright (C) 2006-2018 Oracle Corporation
-+ */
-+
-+#include <linux/mm.h>
-+#include <linux/slab.h>
-+#include <linux/vbox_err.h>
-+#include <linux/vbox_utils.h>
-+#include "vfsmod.h"
-+
-+#define SHFL_REQUEST \
-+ (VMMDEV_REQUESTOR_KERNEL | VMMDEV_REQUESTOR_USR_DRV_OTHER | \
-+ VMMDEV_REQUESTOR_CON_DONT_KNOW | VMMDEV_REQUESTOR_TRUST_NOT_GIVEN)
-+
-+static u32 vboxsf_client_id;
-+
-+int vboxsf_connect(void)
-+{
-+ struct vbg_dev *gdev;
-+ struct vmmdev_hgcm_service_location loc;
-+ int err, vbox_status;
-+
-+ loc.type = VMMDEV_HGCM_LOC_LOCALHOST_EXISTING;
-+ strcpy(loc.u.localhost.service_name, "VBoxSharedFolders");
-+
-+ gdev = vbg_get_gdev();
-+ if (IS_ERR(gdev))
-+ return -ENODEV; /* No guest-device */
-+
-+ err = vbg_hgcm_connect(gdev, SHFL_REQUEST, &loc,
-+ &vboxsf_client_id, &vbox_status);
-+ vbg_put_gdev(gdev);
-+
-+ return err ? err : vbg_status_code_to_errno(vbox_status);
-+}
-+
-+void vboxsf_disconnect(void)
-+{
-+ struct vbg_dev *gdev;
-+ int vbox_status;
-+
-+ gdev = vbg_get_gdev();
-+ if (IS_ERR(gdev))
-+ return; /* guest-device is gone, already disconnected */
-+
-+ vbg_hgcm_disconnect(gdev, SHFL_REQUEST, vboxsf_client_id, &vbox_status);
-+ vbg_put_gdev(gdev);
-+}
-+
-+static int vboxsf_call(u32 function, void *parms, u32 parm_count, int *status)
-+{
-+ struct vbg_dev *gdev;
-+ int err, vbox_status;
-+
-+ gdev = vbg_get_gdev();
-+ if (IS_ERR(gdev))
-+ return -ESHUTDOWN; /* guest-dev removed underneath us */
-+
-+ err = vbg_hgcm_call(gdev, SHFL_REQUEST, vboxsf_client_id, function,
-+ U32_MAX, parms, parm_count, &vbox_status);
-+ vbg_put_gdev(gdev);
-+
-+ if (err < 0)
-+ return err;
-+
-+ if (status)
-+ *status = vbox_status;
-+
-+ return vbg_status_code_to_errno(vbox_status);
-+}
-+
-+int vboxsf_map_folder(struct shfl_string *folder_name, u32 *root)
-+{
-+ struct shfl_map_folder parms;
-+ int err, status;
-+
-+ parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL;
-+ parms.path.u.pointer.size = shfl_string_buf_size(folder_name);
-+ parms.path.u.pointer.u.linear_addr = (uintptr_t)folder_name;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = 0;
-+
-+ parms.delimiter.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.delimiter.u.value32 = '/';
-+
-+ parms.case_sensitive.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.case_sensitive.u.value32 = 1;
-+
-+ err = vboxsf_call(SHFL_FN_MAP_FOLDER, &parms, SHFL_CPARMS_MAP_FOLDER,
-+ &status);
-+ if (err == -ENOSYS && status == VERR_NOT_IMPLEMENTED)
-+ vbg_err("%s: Error host is too old\n", __func__);
-+
-+ *root = parms.root.u.value32;
-+ return err;
-+}
-+
-+int vboxsf_unmap_folder(u32 root)
-+{
-+ struct shfl_unmap_folder parms;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ return vboxsf_call(SHFL_FN_UNMAP_FOLDER, &parms,
-+ SHFL_CPARMS_UNMAP_FOLDER, NULL);
-+}
-+
-+/**
-+ * vboxsf_create - Create a new file or folder
-+ * @root: Root of the shared folder in which to create the file
-+ * @parsed_path: The path of the file or folder relative to the shared folder
-+ * @param: create_parms Parameters for file/folder creation.
-+ *
-+ * Create a new file or folder or open an existing one in a shared folder.
-+ * Note this function always returns 0 / success unless an exceptional condition
-+ * occurs - out of memory, invalid arguments, etc. If the file or folder could
-+ * not be opened or created, create_parms->handle will be set to
-+ * SHFL_HANDLE_NIL on return. In this case the value in create_parms->result
-+ * provides information as to why (e.g. SHFL_FILE_EXISTS), create_parms->result
-+ * is also set on success as additional information.
-+ *
-+ * Returns:
-+ * 0 or negative errno value.
-+ */
-+int vboxsf_create(u32 root, struct shfl_string *parsed_path,
-+ struct shfl_createparms *create_parms)
-+{
-+ struct shfl_create parms;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL;
-+ parms.path.u.pointer.size = shfl_string_buf_size(parsed_path);
-+ parms.path.u.pointer.u.linear_addr = (uintptr_t)parsed_path;
-+
-+ parms.parms.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL;
-+ parms.parms.u.pointer.size = sizeof(struct shfl_createparms);
-+ parms.parms.u.pointer.u.linear_addr = (uintptr_t)create_parms;
-+
-+ return vboxsf_call(SHFL_FN_CREATE, &parms, SHFL_CPARMS_CREATE, NULL);
-+}
-+
-+int vboxsf_close(u32 root, u64 handle)
-+{
-+ struct shfl_close parms;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT;
-+ parms.handle.u.value64 = handle;
-+
-+ return vboxsf_call(SHFL_FN_CLOSE, &parms, SHFL_CPARMS_CLOSE, NULL);
-+}
-+
-+int vboxsf_remove(u32 root, struct shfl_string *parsed_path, u32 flags)
-+{
-+ struct shfl_remove parms;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN;
-+ parms.path.u.pointer.size = shfl_string_buf_size(parsed_path);
-+ parms.path.u.pointer.u.linear_addr = (uintptr_t)parsed_path;
-+
-+ parms.flags.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.flags.u.value32 = flags;
-+
-+ return vboxsf_call(SHFL_FN_REMOVE, &parms, SHFL_CPARMS_REMOVE, NULL);
-+}
-+
-+int vboxsf_rename(u32 root, struct shfl_string *src_path,
-+ struct shfl_string *dest_path, u32 flags)
-+{
-+ struct shfl_rename parms;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.src.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN;
-+ parms.src.u.pointer.size = shfl_string_buf_size(src_path);
-+ parms.src.u.pointer.u.linear_addr = (uintptr_t)src_path;
-+
-+ parms.dest.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN;
-+ parms.dest.u.pointer.size = shfl_string_buf_size(dest_path);
-+ parms.dest.u.pointer.u.linear_addr = (uintptr_t)dest_path;
-+
-+ parms.flags.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.flags.u.value32 = flags;
-+
-+ return vboxsf_call(SHFL_FN_RENAME, &parms, SHFL_CPARMS_RENAME, NULL);
-+}
-+
-+int vboxsf_read(u32 root, u64 handle, u64 offset, u32 *buf_len, u8 *buf)
-+{
-+ struct shfl_read parms;
-+ int err;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT;
-+ parms.handle.u.value64 = handle;
-+ parms.offset.type = VMMDEV_HGCM_PARM_TYPE_64BIT;
-+ parms.offset.u.value64 = offset;
-+ parms.cb.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.cb.u.value32 = *buf_len;
-+ parms.buffer.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_OUT;
-+ parms.buffer.u.pointer.size = *buf_len;
-+ parms.buffer.u.pointer.u.linear_addr = (uintptr_t)buf;
-+
-+ err = vboxsf_call(SHFL_FN_READ, &parms, SHFL_CPARMS_READ, NULL);
-+
-+ *buf_len = parms.cb.u.value32;
-+ return err;
-+}
-+
-+int vboxsf_write(u32 root, u64 handle, u64 offset, u32 *buf_len, u8 *buf)
-+{
-+ struct shfl_write parms;
-+ int err;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT;
-+ parms.handle.u.value64 = handle;
-+ parms.offset.type = VMMDEV_HGCM_PARM_TYPE_64BIT;
-+ parms.offset.u.value64 = offset;
-+ parms.cb.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.cb.u.value32 = *buf_len;
-+ parms.buffer.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN;
-+ parms.buffer.u.pointer.size = *buf_len;
-+ parms.buffer.u.pointer.u.linear_addr = (uintptr_t)buf;
-+
-+ err = vboxsf_call(SHFL_FN_WRITE, &parms, SHFL_CPARMS_WRITE, NULL);
-+
-+ *buf_len = parms.cb.u.value32;
-+ return err;
-+}
-+
-+/* Returns 0 on success, 1 on end-of-dir, negative errno otherwise */
-+int vboxsf_dirinfo(u32 root, u64 handle,
-+ struct shfl_string *parsed_path, u32 flags, u32 index,
-+ u32 *buf_len, struct shfl_dirinfo *buf, u32 *file_count)
-+{
-+ struct shfl_list parms;
-+ int err, status;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT;
-+ parms.handle.u.value64 = handle;
-+ parms.flags.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.flags.u.value32 = flags;
-+ parms.cb.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.cb.u.value32 = *buf_len;
-+ if (parsed_path) {
-+ parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN;
-+ parms.path.u.pointer.size = shfl_string_buf_size(parsed_path);
-+ parms.path.u.pointer.u.linear_addr = (uintptr_t)parsed_path;
-+ } else {
-+ parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_IN;
-+ parms.path.u.pointer.size = 0;
-+ parms.path.u.pointer.u.linear_addr = 0;
-+ }
-+
-+ parms.buffer.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_OUT;
-+ parms.buffer.u.pointer.size = *buf_len;
-+ parms.buffer.u.pointer.u.linear_addr = (uintptr_t)buf;
-+
-+ parms.resume_point.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.resume_point.u.value32 = index;
-+ parms.file_count.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.file_count.u.value32 = 0; /* out parameter only */
-+
-+ err = vboxsf_call(SHFL_FN_LIST, &parms, SHFL_CPARMS_LIST, &status);
-+ if (err == -ENODATA && status == VERR_NO_MORE_FILES)
-+ err = 1;
-+
-+ *buf_len = parms.cb.u.value32;
-+ *file_count = parms.file_count.u.value32;
-+ return err;
-+}
-+
-+int vboxsf_fsinfo(u32 root, u64 handle, u32 flags,
-+ u32 *buf_len, void *buf)
-+{
-+ struct shfl_information parms;
-+ int err;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT;
-+ parms.handle.u.value64 = handle;
-+ parms.flags.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.flags.u.value32 = flags;
-+ parms.cb.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.cb.u.value32 = *buf_len;
-+ parms.info.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL;
-+ parms.info.u.pointer.size = *buf_len;
-+ parms.info.u.pointer.u.linear_addr = (uintptr_t)buf;
-+
-+ err = vboxsf_call(SHFL_FN_INFORMATION, &parms, SHFL_CPARMS_INFORMATION,
-+ NULL);
-+
-+ *buf_len = parms.cb.u.value32;
-+ return err;
-+}
-+
-+int vboxsf_readlink(u32 root, struct shfl_string *parsed_path,
-+ u32 buf_len, u8 *buf)
-+{
-+ struct shfl_readLink parms;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN;
-+ parms.path.u.pointer.size = shfl_string_buf_size(parsed_path);
-+ parms.path.u.pointer.u.linear_addr = (uintptr_t)parsed_path;
-+
-+ parms.buffer.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_OUT;
-+ parms.buffer.u.pointer.size = buf_len;
-+ parms.buffer.u.pointer.u.linear_addr = (uintptr_t)buf;
-+
-+ return vboxsf_call(SHFL_FN_READLINK, &parms, SHFL_CPARMS_READLINK,
-+ NULL);
-+}
-+
-+int vboxsf_symlink(u32 root, struct shfl_string *new_path,
-+ struct shfl_string *old_path, struct shfl_fsobjinfo *buf)
-+{
-+ struct shfl_symlink parms;
-+
-+ parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT;
-+ parms.root.u.value32 = root;
-+
-+ parms.new_path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN;
-+ parms.new_path.u.pointer.size = shfl_string_buf_size(new_path);
-+ parms.new_path.u.pointer.u.linear_addr = (uintptr_t)new_path;
-+
-+ parms.old_path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN;
-+ parms.old_path.u.pointer.size = shfl_string_buf_size(old_path);
-+ parms.old_path.u.pointer.u.linear_addr = (uintptr_t)old_path;
-+
-+ parms.info.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_OUT;
-+ parms.info.u.pointer.size = sizeof(struct shfl_fsobjinfo);
-+ parms.info.u.pointer.u.linear_addr = (uintptr_t)buf;
-+
-+ return vboxsf_call(SHFL_FN_SYMLINK, &parms, SHFL_CPARMS_SYMLINK, NULL);
-+}
-+
-+int vboxsf_set_utf8(void)
-+{
-+ return vboxsf_call(SHFL_FN_SET_UTF8, NULL, 0, NULL);
-+}
-+
-+int vboxsf_set_symlinks(void)
-+{
-+ return vboxsf_call(SHFL_FN_SET_SYMLINKS, NULL, 0, NULL);
-+}
-diff --git a/fs/vboxsf/vfsmod.h b/fs/vboxsf/vfsmod.h
-new file mode 100644
-index 000000000000..18f95b00fc33
---- /dev/null
-+++ b/fs/vboxsf/vfsmod.h
-@@ -0,0 +1,137 @@
-+/* SPDX-License-Identifier: MIT */
-+/*
-+ * VirtualBox Guest Shared Folders support: module header.
-+ *
-+ * Copyright (C) 2006-2018 Oracle Corporation
-+ */
-+
-+#ifndef VFSMOD_H
-+#define VFSMOD_H
-+
-+#include <linux/backing-dev.h>
-+#include <linux/idr.h>
-+#include "shfl_hostintf.h"
-+
-+#define DIR_BUFFER_SIZE SZ_16K
-+
-+/* The cast is to prevent assignment of void * to pointers of arbitrary type */
-+#define VBOXSF_SBI(sb) ((struct vboxsf_sbi *)(sb)->s_fs_info)
-+#define VBOXSF_I(i) container_of(i, struct vboxsf_inode, vfs_inode)
-+
-+struct vboxsf_options {
-+ unsigned long ttl;
-+ kuid_t uid;
-+ kgid_t gid;
-+ bool dmode_set;
-+ bool fmode_set;
-+ umode_t dmode;
-+ umode_t fmode;
-+ umode_t dmask;
-+ umode_t fmask;
-+};
-+
-+struct vboxsf_fs_context {
-+ struct vboxsf_options o;
-+ char *nls_name;
-+};
-+
-+/* per-shared folder information */
-+struct vboxsf_sbi {
-+ struct vboxsf_options o;
-+ struct shfl_fsobjinfo root_info;
-+ struct idr ino_idr;
-+ spinlock_t ino_idr_lock; /* This protects ino_idr */
-+ struct nls_table *nls;
-+ u32 next_generation;
-+ u32 root;
-+ int bdi_id;
-+};
-+
-+/* per-inode information */
-+struct vboxsf_inode {
-+ /* some information was changed, update data on next revalidate */
-+ int force_restat;
-+ /* list of open handles for this inode + lock protecting it */
-+ struct list_head handle_list;
-+ /* This mutex protects handle_list accesses */
-+ struct mutex handle_list_mutex;
-+ /* The VFS inode struct */
-+ struct inode vfs_inode;
-+};
-+
-+struct vboxsf_dir_info {
-+ struct list_head info_list;
-+};
-+
-+struct vboxsf_dir_buf {
-+ size_t entries;
-+ size_t free;
-+ size_t used;
-+ void *buf;
-+ struct list_head head;
-+};
-+
-+/* globals */
-+extern const struct inode_operations vboxsf_dir_iops;
-+extern const struct inode_operations vboxsf_lnk_iops;
-+extern const struct inode_operations vboxsf_reg_iops;
-+extern const struct file_operations vboxsf_dir_fops;
-+extern const struct file_operations vboxsf_reg_fops;
-+extern const struct address_space_operations vboxsf_reg_aops;
-+extern const struct dentry_operations vboxsf_dentry_ops;
-+
-+/* from utils.c */
-+struct inode *vboxsf_new_inode(struct super_block *sb);
-+void vboxsf_init_inode(struct vboxsf_sbi *sbi, struct inode *inode,
-+ const struct shfl_fsobjinfo *info);
-+int vboxsf_create_at_dentry(struct dentry *dentry,
-+ struct shfl_createparms *params);
-+int vboxsf_stat(struct vboxsf_sbi *sbi, struct shfl_string *path,
-+ struct shfl_fsobjinfo *info);
-+int vboxsf_stat_dentry(struct dentry *dentry, struct shfl_fsobjinfo *info);
-+int vboxsf_inode_revalidate(struct dentry *dentry);
-+int vboxsf_getattr(const struct path *path, struct kstat *kstat,
-+ u32 request_mask, unsigned int query_flags);
-+int vboxsf_setattr(struct dentry *dentry, struct iattr *iattr);
-+struct shfl_string *vboxsf_path_from_dentry(struct vboxsf_sbi *sbi,
-+ struct dentry *dentry);
-+int vboxsf_nlscpy(struct vboxsf_sbi *sbi, char *name, size_t name_bound_len,
-+ const unsigned char *utf8_name, size_t utf8_len);
-+struct vboxsf_dir_info *vboxsf_dir_info_alloc(void);
-+void vboxsf_dir_info_free(struct vboxsf_dir_info *p);
-+int vboxsf_dir_read_all(struct vboxsf_sbi *sbi, struct vboxsf_dir_info *sf_d,
-+ u64 handle);
-+
-+/* from vboxsf_wrappers.c */
-+int vboxsf_connect(void);
-+void vboxsf_disconnect(void);
-+
-+int vboxsf_create(u32 root, struct shfl_string *parsed_path,
-+ struct shfl_createparms *create_parms);
-+
-+int vboxsf_close(u32 root, u64 handle);
-+int vboxsf_remove(u32 root, struct shfl_string *parsed_path, u32 flags);
-+int vboxsf_rename(u32 root, struct shfl_string *src_path,
-+ struct shfl_string *dest_path, u32 flags);
-+
-+int vboxsf_read(u32 root, u64 handle, u64 offset, u32 *buf_len, u8 *buf);
-+int vboxsf_write(u32 root, u64 handle, u64 offset, u32 *buf_len, u8 *buf);
-+
-+int vboxsf_dirinfo(u32 root, u64 handle,
-+ struct shfl_string *parsed_path, u32 flags, u32 index,
-+ u32 *buf_len, struct shfl_dirinfo *buf, u32 *file_count);
-+int vboxsf_fsinfo(u32 root, u64 handle, u32 flags,
-+ u32 *buf_len, void *buf);
-+
-+int vboxsf_map_folder(struct shfl_string *folder_name, u32 *root);
-+int vboxsf_unmap_folder(u32 root);
-+
-+int vboxsf_readlink(u32 root, struct shfl_string *parsed_path,
-+ u32 buf_len, u8 *buf);
-+int vboxsf_symlink(u32 root, struct shfl_string *new_path,
-+ struct shfl_string *old_path, struct shfl_fsobjinfo *buf);
-+
-+int vboxsf_set_utf8(void);
-+int vboxsf_set_symlinks(void);
-+
-+#endif
diff --git a/v2-2-2-x86-Add-support-for-ZSTD-compressed-kernel.patch b/v2-2-2-x86-Add-support-for-ZSTD-compressed-kernel.patch
deleted file mode 100644
index 21e762a..0000000
--- a/v2-2-2-x86-Add-support-for-ZSTD-compressed-kernel.patch
+++ /dev/null
@@ -1,102 +0,0 @@
-diff -up linux-5.2-rc1/arch/x86/boot/compressed/Makefile.20~ linux-5.2-rc1/arch/x86/boot/compressed/Makefile
---- linux-5.2-rc1/arch/x86/boot/compressed/Makefile.20~ 2019-05-20 00:47:09.000000000 +0200
-+++ linux-5.2-rc1/arch/x86/boot/compressed/Makefile 2019-05-21 02:58:19.920070437 +0200
-@@ -24,7 +24,7 @@ OBJECT_FILES_NON_STANDARD := y
- KCOV_INSTRUMENT := n
-
- targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \
-- vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4
-+ vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst
-
- KBUILD_CFLAGS := -m$(BITS) -O2
- KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC)
-@@ -144,6 +144,8 @@ $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.al
- $(call if_changed,lzo)
- $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE
- $(call if_changed,lz4)
-+$(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE
-+ $(call if_changed,zstd)
-
- suffix-$(CONFIG_KERNEL_GZIP) := gz
- suffix-$(CONFIG_KERNEL_BZIP2) := bz2
-@@ -151,6 +153,7 @@ suffix-$(CONFIG_KERNEL_LZMA) := lzma
- suffix-$(CONFIG_KERNEL_XZ) := xz
- suffix-$(CONFIG_KERNEL_LZO) := lzo
- suffix-$(CONFIG_KERNEL_LZ4) := lz4
-+suffix-$(CONFIG_KERNEL_ZSTD) := zst
-
- quiet_cmd_mkpiggy = MKPIGGY $@
- cmd_mkpiggy = $(obj)/mkpiggy $< > $@
-diff -up linux-5.2-rc1/arch/x86/boot/compressed/misc.c.20~ linux-5.2-rc1/arch/x86/boot/compressed/misc.c
---- linux-5.2-rc1/arch/x86/boot/compressed/misc.c.20~ 2019-05-20 00:47:09.000000000 +0200
-+++ linux-5.2-rc1/arch/x86/boot/compressed/misc.c 2019-05-21 02:58:19.920070437 +0200
-@@ -76,6 +76,10 @@ static int lines, cols;
- #ifdef CONFIG_KERNEL_LZ4
- #include "../../../../lib/decompress_unlz4.c"
- #endif
-+
-+#ifdef CONFIG_KERNEL_ZSTD
-+#include "../../../../lib/decompress_unzstd.c"
-+#endif
- /*
- * NOTE: When adding a new decompressor, please update the analysis in
- * ../header.S.
-diff -up linux-5.2-rc1/arch/x86/boot/header.S.20~ linux-5.2-rc1/arch/x86/boot/header.S
---- linux-5.2-rc1/arch/x86/boot/header.S.20~ 2019-05-20 00:47:09.000000000 +0200
-+++ linux-5.2-rc1/arch/x86/boot/header.S 2019-05-21 02:58:19.920070437 +0200
-@@ -526,8 +526,14 @@ pref_address: .quad LOAD_PHYSICAL_ADDR
- # the size-dependent part now grows so fast.
- #
- # extra_bytes = (uncompressed_size >> 8) + 65536
-+#
-+# ZSTD compressed data grows by at most 3 bytes per 128K, and only has a 22
-+# byte fixed overhead but has a maximum block size of 128K, so it needs a
-+# larger margin.
-+#
-+# extra_bytes = (uncompressed_size >> 8) + 131072
-
--#define ZO_z_extra_bytes ((ZO_z_output_len >> 8) + 65536)
-+#define ZO_z_extra_bytes ((ZO_z_output_len >> 8) + 131072)
- #if ZO_z_output_len > ZO_z_input_len
- # define ZO_z_extract_offset (ZO_z_output_len + ZO_z_extra_bytes - \
- ZO_z_input_len)
-diff -up linux-5.2-rc1/arch/x86/include/asm/boot.h.20~ linux-5.2-rc1/arch/x86/include/asm/boot.h
---- linux-5.2-rc1/arch/x86/include/asm/boot.h.20~ 2019-05-20 00:47:09.000000000 +0200
-+++ linux-5.2-rc1/arch/x86/include/asm/boot.h 2019-05-21 02:58:19.920070437 +0200
-@@ -24,9 +24,11 @@
- # error "Invalid value for CONFIG_PHYSICAL_ALIGN"
- #endif
-
--#ifdef CONFIG_KERNEL_BZIP2
-+#if defined(CONFIG_KERNEL_BZIP2)
- # define BOOT_HEAP_SIZE 0x400000
--#else /* !CONFIG_KERNEL_BZIP2 */
-+#elif defined(CONFIG_KERNEL_ZSTD)
-+# define BOOT_HEAP_SIZE 0x30000
-+#else
- # define BOOT_HEAP_SIZE 0x10000
- #endif
-
-diff -up linux-5.2-rc1/arch/x86/Kconfig.20~ linux-5.2-rc1/arch/x86/Kconfig
---- linux-5.2-rc1/arch/x86/Kconfig.20~ 2019-05-20 00:47:09.000000000 +0200
-+++ linux-5.2-rc1/arch/x86/Kconfig 2019-05-21 02:58:19.921070428 +0200
-@@ -172,6 +172,7 @@ config X86
- select HAVE_KERNEL_LZMA
- select HAVE_KERNEL_LZO
- select HAVE_KERNEL_XZ
-+ select HAVE_KERNEL_ZSTD
- select HAVE_KPROBES
- select HAVE_KPROBES_ON_FTRACE
- select HAVE_FUNCTION_ERROR_INJECTION
-diff -up linux-5.2-rc1/Documentation/x86/boot.rst.20~ linux-5.2-rc1/Documentation/x86/boot.rst
---- linux-5.2-rc1/Documentation/x86/boot.rst.20~ 2019-05-21 02:58:19.921070428 +0200
-+++ linux-5.2-rc1/Documentation/x86/boot.rst 2019-05-21 02:59:27.597466111 +0200
-@@ -768,7 +768,7 @@ Protocol: 2.08+
- numbers. The currently supported compression formats are gzip
- (magic numbers 1F 8B or 1F 9E), bzip2 (magic number 42 5A), LZMA
- (magic number 5D 00), XZ (magic number FD 37), and LZ4 (magic number
-- 02 21). The uncompressed payload is currently always ELF (magic
-+ 02 21) and ZSTD (magic number 28 B5). The uncompressed payload is currently always ELF (magic
- number 7F 45 4C 46).
-
- ============ ==============
diff --git a/vbox-6.1-fix-build-on-znver1-hosts.patch b/vbox-6.1-fix-build-on-znver1-hosts.patch
index 5b445f3..e0b51ab 100644
--- a/vbox-6.1-fix-build-on-znver1-hosts.patch
+++ b/vbox-6.1-fix-build-on-znver1-hosts.patch
@@ -36,9 +36,9 @@ diff -up linux-5.4/drivers/pci/vboxpci/Makefile-header.gmk.omv~ linux-5.4/driver
BUILD_TARGET_ARCH_DEF := amd64
else
BUILD_TARGET_ARCH_DEF := x86
-diff -up linux-5.4/drivers/staging/vboxvideo/Makefile-header.gmk.omv~ linux-5.4/drivers/staging/vboxvideo/Makefile-header.gmk
---- linux-5.4/drivers/staging/vboxvideo/Makefile-header.gmk.omv~ 2020-01-10 05:06:12.900953886 +0100
-+++ linux-5.4/drivers/staging/vboxvideo/Makefile-header.gmk 2020-01-10 05:06:19.168965103 +0100
+diff -up linux-5.4/drivers/gpu/drm/vboxvideo/Makefile-header.gmk.omv~ linux-5.4/drivers/gpu/drm/vboxvideo/Makefile-header.gmk
+--- linux-5.4/drivers/gpu/drm/vboxvideo/Makefile-header.gmk.omv~ 2020-01-10 05:06:12.900953886 +0100
++++ linux-5.4/drivers/gpu/drm/vboxvideo/Makefile-header.gmk 2020-01-10 05:06:19.168965103 +0100
@@ -45,7 +45,7 @@
# (We have to support basic cross building (ARCH=i386|x86_64).)
# While at it, warn about BUILD_* vars found to help with user problems.
diff --git a/x86_64-common.config b/x86_64-common.config
index 7805f86..5d9b29e 100644
--- a/x86_64-common.config
+++ b/x86_64-common.config
@@ -23,7 +23,6 @@ CONFIG_SMT_NICE=y
CONFIG_PREEMPT=y
CONFIG_PREEMPT_COUNT=y
CONFIG_PREEMPT_RCU=y
-# CONFIG_X86_VERBOSE_BOOTUP is not set
# CONFIG_X86_PTDUMP_CORE is not set
# CONFIG_X86_DEBUG_FPU is not set
CONFIG_FAIR_GROUP_SCHED=y
@@ -82,6 +81,7 @@ CONFIG_SND_SOC_INTEL_BYT_MAX98090_MACH=m
CONFIG_SND_SOC_INTEL_BYT_RT5640_MACH=m
CONFIG_CRYPTO_GLUE_HELPER_X86=m
CONFIG_CRYPTO_CHACHA20_X86_64=m
+CONFIG_CRYPTO_POLY1305_X86_64=m
CONFIG_CRYPTO_AES_X86_64=m
CONFIG_CRYPTO_BLOWFISH_X86_64=m
CONFIG_CRYPTO_CAMELLIA_X86_64=m
@@ -96,6 +96,5 @@ CONFIG_CRYPTO_SERPENT=AVX_X86_64=m
CONFIG_CRYPTO_TWOFISH_X86_64=m
CONFIG_CRYPTO_TWOFISH_X86_64_3WAY=m
CONFIG_CRYPTO_TWOFISH_AVX_X86_64=m
-# CONFIG_X86_VERBOSE_BOOTUP is not set
# CONFIG_X86_DEBUG_FPU is not set
CONFIG_UEFI_CPER_X86=y