summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/842/842.h1
-rw-r--r--lib/842/842_debugfs.h1
-rw-r--r--lib/Kconfig24
-rw-r--r--lib/Kconfig.debug482
-rw-r--r--lib/Kconfig.kgdb2
-rw-r--r--lib/Makefile13
-rw-r--r--lib/argv_split.c1
-rw-r--r--lib/asn1_decoder.c7
-rw-r--r--lib/assoc_array.c73
-rw-r--r--lib/atomic64_test.c7
-rw-r--r--lib/audit.c1
-rw-r--r--lib/bcd.c1
-rw-r--r--lib/bitmap.c30
-rw-r--r--lib/bsearch.c22
-rw-r--r--lib/bug.c1
-rw-r--r--lib/bust_spinlocks.c1
-rw-r--r--lib/check_signature.c1
-rw-r--r--lib/clz_tab.c1
-rw-r--r--lib/cmdline.c7
-rw-r--r--lib/compat_audit.c1
-rw-r--r--lib/cpumask.c49
-rw-r--r--lib/crc32.c2
-rw-r--r--lib/crc32defs.h1
-rw-r--r--lib/crc4.c46
-rw-r--r--lib/crc8.c22
-rw-r--r--lib/ctype.c1
-rw-r--r--lib/debug_info.c1
-rw-r--r--lib/debugobjects.c3
-rw-r--r--lib/dec_and_lock.c1
-rw-r--r--lib/decompress.c1
-rw-r--r--lib/decompress_inflate.c1
-rw-r--r--lib/devres.c1
-rw-r--r--lib/digsig.c6
-rw-r--r--lib/div64.c7
-rw-r--r--lib/dma-noop.c22
-rw-r--r--lib/dma-virt.c13
-rw-r--r--lib/dump_stack.c1
-rw-r--r--lib/dynamic_queue_limits.c3
-rw-r--r--lib/errseq.c206
-rw-r--r--lib/extable.c41
-rw-r--r--lib/fault-inject.c12
-rw-r--r--lib/flex_proportions.c7
-rw-r--r--lib/fonts/Makefile1
-rw-r--r--lib/fonts/font_10x18.c1
-rw-r--r--lib/fonts/font_6x10.c1
-rw-r--r--lib/fonts/font_6x11.c1
-rw-r--r--lib/fonts/font_7x14.c1
-rw-r--r--lib/fonts/font_8x16.c1
-rw-r--r--lib/fonts/font_8x8.c1
-rw-r--r--lib/fonts/font_acorn_8x8.c1
-rw-r--r--lib/fonts/font_pearl_8x8.c1
-rw-r--r--lib/fonts/font_sun12x22.c1
-rw-r--r--lib/fonts/font_sun8x16.c1
-rw-r--r--lib/gcd.c6
-rw-r--r--lib/gen_crc32table.c1
-rw-r--r--lib/hexdump.c5
-rw-r--r--lib/hweight.c1
-rw-r--r--lib/idr.c70
-rw-r--r--lib/inflate.c1
-rw-r--r--lib/int_sqrt.c1
-rw-r--r--lib/interval_tree_test.c95
-rw-r--r--lib/iomap.c1
-rw-r--r--lib/iommu-common.c1
-rw-r--r--lib/iommu-helper.c1
-rw-r--r--lib/ioremap.c2
-rw-r--r--lib/iov_iter.c118
-rw-r--r--lib/irq_poll.c1
-rw-r--r--lib/kasprintf.c1
-rw-r--r--lib/kobject_uevent.c218
-rw-r--r--lib/kstrtox.c13
-rw-r--r--lib/kstrtox.h1
-rw-r--r--lib/list_sort.c1
-rw-r--r--lib/llist.c2
-rw-r--r--lib/locking-selftest-hardirq.h1
-rw-r--r--lib/locking-selftest-mutex.h1
-rw-r--r--lib/locking-selftest-rlock.h1
-rw-r--r--lib/locking-selftest-rsem.h1
-rw-r--r--lib/locking-selftest-rtmutex.h12
-rw-r--r--lib/locking-selftest-softirq.h1
-rw-r--r--lib/locking-selftest-spin.h1
-rw-r--r--lib/locking-selftest-wlock.h1
-rw-r--r--lib/locking-selftest-wsem.h1
-rw-r--r--lib/locking-selftest.c259
-rw-r--r--lib/lockref.c1
-rw-r--r--lib/lz4/lz4_decompress.c4
-rw-r--r--lib/lzo/lzodefs.h1
-rw-r--r--lib/memweight.c1
-rw-r--r--lib/mpi/Makefile1
-rw-r--r--lib/mpi/longlong.h24
-rw-r--r--lib/mpi/mpi-pow.c2
-rw-r--r--lib/mpi/mpicoder.c4
-rw-r--r--lib/net_utils.c1
-rw-r--r--lib/nlattr.c66
-rw-r--r--lib/nmi_backtrace.c4
-rw-r--r--lib/nodemask.c1
-rw-r--r--lib/notifier-error-inject.h1
-rw-r--r--lib/oid_registry.c4
-rw-r--r--lib/once.c1
-rw-r--r--lib/percpu_counter.c12
-rw-r--r--lib/radix-tree.c16
-rw-r--r--lib/raid6/Makefile5
-rw-r--r--lib/raid6/algos.c3
-rw-r--r--lib/raid6/avx512.c2
-rw-r--r--lib/raid6/mktables.c20
-rw-r--r--lib/raid6/neon.uc33
-rw-r--r--lib/raid6/recov_neon.c110
-rw-r--r--lib/raid6/recov_neon_inner.c117
-rw-r--r--lib/raid6/recov_s390xc.c1
-rw-r--r--lib/raid6/s390vx.uc1
-rw-r--r--lib/raid6/test/Makefile1
-rw-r--r--lib/random32.c3
-rw-r--r--lib/ratelimit.c4
-rw-r--r--lib/rational.c1
-rw-r--r--lib/rbtree.c65
-rw-r--r--lib/rbtree_test.c230
-rw-r--r--lib/reciprocal_div.c1
-rw-r--r--lib/refcount.c4
-rw-r--r--lib/rhashtable.c18
-rw-r--r--lib/scatterlist.c35
-rw-r--r--lib/seq_buf.c1
-rw-r--r--lib/sha1.c1
-rw-r--r--lib/smp_processor_id.c3
-rw-r--r--lib/sort.c1
-rw-r--r--lib/string.c215
-rw-r--r--lib/string_helpers.c4
-rw-r--r--lib/strncpy_from_user.c1
-rw-r--r--lib/strnlen_user.c35
-rw-r--r--lib/swiotlb.c58
-rw-r--r--lib/syscall.c1
-rw-r--r--lib/test_bitmap.c120
-rw-r--r--lib/test_bpf.c423
-rw-r--r--lib/test_debug_virtual.c49
-rw-r--r--lib/test_firmware.c710
-rw-r--r--lib/test_kmod.c1246
-rw-r--r--lib/test_rhashtable.c57
-rw-r--r--lib/test_sysctl.c148
-rw-r--r--lib/test_uuid.c36
-rw-r--r--lib/ts_fsm.c2
-rw-r--r--lib/ts_kmp.c2
-rw-r--r--lib/ubsan.h1
-rw-r--r--lib/ucs2_string.c1
-rw-r--r--lib/usercopy.c11
-rw-r--r--lib/uuid.c33
-rw-r--r--lib/vsprintf.c144
-rw-r--r--lib/win_minmax.c1
-rw-r--r--lib/xxhash.c500
-rw-r--r--lib/xz/xz_dec_stream.c16
-rw-r--r--lib/zstd/Makefile18
-rw-r--r--lib/zstd/bitstream.h374
-rw-r--r--lib/zstd/compress.c3484
-rw-r--r--lib/zstd/decompress.c2528
-rw-r--r--lib/zstd/entropy_common.c243
-rw-r--r--lib/zstd/error_private.h53
-rw-r--r--lib/zstd/fse.h575
-rw-r--r--lib/zstd/fse_compress.c795
-rw-r--r--lib/zstd/fse_decompress.c332
-rw-r--r--lib/zstd/huf.h212
-rw-r--r--lib/zstd/huf_compress.c770
-rw-r--r--lib/zstd/huf_decompress.c960
-rw-r--r--lib/zstd/mem.h151
-rw-r--r--lib/zstd/zstd_common.c75
-rw-r--r--lib/zstd/zstd_internal.h263
-rw-r--r--lib/zstd/zstd_opt.h1014
163 files changed, 17644 insertions, 775 deletions
diff --git a/lib/842/842.h b/lib/842/842.h
index e0a122bc1cdb..7b1f581a2907 100644
--- a/lib/842/842.h
+++ b/lib/842/842.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __842_H__
#define __842_H__
diff --git a/lib/842/842_debugfs.h b/lib/842/842_debugfs.h
index e7f3bffaf255..277e403e8701 100644
--- a/lib/842/842_debugfs.h
+++ b/lib/842/842_debugfs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __842_DEBUGFS_H__
#define __842_DEBUGFS_H__
diff --git a/lib/Kconfig b/lib/Kconfig
index 0c8b78a9ae2e..b1445b22a6de 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -158,6 +158,14 @@ config CRC32_BIT
endchoice
+config CRC4
+ tristate "CRC4 functions"
+ help
+ This option is provided for the case where no in-kernel-tree
+ modules require CRC4 functions, but a module built outside
+ the kernel tree does. Such modules that use library CRC4
+ functions require M here.
+
config CRC7
tristate "CRC7 functions"
help
@@ -184,6 +192,9 @@ config CRC8
when they need to do cyclic redundancy check according CRC8
algorithm. Module will be called crc8.
+config XXHASH
+ tristate
+
config AUDIT_GENERIC
bool
depends on AUDIT && !AUDIT_ARCH
@@ -238,6 +249,14 @@ config LZ4HC_COMPRESS
config LZ4_DECOMPRESS
tristate
+config ZSTD_COMPRESS
+ select XXHASH
+ tristate
+
+config ZSTD_DECOMPRESS
+ select XXHASH
+ tristate
+
source "lib/xz/Kconfig"
#
@@ -548,7 +567,7 @@ config ARCH_HAS_SG_CHAIN
config ARCH_HAS_PMEM_API
bool
-config ARCH_HAS_MMIO_FLUSH
+config ARCH_HAS_UACCESS_FLUSHCACHE
bool
config STACKDEPOT
@@ -564,4 +583,7 @@ config PARMAN
config PRIME_NUMBERS
tristate
+config STRING_SELFTEST
+ bool "Test string functions"
+
endmenu
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index e4587ebe52c7..07ce7449765a 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -219,7 +219,8 @@ config FRAME_WARN
range 0 8192
default 0 if KASAN
default 2048 if GCC_PLUGIN_LATENT_ENTROPY
- default 1024 if !64BIT
+ default 1280 if (!64BIT && PARISC)
+ default 1024 if (!64BIT && !PARISC)
default 2048 if 64BIT
help
Tell gcc to warn at build time for stack frames larger than this.
@@ -286,7 +287,7 @@ config DEBUG_FS
write to these files.
For detailed documentation on the debugfs API, see
- Documentation/DocBook/filesystems.
+ Documentation/filesystems/.
If unsure, say N.
@@ -374,6 +375,9 @@ config STACK_VALIDATION
pointers (if CONFIG_FRAME_POINTER is enabled). This helps ensure
that runtime stack traces are more reliable.
+ This is also a prerequisite for generation of ORC unwind data, which
+ is needed for CONFIG_UNWINDER_ORC.
+
For more information, see
tools/objtool/Documentation/stack-validation.txt.
@@ -778,34 +782,52 @@ config DEBUG_SHIRQ
menu "Debug Lockups and Hangs"
config LOCKUP_DETECTOR
- bool "Detect Hard and Soft Lockups"
+ bool
+
+config SOFTLOCKUP_DETECTOR
+ bool "Detect Soft Lockups"
depends on DEBUG_KERNEL && !S390
+ select LOCKUP_DETECTOR
help
Say Y here to enable the kernel to act as a watchdog to detect
- hard and soft lockups.
+ soft lockups.
Softlockups are bugs that cause the kernel to loop in kernel
mode for more than 20 seconds, without giving other tasks a
chance to run. The current stack trace is displayed upon
detection and the system will stay locked up.
+config HARDLOCKUP_DETECTOR_PERF
+ bool
+ select SOFTLOCKUP_DETECTOR
+
+#
+# Enables a timestamp based low pass filter to compensate for perf based
+# hard lockup detection which runs too fast due to turbo modes.
+#
+config HARDLOCKUP_CHECK_TIMESTAMP
+ bool
+
+#
+# arch/ can define HAVE_HARDLOCKUP_DETECTOR_ARCH to provide their own hard
+# lockup detector rather than the perf based detector.
+#
+config HARDLOCKUP_DETECTOR
+ bool "Detect Hard Lockups"
+ depends on DEBUG_KERNEL && !S390
+ depends on HAVE_HARDLOCKUP_DETECTOR_PERF || HAVE_HARDLOCKUP_DETECTOR_ARCH
+ select LOCKUP_DETECTOR
+ select HARDLOCKUP_DETECTOR_PERF if HAVE_HARDLOCKUP_DETECTOR_PERF
+ select HARDLOCKUP_DETECTOR_ARCH if HAVE_HARDLOCKUP_DETECTOR_ARCH
+ help
+ Say Y here to enable the kernel to act as a watchdog to detect
+ hard lockups.
+
Hardlockups are bugs that cause the CPU to loop in kernel mode
for more than 10 seconds, without letting other interrupts have a
chance to run. The current stack trace is displayed upon detection
and the system will stay locked up.
- The overhead should be minimal. A periodic hrtimer runs to
- generate interrupts and kick the watchdog task every 4 seconds.
- An NMI is generated every 10 seconds or so to check for hardlockups.
-
- The frequency of hrtimer and NMI events and the soft and hard lockup
- thresholds can be controlled through the sysctl watchdog_thresh.
-
-config HARDLOCKUP_DETECTOR
- def_bool y
- depends on LOCKUP_DETECTOR && !HAVE_NMI_WATCHDOG
- depends on PERF_EVENTS && HAVE_PERF_EVENTS_NMI
-
config BOOTPARAM_HARDLOCKUP_PANIC
bool "Panic (Reboot) On Hard Lockups"
depends on HARDLOCKUP_DETECTOR
@@ -826,7 +848,7 @@ config BOOTPARAM_HARDLOCKUP_PANIC_VALUE
config BOOTPARAM_SOFTLOCKUP_PANIC
bool "Panic (Reboot) On Soft Lockups"
- depends on LOCKUP_DETECTOR
+ depends on SOFTLOCKUP_DETECTOR
help
Say Y here to enable the kernel to panic on "soft lockups",
which are bugs that cause the kernel to loop in kernel
@@ -843,7 +865,7 @@ config BOOTPARAM_SOFTLOCKUP_PANIC
config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE
int
- depends on LOCKUP_DETECTOR
+ depends on SOFTLOCKUP_DETECTOR
range 0 1
default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC
default 1 if BOOTPARAM_SOFTLOCKUP_PANIC
@@ -851,7 +873,7 @@ config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE
config DETECT_HUNG_TASK
bool "Detect Hung Tasks"
depends on DEBUG_KERNEL
- default LOCKUP_DETECTOR
+ default SOFTLOCKUP_DETECTOR
help
Say Y here to enable the kernel to detect "hung tasks",
which are bugs that cause the task to be stuck in
@@ -1052,6 +1074,7 @@ config DEBUG_LOCK_ALLOC
depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
select DEBUG_SPINLOCK
select DEBUG_MUTEXES
+ select DEBUG_RT_MUTEXES if RT_MUTEXES
select LOCKDEP
help
This feature will check whether any held lock (spinlock, rwlock,
@@ -1067,7 +1090,10 @@ config PROVE_LOCKING
select LOCKDEP
select DEBUG_SPINLOCK
select DEBUG_MUTEXES
+ select DEBUG_RT_MUTEXES if RT_MUTEXES
select DEBUG_LOCK_ALLOC
+ select LOCKDEP_CROSSRELEASE
+ select LOCKDEP_COMPLETIONS
select TRACE_IRQFLAGS
default n
help
@@ -1108,7 +1134,7 @@ config LOCKDEP
bool
depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
select STACKTRACE
- select FRAME_POINTER if !MIPS && !PPC && !ARM_UNWIND && !S390 && !MICROBLAZE && !ARC && !SCORE
+ select FRAME_POINTER if !MIPS && !PPC && !ARM_UNWIND && !S390 && !MICROBLAZE && !ARC && !SCORE && !X86
select KALLSYMS
select KALLSYMS_ALL
@@ -1121,6 +1147,7 @@ config LOCK_STAT
select LOCKDEP
select DEBUG_SPINLOCK
select DEBUG_MUTEXES
+ select DEBUG_RT_MUTEXES if RT_MUTEXES
select DEBUG_LOCK_ALLOC
default n
help
@@ -1136,6 +1163,37 @@ config LOCK_STAT
CONFIG_LOCK_STAT defines "contended" and "acquired" lock events.
(CONFIG_LOCKDEP defines "acquire" and "release" events.)
+config LOCKDEP_CROSSRELEASE
+ bool
+ help
+ This makes lockdep work for crosslock which is a lock allowed to
+ be released in a different context from the acquisition context.
+ Normally a lock must be released in the context acquiring the lock.
+ However, relexing this constraint helps synchronization primitives
+ such as page locks or completions can use the lock correctness
+ detector, lockdep.
+
+config LOCKDEP_COMPLETIONS
+ bool
+ help
+ A deadlock caused by wait_for_completion() and complete() can be
+ detected by lockdep using crossrelease feature.
+
+config BOOTPARAM_LOCKDEP_CROSSRELEASE_FULLSTACK
+ bool "Enable the boot parameter, crossrelease_fullstack"
+ depends on LOCKDEP_CROSSRELEASE
+ default n
+ help
+ The lockdep "cross-release" feature needs to record stack traces
+ (of calling functions) for all acquisitions, for eventual later
+ use during analysis. By default only a single caller is recorded,
+ because the unwind operation can be very expensive with deeper
+ stack chains.
+
+ However a boot parameter, crossrelease_fullstack, was
+ introduced since sometimes deeper traces are required for full
+ analysis. This option turns on the boot parameter.
+
config DEBUG_LOCKDEP
bool "Lock dependency engine debugging"
depends on DEBUG_KERNEL && LOCKDEP
@@ -1209,6 +1267,34 @@ config STACKTRACE
It is also used by various kernel debugging features that require
stack trace generation.
+config WARN_ALL_UNSEEDED_RANDOM
+ bool "Warn for all uses of unseeded randomness"
+ default n
+ help
+ Some parts of the kernel contain bugs relating to their use of
+ cryptographically secure random numbers before it's actually possible
+ to generate those numbers securely. This setting ensures that these
+ flaws don't go unnoticed, by enabling a message, should this ever
+ occur. This will allow people with obscure setups to know when things
+ are going wrong, so that they might contact developers about fixing
+ it.
+
+ Unfortunately, on some models of some architectures getting
+ a fully seeded CRNG is extremely difficult, and so this can
+ result in dmesg getting spammed for a surprisingly long
+ time. This is really bad from a security perspective, and
+ so architecture maintainers really need to do what they can
+ to get the CRNG seeded sooner after the system is booted.
+ However, since users can not do anything actionble to
+ address this, by default the kernel will issue only a single
+ warning for the first use of unseeded randomness.
+
+ Say Y here if you want to receive warnings for all uses of
+ unseeded randomness. This will be of use primarily for
+ those developers interersted in improving the security of
+ Linux kernels running on their architecture (or
+ subarchitecture).
+
config DEBUG_KOBJECT
bool "kobject debugging"
depends on DEBUG_KERNEL
@@ -1301,189 +1387,7 @@ config DEBUG_CREDENTIALS
If unsure, say N.
-menu "RCU Debugging"
-
-config PROVE_RCU
- def_bool PROVE_LOCKING
-
-config PROVE_RCU_REPEATEDLY
- bool "RCU debugging: don't disable PROVE_RCU on first splat"
- depends on PROVE_RCU
- default n
- help
- By itself, PROVE_RCU will disable checking upon issuing the
- first warning (or "splat"). This feature prevents such
- disabling, allowing multiple RCU-lockdep warnings to be printed
- on a single reboot.
-
- Say Y to allow multiple RCU-lockdep warnings per boot.
-
- Say N if you are unsure.
-
-config SPARSE_RCU_POINTER
- bool "RCU debugging: sparse-based checks for pointer usage"
- default n
- help
- This feature enables the __rcu sparse annotation for
- RCU-protected pointers. This annotation will cause sparse
- to flag any non-RCU used of annotated pointers. This can be
- helpful when debugging RCU usage. Please note that this feature
- is not intended to enforce code cleanliness; it is instead merely
- a debugging aid.
-
- Say Y to make sparse flag questionable use of RCU-protected pointers
-
- Say N if you are unsure.
-
-config TORTURE_TEST
- tristate
- default n
-
-config RCU_PERF_TEST
- tristate "performance tests for RCU"
- depends on DEBUG_KERNEL
- select TORTURE_TEST
- select SRCU
- select TASKS_RCU
- default n
- help
- This option provides a kernel module that runs performance
- tests on the RCU infrastructure. The kernel module may be built
- after the fact on the running kernel to be tested, if desired.
-
- Say Y here if you want RCU performance tests to be built into
- the kernel.
- Say M if you want the RCU performance tests to build as a module.
- Say N if you are unsure.
-
-config RCU_TORTURE_TEST
- tristate "torture tests for RCU"
- depends on DEBUG_KERNEL
- select TORTURE_TEST
- select SRCU
- select TASKS_RCU
- default n
- help
- This option provides a kernel module that runs torture tests
- on the RCU infrastructure. The kernel module may be built
- after the fact on the running kernel to be tested, if desired.
-
- Say Y here if you want RCU torture tests to be built into
- the kernel.
- Say M if you want the RCU torture tests to build as a module.
- Say N if you are unsure.
-
-config RCU_TORTURE_TEST_SLOW_PREINIT
- bool "Slow down RCU grace-period pre-initialization to expose races"
- depends on RCU_TORTURE_TEST
- help
- This option delays grace-period pre-initialization (the
- propagation of CPU-hotplug changes up the rcu_node combining
- tree) for a few jiffies between initializing each pair of
- consecutive rcu_node structures. This helps to expose races
- involving grace-period pre-initialization, in other words, it
- makes your kernel less stable. It can also greatly increase
- grace-period latency, especially on systems with large numbers
- of CPUs. This is useful when torture-testing RCU, but in
- almost no other circumstance.
-
- Say Y here if you want your system to crash and hang more often.
- Say N if you want a sane system.
-
-config RCU_TORTURE_TEST_SLOW_PREINIT_DELAY
- int "How much to slow down RCU grace-period pre-initialization"
- range 0 5
- default 3
- depends on RCU_TORTURE_TEST_SLOW_PREINIT
- help
- This option specifies the number of jiffies to wait between
- each rcu_node structure pre-initialization step.
-
-config RCU_TORTURE_TEST_SLOW_INIT
- bool "Slow down RCU grace-period initialization to expose races"
- depends on RCU_TORTURE_TEST
- help
- This option delays grace-period initialization for a few
- jiffies between initializing each pair of consecutive
- rcu_node structures. This helps to expose races involving
- grace-period initialization, in other words, it makes your
- kernel less stable. It can also greatly increase grace-period
- latency, especially on systems with large numbers of CPUs.
- This is useful when torture-testing RCU, but in almost no
- other circumstance.
-
- Say Y here if you want your system to crash and hang more often.
- Say N if you want a sane system.
-
-config RCU_TORTURE_TEST_SLOW_INIT_DELAY
- int "How much to slow down RCU grace-period initialization"
- range 0 5
- default 3
- depends on RCU_TORTURE_TEST_SLOW_INIT
- help
- This option specifies the number of jiffies to wait between
- each rcu_node structure initialization.
-
-config RCU_TORTURE_TEST_SLOW_CLEANUP
- bool "Slow down RCU grace-period cleanup to expose races"
- depends on RCU_TORTURE_TEST
- help
- This option delays grace-period cleanup for a few jiffies
- between cleaning up each pair of consecutive rcu_node
- structures. This helps to expose races involving grace-period
- cleanup, in other words, it makes your kernel less stable.
- It can also greatly increase grace-period latency, especially
- on systems with large numbers of CPUs. This is useful when
- torture-testing RCU, but in almost no other circumstance.
-
- Say Y here if you want your system to crash and hang more often.
- Say N if you want a sane system.
-
-config RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY
- int "How much to slow down RCU grace-period cleanup"
- range 0 5
- default 3
- depends on RCU_TORTURE_TEST_SLOW_CLEANUP
- help
- This option specifies the number of jiffies to wait between
- each rcu_node structure cleanup operation.
-
-config RCU_CPU_STALL_TIMEOUT
- int "RCU CPU stall timeout in seconds"
- depends on RCU_STALL_COMMON
- range 3 300
- default 21
- help
- If a given RCU grace period extends more than the specified
- number of seconds, a CPU stall warning is printed. If the
- RCU grace period persists, additional CPU stall warnings are
- printed at more widely spaced intervals.
-
-config RCU_TRACE
- bool "Enable tracing for RCU"
- depends on DEBUG_KERNEL
- default y if TREE_RCU
- select TRACE_CLOCK
- help
- This option provides tracing in RCU which presents stats
- in debugfs for debugging RCU implementation. It also enables
- additional tracepoints for ftrace-style event tracing.
-
- Say Y here if you want to enable RCU tracing
- Say N if you are unsure.
-
-config RCU_EQS_DEBUG
- bool "Provide debugging asserts for adding NO_HZ support to an arch"
- depends on DEBUG_KERNEL
- help
- This option provides consistency checks in RCU's handling of
- NO_HZ. These checks have proven quite helpful in detecting
- bugs in arch-specific NO_HZ code.
-
- Say N here if you need ultimate kernel/user switch latencies
- Say Y if you are unsure
-
-endmenu # "RCU Debugging"
+source "kernel/rcu/Kconfig.debug"
config DEBUG_WQ_FORCE_RR_CPU
bool "Force round-robin CPU selection for unbound work items"
@@ -1680,7 +1584,7 @@ config FAULT_INJECTION_STACKTRACE_FILTER
depends on FAULT_INJECTION_DEBUG_FS && STACKTRACE_SUPPORT
depends on !X86_64
select STACKTRACE
- select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM_UNWIND && !ARC && !SCORE
+ select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM_UNWIND && !ARC && !SCORE && !X86
help
Provide stacktrace filter for fault-injection capabilities
@@ -1689,7 +1593,7 @@ config LATENCYTOP
depends on DEBUG_KERNEL
depends on STACKTRACE_SUPPORT
depends on PROC_FS
- select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM_UNWIND && !ARC
+ select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM_UNWIND && !ARC && !X86
select KALLSYMS
select KALLSYMS_ALL
select STACKTRACE
@@ -1701,6 +1605,54 @@ config LATENCYTOP
source kernel/trace/Kconfig
+config PROVIDE_OHCI1394_DMA_INIT
+ bool "Remote debugging over FireWire early on boot"
+ depends on PCI && X86
+ help
+ If you want to debug problems which hang or crash the kernel early
+ on boot and the crashing machine has a FireWire port, you can use
+ this feature to remotely access the memory of the crashed machine
+ over FireWire. This employs remote DMA as part of the OHCI1394
+ specification which is now the standard for FireWire controllers.
+
+ With remote DMA, you can monitor the printk buffer remotely using
+ firescope and access all memory below 4GB using fireproxy from gdb.
+ Even controlling a kernel debugger is possible using remote DMA.
+
+ Usage:
+
+ If ohci1394_dma=early is used as boot parameter, it will initialize
+ all OHCI1394 controllers which are found in the PCI config space.
+
+ As all changes to the FireWire bus such as enabling and disabling
+ devices cause a bus reset and thereby disable remote DMA for all
+ devices, be sure to have the cable plugged and FireWire enabled on
+ the debugging host before booting the debug target for debugging.
+
+ This code (~1k) is freed after boot. By then, the firewire stack
+ in charge of the OHCI-1394 controllers should be used instead.
+
+ See Documentation/debugging-via-ohci1394.txt for more information.
+
+config DMA_API_DEBUG
+ bool "Enable debugging of DMA-API usage"
+ depends on HAVE_DMA_API_DEBUG
+ help
+ Enable this option to debug the use of the DMA API by device drivers.
+ With this option you will be able to detect common bugs in device
+ drivers like double-freeing of DMA mappings or freeing mappings that
+ were never allocated.
+
+ This also attempts to catch cases where a page owned by DMA is
+ accessed by the cpu in a way that could cause data corruption. For
+ example, this enables cow_user_page() to check that the source page is
+ not undergoing DMA.
+
+ This option causes a performance degradation. Use only if you want to
+ debug device drivers and dma interactions.
+
+ If unsure, say N.
+
menu "Runtime Testing"
config LKDTM
@@ -1773,7 +1725,7 @@ config RBTREE_TEST
config INTERVAL_TREE_TEST
tristate "Interval tree test"
- depends on m && DEBUG_KERNEL
+ depends on DEBUG_KERNEL
select INTERVAL_TREE
help
A benchmark measuring the performance of the interval tree library
@@ -1860,56 +1812,6 @@ config TEST_PARMAN
If unsure, say N.
-endmenu # runtime tests
-
-config PROVIDE_OHCI1394_DMA_INIT
- bool "Remote debugging over FireWire early on boot"
- depends on PCI && X86
- help
- If you want to debug problems which hang or crash the kernel early
- on boot and the crashing machine has a FireWire port, you can use
- this feature to remotely access the memory of the crashed machine
- over FireWire. This employs remote DMA as part of the OHCI1394
- specification which is now the standard for FireWire controllers.
-
- With remote DMA, you can monitor the printk buffer remotely using
- firescope and access all memory below 4GB using fireproxy from gdb.
- Even controlling a kernel debugger is possible using remote DMA.
-
- Usage:
-
- If ohci1394_dma=early is used as boot parameter, it will initialize
- all OHCI1394 controllers which are found in the PCI config space.
-
- As all changes to the FireWire bus such as enabling and disabling
- devices cause a bus reset and thereby disable remote DMA for all
- devices, be sure to have the cable plugged and FireWire enabled on
- the debugging host before booting the debug target for debugging.
-
- This code (~1k) is freed after boot. By then, the firewire stack
- in charge of the OHCI-1394 controllers should be used instead.
-
- See Documentation/debugging-via-ohci1394.txt for more information.
-
-config DMA_API_DEBUG
- bool "Enable debugging of DMA-API usage"
- depends on HAVE_DMA_API_DEBUG
- help
- Enable this option to debug the use of the DMA API by device drivers.
- With this option you will be able to detect common bugs in device
- drivers like double-freeing of DMA mappings or freeing mappings that
- were never allocated.
-
- This also attempts to catch cases where a page owned by DMA is
- accessed by the cpu in a way that could cause data corruption. For
- example, this enables cow_user_page() to check that the source page is
- not undergoing DMA.
-
- This option causes a performance degradation. Use only if you want to
- debug device drivers and dma interactions.
-
- If unsure, say N.
-
config TEST_LKM
tristate "Test module loading with 'hello world' module"
default n
@@ -1964,6 +1866,17 @@ config TEST_FIRMWARE
If unsure, say N.
+config TEST_SYSCTL
+ tristate "sysctl test driver"
+ default n
+ depends on PROC_SYSCTL
+ help
+ This builds the "test_sysctl" module. This driver enables to test the
+ proc sysctl interfaces available to drivers safely without affecting
+ production knobs which might alter system functionality.
+
+ If unsure, say N.
+
config TEST_UDELAY
tristate "udelay test driver"
default n
@@ -1973,6 +1886,54 @@ config TEST_UDELAY
If unsure, say N.
+config TEST_STATIC_KEYS
+ tristate "Test static keys"
+ default n
+ depends on m
+ help
+ Test the static key interfaces.
+
+ If unsure, say N.
+
+config TEST_KMOD
+ tristate "kmod stress tester"
+ default n
+ depends on m
+ depends on BLOCK && (64BIT || LBDAF) # for XFS, BTRFS
+ depends on NETDEVICES && NET_CORE && INET # for TUN
+ select TEST_LKM
+ select XFS_FS
+ select TUN
+ select BTRFS_FS
+ help
+ Test the kernel's module loading mechanism: kmod. kmod implements
+ support to load modules using the Linux kernel's usermode helper.
+ This test provides a series of tests against kmod.
+
+ Although technically you can either build test_kmod as a module or
+ into the kernel we disallow building it into the kernel since
+ it stress tests request_module() and this will very likely cause
+ some issues by taking over precious threads available from other
+ module load requests, ultimately this could be fatal.
+
+ To run tests run:
+
+ tools/testing/selftests/kmod/kmod.sh --help
+
+ If unsure, say N.
+
+config TEST_DEBUG_VIRTUAL
+ tristate "Test CONFIG_DEBUG_VIRTUAL feature"
+ depends on DEBUG_VIRTUAL
+ help
+ Test the kernel's ability to detect incorrect calls to
+ virt_to_phys() done against the non-linear part of the
+ kernel's virtual address map.
+
+ If unsure, say N.
+
+endmenu # runtime tests
+
config MEMTEST
bool "Memtest"
depends on HAVE_MEMBLOCK
@@ -1985,15 +1946,6 @@ config MEMTEST
memtest=17, mean do 17 test patterns.
If you are unsure how to answer this question, answer N.
-config TEST_STATIC_KEYS
- tristate "Test static keys"
- default n
- depends on m
- help
- Test the static key interfaces.
-
- If unsure, say N.
-
config BUG_ON_DATA_CORRUPTION
bool "Trigger a BUG when data corruption is detected"
select DEBUG_LIST
diff --git a/lib/Kconfig.kgdb b/lib/Kconfig.kgdb
index 533f912638ed..ab4ff0eea776 100644
--- a/lib/Kconfig.kgdb
+++ b/lib/Kconfig.kgdb
@@ -13,7 +13,7 @@ menuconfig KGDB
CONFIG_FRAME_POINTER to aid in producing more reliable stack
backtraces in the external debugger. Documentation of
kernel debugger is available at http://kgdb.sourceforge.net
- as well as in DocBook form in Documentation/DocBook/. If
+ as well as in Documentation/dev-tools/kgdb.rst. If
unsure, say N.
if KGDB
diff --git a/lib/Makefile b/lib/Makefile
index 0166fbc0fa81..b8f2c16fccaa 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
#
# Makefile for some libs needed in the kernel.
#
@@ -25,9 +26,6 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \
earlycpio.o seq_buf.o siphash.o \
nmi_backtrace.o nodemask.o win_minmax.o
-CFLAGS_radix-tree.o += -DCONFIG_SPARSE_RCU_POINTER
-CFLAGS_idr.o += -DCONFIG_SPARSE_RCU_POINTER
-
lib-$(CONFIG_MMU) += ioremap.o
lib-$(CONFIG_SMP) += cpumask.o
lib-$(CONFIG_DMA_NOOP_OPS) += dma-noop.o
@@ -41,7 +39,7 @@ obj-y += bcd.o div64.o sort.o parser.o debug_locks.o random32.o \
gcd.o lcm.o list_sort.o uuid.o flex_array.o iov_iter.o clz_ctz.o \
bsearch.o find_bit.o llist.o memweight.o kfifo.o \
percpu-refcount.o percpu_ida.o rhashtable.o reciprocal_div.o \
- once.o refcount.o usercopy.o
+ once.o refcount.o usercopy.o errseq.o
obj-y += string_helpers.o
obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o
obj-y += hexdump.o
@@ -49,6 +47,7 @@ obj-$(CONFIG_TEST_HEXDUMP) += test_hexdump.o
obj-y += kstrtox.o
obj-$(CONFIG_TEST_BPF) += test_bpf.o
obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o
+obj-$(CONFIG_TEST_SYSCTL) += test_sysctl.o
obj-$(CONFIG_TEST_HASH) += test_hash.o test_siphash.o
obj-$(CONFIG_TEST_KASAN) += test_kasan.o
obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o
@@ -63,6 +62,8 @@ obj-$(CONFIG_TEST_PRINTF) += test_printf.o
obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o
obj-$(CONFIG_TEST_UUID) += test_uuid.o
obj-$(CONFIG_TEST_PARMAN) += test_parman.o
+obj-$(CONFIG_TEST_KMOD) += test_kmod.o
+obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o
ifeq ($(CONFIG_DEBUG_KOBJECT),y)
CFLAGS_kobject.o += -DDEBUG
@@ -99,9 +100,11 @@ obj-$(CONFIG_CRC_T10DIF)+= crc-t10dif.o
obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o
obj-$(CONFIG_CRC32) += crc32.o
obj-$(CONFIG_CRC32_SELFTEST) += crc32test.o
+obj-$(CONFIG_CRC4) += crc4.o
obj-$(CONFIG_CRC7) += crc7.o
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
obj-$(CONFIG_CRC8) += crc8.o
+obj-$(CONFIG_XXHASH) += xxhash.o
obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
obj-$(CONFIG_842_COMPRESS) += 842/
@@ -115,6 +118,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
obj-$(CONFIG_LZ4_COMPRESS) += lz4/
obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/
obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
+obj-$(CONFIG_ZSTD_COMPRESS) += zstd/
+obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/
obj-$(CONFIG_XZ_DEC) += xz/
obj-$(CONFIG_RAID6_PQ) += raid6/
diff --git a/lib/argv_split.c b/lib/argv_split.c
index e927ed0e18a8..5c35752a9414 100644
--- a/lib/argv_split.c
+++ b/lib/argv_split.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Helper function for splitting a string into an argv-like array.
*/
diff --git a/lib/asn1_decoder.c b/lib/asn1_decoder.c
index 0bd8a611eb83..1ef0cec38d78 100644
--- a/lib/asn1_decoder.c
+++ b/lib/asn1_decoder.c
@@ -228,7 +228,7 @@ next_op:
hdr = 2;
/* Extract a tag from the data */
- if (unlikely(dp >= datalen - 1))
+ if (unlikely(datalen - dp < 2))
goto data_overrun_error;
tag = data[dp++];
if (unlikely((tag & 0x1f) == ASN1_LONG_TAG))
@@ -274,7 +274,7 @@ next_op:
int n = len - 0x80;
if (unlikely(n > 2))
goto length_too_long;
- if (unlikely(dp >= datalen - n))
+ if (unlikely(n > datalen - dp))
goto data_overrun_error;
hdr += n;
for (len = 0; n > 0; n--) {
@@ -284,6 +284,9 @@ next_op:
if (unlikely(len > datalen - dp))
goto data_overrun_error;
}
+ } else {
+ if (unlikely(len > datalen - dp))
+ goto data_overrun_error;
}
if (flags & FLAG_CONS) {
diff --git a/lib/assoc_array.c b/lib/assoc_array.c
index 59fd7c0b119c..b77d51da8c73 100644
--- a/lib/assoc_array.c
+++ b/lib/assoc_array.c
@@ -1,6 +1,6 @@
/* Generic associative array implementation.
*
- * See Documentation/assoc_array.txt for information.
+ * See Documentation/core-api/assoc_array.rst for information.
*
* Copyright (C) 2013 Red Hat, Inc. All Rights Reserved.
* Written by David Howells (dhowells@redhat.com)
@@ -39,7 +39,7 @@ begin_node:
/* Descend through a shortcut */
shortcut = assoc_array_ptr_to_shortcut(cursor);
smp_read_barrier_depends();
- cursor = ACCESS_ONCE(shortcut->next_node);
+ cursor = READ_ONCE(shortcut->next_node);
}
node = assoc_array_ptr_to_node(cursor);
@@ -55,7 +55,7 @@ begin_node:
*/
has_meta = 0;
for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
- ptr = ACCESS_ONCE(node->slots[slot]);
+ ptr = READ_ONCE(node->slots[slot]);
has_meta |= (unsigned long)ptr;
if (ptr && assoc_array_ptr_is_leaf(ptr)) {
/* We need a barrier between the read of the pointer
@@ -89,7 +89,7 @@ continue_node:
smp_read_barrier_depends();
for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
- ptr = ACCESS_ONCE(node->slots[slot]);
+ ptr = READ_ONCE(node->slots[slot]);
if (assoc_array_ptr_is_meta(ptr)) {
cursor = ptr;
goto begin_node;
@@ -98,7 +98,7 @@ continue_node:
finished_node:
/* Move up to the parent (may need to skip back over a shortcut) */
- parent = ACCESS_ONCE(node->back_pointer);
+ parent = READ_ONCE(node->back_pointer);
slot = node->parent_slot;
if (parent == stop)
return 0;
@@ -107,7 +107,7 @@ finished_node:
shortcut = assoc_array_ptr_to_shortcut(parent);
smp_read_barrier_depends();
cursor = parent;
- parent = ACCESS_ONCE(shortcut->back_pointer);
+ parent = READ_ONCE(shortcut->back_pointer);
slot = shortcut->parent_slot;
if (parent == stop)
return 0;
@@ -147,7 +147,7 @@ int assoc_array_iterate(const struct assoc_array *array,
void *iterator_data),
void *iterator_data)
{
- struct assoc_array_ptr *root = ACCESS_ONCE(array->root);
+ struct assoc_array_ptr *root = READ_ONCE(array->root);
if (!root)
return 0;
@@ -194,7 +194,7 @@ assoc_array_walk(const struct assoc_array *array,
pr_devel("-->%s()\n", __func__);
- cursor = ACCESS_ONCE(array->root);
+ cursor = READ_ONCE(array->root);
if (!cursor)
return assoc_array_walk_tree_empty;
@@ -220,7 +220,7 @@ consider_node:
slot = segments >> (level & ASSOC_ARRAY_KEY_CHUNK_MASK);
slot &= ASSOC_ARRAY_FAN_MASK;
- ptr = ACCESS_ONCE(node->slots[slot]);
+ ptr = READ_ONCE(node->slots[slot]);
pr_devel("consider slot %x [ix=%d type=%lu]\n",
slot, level, (unsigned long)ptr & 3);
@@ -294,7 +294,7 @@ follow_shortcut:
} while (sc_level < shortcut->skip_to_level);
/* The shortcut matches the leaf's index to this point. */
- cursor = ACCESS_ONCE(shortcut->next_node);
+ cursor = READ_ONCE(shortcut->next_node);
if (((level ^ sc_level) & ~ASSOC_ARRAY_KEY_CHUNK_MASK) != 0) {
level = sc_level;
goto jumped;
@@ -337,7 +337,7 @@ void *assoc_array_find(const struct assoc_array *array,
* the terminal node.
*/
for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
- ptr = ACCESS_ONCE(node->slots[slot]);
+ ptr = READ_ONCE(node->slots[slot]);
if (ptr && assoc_array_ptr_is_leaf(ptr)) {
/* We need a barrier between the read of the pointer
* and dereferencing the pointer - but only if we are
@@ -598,21 +598,31 @@ static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit,
if ((edit->segment_cache[ASSOC_ARRAY_FAN_OUT] ^ base_seg) == 0)
goto all_leaves_cluster_together;
- /* Otherwise we can just insert a new node ahead of the old
- * one.
+ /* Otherwise all the old leaves cluster in the same slot, but
+ * the new leaf wants to go into a different slot - so we
+ * create a new node (n0) to hold the new leaf and a pointer to
+ * a new node (n1) holding all the old leaves.
+ *
+ * This can be done by falling through to the node splitting
+ * path.
*/
- goto present_leaves_cluster_but_not_new_leaf;
+ pr_devel("present leaves cluster but not new leaf\n");
}
split_node:
pr_devel("split node\n");
- /* We need to split the current node; we know that the node doesn't
- * simply contain a full set of leaves that cluster together (it
- * contains meta pointers and/or non-clustering leaves).
+ /* We need to split the current node. The node must contain anything
+ * from a single leaf (in the one leaf case, this leaf will cluster
+ * with the new leaf) and the rest meta-pointers, to all leaves, some
+ * of which may cluster.
+ *
+ * It won't contain the case in which all the current leaves plus the
+ * new leaves want to cluster in the same slot.
*
* We need to expel at least two leaves out of a set consisting of the
- * leaves in the node and the new leaf.
+ * leaves in the node and the new leaf. The current meta pointers can
+ * just be copied as they shouldn't cluster with any of the leaves.
*
* We need a new node (n0) to replace the current one and a new node to
* take the expelled nodes (n1).
@@ -717,33 +727,6 @@ found_slot_for_multiple_occupancy:
pr_devel("<--%s() = ok [split node]\n", __func__);
return true;
-present_leaves_cluster_but_not_new_leaf:
- /* All the old leaves cluster in the same slot, but the new leaf wants
- * to go into a different slot, so we create a new node to hold the new
- * leaf and a pointer to a new node holding all the old leaves.
- */
- pr_devel("present leaves cluster but not new leaf\n");
-
- new_n0->back_pointer = node->back_pointer;
- new_n0->parent_slot = node->parent_slot;
- new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
- new_n1->back_pointer = assoc_array_node_to_ptr(new_n0);
- new_n1->parent_slot = edit->segment_cache[0];
- new_n1->nr_leaves_on_branch = node->nr_leaves_on_branch;
- edit->adjust_count_on = new_n0;
-
- for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++)
- new_n1->slots[i] = node->slots[i];
-
- new_n0->slots[edit->segment_cache[0]] = assoc_array_node_to_ptr(new_n0);
- edit->leaf_p = &new_n0->slots[edit->segment_cache[ASSOC_ARRAY_FAN_OUT]];
-
- edit->set[0].ptr = &assoc_array_ptr_to_node(node->back_pointer)->slots[node->parent_slot];
- edit->set[0].to = assoc_array_node_to_ptr(new_n0);
- edit->excised_meta[0] = assoc_array_node_to_ptr(node);
- pr_devel("<--%s() = ok [insert node before]\n", __func__);
- return true;
-
all_leaves_cluster_together:
/* All the leaves, new and old, want to cluster together in this node
* in the same slot, so we have to replace this node with a shortcut to
diff --git a/lib/atomic64_test.c b/lib/atomic64_test.c
index fd70c0e0e673..62ab629f51ca 100644
--- a/lib/atomic64_test.c
+++ b/lib/atomic64_test.c
@@ -153,8 +153,10 @@ static __init void test_atomic64(void)
long long v0 = 0xaaa31337c001d00dLL;
long long v1 = 0xdeadbeefdeafcafeLL;
long long v2 = 0xfaceabadf00df001LL;
+ long long v3 = 0x8000000000000000LL;
long long onestwos = 0x1111111122222222LL;
long long one = 1LL;
+ int r_int;
atomic64_t v = ATOMIC64_INIT(v0);
long long r = v0;
@@ -240,6 +242,11 @@ static __init void test_atomic64(void)
BUG_ON(!atomic64_inc_not_zero(&v));
r += one;
BUG_ON(v.counter != r);
+
+ /* Confirm the return value fits in an int, even if the value doesn't */
+ INIT(v3);
+ r_int = atomic64_inc_not_zero(&v);
+ BUG_ON(!r_int);
}
static __init int test_atomics_init(void)
diff --git a/lib/audit.c b/lib/audit.c
index b8fb5ee81e26..5004bff928a7 100644
--- a/lib/audit.c
+++ b/lib/audit.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/init.h>
#include <linux/types.h>
#include <linux/audit.h>
diff --git a/lib/bcd.c b/lib/bcd.c
index 40d304efe272..7e4750b6e801 100644
--- a/lib/bcd.c
+++ b/lib/bcd.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/bcd.h>
#include <linux/export.h>
diff --git a/lib/bitmap.c b/lib/bitmap.c
index 08c6ef3a2b6f..d8f0c094b18e 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -18,7 +18,9 @@
#include <asm/page.h>
-/*
+/**
+ * DOC: bitmap introduction
+ *
* bitmaps provide an array of bits, implemented using an an
* array of unsigned longs. The number of valid bits in a
* given bitmap does _not_ need to be an exact multiple of
@@ -251,7 +253,7 @@ int __bitmap_weight(const unsigned long *bitmap, unsigned int bits)
}
EXPORT_SYMBOL(__bitmap_weight);
-void bitmap_set(unsigned long *map, unsigned int start, int len)
+void __bitmap_set(unsigned long *map, unsigned int start, int len)
{
unsigned long *p = map + BIT_WORD(start);
const unsigned int size = start + len;
@@ -270,9 +272,9 @@ void bitmap_set(unsigned long *map, unsigned int start, int len)
*p |= mask_to_set;
}
}
-EXPORT_SYMBOL(bitmap_set);
+EXPORT_SYMBOL(__bitmap_set);
-void bitmap_clear(unsigned long *map, unsigned int start, int len)
+void __bitmap_clear(unsigned long *map, unsigned int start, int len)
{
unsigned long *p = map + BIT_WORD(start);
const unsigned int size = start + len;
@@ -291,7 +293,7 @@ void bitmap_clear(unsigned long *map, unsigned int start, int len)
*p &= ~mask_to_clear;
}
}
-EXPORT_SYMBOL(bitmap_clear);
+EXPORT_SYMBOL(__bitmap_clear);
/**
* bitmap_find_next_zero_area_off - find a contiguous aligned zero area
@@ -513,7 +515,7 @@ static int __bitmap_parselist(const char *buf, unsigned int buflen,
int nmaskbits)
{
unsigned int a, b, old_a, old_b;
- unsigned int group_size, used_size;
+ unsigned int group_size, used_size, off;
int c, old_c, totaldigits, ndigits;
const char __user __force *ubuf = (const char __user __force *)buf;
int at_start, in_range, in_partial_range;
@@ -599,6 +601,8 @@ static int __bitmap_parselist(const char *buf, unsigned int buflen,
a = old_a;
b = old_b;
old_a = old_b = 0;
+ } else {
+ used_size = group_size = b - a + 1;
}
/* if no digit is after '-', it's wrong*/
if (at_start && in_range)
@@ -608,17 +612,9 @@ static int __bitmap_parselist(const char *buf, unsigned int buflen,
if (b >= nmaskbits)
return -ERANGE;
while (a <= b) {
- if (in_partial_range) {
- static int pos_in_group = 1;
-
- if (pos_in_group <= used_size)
- set_bit(a, maskp);
-
- if (a == b || ++pos_in_group > group_size)
- pos_in_group = 1;
- } else
- set_bit(a, maskp);
- a++;
+ off = min(b - a + 1, used_size);
+ bitmap_set(maskp, a, off);
+ a += group_size;
}
} while (buflen && c == ',');
return 0;
diff --git a/lib/bsearch.c b/lib/bsearch.c
index e33c179089db..18b445b010c3 100644
--- a/lib/bsearch.c
+++ b/lib/bsearch.c
@@ -33,19 +33,21 @@
void *bsearch(const void *key, const void *base, size_t num, size_t size,
int (*cmp)(const void *key, const void *elt))
{
- size_t start = 0, end = num;
+ const char *pivot;
int result;
- while (start < end) {
- size_t mid = start + (end - start) / 2;
+ while (num > 0) {
+ pivot = base + (num >> 1) * size;
+ result = cmp(key, pivot);
- result = cmp(key, base + mid * size);
- if (result < 0)
- end = mid;
- else if (result > 0)
- start = mid + 1;
- else
- return (void *)base + mid * size;
+ if (result == 0)
+ return (void *)pivot;
+
+ if (result > 0) {
+ base = pivot + size;
+ num--;
+ }
+ num >>= 1;
}
return NULL;
diff --git a/lib/bug.c b/lib/bug.c
index a6a1137d06db..1e094408c893 100644
--- a/lib/bug.c
+++ b/lib/bug.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
Generic support for BUG()
diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c
index f8e0e5367398..ab719495e2cb 100644
--- a/lib/bust_spinlocks.c
+++ b/lib/bust_spinlocks.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* lib/bust_spinlocks.c
*
diff --git a/lib/check_signature.c b/lib/check_signature.c
index 6b49797980c4..43a7301da7ab 100644
--- a/lib/check_signature.c
+++ b/lib/check_signature.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/io.h>
#include <linux/export.h>
diff --git a/lib/clz_tab.c b/lib/clz_tab.c
index 7287b4a991a7..b6118d09f244 100644
--- a/lib/clz_tab.c
+++ b/lib/clz_tab.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
const unsigned char __clz_tab[] = {
0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5,
diff --git a/lib/cmdline.c b/lib/cmdline.c
index 3c6432df7e63..171c19b6888e 100644
--- a/lib/cmdline.c
+++ b/lib/cmdline.c
@@ -23,14 +23,14 @@
* the values[M, M+1, ..., N] into the ints array in get_options.
*/
-static int get_range(char **str, int *pint)
+static int get_range(char **str, int *pint, int n)
{
int x, inc_counter, upper_range;
(*str)++;
upper_range = simple_strtol((*str), NULL, 0);
inc_counter = upper_range - *pint;
- for (x = *pint; x < upper_range; x++)
+ for (x = *pint; n && x < upper_range; x++, n--)
*pint++ = x;
return inc_counter;
}
@@ -97,7 +97,7 @@ char *get_options(const char *str, int nints, int *ints)
break;
if (res == 3) {
int range_nums;
- range_nums = get_range((char **)&str, ints + i);
+ range_nums = get_range((char **)&str, ints + i, nints - i);
if (range_nums < 0)
break;
/*
@@ -244,5 +244,4 @@ char *next_arg(char *args, char **param, char **val)
/* Chew up trailing spaces. */
return skip_spaces(next);
- //return next;
}
diff --git a/lib/compat_audit.c b/lib/compat_audit.c
index 873f75b640ab..77eabad69b4a 100644
--- a/lib/compat_audit.c
+++ b/lib/compat_audit.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/init.h>
#include <linux/types.h>
#include <asm/unistd32.h>
diff --git a/lib/cpumask.c b/lib/cpumask.c
index 81dedaab36cc..35fe142ebb5e 100644
--- a/lib/cpumask.c
+++ b/lib/cpumask.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/bitops.h>
@@ -6,6 +7,22 @@
#include <linux/bootmem.h>
/**
+ * cpumask_next - get the next cpu in a cpumask
+ * @n: the cpu prior to the place to search (ie. return will be > @n)
+ * @srcp: the cpumask pointer
+ *
+ * Returns >= nr_cpu_ids if no further cpus set.
+ */
+unsigned int cpumask_next(int n, const struct cpumask *srcp)
+{
+ /* -1 is a legal arg here. */
+ if (n != -1)
+ cpumask_check(n);
+ return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);
+}
+EXPORT_SYMBOL(cpumask_next);
+
+/**
* cpumask_next_and - get the next cpu in *src1p & *src2p
* @n: the cpu prior to the place to search (ie. return will be > @n)
* @src1p: the first cpumask pointer
@@ -43,6 +60,38 @@ int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
}
EXPORT_SYMBOL(cpumask_any_but);
+/**
+ * cpumask_next_wrap - helper to implement for_each_cpu_wrap
+ * @n: the cpu prior to the place to search
+ * @mask: the cpumask pointer
+ * @start: the start point of the iteration
+ * @wrap: assume @n crossing @start terminates the iteration
+ *
+ * Returns >= nr_cpu_ids on completion
+ *
+ * Note: the @wrap argument is required for the start condition when
+ * we cannot assume @start is set in @mask.
+ */
+int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
+{
+ int next;
+
+again:
+ next = cpumask_next(n, mask);
+
+ if (wrap && n < start && next >= start) {
+ return nr_cpumask_bits;
+
+ } else if (next >= nr_cpumask_bits) {
+ wrap = true;
+ n = -1;
+ goto again;
+ }
+
+ return next;
+}
+EXPORT_SYMBOL(cpumask_next_wrap);
+
/* These are not inline because of header tangles. */
#ifdef CONFIG_CPUMASK_OFFSTACK
/**
diff --git a/lib/crc32.c b/lib/crc32.c
index 6ddc92bc1460..2ef20fe84b69 100644
--- a/lib/crc32.c
+++ b/lib/crc32.c
@@ -225,7 +225,7 @@ static u32 __attribute_const__ gf2_multiply(u32 x, u32 y, u32 modulus)
}
/**
- * crc32_generic_shift - Append len 0 bytes to crc, in logarithmic time
+ * crc32_generic_shift - Append @len 0 bytes to crc, in logarithmic time
* @crc: The original little-endian CRC (i.e. lsbit is x^31 coefficient)
* @len: The number of bytes. @crc is multiplied by x^(8*@len)
* @polynomial: The modulus used to reduce the result to 32 bits.
diff --git a/lib/crc32defs.h b/lib/crc32defs.h
index 64cba2c3c700..cb275a28a750 100644
--- a/lib/crc32defs.h
+++ b/lib/crc32defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
* There are multiple 16-bit CRC polynomials in common use, but this is
* *the* standard CRC-32 polynomial, first popularized by Ethernet.
diff --git a/lib/crc4.c b/lib/crc4.c
new file mode 100644
index 000000000000..164ed9444cd3
--- /dev/null
+++ b/lib/crc4.c
@@ -0,0 +1,46 @@
+/*
+ * crc4.c - simple crc-4 calculations.
+ *
+ * This source code is licensed under the GNU General Public License, Version
+ * 2. See the file COPYING for more details.
+ */
+
+#include <linux/crc4.h>
+#include <linux/module.h>
+
+static const uint8_t crc4_tab[] = {
+ 0x0, 0x7, 0xe, 0x9, 0xb, 0xc, 0x5, 0x2,
+ 0x1, 0x6, 0xf, 0x8, 0xa, 0xd, 0x4, 0x3,
+};
+
+/**
+ * crc4 - calculate the 4-bit crc of a value.
+ * @c: starting crc4
+ * @x: value to checksum
+ * @bits: number of bits in @x to checksum
+ *
+ * Returns the crc4 value of @x, using polynomial 0b10111.
+ *
+ * The @x value is treated as left-aligned, and bits above @bits are ignored
+ * in the crc calculations.
+ */
+uint8_t crc4(uint8_t c, uint64_t x, int bits)
+{
+ int i;
+
+ /* mask off anything above the top bit */
+ x &= (1ull << bits) - 1;
+
+ /* Align to 4-bits */
+ bits = (bits + 3) & ~0x3;
+
+ /* Calculate crc4 over four-bit nibbles, starting at the MSbit */
+ for (i = bits - 4; i >= 0; i -= 4)
+ c = crc4_tab[c ^ ((x >> i) & 0xf)];
+
+ return c;
+}
+EXPORT_SYMBOL_GPL(crc4);
+
+MODULE_DESCRIPTION("CRC4 calculations");
+MODULE_LICENSE("GPL");
diff --git a/lib/crc8.c b/lib/crc8.c
index 87b59cafdb83..595a5a75e3cd 100644
--- a/lib/crc8.c
+++ b/lib/crc8.c
@@ -20,11 +20,11 @@
#include <linux/crc8.h>
#include <linux/printk.h>
-/*
+/**
* crc8_populate_msb - fill crc table for given polynomial in reverse bit order.
*
- * table: table to be filled.
- * polynomial: polynomial for which table is to be filled.
+ * @table: table to be filled.
+ * @polynomial: polynomial for which table is to be filled.
*/
void crc8_populate_msb(u8 table[CRC8_TABLE_SIZE], u8 polynomial)
{
@@ -42,11 +42,11 @@ void crc8_populate_msb(u8 table[CRC8_TABLE_SIZE], u8 polynomial)
}
EXPORT_SYMBOL(crc8_populate_msb);
-/*
+/**
* crc8_populate_lsb - fill crc table for given polynomial in regular bit order.
*
- * table: table to be filled.
- * polynomial: polynomial for which table is to be filled.
+ * @table: table to be filled.
+ * @polynomial: polynomial for which table is to be filled.
*/
void crc8_populate_lsb(u8 table[CRC8_TABLE_SIZE], u8 polynomial)
{
@@ -63,13 +63,13 @@ void crc8_populate_lsb(u8 table[CRC8_TABLE_SIZE], u8 polynomial)
}
EXPORT_SYMBOL(crc8_populate_lsb);
-/*
+/**
* crc8 - calculate a crc8 over the given input data.
*
- * table: crc table used for calculation.
- * pdata: pointer to data buffer.
- * nbytes: number of bytes in data buffer.
- * crc: previous returned crc8 value.
+ * @table: crc table used for calculation.
+ * @pdata: pointer to data buffer.
+ * @nbytes: number of bytes in data buffer.
+ * @crc: previous returned crc8 value.
*/
u8 crc8(const u8 table[CRC8_TABLE_SIZE], u8 *pdata, size_t nbytes, u8 crc)
{
diff --git a/lib/ctype.c b/lib/ctype.c
index c646df91a2f7..c819fe269eb2 100644
--- a/lib/ctype.c
+++ b/lib/ctype.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* linux/lib/ctype.c
*
diff --git a/lib/debug_info.c b/lib/debug_info.c
index 2edbe27517ed..36daf753293c 100644
--- a/lib/debug_info.c
+++ b/lib/debug_info.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* This file exists solely to ensure debug information for some core
* data structures is included in the final image even for
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 17afb0430161..2f5349c6e81a 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -18,6 +18,7 @@
#include <linux/debugfs.h>
#include <linux/slab.h>
#include <linux/hash.h>
+#include <linux/kmemleak.h>
#define ODEBUG_HASH_BITS 14
#define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS)
@@ -110,6 +111,7 @@ static void fill_pool(void)
if (!new)
return;
+ kmemleak_ignore(new);
raw_spin_lock_irqsave(&pool_lock, flags);
hlist_add_head(&new->node, &obj_pool);
debug_objects_allocated++;
@@ -1080,6 +1082,7 @@ static int __init debug_objects_replace_static_objects(void)
obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
if (!obj)
goto free;
+ kmemleak_ignore(obj);
hlist_add_head(&obj->node, &objects);
}
diff --git a/lib/dec_and_lock.c b/lib/dec_and_lock.c
index e26278576b31..347fa7ac2e8a 100644
--- a/lib/dec_and_lock.c
+++ b/lib/dec_and_lock.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/atomic.h>
diff --git a/lib/decompress.c b/lib/decompress.c
index 62696dff5730..857ab1af1ef3 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* decompress.c
*
diff --git a/lib/decompress_inflate.c b/lib/decompress_inflate.c
index 555c06bf20da..63b4b7eee138 100644
--- a/lib/decompress_inflate.c
+++ b/lib/decompress_inflate.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#ifdef STATIC
#define PREBOOT
/* Pre-boot environment: included */
diff --git a/lib/devres.c b/lib/devres.c
index 78eca713b1d9..5f2aedd58bc5 100644
--- a/lib/devres.c
+++ b/lib/devres.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/err.h>
#include <linux/pci.h>
#include <linux/io.h>
diff --git a/lib/digsig.c b/lib/digsig.c
index 03d7c63837ae..6ba6fcd92dd1 100644
--- a/lib/digsig.c
+++ b/lib/digsig.c
@@ -87,6 +87,12 @@ static int digsig_verify_rsa(struct key *key,
down_read(&key->sem);
ukp = user_key_payload_locked(key);
+ if (!ukp) {
+ /* key was revoked before we acquired its semaphore */
+ err = -EKEYREVOKED;
+ goto err1;
+ }
+
if (ukp->datalen < sizeof(*pkh))
goto err1;
diff --git a/lib/div64.c b/lib/div64.c
index 7f345259c32f..01c8602bb6ff 100644
--- a/lib/div64.c
+++ b/lib/div64.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2003 Bernardo Innocenti <bernie@develer.com>
*
@@ -60,6 +61,12 @@ uint32_t __attribute__((weak)) __div64_32(uint64_t *n, uint32_t base)
EXPORT_SYMBOL(__div64_32);
#endif
+/**
+ * div_s64_rem - signed 64bit divide with 64bit divisor and remainder
+ * @dividend: 64bit dividend
+ * @divisor: 64bit divisor
+ * @remainder: 64bit remainder
+ */
#ifndef div_s64_rem
s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder)
{
diff --git a/lib/dma-noop.c b/lib/dma-noop.c
index de26c8b68f34..a10185b0c2d4 100644
--- a/lib/dma-noop.c
+++ b/lib/dma-noop.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* lib/dma-noop.c
*
@@ -7,6 +8,7 @@
#include <linux/mm.h>
#include <linux/dma-mapping.h>
#include <linux/scatterlist.h>
+#include <linux/pfn.h>
static void *dma_noop_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp,
@@ -16,7 +18,8 @@ static void *dma_noop_alloc(struct device *dev, size_t size,
ret = (void *)__get_free_pages(gfp, get_order(size));
if (ret)
- *dma_handle = virt_to_phys(ret);
+ *dma_handle = virt_to_phys(ret) - PFN_PHYS(dev->dma_pfn_offset);
+
return ret;
}
@@ -32,7 +35,7 @@ static dma_addr_t dma_noop_map_page(struct device *dev, struct page *page,
enum dma_data_direction dir,
unsigned long attrs)
{
- return page_to_phys(page) + offset;
+ return page_to_phys(page) + offset - PFN_PHYS(dev->dma_pfn_offset);
}
static int dma_noop_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
@@ -43,34 +46,23 @@ static int dma_noop_map_sg(struct device *dev, struct scatterlist *sgl, int nent
struct scatterlist *sg;
for_each_sg(sgl, sg, nents, i) {
+ dma_addr_t offset = PFN_PHYS(dev->dma_pfn_offset);
void *va;
BUG_ON(!sg_page(sg));
va = sg_virt(sg);
- sg_dma_address(sg) = (dma_addr_t)virt_to_phys(va);
+ sg_dma_address(sg) = (dma_addr_t)virt_to_phys(va) - offset;
sg_dma_len(sg) = sg->length;
}
return nents;
}
-static int dma_noop_mapping_error(struct device *dev, dma_addr_t dma_addr)
-{
- return 0;
-}
-
-static int dma_noop_supported(struct device *dev, u64 mask)
-{
- return 1;
-}
-
const struct dma_map_ops dma_noop_ops = {
.alloc = dma_noop_alloc,
.free = dma_noop_free,
.map_page = dma_noop_map_page,
.map_sg = dma_noop_map_sg,
- .mapping_error = dma_noop_mapping_error,
- .dma_supported = dma_noop_supported,
};
EXPORT_SYMBOL(dma_noop_ops);
diff --git a/lib/dma-virt.c b/lib/dma-virt.c
index dcd4df1f7174..8e61a02ef9ca 100644
--- a/lib/dma-virt.c
+++ b/lib/dma-virt.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* lib/dma-virt.c
*
@@ -51,22 +52,10 @@ static int dma_virt_map_sg(struct device *dev, struct scatterlist *sgl,
return nents;
}
-static int dma_virt_mapping_error(struct device *dev, dma_addr_t dma_addr)
-{
- return false;
-}
-
-static int dma_virt_supported(struct device *dev, u64 mask)
-{
- return true;
-}
-
const struct dma_map_ops dma_virt_ops = {
.alloc = dma_virt_alloc,
.free = dma_virt_free,
.map_page = dma_virt_map_page,
.map_sg = dma_virt_map_sg,
- .mapping_error = dma_virt_mapping_error,
- .dma_supported = dma_virt_supported,
};
EXPORT_SYMBOL(dma_virt_ops);
diff --git a/lib/dump_stack.c b/lib/dump_stack.c
index 625375e7f11f..c5edbedd364d 100644
--- a/lib/dump_stack.c
+++ b/lib/dump_stack.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Provide a default dump_stack() function for architectures
* which don't implement their own.
diff --git a/lib/dynamic_queue_limits.c b/lib/dynamic_queue_limits.c
index f346715e2255..da4672a50a54 100644
--- a/lib/dynamic_queue_limits.c
+++ b/lib/dynamic_queue_limits.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Dynamic byte queue limits. See include/linux/dynamic_queue_limits.h
*
@@ -20,7 +21,7 @@ void dql_completed(struct dql *dql, unsigned int count)
unsigned int ovlimit, completed, num_queued;
bool all_prev_completed;
- num_queued = ACCESS_ONCE(dql->num_queued);
+ num_queued = READ_ONCE(dql->num_queued);
/* Can't complete more than what's in queue */
BUG_ON(count > num_queued - dql->num_completed);
diff --git a/lib/errseq.c b/lib/errseq.c
new file mode 100644
index 000000000000..79cc66897db4
--- /dev/null
+++ b/lib/errseq.c
@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/err.h>
+#include <linux/bug.h>
+#include <linux/atomic.h>
+#include <linux/errseq.h>
+
+/*
+ * An errseq_t is a way of recording errors in one place, and allowing any
+ * number of "subscribers" to tell whether it has changed since a previous
+ * point where it was sampled.
+ *
+ * It's implemented as an unsigned 32-bit value. The low order bits are
+ * designated to hold an error code (between 0 and -MAX_ERRNO). The upper bits
+ * are used as a counter. This is done with atomics instead of locking so that
+ * these functions can be called from any context.
+ *
+ * The general idea is for consumers to sample an errseq_t value. That value
+ * can later be used to tell whether any new errors have occurred since that
+ * sampling was done.
+ *
+ * Note that there is a risk of collisions if new errors are being recorded
+ * frequently, since we have so few bits to use as a counter.
+ *
+ * To mitigate this, one bit is used as a flag to tell whether the value has
+ * been sampled since a new value was recorded. That allows us to avoid bumping
+ * the counter if no one has sampled it since the last time an error was
+ * recorded.
+ *
+ * A new errseq_t should always be zeroed out. A errseq_t value of all zeroes
+ * is the special (but common) case where there has never been an error. An all
+ * zero value thus serves as the "epoch" if one wishes to know whether there
+ * has ever been an error set since it was first initialized.
+ */
+
+/* The low bits are designated for error code (max of MAX_ERRNO) */
+#define ERRSEQ_SHIFT ilog2(MAX_ERRNO + 1)
+
+/* This bit is used as a flag to indicate whether the value has been seen */
+#define ERRSEQ_SEEN (1 << ERRSEQ_SHIFT)
+
+/* The lowest bit of the counter */
+#define ERRSEQ_CTR_INC (1 << (ERRSEQ_SHIFT + 1))
+
+/**
+ * errseq_set - set a errseq_t for later reporting
+ * @eseq: errseq_t field that should be set
+ * @err: error to set (must be between -1 and -MAX_ERRNO)
+ *
+ * This function sets the error in *eseq, and increments the sequence counter
+ * if the last sequence was sampled at some point in the past.
+ *
+ * Any error set will always overwrite an existing error.
+ *
+ * We do return the latest value here, primarily for debugging purposes. The
+ * return value should not be used as a previously sampled value in later calls
+ * as it will not have the SEEN flag set.
+ */
+errseq_t errseq_set(errseq_t *eseq, int err)
+{
+ errseq_t cur, old;
+
+ /* MAX_ERRNO must be able to serve as a mask */
+ BUILD_BUG_ON_NOT_POWER_OF_2(MAX_ERRNO + 1);
+
+ /*
+ * Ensure the error code actually fits where we want it to go. If it
+ * doesn't then just throw a warning and don't record anything. We
+ * also don't accept zero here as that would effectively clear a
+ * previous error.
+ */
+ old = READ_ONCE(*eseq);
+
+ if (WARN(unlikely(err == 0 || (unsigned int)-err > MAX_ERRNO),
+ "err = %d\n", err))
+ return old;
+
+ for (;;) {
+ errseq_t new;
+
+ /* Clear out error bits and set new error */
+ new = (old & ~(MAX_ERRNO|ERRSEQ_SEEN)) | -err;
+
+ /* Only increment if someone has looked at it */
+ if (old & ERRSEQ_SEEN)
+ new += ERRSEQ_CTR_INC;
+
+ /* If there would be no change, then call it done */
+ if (new == old) {
+ cur = new;
+ break;
+ }
+
+ /* Try to swap the new value into place */
+ cur = cmpxchg(eseq, old, new);
+
+ /*
+ * Call it success if we did the swap or someone else beat us
+ * to it for the same value.
+ */
+ if (likely(cur == old || cur == new))
+ break;
+
+ /* Raced with an update, try again */
+ old = cur;
+ }
+ return cur;
+}
+EXPORT_SYMBOL(errseq_set);
+
+/**
+ * errseq_sample - grab current errseq_t value
+ * @eseq: pointer to errseq_t to be sampled
+ *
+ * This function allows callers to sample an errseq_t value, marking it as
+ * "seen" if required.
+ */
+errseq_t errseq_sample(errseq_t *eseq)
+{
+ errseq_t old = READ_ONCE(*eseq);
+ errseq_t new = old;
+
+ /*
+ * For the common case of no errors ever having been set, we can skip
+ * marking the SEEN bit. Once an error has been set, the value will
+ * never go back to zero.
+ */
+ if (old != 0) {
+ new |= ERRSEQ_SEEN;
+ if (old != new)
+ cmpxchg(eseq, old, new);
+ }
+ return new;
+}
+EXPORT_SYMBOL(errseq_sample);
+
+/**
+ * errseq_check - has an error occurred since a particular sample point?
+ * @eseq: pointer to errseq_t value to be checked
+ * @since: previously-sampled errseq_t from which to check
+ *
+ * Grab the value that eseq points to, and see if it has changed "since"
+ * the given value was sampled. The "since" value is not advanced, so there
+ * is no need to mark the value as seen.
+ *
+ * Returns the latest error set in the errseq_t or 0 if it hasn't changed.
+ */
+int errseq_check(errseq_t *eseq, errseq_t since)
+{
+ errseq_t cur = READ_ONCE(*eseq);
+
+ if (likely(cur == since))
+ return 0;
+ return -(cur & MAX_ERRNO);
+}
+EXPORT_SYMBOL(errseq_check);
+
+/**
+ * errseq_check_and_advance - check an errseq_t and advance to current value
+ * @eseq: pointer to value being checked and reported
+ * @since: pointer to previously-sampled errseq_t to check against and advance
+ *
+ * Grab the eseq value, and see whether it matches the value that "since"
+ * points to. If it does, then just return 0.
+ *
+ * If it doesn't, then the value has changed. Set the "seen" flag, and try to
+ * swap it into place as the new eseq value. Then, set that value as the new
+ * "since" value, and return whatever the error portion is set to.
+ *
+ * Note that no locking is provided here for concurrent updates to the "since"
+ * value. The caller must provide that if necessary. Because of this, callers
+ * may want to do a lockless errseq_check before taking the lock and calling
+ * this.
+ */
+int errseq_check_and_advance(errseq_t *eseq, errseq_t *since)
+{
+ int err = 0;
+ errseq_t old, new;
+
+ /*
+ * Most callers will want to use the inline wrapper to check this,
+ * so that the common case of no error is handled without needing
+ * to take the lock that protects the "since" value.
+ */
+ old = READ_ONCE(*eseq);
+ if (old != *since) {
+ /*
+ * Set the flag and try to swap it into place if it has
+ * changed.
+ *
+ * We don't care about the outcome of the swap here. If the
+ * swap doesn't occur, then it has either been updated by a
+ * writer who is altering the value in some way (updating
+ * counter or resetting the error), or another reader who is
+ * just setting the "seen" flag. Either outcome is OK, and we
+ * can advance "since" and return an error based on what we
+ * have.
+ */
+ new = old | ERRSEQ_SEEN;
+ if (new != old)
+ cmpxchg(eseq, old, new);
+ *since = new;
+ err = -(new & MAX_ERRNO);
+ }
+ return err;
+}
+EXPORT_SYMBOL(errseq_check_and_advance);
diff --git a/lib/extable.c b/lib/extable.c
index 62968daa66a9..f54996fdd0b8 100644
--- a/lib/extable.c
+++ b/lib/extable.c
@@ -9,6 +9,7 @@
* 2 of the License, or (at your option) any later version.
*/
+#include <linux/bsearch.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/sort.h>
@@ -51,7 +52,7 @@ static void swap_ex(void *a, void *b, int size)
* This is used both for the kernel exception table and for
* the exception tables of modules that get loaded.
*/
-static int cmp_ex(const void *a, const void *b)
+static int cmp_ex_sort(const void *a, const void *b)
{
const struct exception_table_entry *x = a, *y = b;
@@ -67,7 +68,7 @@ void sort_extable(struct exception_table_entry *start,
struct exception_table_entry *finish)
{
sort(start, finish - start, sizeof(struct exception_table_entry),
- cmp_ex, swap_ex);
+ cmp_ex_sort, swap_ex);
}
#ifdef CONFIG_MODULES
@@ -93,6 +94,20 @@ void trim_init_extable(struct module *m)
#endif /* !ARCH_HAS_SORT_EXTABLE */
#ifndef ARCH_HAS_SEARCH_EXTABLE
+
+static int cmp_ex_search(const void *key, const void *elt)
+{
+ const struct exception_table_entry *_elt = elt;
+ unsigned long _key = *(unsigned long *)key;
+
+ /* avoid overflow */
+ if (_key > ex_to_insn(_elt))
+ return 1;
+ if (_key < ex_to_insn(_elt))
+ return -1;
+ return 0;
+}
+
/*
* Search one exception table for an entry corresponding to the
* given instruction address, and return the address of the entry,
@@ -101,25 +116,11 @@ void trim_init_extable(struct module *m)
* already sorted.
*/
const struct exception_table_entry *
-search_extable(const struct exception_table_entry *first,
- const struct exception_table_entry *last,
+search_extable(const struct exception_table_entry *base,
+ const size_t num,
unsigned long value)
{
- while (first <= last) {
- const struct exception_table_entry *mid;
-
- mid = ((last - first) >> 1) + first;
- /*
- * careful, the distance between value and insn
- * can be larger than MAX_LONG:
- */
- if (ex_to_insn(mid) < value)
- first = mid + 1;
- else if (ex_to_insn(mid) > value)
- last = mid - 1;
- else
- return mid;
- }
- return NULL;
+ return bsearch(&value, base, num,
+ sizeof(struct exception_table_entry), cmp_ex_search);
}
#endif
diff --git a/lib/fault-inject.c b/lib/fault-inject.c
index 4ff157159a0d..cf7b129b0b2b 100644
--- a/lib/fault-inject.c
+++ b/lib/fault-inject.c
@@ -107,6 +107,17 @@ static inline bool fail_stacktrace(struct fault_attr *attr)
bool should_fail(struct fault_attr *attr, ssize_t size)
{
+ if (in_task()) {
+ unsigned int fail_nth = READ_ONCE(current->fail_nth);
+
+ if (fail_nth) {
+ if (!WRITE_ONCE(current->fail_nth, fail_nth - 1))
+ goto fail;
+
+ return false;
+ }
+ }
+
/* No need to check any other properties if the probability is 0 */
if (attr->probability == 0)
return false;
@@ -134,6 +145,7 @@ bool should_fail(struct fault_attr *attr, ssize_t size)
if (!fail_stacktrace(attr))
return false;
+fail:
fail_dump(attr);
if (atomic_read(&attr->times) != -1)
diff --git a/lib/flex_proportions.c b/lib/flex_proportions.c
index a71cf1bdd4c9..7852bfff50b1 100644
--- a/lib/flex_proportions.c
+++ b/lib/flex_proportions.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Floating proportions with flexible aging period
*
@@ -207,7 +208,7 @@ static void fprop_reflect_period_percpu(struct fprop_global *p,
if (val < (nr_cpu_ids * PROP_BATCH))
val = percpu_counter_sum(&pl->events);
- __percpu_counter_add(&pl->events,
+ percpu_counter_add_batch(&pl->events,
-val + (val >> (period-pl->period)), PROP_BATCH);
} else
percpu_counter_set(&pl->events, 0);
@@ -219,7 +220,7 @@ static void fprop_reflect_period_percpu(struct fprop_global *p,
void __fprop_inc_percpu(struct fprop_global *p, struct fprop_local_percpu *pl)
{
fprop_reflect_period_percpu(p, pl);
- __percpu_counter_add(&pl->events, 1, PROP_BATCH);
+ percpu_counter_add_batch(&pl->events, 1, PROP_BATCH);
percpu_counter_add(&p->events, 1);
}
@@ -267,6 +268,6 @@ void __fprop_inc_percpu_max(struct fprop_global *p,
return;
} else
fprop_reflect_period_percpu(p, pl);
- __percpu_counter_add(&pl->events, 1, PROP_BATCH);
+ percpu_counter_add_batch(&pl->events, 1, PROP_BATCH);
percpu_counter_add(&p->events, 1);
}
diff --git a/lib/fonts/Makefile b/lib/fonts/Makefile
index e04d010cfbf5..d56f02dea83a 100644
--- a/lib/fonts/Makefile
+++ b/lib/fonts/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
# Font handling
font-objs := fonts.o
diff --git a/lib/fonts/font_10x18.c b/lib/fonts/font_10x18.c
index 6be72bb218ee..532f0ff89a96 100644
--- a/lib/fonts/font_10x18.c
+++ b/lib/fonts/font_10x18.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/********************************
* adapted from font_sun12x22.c *
* by Jurriaan Kalkman 06-2005 *
diff --git a/lib/fonts/font_6x10.c b/lib/fonts/font_6x10.c
index b20620904d31..09b2cc03435b 100644
--- a/lib/fonts/font_6x10.c
+++ b/lib/fonts/font_6x10.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/font.h>
static const unsigned char fontdata_6x10[] = {
diff --git a/lib/fonts/font_6x11.c b/lib/fonts/font_6x11.c
index 46e86e67aa6a..d7136c33f1f0 100644
--- a/lib/fonts/font_6x11.c
+++ b/lib/fonts/font_6x11.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/**********************************************/
/* */
/* Font file generated by rthelen */
diff --git a/lib/fonts/font_7x14.c b/lib/fonts/font_7x14.c
index 3b7dbf9c060b..9ae5b62c8a0d 100644
--- a/lib/fonts/font_7x14.c
+++ b/lib/fonts/font_7x14.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/**************************************/
/* this file adapted from font_8x16.c */
/* by Jurriaan Kalkman 05-2005 */
diff --git a/lib/fonts/font_8x16.c b/lib/fonts/font_8x16.c
index 00a0c67a5c7d..34292cdfaa23 100644
--- a/lib/fonts/font_8x16.c
+++ b/lib/fonts/font_8x16.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/**********************************************/
/* */
/* Font file generated by cpi2fnt */
diff --git a/lib/fonts/font_8x8.c b/lib/fonts/font_8x8.c
index 9f56efe2cee7..751becf3c521 100644
--- a/lib/fonts/font_8x8.c
+++ b/lib/fonts/font_8x8.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/**********************************************/
/* */
/* Font file generated by cpi2fnt */
diff --git a/lib/fonts/font_acorn_8x8.c b/lib/fonts/font_acorn_8x8.c
index 639e31ae1100..0ff0e85d4481 100644
--- a/lib/fonts/font_acorn_8x8.c
+++ b/lib/fonts/font_acorn_8x8.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/* Acorn-like font definition, with PC graphics characters */
#include <linux/font.h>
diff --git a/lib/fonts/font_pearl_8x8.c b/lib/fonts/font_pearl_8x8.c
index dc6ad539ca4e..b0514c0a7445 100644
--- a/lib/fonts/font_pearl_8x8.c
+++ b/lib/fonts/font_pearl_8x8.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/**********************************************/
/* */
/* Font file generated by cpi2fnt */
diff --git a/lib/fonts/font_sun12x22.c b/lib/fonts/font_sun12x22.c
index d3643853c33a..955d6eee3959 100644
--- a/lib/fonts/font_sun12x22.c
+++ b/lib/fonts/font_sun12x22.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/font.h>
#define FONTDATAMAX 11264
diff --git a/lib/fonts/font_sun8x16.c b/lib/fonts/font_sun8x16.c
index 268151325b83..03d71e53954a 100644
--- a/lib/fonts/font_sun8x16.c
+++ b/lib/fonts/font_sun8x16.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/font.h>
#define FONTDATAMAX 4096
diff --git a/lib/gcd.c b/lib/gcd.c
index 135ee6407a5e..227dea924425 100644
--- a/lib/gcd.c
+++ b/lib/gcd.c
@@ -13,6 +13,12 @@
#if !defined(CONFIG_CPU_NO_EFFICIENT_FFS) && !defined(CPU_NO_EFFICIENT_FFS)
/* If __ffs is available, the even/odd algorithm benchmarks slower. */
+
+/**
+ * gcd - calculate and return the greatest common divisor of 2 unsigned longs
+ * @a: first value
+ * @b: second value
+ */
unsigned long gcd(unsigned long a, unsigned long b)
{
unsigned long r = a | b;
diff --git a/lib/gen_crc32table.c b/lib/gen_crc32table.c
index d83a372fa76f..8f26660ea10a 100644
--- a/lib/gen_crc32table.c
+++ b/lib/gen_crc32table.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <stdio.h>
#include "../include/generated/autoconf.h"
#include "crc32defs.h"
diff --git a/lib/hexdump.c b/lib/hexdump.c
index 992457b1284c..81b70ed37209 100644
--- a/lib/hexdump.c
+++ b/lib/hexdump.c
@@ -9,6 +9,7 @@
#include <linux/types.h>
#include <linux/ctype.h>
+#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <asm/unaligned.h>
@@ -42,7 +43,7 @@ EXPORT_SYMBOL(hex_to_bin);
* @src: ascii hexadecimal string
* @count: result length
*
- * Return 0 on success, -1 in case of bad input.
+ * Return 0 on success, -EINVAL in case of bad input.
*/
int hex2bin(u8 *dst, const char *src, size_t count)
{
@@ -51,7 +52,7 @@ int hex2bin(u8 *dst, const char *src, size_t count)
int lo = hex_to_bin(*src++);
if ((hi < 0) || (lo < 0))
- return -1;
+ return -EINVAL;
*dst++ = (hi << 4) | lo;
}
diff --git a/lib/hweight.c b/lib/hweight.c
index 43273a7d83cf..7660d88fd496 100644
--- a/lib/hweight.c
+++ b/lib/hweight.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/bitops.h>
#include <asm/types.h>
diff --git a/lib/idr.c b/lib/idr.c
index b13682bb0a1c..edd9b2be1651 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -7,45 +7,32 @@
DEFINE_PER_CPU(struct ida_bitmap *, ida_bitmap);
static DEFINE_SPINLOCK(simple_ida_lock);
-/**
- * idr_alloc - allocate an id
- * @idr: idr handle
- * @ptr: pointer to be associated with the new id
- * @start: the minimum id (inclusive)
- * @end: the maximum id (exclusive)
- * @gfp: memory allocation flags
- *
- * Allocates an unused ID in the range [start, end). Returns -ENOSPC
- * if there are no unused IDs in that range.
- *
- * Note that @end is treated as max when <= 0. This is to always allow
- * using @start + N as @end as long as N is inside integer range.
- *
- * Simultaneous modifications to the @idr are not allowed and should be
- * prevented by the user, usually with a lock. idr_alloc() may be called
- * concurrently with read-only accesses to the @idr, such as idr_find() and
- * idr_for_each_entry().
- */
-int idr_alloc(struct idr *idr, void *ptr, int start, int end, gfp_t gfp)
+int idr_alloc_cmn(struct idr *idr, void *ptr, unsigned long *index,
+ unsigned long start, unsigned long end, gfp_t gfp,
+ bool ext)
{
- void __rcu **slot;
struct radix_tree_iter iter;
+ void __rcu **slot;
- if (WARN_ON_ONCE(start < 0))
- return -EINVAL;
if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr)))
return -EINVAL;
radix_tree_iter_init(&iter, start);
- slot = idr_get_free(&idr->idr_rt, &iter, gfp, end);
+ if (ext)
+ slot = idr_get_free_ext(&idr->idr_rt, &iter, gfp, end);
+ else
+ slot = idr_get_free(&idr->idr_rt, &iter, gfp, end);
if (IS_ERR(slot))
return PTR_ERR(slot);
radix_tree_iter_replace(&idr->idr_rt, &iter, slot, ptr);
radix_tree_iter_tag_clear(&idr->idr_rt, &iter, IDR_FREE);
- return iter.index;
+
+ if (index)
+ *index = iter.index;
+ return 0;
}
-EXPORT_SYMBOL_GPL(idr_alloc);
+EXPORT_SYMBOL_GPL(idr_alloc_cmn);
/**
* idr_alloc_cyclic - allocate new idr entry in a cyclical fashion
@@ -134,6 +121,20 @@ void *idr_get_next(struct idr *idr, int *nextid)
}
EXPORT_SYMBOL(idr_get_next);
+void *idr_get_next_ext(struct idr *idr, unsigned long *nextid)
+{
+ struct radix_tree_iter iter;
+ void __rcu **slot;
+
+ slot = radix_tree_iter_find(&idr->idr_rt, &iter, *nextid);
+ if (!slot)
+ return NULL;
+
+ *nextid = iter.index;
+ return rcu_dereference_raw(*slot);
+}
+EXPORT_SYMBOL(idr_get_next_ext);
+
/**
* idr_replace - replace pointer for given id
* @idr: idr handle
@@ -145,17 +146,24 @@ EXPORT_SYMBOL(idr_get_next);
* idr_alloc() and idr_remove() (as long as the ID being removed is not
* the one being replaced!).
*
- * Returns: 0 on success. %-ENOENT indicates that @id was not found.
- * %-EINVAL indicates that @id or @ptr were not valid.
+ * Returns: the old value on success. %-ENOENT indicates that @id was not
+ * found. %-EINVAL indicates that @id or @ptr were not valid.
*/
void *idr_replace(struct idr *idr, void *ptr, int id)
{
+ if (id < 0)
+ return ERR_PTR(-EINVAL);
+
+ return idr_replace_ext(idr, ptr, id);
+}
+EXPORT_SYMBOL(idr_replace);
+
+void *idr_replace_ext(struct idr *idr, void *ptr, unsigned long id)
+{
struct radix_tree_node *node;
void __rcu **slot = NULL;
void *entry;
- if (WARN_ON_ONCE(id < 0))
- return ERR_PTR(-EINVAL);
if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr)))
return ERR_PTR(-EINVAL);
@@ -167,7 +175,7 @@ void *idr_replace(struct idr *idr, void *ptr, int id)
return entry;
}
-EXPORT_SYMBOL(idr_replace);
+EXPORT_SYMBOL(idr_replace_ext);
/**
* DOC: IDA description
diff --git a/lib/inflate.c b/lib/inflate.c
index 013a76193481..fbaf03c1748d 100644
--- a/lib/inflate.c
+++ b/lib/inflate.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#define DEBG(x)
#define DEBG1(x)
/* inflate.c -- Not copyrighted 1992 by Mark Adler
diff --git a/lib/int_sqrt.c b/lib/int_sqrt.c
index 1ef4cc344977..db0b5aa071fc 100644
--- a/lib/int_sqrt.c
+++ b/lib/int_sqrt.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2013 Davidlohr Bueso <davidlohr.bueso@hp.com>
*
diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c
index 245900b98c8e..0e343fd29570 100644
--- a/lib/interval_tree_test.c
+++ b/lib/interval_tree_test.c
@@ -1,27 +1,38 @@
#include <linux/module.h>
+#include <linux/moduleparam.h>
#include <linux/interval_tree.h>
#include <linux/random.h>
+#include <linux/slab.h>
#include <asm/timex.h>
-#define NODES 100
-#define PERF_LOOPS 100000
-#define SEARCHES 100
-#define SEARCH_LOOPS 10000
+#define __param(type, name, init, msg) \
+ static type name = init; \
+ module_param(name, type, 0444); \
+ MODULE_PARM_DESC(name, msg);
-static struct rb_root root = RB_ROOT;
-static struct interval_tree_node nodes[NODES];
-static u32 queries[SEARCHES];
+__param(int, nnodes, 100, "Number of nodes in the interval tree");
+__param(int, perf_loops, 100000, "Number of iterations modifying the tree");
+
+__param(int, nsearches, 100, "Number of searches to the interval tree");
+__param(int, search_loops, 10000, "Number of iterations searching the tree");
+__param(bool, search_all, false, "Searches will iterate all nodes in the tree");
+
+__param(uint, max_endpoint, ~0, "Largest value for the interval's endpoint");
+
+static struct rb_root_cached root = RB_ROOT_CACHED;
+static struct interval_tree_node *nodes = NULL;
+static u32 *queries = NULL;
static struct rnd_state rnd;
static inline unsigned long
-search(unsigned long query, struct rb_root *root)
+search(struct rb_root_cached *root, unsigned long start, unsigned long last)
{
struct interval_tree_node *node;
unsigned long results = 0;
- for (node = interval_tree_iter_first(root, query, query); node;
- node = interval_tree_iter_next(node, query, query))
+ for (node = interval_tree_iter_first(root, start, last); node;
+ node = interval_tree_iter_next(node, start, last))
results++;
return results;
}
@@ -29,19 +40,22 @@ search(unsigned long query, struct rb_root *root)
static void init(void)
{
int i;
- for (i = 0; i < NODES; i++) {
- u32 a = prandom_u32_state(&rnd);
- u32 b = prandom_u32_state(&rnd);
- if (a <= b) {
- nodes[i].start = a;
- nodes[i].last = b;
- } else {
- nodes[i].start = b;
- nodes[i].last = a;
- }
+
+ for (i = 0; i < nnodes; i++) {
+ u32 b = (prandom_u32_state(&rnd) >> 4) % max_endpoint;
+ u32 a = (prandom_u32_state(&rnd) >> 4) % b;
+
+ nodes[i].start = a;
+ nodes[i].last = b;
}
- for (i = 0; i < SEARCHES; i++)
- queries[i] = prandom_u32_state(&rnd);
+
+ /*
+ * Limit the search scope to what the user defined.
+ * Otherwise we are merely measuring empty walks,
+ * which is pointless.
+ */
+ for (i = 0; i < nsearches; i++)
+ queries[i] = (prandom_u32_state(&rnd) >> 4) % max_endpoint;
}
static int interval_tree_test_init(void)
@@ -50,6 +64,16 @@ static int interval_tree_test_init(void)
unsigned long results;
cycles_t time1, time2, time;
+ nodes = kmalloc(nnodes * sizeof(struct interval_tree_node), GFP_KERNEL);
+ if (!nodes)
+ return -ENOMEM;
+
+ queries = kmalloc(nsearches * sizeof(int), GFP_KERNEL);
+ if (!queries) {
+ kfree(nodes);
+ return -ENOMEM;
+ }
+
printk(KERN_ALERT "interval tree insert/remove");
prandom_seed_state(&rnd, 3141592653589793238ULL);
@@ -57,39 +81,46 @@ static int interval_tree_test_init(void)
time1 = get_cycles();
- for (i = 0; i < PERF_LOOPS; i++) {
- for (j = 0; j < NODES; j++)
+ for (i = 0; i < perf_loops; i++) {
+ for (j = 0; j < nnodes; j++)
interval_tree_insert(nodes + j, &root);
- for (j = 0; j < NODES; j++)
+ for (j = 0; j < nnodes; j++)
interval_tree_remove(nodes + j, &root);
}
time2 = get_cycles();
time = time2 - time1;
- time = div_u64(time, PERF_LOOPS);
+ time = div_u64(time, perf_loops);
printk(" -> %llu cycles\n", (unsigned long long)time);
printk(KERN_ALERT "interval tree search");
- for (j = 0; j < NODES; j++)
+ for (j = 0; j < nnodes; j++)
interval_tree_insert(nodes + j, &root);
time1 = get_cycles();
results = 0;
- for (i = 0; i < SEARCH_LOOPS; i++)
- for (j = 0; j < SEARCHES; j++)
- results += search(queries[j], &root);
+ for (i = 0; i < search_loops; i++)
+ for (j = 0; j < nsearches; j++) {
+ unsigned long start = search_all ? 0 : queries[j];
+ unsigned long last = search_all ? max_endpoint : queries[j];
+
+ results += search(&root, start, last);
+ }
time2 = get_cycles();
time = time2 - time1;
- time = div_u64(time, SEARCH_LOOPS);
- results = div_u64(results, SEARCH_LOOPS);
+ time = div_u64(time, search_loops);
+ results = div_u64(results, search_loops);
printk(" -> %llu cycles (%lu results)\n",
(unsigned long long)time, results);
+ kfree(queries);
+ kfree(nodes);
+
return -EAGAIN; /* Fail will directly unload the module */
}
diff --git a/lib/iomap.c b/lib/iomap.c
index fc3dcb4b238e..541d926da95e 100644
--- a/lib/iomap.c
+++ b/lib/iomap.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Implement the default iomap interfaces
*
diff --git a/lib/iommu-common.c b/lib/iommu-common.c
index 858dc1aae478..55b00de106b5 100644
--- a/lib/iommu-common.c
+++ b/lib/iommu-common.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* IOMMU mmap management and range allocation functions.
* Based almost entirely upon the powerpc iommu allocator.
diff --git a/lib/iommu-helper.c b/lib/iommu-helper.c
index a816f3a80625..23633c0fda4a 100644
--- a/lib/iommu-helper.c
+++ b/lib/iommu-helper.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* IOMMU helper functions for the free area management
*/
diff --git a/lib/ioremap.c b/lib/ioremap.c
index 4bb30206b942..b808a390e4c3 100644
--- a/lib/ioremap.c
+++ b/lib/ioremap.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Re-map IO memory to kernel address space so that we can access it.
* This is needed for high PCI addresses that aren't mapped in the
@@ -161,6 +162,7 @@ int ioremap_page_range(unsigned long addr,
unsigned long next;
int err;
+ might_sleep();
BUG_ON(addr >= end);
start = addr;
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index f835964c9485..1c1c06ddc20a 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -130,6 +130,24 @@
} \
}
+static int copyout(void __user *to, const void *from, size_t n)
+{
+ if (access_ok(VERIFY_WRITE, to, n)) {
+ kasan_check_read(from, n);
+ n = raw_copy_to_user(to, from, n);
+ }
+ return n;
+}
+
+static int copyin(void *to, const void __user *from, size_t n)
+{
+ if (access_ok(VERIFY_READ, from, n)) {
+ kasan_check_write(to, n);
+ n = raw_copy_from_user(to, from, n);
+ }
+ return n;
+}
+
static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i)
{
@@ -144,6 +162,7 @@ static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t b
if (unlikely(!bytes))
return 0;
+ might_fault();
wanted = bytes;
iov = i->iov;
skip = i->iov_offset;
@@ -155,7 +174,7 @@ static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t b
from = kaddr + offset;
/* first chunk, usually the only one */
- left = __copy_to_user_inatomic(buf, from, copy);
+ left = copyout(buf, from, copy);
copy -= left;
skip += copy;
from += copy;
@@ -165,7 +184,7 @@ static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t b
iov++;
buf = iov->iov_base;
copy = min(bytes, iov->iov_len);
- left = __copy_to_user_inatomic(buf, from, copy);
+ left = copyout(buf, from, copy);
copy -= left;
skip = copy;
from += copy;
@@ -184,7 +203,7 @@ static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t b
kaddr = kmap(page);
from = kaddr + offset;
- left = __copy_to_user(buf, from, copy);
+ left = copyout(buf, from, copy);
copy -= left;
skip += copy;
from += copy;
@@ -193,7 +212,7 @@ static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t b
iov++;
buf = iov->iov_base;
copy = min(bytes, iov->iov_len);
- left = __copy_to_user(buf, from, copy);
+ left = copyout(buf, from, copy);
copy -= left;
skip = copy;
from += copy;
@@ -227,6 +246,7 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t
if (unlikely(!bytes))
return 0;
+ might_fault();
wanted = bytes;
iov = i->iov;
skip = i->iov_offset;
@@ -238,7 +258,7 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t
to = kaddr + offset;
/* first chunk, usually the only one */
- left = __copy_from_user_inatomic(to, buf, copy);
+ left = copyin(to, buf, copy);
copy -= left;
skip += copy;
to += copy;
@@ -248,7 +268,7 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t
iov++;
buf = iov->iov_base;
copy = min(bytes, iov->iov_len);
- left = __copy_from_user_inatomic(to, buf, copy);
+ left = copyin(to, buf, copy);
copy -= left;
skip = copy;
to += copy;
@@ -267,7 +287,7 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t
kaddr = kmap(page);
to = kaddr + offset;
- left = __copy_from_user(to, buf, copy);
+ left = copyin(to, buf, copy);
copy -= left;
skip += copy;
to += copy;
@@ -276,7 +296,7 @@ static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t
iov++;
buf = iov->iov_base;
copy = min(bytes, iov->iov_len);
- left = __copy_from_user(to, buf, copy);
+ left = copyin(to, buf, copy);
copy -= left;
skip = copy;
to += copy;
@@ -535,14 +555,15 @@ static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
return bytes;
}
-size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
+size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
{
const char *from = addr;
if (unlikely(i->type & ITER_PIPE))
return copy_pipe_to_iter(addr, bytes, i);
+ if (iter_is_iovec(i))
+ might_fault();
iterate_and_advance(i, bytes, v,
- __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
- v.iov_len),
+ copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
memcpy_to_page(v.bv_page, v.bv_offset,
(from += v.bv_len) - v.bv_len, v.bv_len),
memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
@@ -550,18 +571,19 @@ size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
return bytes;
}
-EXPORT_SYMBOL(copy_to_iter);
+EXPORT_SYMBOL(_copy_to_iter);
-size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
+size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
{
char *to = addr;
if (unlikely(i->type & ITER_PIPE)) {
WARN_ON(1);
return 0;
}
+ if (iter_is_iovec(i))
+ might_fault();
iterate_and_advance(i, bytes, v,
- __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
- v.iov_len),
+ copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
v.bv_offset, v.bv_len),
memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
@@ -569,9 +591,9 @@ size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
return bytes;
}
-EXPORT_SYMBOL(copy_from_iter);
+EXPORT_SYMBOL(_copy_from_iter);
-bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
+bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
{
char *to = addr;
if (unlikely(i->type & ITER_PIPE)) {
@@ -581,8 +603,10 @@ bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
if (unlikely(i->count < bytes))
return false;
+ if (iter_is_iovec(i))
+ might_fault();
iterate_all_kinds(i, bytes, v, ({
- if (__copy_from_user((to += v.iov_len) - v.iov_len,
+ if (copyin((to += v.iov_len) - v.iov_len,
v.iov_base, v.iov_len))
return false;
0;}),
@@ -594,9 +618,9 @@ bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
iov_iter_advance(i, bytes);
return true;
}
-EXPORT_SYMBOL(copy_from_iter_full);
+EXPORT_SYMBOL(_copy_from_iter_full);
-size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
+size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
{
char *to = addr;
if (unlikely(i->type & ITER_PIPE)) {
@@ -613,9 +637,31 @@ size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
return bytes;
}
-EXPORT_SYMBOL(copy_from_iter_nocache);
+EXPORT_SYMBOL(_copy_from_iter_nocache);
+
+#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
+size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ if (unlikely(i->type & ITER_PIPE)) {
+ WARN_ON(1);
+ return 0;
+ }
+ iterate_and_advance(i, bytes, v,
+ __copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
+ v.iov_base, v.iov_len),
+ memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
+ v.bv_offset, v.bv_len),
+ memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
+ v.iov_len)
+ )
+
+ return bytes;
+}
+EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
+#endif
-bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
+bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
{
char *to = addr;
if (unlikely(i->type & ITER_PIPE)) {
@@ -637,11 +683,24 @@ bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
iov_iter_advance(i, bytes);
return true;
}
-EXPORT_SYMBOL(copy_from_iter_full_nocache);
+EXPORT_SYMBOL(_copy_from_iter_full_nocache);
+
+static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
+{
+ struct page *head = compound_head(page);
+ size_t v = n + offset + page_address(page) - page_address(head);
+
+ if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head))))
+ return true;
+ WARN_ON(1);
+ return false;
+}
size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i)
{
+ if (unlikely(!page_copy_sane(page, offset, bytes)))
+ return 0;
if (i->type & (ITER_BVEC|ITER_KVEC)) {
void *kaddr = kmap_atomic(page);
size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
@@ -657,13 +716,15 @@ EXPORT_SYMBOL(copy_page_to_iter);
size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i)
{
+ if (unlikely(!page_copy_sane(page, offset, bytes)))
+ return 0;
if (unlikely(i->type & ITER_PIPE)) {
WARN_ON(1);
return 0;
}
if (i->type & (ITER_BVEC|ITER_KVEC)) {
void *kaddr = kmap_atomic(page);
- size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
+ size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
kunmap_atomic(kaddr);
return wanted;
} else
@@ -700,7 +761,7 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
if (unlikely(i->type & ITER_PIPE))
return pipe_zero(bytes, i);
iterate_and_advance(i, bytes, v,
- __clear_user(v.iov_base, v.iov_len),
+ clear_user(v.iov_base, v.iov_len),
memzero_page(v.bv_page, v.bv_offset, v.bv_len),
memset(v.iov_base, 0, v.iov_len)
)
@@ -713,14 +774,17 @@ size_t iov_iter_copy_from_user_atomic(struct page *page,
struct iov_iter *i, unsigned long offset, size_t bytes)
{
char *kaddr = kmap_atomic(page), *p = kaddr + offset;
+ if (unlikely(!page_copy_sane(page, offset, bytes))) {
+ kunmap_atomic(kaddr);
+ return 0;
+ }
if (unlikely(i->type & ITER_PIPE)) {
kunmap_atomic(kaddr);
WARN_ON(1);
return 0;
}
iterate_all_kinds(i, bytes, v,
- __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
- v.iov_base, v.iov_len),
+ copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
v.bv_offset, v.bv_len),
memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
diff --git a/lib/irq_poll.c b/lib/irq_poll.c
index 1d6565e81030..86a709954f5a 100644
--- a/lib/irq_poll.c
+++ b/lib/irq_poll.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Functions related to interrupt-poll handling in the block layer. This
* is similar to NAPI for network devices.
diff --git a/lib/kasprintf.c b/lib/kasprintf.c
index 7f6c506a4942..bacf7b83ccf0 100644
--- a/lib/kasprintf.c
+++ b/lib/kasprintf.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* linux/lib/kasprintf.c
*
diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c
index 9a2b811966eb..f237a09a5862 100644
--- a/lib/kobject_uevent.c
+++ b/lib/kobject_uevent.c
@@ -23,6 +23,8 @@
#include <linux/socket.h>
#include <linux/skbuff.h>
#include <linux/netlink.h>
+#include <linux/uuid.h>
+#include <linux/ctype.h>
#include <net/sock.h>
#include <net/net_namespace.h>
@@ -50,21 +52,17 @@ static const char *kobject_actions[] = {
[KOBJ_MOVE] = "move",
[KOBJ_ONLINE] = "online",
[KOBJ_OFFLINE] = "offline",
+ [KOBJ_BIND] = "bind",
+ [KOBJ_UNBIND] = "unbind",
};
-/**
- * kobject_action_type - translate action string to numeric type
- *
- * @buf: buffer containing the action string, newline is ignored
- * @count: length of buffer
- * @type: pointer to the location to store the action type
- *
- * Returns 0 if the action string was recognized.
- */
-int kobject_action_type(const char *buf, size_t count,
- enum kobject_action *type)
+static int kobject_action_type(const char *buf, size_t count,
+ enum kobject_action *type,
+ const char **args)
{
enum kobject_action action;
+ size_t count_first;
+ const char *args_start;
int ret = -EINVAL;
if (count && (buf[count-1] == '\n' || buf[count-1] == '\0'))
@@ -73,11 +71,20 @@ int kobject_action_type(const char *buf, size_t count,
if (!count)
goto out;
+ args_start = strnchr(buf, count, ' ');
+ if (args_start) {
+ count_first = args_start - buf;
+ args_start = args_start + 1;
+ } else
+ count_first = count;
+
for (action = 0; action < ARRAY_SIZE(kobject_actions); action++) {
- if (strncmp(kobject_actions[action], buf, count) != 0)
+ if (strncmp(kobject_actions[action], buf, count_first) != 0)
continue;
- if (kobject_actions[action][count] != '\0')
+ if (kobject_actions[action][count_first] != '\0')
continue;
+ if (args)
+ *args = args_start;
*type = action;
ret = 0;
break;
@@ -86,6 +93,142 @@ out:
return ret;
}
+static const char *action_arg_word_end(const char *buf, const char *buf_end,
+ char delim)
+{
+ const char *next = buf;
+
+ while (next <= buf_end && *next != delim)
+ if (!isalnum(*next++))
+ return NULL;
+
+ if (next == buf)
+ return NULL;
+
+ return next;
+}
+
+static int kobject_action_args(const char *buf, size_t count,
+ struct kobj_uevent_env **ret_env)
+{
+ struct kobj_uevent_env *env = NULL;
+ const char *next, *buf_end, *key;
+ int key_len;
+ int r = -EINVAL;
+
+ if (count && (buf[count - 1] == '\n' || buf[count - 1] == '\0'))
+ count--;
+
+ if (!count)
+ return -EINVAL;
+
+ env = kzalloc(sizeof(*env), GFP_KERNEL);
+ if (!env)
+ return -ENOMEM;
+
+ /* first arg is UUID */
+ if (count < UUID_STRING_LEN || !uuid_is_valid(buf) ||
+ add_uevent_var(env, "SYNTH_UUID=%.*s", UUID_STRING_LEN, buf))
+ goto out;
+
+ /*
+ * the rest are custom environment variables in KEY=VALUE
+ * format with ' ' delimiter between each KEY=VALUE pair
+ */
+ next = buf + UUID_STRING_LEN;
+ buf_end = buf + count - 1;
+
+ while (next <= buf_end) {
+ if (*next != ' ')
+ goto out;
+
+ /* skip the ' ', key must follow */
+ key = ++next;
+ if (key > buf_end)
+ goto out;
+
+ buf = next;
+ next = action_arg_word_end(buf, buf_end, '=');
+ if (!next || next > buf_end || *next != '=')
+ goto out;
+ key_len = next - buf;
+
+ /* skip the '=', value must follow */
+ if (++next > buf_end)
+ goto out;
+
+ buf = next;
+ next = action_arg_word_end(buf, buf_end, ' ');
+ if (!next)
+ goto out;
+
+ if (add_uevent_var(env, "SYNTH_ARG_%.*s=%.*s",
+ key_len, key, (int) (next - buf), buf))
+ goto out;
+ }
+
+ r = 0;
+out:
+ if (r)
+ kfree(env);
+ else
+ *ret_env = env;
+ return r;
+}
+
+/**
+ * kobject_synth_uevent - send synthetic uevent with arguments
+ *
+ * @kobj: struct kobject for which synthetic uevent is to be generated
+ * @buf: buffer containing action type and action args, newline is ignored
+ * @count: length of buffer
+ *
+ * Returns 0 if kobject_synthetic_uevent() is completed with success or the
+ * corresponding error when it fails.
+ */
+int kobject_synth_uevent(struct kobject *kobj, const char *buf, size_t count)
+{
+ char *no_uuid_envp[] = { "SYNTH_UUID=0", NULL };
+ enum kobject_action action;
+ const char *action_args;
+ struct kobj_uevent_env *env;
+ const char *msg = NULL, *devpath;
+ int r;
+
+ r = kobject_action_type(buf, count, &action, &action_args);
+ if (r) {
+ msg = "unknown uevent action string\n";
+ goto out;
+ }
+
+ if (!action_args) {
+ r = kobject_uevent_env(kobj, action, no_uuid_envp);
+ goto out;
+ }
+
+ r = kobject_action_args(action_args,
+ count - (action_args - buf), &env);
+ if (r == -EINVAL) {
+ msg = "incorrect uevent action arguments\n";
+ goto out;
+ }
+
+ if (r)
+ goto out;
+
+ r = kobject_uevent_env(kobj, action, env->envp);
+ kfree(env);
+out:
+ if (r) {
+ devpath = kobject_get_path(kobj, GFP_KERNEL);
+ printk(KERN_WARNING "synth uevent: %s: %s",
+ devpath ?: "unknown device",
+ msg ?: "failed to send uevent");
+ kfree(devpath);
+ }
+ return r;
+}
+
#ifdef CONFIG_NET
static int kobj_bcast_filter(struct sock *dsk, struct sk_buff *skb, void *data)
{
@@ -151,6 +294,26 @@ static void cleanup_uevent_env(struct subprocess_info *info)
}
#endif
+static void zap_modalias_env(struct kobj_uevent_env *env)
+{
+ static const char modalias_prefix[] = "MODALIAS=";
+ int i;
+
+ for (i = 0; i < env->envp_idx;) {
+ if (strncmp(env->envp[i], modalias_prefix,
+ sizeof(modalias_prefix) - 1)) {
+ i++;
+ continue;
+ }
+
+ if (i != env->envp_idx - 1)
+ memmove(&env->envp[i], &env->envp[i + 1],
+ sizeof(env->envp[i]) * env->envp_idx - 1);
+
+ env->envp_idx--;
+ }
+}
+
/**
* kobject_uevent_env - send an uevent with environmental data
*
@@ -266,16 +429,29 @@ int kobject_uevent_env(struct kobject *kobj, enum kobject_action action,
}
}
- /*
- * Mark "add" and "remove" events in the object to ensure proper
- * events to userspace during automatic cleanup. If the object did
- * send an "add" event, "remove" will automatically generated by
- * the core, if not already done by the caller.
- */
- if (action == KOBJ_ADD)
+ switch (action) {
+ case KOBJ_ADD:
+ /*
+ * Mark "add" event so we can make sure we deliver "remove"
+ * event to userspace during automatic cleanup. If
+ * the object did send an "add" event, "remove" will
+ * automatically generated by the core, if not already done
+ * by the caller.
+ */
kobj->state_add_uevent_sent = 1;
- else if (action == KOBJ_REMOVE)
+ break;
+
+ case KOBJ_REMOVE:
kobj->state_remove_uevent_sent = 1;
+ break;
+
+ case KOBJ_UNBIND:
+ zap_modalias_env(env);
+ break;
+
+ default:
+ break;
+ }
mutex_lock(&uevent_sock_mutex);
/* we will send an event, so request a new sequence number */
diff --git a/lib/kstrtox.c b/lib/kstrtox.c
index bf85e05ce858..661a1e807bd1 100644
--- a/lib/kstrtox.c
+++ b/lib/kstrtox.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Convert integer string representation to an integer.
* If an integer doesn't fit into specified type, -E is returned.
@@ -51,13 +52,15 @@ unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long
res = 0;
rv = 0;
- while (*s) {
+ while (1) {
+ unsigned int c = *s;
+ unsigned int lc = c | 0x20; /* don't tolower() this line */
unsigned int val;
- if ('0' <= *s && *s <= '9')
- val = *s - '0';
- else if ('a' <= _tolower(*s) && _tolower(*s) <= 'f')
- val = _tolower(*s) - 'a' + 10;
+ if ('0' <= c && c <= '9')
+ val = c - '0';
+ else if ('a' <= lc && lc <= 'f')
+ val = lc - 'a' + 10;
else
break;
diff --git a/lib/kstrtox.h b/lib/kstrtox.h
index f13eeeaf441d..3b4637bcd254 100644
--- a/lib/kstrtox.h
+++ b/lib/kstrtox.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LIB_KSTRTOX_H
#define _LIB_KSTRTOX_H
diff --git a/lib/list_sort.c b/lib/list_sort.c
index 9e9acc37652f..85759928215b 100644
--- a/lib/list_sort.c
+++ b/lib/list_sort.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/compiler.h>
diff --git a/lib/llist.c b/lib/llist.c
index ae5872b1df0c..7062e931a7bb 100644
--- a/lib/llist.c
+++ b/lib/llist.c
@@ -41,7 +41,7 @@ bool llist_add_batch(struct llist_node *new_first, struct llist_node *new_last,
struct llist_node *first;
do {
- new_last->next = first = ACCESS_ONCE(head->first);
+ new_last->next = first = READ_ONCE(head->first);
} while (cmpxchg(&head->first, first, new_first) != first);
return !first;
diff --git a/lib/locking-selftest-hardirq.h b/lib/locking-selftest-hardirq.h
index 10d4a150b259..0d144a6d6a96 100644
--- a/lib/locking-selftest-hardirq.h
+++ b/lib/locking-selftest-hardirq.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#undef IRQ_DISABLE
#undef IRQ_ENABLE
#undef IRQ_ENTER
diff --git a/lib/locking-selftest-mutex.h b/lib/locking-selftest-mutex.h
index 68601b6f584b..7526c7746fb2 100644
--- a/lib/locking-selftest-mutex.h
+++ b/lib/locking-selftest-mutex.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#undef LOCK
#define LOCK ML
diff --git a/lib/locking-selftest-rlock.h b/lib/locking-selftest-rlock.h
index 6789044f4d0e..eccab18f5584 100644
--- a/lib/locking-selftest-rlock.h
+++ b/lib/locking-selftest-rlock.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#undef LOCK
#define LOCK RL
diff --git a/lib/locking-selftest-rsem.h b/lib/locking-selftest-rsem.h
index 62da886680c7..4544858f922e 100644
--- a/lib/locking-selftest-rsem.h
+++ b/lib/locking-selftest-rsem.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#undef LOCK
#define LOCK RSL
diff --git a/lib/locking-selftest-rtmutex.h b/lib/locking-selftest-rtmutex.h
new file mode 100644
index 000000000000..fce8714c4170
--- /dev/null
+++ b/lib/locking-selftest-rtmutex.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef LOCK
+#define LOCK RTL
+
+#undef UNLOCK
+#define UNLOCK RTU
+
+#undef RLOCK
+#undef WLOCK
+
+#undef INIT
+#define INIT RTI
diff --git a/lib/locking-selftest-softirq.h b/lib/locking-selftest-softirq.h
index a83de2a04ace..6adde4867cb8 100644
--- a/lib/locking-selftest-softirq.h
+++ b/lib/locking-selftest-softirq.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#undef IRQ_DISABLE
#undef IRQ_ENABLE
#undef IRQ_ENTER
diff --git a/lib/locking-selftest-spin.h b/lib/locking-selftest-spin.h
index ccd1b4b09757..6b24d699e502 100644
--- a/lib/locking-selftest-spin.h
+++ b/lib/locking-selftest-spin.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#undef LOCK
#define LOCK L
diff --git a/lib/locking-selftest-wlock.h b/lib/locking-selftest-wlock.h
index 0815322d99ed..0bc51c8cf3c5 100644
--- a/lib/locking-selftest-wlock.h
+++ b/lib/locking-selftest-wlock.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#undef LOCK
#define LOCK WL
diff --git a/lib/locking-selftest-wsem.h b/lib/locking-selftest-wsem.h
index b88c5f2dc5f0..5ef18f931c9b 100644
--- a/lib/locking-selftest-wsem.h
+++ b/lib/locking-selftest-wsem.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#undef LOCK
#define LOCK WSL
diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index f3a217ea0388..b5c1293ce147 100644
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* lib/locking-selftest.c
*
@@ -21,6 +22,7 @@
#include <linux/interrupt.h>
#include <linux/debug_locks.h>
#include <linux/irqflags.h>
+#include <linux/rtmutex.h>
/*
* Change this to 1 if you want to see the failure printouts:
@@ -46,6 +48,7 @@ __setup("debug_locks_verbose=", setup_debug_locks_verbose);
#define LOCKTYPE_MUTEX 0x4
#define LOCKTYPE_RWSEM 0x8
#define LOCKTYPE_WW 0x10
+#define LOCKTYPE_RTMUTEX 0x20
static struct ww_acquire_ctx t, t2;
static struct ww_mutex o, o2, o3;
@@ -74,6 +77,15 @@ static DECLARE_RWSEM(rwsem_B);
static DECLARE_RWSEM(rwsem_C);
static DECLARE_RWSEM(rwsem_D);
+#ifdef CONFIG_RT_MUTEXES
+
+static DEFINE_RT_MUTEX(rtmutex_A);
+static DEFINE_RT_MUTEX(rtmutex_B);
+static DEFINE_RT_MUTEX(rtmutex_C);
+static DEFINE_RT_MUTEX(rtmutex_D);
+
+#endif
+
/*
* Locks that we initialize dynamically as well so that
* e.g. X1 and X2 becomes two instances of the same class,
@@ -108,6 +120,17 @@ static DECLARE_RWSEM(rwsem_Y2);
static DECLARE_RWSEM(rwsem_Z1);
static DECLARE_RWSEM(rwsem_Z2);
+#ifdef CONFIG_RT_MUTEXES
+
+static DEFINE_RT_MUTEX(rtmutex_X1);
+static DEFINE_RT_MUTEX(rtmutex_X2);
+static DEFINE_RT_MUTEX(rtmutex_Y1);
+static DEFINE_RT_MUTEX(rtmutex_Y2);
+static DEFINE_RT_MUTEX(rtmutex_Z1);
+static DEFINE_RT_MUTEX(rtmutex_Z2);
+
+#endif
+
/*
* non-inlined runtime initializers, to let separate locks share
* the same lock-class:
@@ -129,6 +152,17 @@ INIT_CLASS_FUNC(Z)
static void init_shared_classes(void)
{
+#ifdef CONFIG_RT_MUTEXES
+ static struct lock_class_key rt_X, rt_Y, rt_Z;
+
+ __rt_mutex_init(&rtmutex_X1, __func__, &rt_X);
+ __rt_mutex_init(&rtmutex_X2, __func__, &rt_X);
+ __rt_mutex_init(&rtmutex_Y1, __func__, &rt_Y);
+ __rt_mutex_init(&rtmutex_Y2, __func__, &rt_Y);
+ __rt_mutex_init(&rtmutex_Z1, __func__, &rt_Z);
+ __rt_mutex_init(&rtmutex_Z2, __func__, &rt_Z);
+#endif
+
init_class_X(&lock_X1, &rwlock_X1, &mutex_X1, &rwsem_X1);
init_class_X(&lock_X2, &rwlock_X2, &mutex_X2, &rwsem_X2);
@@ -193,6 +227,10 @@ static void init_shared_classes(void)
#define MU(x) mutex_unlock(&mutex_##x)
#define MI(x) mutex_init(&mutex_##x)
+#define RTL(x) rt_mutex_lock(&rtmutex_##x)
+#define RTU(x) rt_mutex_unlock(&rtmutex_##x)
+#define RTI(x) rt_mutex_init(&rtmutex_##x)
+
#define WSL(x) down_write(&rwsem_##x)
#define WSU(x) up_write(&rwsem_##x)
@@ -264,6 +302,11 @@ GENERATE_TESTCASE(AA_wsem)
#include "locking-selftest-rsem.h"
GENERATE_TESTCASE(AA_rsem)
+#ifdef CONFIG_RT_MUTEXES
+#include "locking-selftest-rtmutex.h"
+GENERATE_TESTCASE(AA_rtmutex);
+#endif
+
#undef E
/*
@@ -321,6 +364,103 @@ static void rsem_AA3(void)
}
/*
+ * read_lock(A)
+ * spin_lock(B)
+ * spin_lock(B)
+ * write_lock(A)
+ */
+static void rlock_ABBA1(void)
+{
+ RL(X1);
+ L(Y1);
+ U(Y1);
+ RU(X1);
+
+ L(Y1);
+ WL(X1);
+ WU(X1);
+ U(Y1); // should fail
+}
+
+static void rwsem_ABBA1(void)
+{
+ RSL(X1);
+ ML(Y1);
+ MU(Y1);
+ RSU(X1);
+
+ ML(Y1);
+ WSL(X1);
+ WSU(X1);
+ MU(Y1); // should fail
+}
+
+/*
+ * read_lock(A)
+ * spin_lock(B)
+ * spin_lock(B)
+ * read_lock(A)
+ */
+static void rlock_ABBA2(void)
+{
+ RL(X1);
+ L(Y1);
+ U(Y1);
+ RU(X1);
+
+ L(Y1);
+ RL(X1);
+ RU(X1);
+ U(Y1); // should NOT fail
+}
+
+static void rwsem_ABBA2(void)
+{
+ RSL(X1);
+ ML(Y1);
+ MU(Y1);
+ RSU(X1);
+
+ ML(Y1);
+ RSL(X1);
+ RSU(X1);
+ MU(Y1); // should fail
+}
+
+
+/*
+ * write_lock(A)
+ * spin_lock(B)
+ * spin_lock(B)
+ * write_lock(A)
+ */
+static void rlock_ABBA3(void)
+{
+ WL(X1);
+ L(Y1);
+ U(Y1);
+ WU(X1);
+
+ L(Y1);
+ WL(X1);
+ WU(X1);
+ U(Y1); // should fail
+}
+
+static void rwsem_ABBA3(void)
+{
+ WSL(X1);
+ ML(Y1);
+ MU(Y1);
+ WSU(X1);
+
+ ML(Y1);
+ WSL(X1);
+ WSU(X1);
+ MU(Y1); // should fail
+}
+
+/*
* ABBA deadlock:
*/
@@ -345,6 +485,11 @@ GENERATE_TESTCASE(ABBA_wsem)
#include "locking-selftest-rsem.h"
GENERATE_TESTCASE(ABBA_rsem)
+#ifdef CONFIG_RT_MUTEXES
+#include "locking-selftest-rtmutex.h"
+GENERATE_TESTCASE(ABBA_rtmutex);
+#endif
+
#undef E
/*
@@ -373,6 +518,11 @@ GENERATE_TESTCASE(ABBCCA_wsem)
#include "locking-selftest-rsem.h"
GENERATE_TESTCASE(ABBCCA_rsem)
+#ifdef CONFIG_RT_MUTEXES
+#include "locking-selftest-rtmutex.h"
+GENERATE_TESTCASE(ABBCCA_rtmutex);
+#endif
+
#undef E
/*
@@ -401,6 +551,11 @@ GENERATE_TESTCASE(ABCABC_wsem)
#include "locking-selftest-rsem.h"
GENERATE_TESTCASE(ABCABC_rsem)
+#ifdef CONFIG_RT_MUTEXES
+#include "locking-selftest-rtmutex.h"
+GENERATE_TESTCASE(ABCABC_rtmutex);
+#endif
+
#undef E
/*
@@ -430,6 +585,11 @@ GENERATE_TESTCASE(ABBCCDDA_wsem)
#include "locking-selftest-rsem.h"
GENERATE_TESTCASE(ABBCCDDA_rsem)
+#ifdef CONFIG_RT_MUTEXES
+#include "locking-selftest-rtmutex.h"
+GENERATE_TESTCASE(ABBCCDDA_rtmutex);
+#endif
+
#undef E
/*
@@ -458,6 +618,11 @@ GENERATE_TESTCASE(ABCDBDDA_wsem)
#include "locking-selftest-rsem.h"
GENERATE_TESTCASE(ABCDBDDA_rsem)
+#ifdef CONFIG_RT_MUTEXES
+#include "locking-selftest-rtmutex.h"
+GENERATE_TESTCASE(ABCDBDDA_rtmutex);
+#endif
+
#undef E
/*
@@ -486,6 +651,11 @@ GENERATE_TESTCASE(ABCDBCDA_wsem)
#include "locking-selftest-rsem.h"
GENERATE_TESTCASE(ABCDBCDA_rsem)
+#ifdef CONFIG_RT_MUTEXES
+#include "locking-selftest-rtmutex.h"
+GENERATE_TESTCASE(ABCDBCDA_rtmutex);
+#endif
+
#undef E
/*
@@ -513,33 +683,10 @@ GENERATE_TESTCASE(double_unlock_wsem)
#include "locking-selftest-rsem.h"
GENERATE_TESTCASE(double_unlock_rsem)
-#undef E
-
-/*
- * Bad unlock ordering:
- */
-#define E() \
- \
- LOCK(A); \
- LOCK(B); \
- UNLOCK(A); /* fail */ \
- UNLOCK(B);
-
-/*
- * 6 testcases:
- */
-#include "locking-selftest-spin.h"
-GENERATE_TESTCASE(bad_unlock_order_spin)
-#include "locking-selftest-wlock.h"
-GENERATE_TESTCASE(bad_unlock_order_wlock)
-#include "locking-selftest-rlock.h"
-GENERATE_TESTCASE(bad_unlock_order_rlock)
-#include "locking-selftest-mutex.h"
-GENERATE_TESTCASE(bad_unlock_order_mutex)
-#include "locking-selftest-wsem.h"
-GENERATE_TESTCASE(bad_unlock_order_wsem)
-#include "locking-selftest-rsem.h"
-GENERATE_TESTCASE(bad_unlock_order_rsem)
+#ifdef CONFIG_RT_MUTEXES
+#include "locking-selftest-rtmutex.h"
+GENERATE_TESTCASE(double_unlock_rtmutex);
+#endif
#undef E
@@ -567,6 +714,11 @@ GENERATE_TESTCASE(init_held_wsem)
#include "locking-selftest-rsem.h"
GENERATE_TESTCASE(init_held_rsem)
+#ifdef CONFIG_RT_MUTEXES
+#include "locking-selftest-rtmutex.h"
+GENERATE_TESTCASE(init_held_rtmutex);
+#endif
+
#undef E
/*
@@ -916,6 +1068,9 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft)
# define I_MUTEX(x) lockdep_reset_lock(&mutex_##x.dep_map)
# define I_RWSEM(x) lockdep_reset_lock(&rwsem_##x.dep_map)
# define I_WW(x) lockdep_reset_lock(&x.dep_map)
+#ifdef CONFIG_RT_MUTEXES
+# define I_RTMUTEX(x) lockdep_reset_lock(&rtmutex_##x.dep_map)
+#endif
#else
# define I_SPINLOCK(x)
# define I_RWLOCK(x)
@@ -924,12 +1079,23 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft)
# define I_WW(x)
#endif
+#ifndef I_RTMUTEX
+# define I_RTMUTEX(x)
+#endif
+
+#ifdef CONFIG_RT_MUTEXES
+#define I2_RTMUTEX(x) rt_mutex_init(&rtmutex_##x)
+#else
+#define I2_RTMUTEX(x)
+#endif
+
#define I1(x) \
do { \
I_SPINLOCK(x); \
I_RWLOCK(x); \
I_MUTEX(x); \
I_RWSEM(x); \
+ I_RTMUTEX(x); \
} while (0)
#define I2(x) \
@@ -938,6 +1104,7 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft)
rwlock_init(&rwlock_##x); \
mutex_init(&mutex_##x); \
init_rwsem(&rwsem_##x); \
+ I2_RTMUTEX(x); \
} while (0)
static void reset_locks(void)
@@ -987,8 +1154,6 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
if (debug_locks != expected) {
unexpected_testcase_failures++;
pr_cont("FAILED|");
-
- dump_stack();
} else {
testcase_successes++;
pr_cont(" ok |");
@@ -1013,6 +1178,12 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
reset_locks();
}
+#ifdef CONFIG_RT_MUTEXES
+#define dotest_rt(fn, e, m) dotest((fn), (e), (m))
+#else
+#define dotest_rt(fn, e, m)
+#endif
+
static inline void print_testname(const char *testname)
{
printk("%33s:", testname);
@@ -1050,6 +1221,7 @@ static inline void print_testname(const char *testname)
dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX); \
dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM); \
dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM); \
+ dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX); \
pr_cont("\n");
#define DO_TESTCASE_6_SUCCESS(desc, name) \
@@ -1060,6 +1232,7 @@ static inline void print_testname(const char *testname)
dotest(name##_mutex, SUCCESS, LOCKTYPE_MUTEX); \
dotest(name##_wsem, SUCCESS, LOCKTYPE_RWSEM); \
dotest(name##_rsem, SUCCESS, LOCKTYPE_RWSEM); \
+ dotest_rt(name##_rtmutex, SUCCESS, LOCKTYPE_RTMUTEX); \
pr_cont("\n");
/*
@@ -1073,6 +1246,7 @@ static inline void print_testname(const char *testname)
dotest(name##_mutex, FAILURE, LOCKTYPE_MUTEX); \
dotest(name##_wsem, FAILURE, LOCKTYPE_RWSEM); \
dotest(name##_rsem, FAILURE, LOCKTYPE_RWSEM); \
+ dotest_rt(name##_rtmutex, FAILURE, LOCKTYPE_RTMUTEX); \
pr_cont("\n");
#define DO_TESTCASE_2I(desc, name, nr) \
@@ -1825,7 +1999,6 @@ void locking_selftest(void)
DO_TESTCASE_6R("A-B-C-D-B-C-D-A deadlock", ABCDBCDA);
DO_TESTCASE_6("double unlock", double_unlock);
DO_TESTCASE_6("initialize held", init_held);
- DO_TESTCASE_6_SUCCESS("bad unlock order", bad_unlock_order);
printk(" --------------------------------------------------------------------------\n");
print_testname("recursive read-lock");
@@ -1856,6 +2029,32 @@ void locking_selftest(void)
dotest(rsem_AA3, FAILURE, LOCKTYPE_RWSEM);
pr_cont("\n");
+ print_testname("mixed read-lock/lock-write ABBA");
+ pr_cont(" |");
+ dotest(rlock_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
+#ifdef CONFIG_PROVE_LOCKING
+ /*
+ * Lockdep does indeed fail here, but there's nothing we can do about
+ * that now. Don't kill lockdep for it.
+ */
+ unexpected_testcase_failures--;
+#endif
+
+ pr_cont(" |");
+ dotest(rwsem_ABBA1, FAILURE, LOCKTYPE_RWSEM);
+
+ print_testname("mixed read-lock/lock-read ABBA");
+ pr_cont(" |");
+ dotest(rlock_ABBA2, SUCCESS, LOCKTYPE_RWLOCK);
+ pr_cont(" |");
+ dotest(rwsem_ABBA2, FAILURE, LOCKTYPE_RWSEM);
+
+ print_testname("mixed write-lock/lock-write ABBA");
+ pr_cont(" |");
+ dotest(rlock_ABBA3, FAILURE, LOCKTYPE_RWLOCK);
+ pr_cont(" |");
+ dotest(rwsem_ABBA3, FAILURE, LOCKTYPE_RWSEM);
+
printk(" --------------------------------------------------------------------------\n");
/*
diff --git a/lib/lockref.c b/lib/lockref.c
index c4bfcb8836cd..47169ed7e964 100644
--- a/lib/lockref.c
+++ b/lib/lockref.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/lockref.h>
diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c
index bd3574312b82..141734d255e4 100644
--- a/lib/lz4/lz4_decompress.c
+++ b/lib/lz4/lz4_decompress.c
@@ -85,8 +85,8 @@ static FORCE_INLINE int LZ4_decompress_generic(
const BYTE * const lowLimit = lowPrefix - dictSize;
const BYTE * const dictEnd = (const BYTE *)dictStart + dictSize;
- const unsigned int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };
- const int dec64table[] = { 0, 0, 0, -1, 0, 1, 2, 3 };
+ static const unsigned int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 };
+ static const int dec64table[] = { 0, 0, 0, -1, 0, 1, 2, 3 };
const int safeDecode = (endOnInput == endOnInputSize);
const int checkOffset = ((safeDecode) && (dictSize < (int)(64 * KB)));
diff --git a/lib/lzo/lzodefs.h b/lib/lzo/lzodefs.h
index 6710b83ce72e..4edefd2f540c 100644
--- a/lib/lzo/lzodefs.h
+++ b/lib/lzo/lzodefs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
* lzodefs.h -- architecture, OS and compiler specific defines
*
diff --git a/lib/memweight.c b/lib/memweight.c
index e35fc8771893..94dd72ccaa7f 100644
--- a/lib/memweight.c
+++ b/lib/memweight.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/export.h>
#include <linux/bug.h>
#include <linux/bitmap.h>
diff --git a/lib/mpi/Makefile b/lib/mpi/Makefile
index 019a68c90144..d5874a7f5ff9 100644
--- a/lib/mpi/Makefile
+++ b/lib/mpi/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
#
# MPI multiprecision maths library (from gpg)
#
diff --git a/lib/mpi/longlong.h b/lib/mpi/longlong.h
index 93336502af08..57fd45ab7af1 100644
--- a/lib/mpi/longlong.h
+++ b/lib/mpi/longlong.h
@@ -176,8 +176,8 @@ extern UDItype __udiv_qrnnd(UDItype *, UDItype, UDItype, UDItype);
#define add_ssaaaa(sh, sl, ah, al, bh, bl) \
__asm__ ("adds %1, %4, %5\n" \
"adc %0, %2, %3" \
- : "=r" ((USItype)(sh)), \
- "=&r" ((USItype)(sl)) \
+ : "=r" (sh), \
+ "=&r" (sl) \
: "%r" ((USItype)(ah)), \
"rI" ((USItype)(bh)), \
"%r" ((USItype)(al)), \
@@ -185,15 +185,15 @@ extern UDItype __udiv_qrnnd(UDItype *, UDItype, UDItype, UDItype);
#define sub_ddmmss(sh, sl, ah, al, bh, bl) \
__asm__ ("subs %1, %4, %5\n" \
"sbc %0, %2, %3" \
- : "=r" ((USItype)(sh)), \
- "=&r" ((USItype)(sl)) \
+ : "=r" (sh), \
+ "=&r" (sl) \
: "r" ((USItype)(ah)), \
"rI" ((USItype)(bh)), \
"r" ((USItype)(al)), \
"rI" ((USItype)(bl)))
#if defined __ARM_ARCH_2__ || defined __ARM_ARCH_3__
#define umul_ppmm(xh, xl, a, b) \
- __asm__ ("%@ Inlined umul_ppmm\n" \
+ __asm__ ("@ Inlined umul_ppmm\n" \
"mov %|r0, %2, lsr #16 @ AAAA\n" \
"mov %|r2, %3, lsr #16 @ BBBB\n" \
"bic %|r1, %2, %|r0, lsl #16 @ aaaa\n" \
@@ -206,19 +206,19 @@ extern UDItype __udiv_qrnnd(UDItype *, UDItype, UDItype, UDItype);
"addcs %|r2, %|r2, #65536\n" \
"adds %1, %|r1, %|r0, lsl #16\n" \
"adc %0, %|r2, %|r0, lsr #16" \
- : "=&r" ((USItype)(xh)), \
- "=r" ((USItype)(xl)) \
+ : "=&r" (xh), \
+ "=r" (xl) \
: "r" ((USItype)(a)), \
"r" ((USItype)(b)) \
: "r0", "r1", "r2")
#else
#define umul_ppmm(xh, xl, a, b) \
- __asm__ ("%@ Inlined umul_ppmm\n" \
- "umull %r1, %r0, %r2, %r3" \
- : "=&r" ((USItype)(xh)), \
- "=&r" ((USItype)(xl)) \
+ __asm__ ("@ Inlined umul_ppmm\n" \
+ "umull %1, %0, %2, %3" \
+ : "=&r" (xh), \
+ "=&r" (xl) \
: "r" ((USItype)(a)), \
- "r" ((USItype)(b)) \
+ "r" ((USItype)(b)) \
: "r0", "r1")
#endif
#define UMUL_TIME 20
diff --git a/lib/mpi/mpi-pow.c b/lib/mpi/mpi-pow.c
index e24388a863a7..468fb7cd1221 100644
--- a/lib/mpi/mpi-pow.c
+++ b/lib/mpi/mpi-pow.c
@@ -26,6 +26,7 @@
* however I decided to publish this code under the plain GPL.
*/
+#include <linux/sched.h>
#include <linux/string.h>
#include "mpi-internal.h"
#include "longlong.h"
@@ -256,6 +257,7 @@ int mpi_powm(MPI res, MPI base, MPI exp, MPI mod)
}
e <<= 1;
c--;
+ cond_resched();
}
i--;
diff --git a/lib/mpi/mpicoder.c b/lib/mpi/mpicoder.c
index 5a0f75a3bf01..eead4b339466 100644
--- a/lib/mpi/mpicoder.c
+++ b/lib/mpi/mpicoder.c
@@ -364,11 +364,11 @@ MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int nbytes)
}
miter.consumed = lzeros;
- sg_miter_stop(&miter);
nbytes -= lzeros;
nbits = nbytes * 8;
if (nbits > MAX_EXTERN_MPI_BITS) {
+ sg_miter_stop(&miter);
pr_info("MPI: mpi too large (%u bits)\n", nbits);
return NULL;
}
@@ -376,6 +376,8 @@ MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int nbytes)
if (nbytes > 0)
nbits -= count_leading_zeros(*buff) - (BITS_PER_LONG - 8);
+ sg_miter_stop(&miter);
+
nlimbs = DIV_ROUND_UP(nbytes, BYTES_PER_MPI_LIMB);
val = mpi_alloc(nlimbs);
if (!val)
diff --git a/lib/net_utils.c b/lib/net_utils.c
index 148fc6e99ef6..af525353395d 100644
--- a/lib/net_utils.c
+++ b/lib/net_utils.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/string.h>
#include <linux/if_ether.h>
#include <linux/ctype.h>
diff --git a/lib/nlattr.c b/lib/nlattr.c
index a7e0b16078df..3d8295c85505 100644
--- a/lib/nlattr.c
+++ b/lib/nlattr.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* NETLINK Netlink attributes
*
@@ -27,6 +28,30 @@ static const u8 nla_attr_minlen[NLA_TYPE_MAX+1] = {
[NLA_S64] = sizeof(s64),
};
+static int validate_nla_bitfield32(const struct nlattr *nla,
+ u32 *valid_flags_allowed)
+{
+ const struct nla_bitfield32 *bf = nla_data(nla);
+ u32 *valid_flags_mask = valid_flags_allowed;
+
+ if (!valid_flags_allowed)
+ return -EINVAL;
+
+ /*disallow invalid bit selector */
+ if (bf->selector & ~*valid_flags_mask)
+ return -EINVAL;
+
+ /*disallow invalid bit values */
+ if (bf->value & ~*valid_flags_mask)
+ return -EINVAL;
+
+ /*disallow valid bit values that are not selected*/
+ if (bf->value & ~bf->selector)
+ return -EINVAL;
+
+ return 0;
+}
+
static int validate_nla(const struct nlattr *nla, int maxtype,
const struct nla_policy *policy)
{
@@ -46,6 +71,12 @@ static int validate_nla(const struct nlattr *nla, int maxtype,
return -ERANGE;
break;
+ case NLA_BITFIELD32:
+ if (attrlen != sizeof(struct nla_bitfield32))
+ return -ERANGE;
+
+ return validate_nla_bitfield32(nla, pt->validation_data);
+
case NLA_NUL_STRING:
if (pt->len)
minlen = min_t(int, attrlen, pt->len + 1);
@@ -272,6 +303,30 @@ size_t nla_strlcpy(char *dst, const struct nlattr *nla, size_t dstsize)
EXPORT_SYMBOL(nla_strlcpy);
/**
+ * nla_strdup - Copy string attribute payload into a newly allocated buffer
+ * @nla: attribute to copy the string from
+ * @flags: the type of memory to allocate (see kmalloc).
+ *
+ * Returns a pointer to the allocated buffer or NULL on error.
+ */
+char *nla_strdup(const struct nlattr *nla, gfp_t flags)
+{
+ size_t srclen = nla_len(nla);
+ char *src = nla_data(nla), *dst;
+
+ if (srclen > 0 && src[srclen - 1] == '\0')
+ srclen--;
+
+ dst = kmalloc(srclen + 1, flags);
+ if (dst != NULL) {
+ memcpy(dst, src, srclen);
+ dst[srclen] = '\0';
+ }
+ return dst;
+}
+EXPORT_SYMBOL(nla_strdup);
+
+/**
* nla_memcpy - Copy a netlink attribute into another memory area
* @dest: where to copy to memcpy
* @src: netlink attribute to copy from
@@ -352,7 +407,7 @@ struct nlattr *__nla_reserve(struct sk_buff *skb, int attrtype, int attrlen)
{
struct nlattr *nla;
- nla = (struct nlattr *) skb_put(skb, nla_total_size(attrlen));
+ nla = skb_put(skb, nla_total_size(attrlen));
nla->nla_type = attrtype;
nla->nla_len = nla_attr_size(attrlen);
@@ -398,12 +453,7 @@ EXPORT_SYMBOL(__nla_reserve_64bit);
*/
void *__nla_reserve_nohdr(struct sk_buff *skb, int attrlen)
{
- void *start;
-
- start = skb_put(skb, NLA_ALIGN(attrlen));
- memset(start, 0, NLA_ALIGN(attrlen));
-
- return start;
+ return skb_put_zero(skb, NLA_ALIGN(attrlen));
}
EXPORT_SYMBOL(__nla_reserve_nohdr);
@@ -617,7 +667,7 @@ int nla_append(struct sk_buff *skb, int attrlen, const void *data)
if (unlikely(skb_tailroom(skb) < NLA_ALIGN(attrlen)))
return -EMSGSIZE;
- memcpy(skb_put(skb, attrlen), data, attrlen);
+ skb_put_data(skb, data, attrlen);
return 0;
}
EXPORT_SYMBOL(nla_append);
diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c
index 4e8a30d1c22f..46e4c749e4eb 100644
--- a/lib/nmi_backtrace.c
+++ b/lib/nmi_backtrace.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* NMI backtrace support
*
@@ -86,9 +87,11 @@ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask,
bool nmi_cpu_backtrace(struct pt_regs *regs)
{
+ static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
int cpu = smp_processor_id();
if (cpumask_test_cpu(cpu, to_cpumask(backtrace_mask))) {
+ arch_spin_lock(&lock);
if (regs && cpu_in_idle(instruction_pointer(regs))) {
pr_warn("NMI backtrace for cpu %d skipped: idling at pc %#lx\n",
cpu, instruction_pointer(regs));
@@ -99,6 +102,7 @@ bool nmi_cpu_backtrace(struct pt_regs *regs)
else
dump_stack();
}
+ arch_spin_unlock(&lock);
cpumask_clear_cpu(cpu, to_cpumask(backtrace_mask));
return true;
}
diff --git a/lib/nodemask.c b/lib/nodemask.c
index e42a5bf44d33..3aa454c54c0d 100644
--- a/lib/nodemask.c
+++ b/lib/nodemask.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/nodemask.h>
#include <linux/module.h>
#include <linux/random.h>
diff --git a/lib/notifier-error-inject.h b/lib/notifier-error-inject.h
index 99b3b6fc470b..fafff5f2ac45 100644
--- a/lib/notifier-error-inject.h
+++ b/lib/notifier-error-inject.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/atomic.h>
#include <linux/debugfs.h>
#include <linux/notifier.h>
diff --git a/lib/oid_registry.c b/lib/oid_registry.c
index 318f382a010d..41b9e50711a7 100644
--- a/lib/oid_registry.c
+++ b/lib/oid_registry.c
@@ -142,9 +142,9 @@ int sprint_oid(const void *data, size_t datasize, char *buffer, size_t bufsize)
}
ret += count = snprintf(buffer, bufsize, ".%lu", num);
buffer += count;
- bufsize -= count;
- if (bufsize == 0)
+ if (bufsize <= count)
return -ENOBUFS;
+ bufsize -= count;
}
return ret;
diff --git a/lib/once.c b/lib/once.c
index 05c8604627eb..bfb7420d0de3 100644
--- a/lib/once.c
+++ b/lib/once.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/once.h>
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 9c21000df0b5..c72577e472f2 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Fast batching percpu counters.
*/
@@ -72,7 +73,14 @@ void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
}
EXPORT_SYMBOL(percpu_counter_set);
-void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
+/**
+ * This function is both preempt and irq safe. The former is due to explicit
+ * preemption disable. The latter is guaranteed by the fact that the slow path
+ * is explicitly protected by an irq-safe spinlock whereas the fast patch uses
+ * this_cpu_add which is irq-safe by definition. Hence there is no need muck
+ * with irq state before calling this one
+ */
+void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
{
s64 count;
@@ -89,7 +97,7 @@ void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
}
preempt_enable();
}
-EXPORT_SYMBOL(__percpu_counter_add);
+EXPORT_SYMBOL(percpu_counter_add_batch);
/*
* Add up all the per-cpu counts, return the result. This is a more accurate
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 898e87998417..8b1feca1230a 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -463,7 +463,7 @@ radix_tree_node_free(struct radix_tree_node *node)
* To make use of this facility, the radix tree must be initialised without
* __GFP_DIRECT_RECLAIM being passed to INIT_RADIX_TREE().
*/
-static int __radix_tree_preload(gfp_t gfp_mask, unsigned nr)
+static __must_check int __radix_tree_preload(gfp_t gfp_mask, unsigned nr)
{
struct radix_tree_preload *rtp;
struct radix_tree_node *node;
@@ -2022,6 +2022,7 @@ void radix_tree_iter_delete(struct radix_tree_root *root,
if (__radix_tree_delete(root, iter->node, slot))
iter->index = iter->next_index;
}
+EXPORT_SYMBOL(radix_tree_iter_delete);
/**
* radix_tree_delete_item - delete an item from a radix tree
@@ -2103,7 +2104,8 @@ EXPORT_SYMBOL(radix_tree_tagged);
*/
void idr_preload(gfp_t gfp_mask)
{
- __radix_tree_preload(gfp_mask, IDR_PRELOAD_SIZE);
+ if (__radix_tree_preload(gfp_mask, IDR_PRELOAD_SIZE))
+ preempt_disable();
}
EXPORT_SYMBOL(idr_preload);
@@ -2117,13 +2119,13 @@ EXPORT_SYMBOL(idr_preload);
*/
int ida_pre_get(struct ida *ida, gfp_t gfp)
{
- __radix_tree_preload(gfp, IDA_PRELOAD_SIZE);
/*
* The IDA API has no preload_end() equivalent. Instead,
* ida_get_new() can return -EAGAIN, prompting the caller
* to return to the ida_pre_get() step.
*/
- preempt_enable();
+ if (!__radix_tree_preload(gfp, IDA_PRELOAD_SIZE))
+ preempt_enable();
if (!this_cpu_read(ida_bitmap)) {
struct ida_bitmap *bitmap = kmalloc(sizeof(*bitmap), gfp);
@@ -2137,13 +2139,13 @@ int ida_pre_get(struct ida *ida, gfp_t gfp)
}
EXPORT_SYMBOL(ida_pre_get);
-void __rcu **idr_get_free(struct radix_tree_root *root,
- struct radix_tree_iter *iter, gfp_t gfp, int end)
+void __rcu **idr_get_free_cmn(struct radix_tree_root *root,
+ struct radix_tree_iter *iter, gfp_t gfp,
+ unsigned long max)
{
struct radix_tree_node *node = NULL, *child;
void __rcu **slot = (void __rcu **)&root->rnode;
unsigned long maxindex, start = iter->next_index;
- unsigned long max = end > 0 ? end - 1 : INT_MAX;
unsigned int shift, offset = 0;
grow:
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
index 3057011f5599..4add700ddfe3 100644
--- a/lib/raid6/Makefile
+++ b/lib/raid6/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_RAID6_PQ) += raid6_pq.o
raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \
@@ -5,7 +6,7 @@ raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \
raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o
raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o
-raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o
+raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
raid6_pq-$(CONFIG_TILEGX) += tilegx8.o
raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o
@@ -26,7 +27,9 @@ NEON_FLAGS := -ffreestanding
ifeq ($(ARCH),arm)
NEON_FLAGS += -mfloat-abi=softfp -mfpu=neon
endif
+CFLAGS_recov_neon_inner.o += $(NEON_FLAGS)
ifeq ($(ARCH),arm64)
+CFLAGS_REMOVE_recov_neon_inner.o += -mgeneral-regs-only
CFLAGS_REMOVE_neon1.o += -mgeneral-regs-only
CFLAGS_REMOVE_neon2.o += -mgeneral-regs-only
CFLAGS_REMOVE_neon4.o += -mgeneral-regs-only
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 7857049fd7d3..476994723258 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -113,6 +113,9 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = {
#ifdef CONFIG_S390
&raid6_recov_s390xc,
#endif
+#if defined(CONFIG_KERNEL_MODE_NEON)
+ &raid6_recov_neon,
+#endif
&raid6_recov_intx1,
NULL
};
diff --git a/lib/raid6/avx512.c b/lib/raid6/avx512.c
index f524a7972006..46df7977b971 100644
--- a/lib/raid6/avx512.c
+++ b/lib/raid6/avx512.c
@@ -29,7 +29,7 @@
static const struct raid6_avx512_constants {
u64 x1d[8];
-} raid6_avx512_constants __aligned(512) = {
+} raid6_avx512_constants __aligned(512/8) = {
{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
diff --git a/lib/raid6/mktables.c b/lib/raid6/mktables.c
index 39787db588b0..e824d088f72c 100644
--- a/lib/raid6/mktables.c
+++ b/lib/raid6/mktables.c
@@ -125,6 +125,26 @@ int main(int argc, char *argv[])
printf("EXPORT_SYMBOL(raid6_gfexp);\n");
printf("#endif\n");
+ /* Compute log-of-2 table */
+ printf("\nconst u8 __attribute__((aligned(256)))\n"
+ "raid6_gflog[256] =\n" "{\n");
+ for (i = 0; i < 256; i += 8) {
+ printf("\t");
+ for (j = 0; j < 8; j++) {
+ v = 255;
+ for (k = 0; k < 256; k++)
+ if (exptbl[k] == (i + j)) {
+ v = k;
+ break;
+ }
+ printf("0x%02x,%c", v, (j == 7) ? '\n' : ' ');
+ }
+ }
+ printf("};\n");
+ printf("#ifdef __KERNEL__\n");
+ printf("EXPORT_SYMBOL(raid6_gflog);\n");
+ printf("#endif\n");
+
/* Compute inverse table x^-1 == x^254 */
printf("\nconst u8 __attribute__((aligned(256)))\n"
"raid6_gfinv[256] =\n" "{\n");
diff --git a/lib/raid6/neon.uc b/lib/raid6/neon.uc
index 4fa51b761dd0..d5242f544551 100644
--- a/lib/raid6/neon.uc
+++ b/lib/raid6/neon.uc
@@ -46,8 +46,12 @@ static inline unative_t SHLBYTE(unative_t v)
*/
static inline unative_t MASK(unative_t v)
{
- const uint8x16_t temp = NBYTES(0);
- return (unative_t)vcltq_s8((int8x16_t)v, (int8x16_t)temp);
+ return (unative_t)vshrq_n_s8((int8x16_t)v, 7);
+}
+
+static inline unative_t PMUL(unative_t v, unative_t u)
+{
+ return (unative_t)vmulq_p8((poly8x16_t)v, (poly8x16_t)u);
}
void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
@@ -110,7 +114,30 @@ void raid6_neon$#_xor_syndrome_real(int disks, int start, int stop,
wq$$ = veorq_u8(w1$$, wd$$);
}
/* P/Q left side optimization */
- for ( z = start-1 ; z >= 0 ; z-- ) {
+ for ( z = start-1 ; z >= 3 ; z -= 4 ) {
+ w2$$ = vshrq_n_u8(wq$$, 4);
+ w1$$ = vshlq_n_u8(wq$$, 4);
+
+ w2$$ = PMUL(w2$$, x1d);
+ wq$$ = veorq_u8(w1$$, w2$$);
+ }
+
+ switch (z) {
+ case 2:
+ w2$$ = vshrq_n_u8(wq$$, 5);
+ w1$$ = vshlq_n_u8(wq$$, 3);
+
+ w2$$ = PMUL(w2$$, x1d);
+ wq$$ = veorq_u8(w1$$, w2$$);
+ break;
+ case 1:
+ w2$$ = vshrq_n_u8(wq$$, 6);
+ w1$$ = vshlq_n_u8(wq$$, 2);
+
+ w2$$ = PMUL(w2$$, x1d);
+ wq$$ = veorq_u8(w1$$, w2$$);
+ break;
+ case 0:
w2$$ = MASK(wq$$);
w1$$ = SHLBYTE(wq$$);
diff --git a/lib/raid6/recov_neon.c b/lib/raid6/recov_neon.c
new file mode 100644
index 000000000000..eeb5c4065b92
--- /dev/null
+++ b/lib/raid6/recov_neon.c
@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2012 Intel Corporation
+ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+
+#include <linux/raid/pq.h>
+
+#ifdef __KERNEL__
+#include <asm/neon.h>
+#else
+#define kernel_neon_begin()
+#define kernel_neon_end()
+#define cpu_has_neon() (1)
+#endif
+
+static int raid6_has_neon(void)
+{
+ return cpu_has_neon();
+}
+
+void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
+ uint8_t *dq, const uint8_t *pbmul,
+ const uint8_t *qmul);
+
+void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
+ const uint8_t *qmul);
+
+static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
+ int failb, void **ptrs)
+{
+ u8 *p, *q, *dp, *dq;
+ const u8 *pbmul; /* P multiplier table for B data */
+ const u8 *qmul; /* Q multiplier table (for both) */
+
+ p = (u8 *)ptrs[disks - 2];
+ q = (u8 *)ptrs[disks - 1];
+
+ /*
+ * Compute syndrome with zero for the missing data pages
+ * Use the dead data pages as temporary storage for
+ * delta p and delta q
+ */
+ dp = (u8 *)ptrs[faila];
+ ptrs[faila] = (void *)raid6_empty_zero_page;
+ ptrs[disks - 2] = dp;
+ dq = (u8 *)ptrs[failb];
+ ptrs[failb] = (void *)raid6_empty_zero_page;
+ ptrs[disks - 1] = dq;
+
+ raid6_call.gen_syndrome(disks, bytes, ptrs);
+
+ /* Restore pointer table */
+ ptrs[faila] = dp;
+ ptrs[failb] = dq;
+ ptrs[disks - 2] = p;
+ ptrs[disks - 1] = q;
+
+ /* Now, pick the proper data tables */
+ pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
+ qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
+ raid6_gfexp[failb]]];
+
+ kernel_neon_begin();
+ __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
+ kernel_neon_end();
+}
+
+static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
+ void **ptrs)
+{
+ u8 *p, *q, *dq;
+ const u8 *qmul; /* Q multiplier table */
+
+ p = (u8 *)ptrs[disks - 2];
+ q = (u8 *)ptrs[disks - 1];
+
+ /*
+ * Compute syndrome with zero for the missing data page
+ * Use the dead data page as temporary storage for delta q
+ */
+ dq = (u8 *)ptrs[faila];
+ ptrs[faila] = (void *)raid6_empty_zero_page;
+ ptrs[disks - 1] = dq;
+
+ raid6_call.gen_syndrome(disks, bytes, ptrs);
+
+ /* Restore pointer table */
+ ptrs[faila] = dq;
+ ptrs[disks - 1] = q;
+
+ /* Now, pick the proper data tables */
+ qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+ kernel_neon_begin();
+ __raid6_datap_recov_neon(bytes, p, q, dq, qmul);
+ kernel_neon_end();
+}
+
+const struct raid6_recov_calls raid6_recov_neon = {
+ .data2 = raid6_2data_recov_neon,
+ .datap = raid6_datap_recov_neon,
+ .valid = raid6_has_neon,
+ .name = "neon",
+ .priority = 10,
+};
diff --git a/lib/raid6/recov_neon_inner.c b/lib/raid6/recov_neon_inner.c
new file mode 100644
index 000000000000..8cd20c9f834a
--- /dev/null
+++ b/lib/raid6/recov_neon_inner.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2012 Intel Corporation
+ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+
+#include <arm_neon.h>
+
+static const uint8x16_t x0f = {
+ 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
+ 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
+};
+
+#ifdef CONFIG_ARM
+/*
+ * AArch32 does not provide this intrinsic natively because it does not
+ * implement the underlying instruction. AArch32 only provides a 64-bit
+ * wide vtbl.8 instruction, so use that instead.
+ */
+static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b)
+{
+ union {
+ uint8x16_t val;
+ uint8x8x2_t pair;
+ } __a = { a };
+
+ return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)),
+ vtbl2_u8(__a.pair, vget_high_u8(b)));
+}
+#endif
+
+void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
+ uint8_t *dq, const uint8_t *pbmul,
+ const uint8_t *qmul)
+{
+ uint8x16_t pm0 = vld1q_u8(pbmul);
+ uint8x16_t pm1 = vld1q_u8(pbmul + 16);
+ uint8x16_t qm0 = vld1q_u8(qmul);
+ uint8x16_t qm1 = vld1q_u8(qmul + 16);
+
+ /*
+ * while ( bytes-- ) {
+ * uint8_t px, qx, db;
+ *
+ * px = *p ^ *dp;
+ * qx = qmul[*q ^ *dq];
+ * *dq++ = db = pbmul[px] ^ qx;
+ * *dp++ = db ^ px;
+ * p++; q++;
+ * }
+ */
+
+ while (bytes) {
+ uint8x16_t vx, vy, px, qx, db;
+
+ px = veorq_u8(vld1q_u8(p), vld1q_u8(dp));
+ vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
+
+ vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4);
+ vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
+ vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f));
+ qx = veorq_u8(vx, vy);
+
+ vy = (uint8x16_t)vshrq_n_s16((int16x8_t)px, 4);
+ vx = vqtbl1q_u8(pm0, vandq_u8(px, x0f));
+ vy = vqtbl1q_u8(pm1, vandq_u8(vy, x0f));
+ vx = veorq_u8(vx, vy);
+ db = veorq_u8(vx, qx);
+
+ vst1q_u8(dq, db);
+ vst1q_u8(dp, veorq_u8(db, px));
+
+ bytes -= 16;
+ p += 16;
+ q += 16;
+ dp += 16;
+ dq += 16;
+ }
+}
+
+void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
+ const uint8_t *qmul)
+{
+ uint8x16_t qm0 = vld1q_u8(qmul);
+ uint8x16_t qm1 = vld1q_u8(qmul + 16);
+
+ /*
+ * while (bytes--) {
+ * *p++ ^= *dq = qmul[*q ^ *dq];
+ * q++; dq++;
+ * }
+ */
+
+ while (bytes) {
+ uint8x16_t vx, vy;
+
+ vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
+
+ vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4);
+ vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
+ vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f));
+ vx = veorq_u8(vx, vy);
+ vy = veorq_u8(vx, vld1q_u8(p));
+
+ vst1q_u8(dq, vx);
+ vst1q_u8(p, vy);
+
+ bytes -= 16;
+ p += 16;
+ q += 16;
+ dq += 16;
+ }
+}
diff --git a/lib/raid6/recov_s390xc.c b/lib/raid6/recov_s390xc.c
index b042dac826cc..179eec900cea 100644
--- a/lib/raid6/recov_s390xc.c
+++ b/lib/raid6/recov_s390xc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* RAID-6 data recovery in dual failure mode based on the XC instruction.
*
diff --git a/lib/raid6/s390vx.uc b/lib/raid6/s390vx.uc
index 7b45191a655f..140fa8bb5c23 100644
--- a/lib/raid6/s390vx.uc
+++ b/lib/raid6/s390vx.uc
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* raid6_vx$#.c
*
diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
index 2c7b60edea04..be1010bdc435 100644
--- a/lib/raid6/test/Makefile
+++ b/lib/raid6/test/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
#
# This is a simple Makefile to test some of the RAID-6 code
# from userspace.
diff --git a/lib/random32.c b/lib/random32.c
index fa594b1140e6..65cc018fef40 100644
--- a/lib/random32.c
+++ b/lib/random32.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* This is a maximally equidistributed combined Tausworthe generator
* based on code from GNU Scientific Library 1.5 (30 Jun 2004)
@@ -214,7 +215,7 @@ core_initcall(prandom_init);
static void __prandom_timer(unsigned long dontcare);
-static DEFINE_TIMER(seed_timer, __prandom_timer, 0, 0);
+static DEFINE_TIMER(seed_timer, __prandom_timer);
static void __prandom_timer(unsigned long dontcare)
{
diff --git a/lib/ratelimit.c b/lib/ratelimit.c
index 08f8043cac61..d01f47135239 100644
--- a/lib/ratelimit.c
+++ b/lib/ratelimit.c
@@ -48,7 +48,9 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)
if (time_is_before_jiffies(rs->begin + rs->interval)) {
if (rs->missed) {
if (!(rs->flags & RATELIMIT_MSG_ON_RELEASE)) {
- pr_warn("%s: %d callbacks suppressed\n", func, rs->missed);
+ printk_deferred(KERN_WARNING
+ "%s: %d callbacks suppressed\n",
+ func, rs->missed);
rs->missed = 0;
}
}
diff --git a/lib/rational.c b/lib/rational.c
index f0aa21c2a762..ba7443677c90 100644
--- a/lib/rational.c
+++ b/lib/rational.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* rational fractions
*
diff --git a/lib/rbtree.c b/lib/rbtree.c
index 4ba2828a67c0..ba4a9d165f1b 100644
--- a/lib/rbtree.c
+++ b/lib/rbtree.c
@@ -95,22 +95,35 @@ __rb_rotate_set_parents(struct rb_node *old, struct rb_node *new,
static __always_inline void
__rb_insert(struct rb_node *node, struct rb_root *root,
+ bool newleft, struct rb_node **leftmost,
void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
{
struct rb_node *parent = rb_red_parent(node), *gparent, *tmp;
+ if (newleft)
+ *leftmost = node;
+
while (true) {
/*
- * Loop invariant: node is red
- *
- * If there is a black parent, we are done.
- * Otherwise, take some corrective action as we don't
- * want a red root or two consecutive red nodes.
+ * Loop invariant: node is red.
*/
- if (!parent) {
+ if (unlikely(!parent)) {
+ /*
+ * The inserted node is root. Either this is the
+ * first node, or we recursed at Case 1 below and
+ * are no longer violating 4).
+ */
rb_set_parent_color(node, NULL, RB_BLACK);
break;
- } else if (rb_is_black(parent))
+ }
+
+ /*
+ * If there is a black parent, we are done.
+ * Otherwise, take some corrective action as,
+ * per 4), we don't want a red root or two
+ * consecutive red nodes.
+ */
+ if(rb_is_black(parent))
break;
gparent = rb_red_parent(parent);
@@ -119,7 +132,7 @@ __rb_insert(struct rb_node *node, struct rb_root *root,
if (parent != tmp) { /* parent == gparent->rb_left */
if (tmp && rb_is_red(tmp)) {
/*
- * Case 1 - color flips
+ * Case 1 - node's uncle is red (color flips).
*
* G g
* / \ / \
@@ -142,7 +155,8 @@ __rb_insert(struct rb_node *node, struct rb_root *root,
tmp = parent->rb_right;
if (node == tmp) {
/*
- * Case 2 - left rotate at parent
+ * Case 2 - node's uncle is black and node is
+ * the parent's right child (left rotate at parent).
*
* G G
* / \ / \
@@ -166,7 +180,8 @@ __rb_insert(struct rb_node *node, struct rb_root *root,
}
/*
- * Case 3 - right rotate at gparent
+ * Case 3 - node's uncle is black and node is
+ * the parent's left child (right rotate at gparent).
*
* G P
* / \ / \
@@ -434,19 +449,38 @@ static const struct rb_augment_callbacks dummy_callbacks = {
void rb_insert_color(struct rb_node *node, struct rb_root *root)
{
- __rb_insert(node, root, dummy_rotate);
+ __rb_insert(node, root, false, NULL, dummy_rotate);
}
EXPORT_SYMBOL(rb_insert_color);
void rb_erase(struct rb_node *node, struct rb_root *root)
{
struct rb_node *rebalance;
- rebalance = __rb_erase_augmented(node, root, &dummy_callbacks);
+ rebalance = __rb_erase_augmented(node, root,
+ NULL, &dummy_callbacks);
if (rebalance)
____rb_erase_color(rebalance, root, dummy_rotate);
}
EXPORT_SYMBOL(rb_erase);
+void rb_insert_color_cached(struct rb_node *node,
+ struct rb_root_cached *root, bool leftmost)
+{
+ __rb_insert(node, &root->rb_root, leftmost,
+ &root->rb_leftmost, dummy_rotate);
+}
+EXPORT_SYMBOL(rb_insert_color_cached);
+
+void rb_erase_cached(struct rb_node *node, struct rb_root_cached *root)
+{
+ struct rb_node *rebalance;
+ rebalance = __rb_erase_augmented(node, &root->rb_root,
+ &root->rb_leftmost, &dummy_callbacks);
+ if (rebalance)
+ ____rb_erase_color(rebalance, &root->rb_root, dummy_rotate);
+}
+EXPORT_SYMBOL(rb_erase_cached);
+
/*
* Augmented rbtree manipulation functions.
*
@@ -455,9 +489,10 @@ EXPORT_SYMBOL(rb_erase);
*/
void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
+ bool newleft, struct rb_node **leftmost,
void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
{
- __rb_insert(node, root, augment_rotate);
+ __rb_insert(node, root, newleft, leftmost, augment_rotate);
}
EXPORT_SYMBOL(__rb_insert_augmented);
@@ -502,7 +537,7 @@ struct rb_node *rb_next(const struct rb_node *node)
* as we can.
*/
if (node->rb_right) {
- node = node->rb_right;
+ node = node->rb_right;
while (node->rb_left)
node=node->rb_left;
return (struct rb_node *)node;
@@ -534,7 +569,7 @@ struct rb_node *rb_prev(const struct rb_node *node)
* as we can.
*/
if (node->rb_left) {
- node = node->rb_left;
+ node = node->rb_left;
while (node->rb_right)
node=node->rb_right;
return (struct rb_node *)node;
diff --git a/lib/rbtree_test.c b/lib/rbtree_test.c
index 8b3c9dc88262..191a238e5a9d 100644
--- a/lib/rbtree_test.c
+++ b/lib/rbtree_test.c
@@ -1,11 +1,18 @@
#include <linux/module.h>
+#include <linux/moduleparam.h>
#include <linux/rbtree_augmented.h>
#include <linux/random.h>
+#include <linux/slab.h>
#include <asm/timex.h>
-#define NODES 100
-#define PERF_LOOPS 100000
-#define CHECK_LOOPS 100
+#define __param(type, name, init, msg) \
+ static type name = init; \
+ module_param(name, type, 0444); \
+ MODULE_PARM_DESC(name, msg);
+
+__param(int, nnodes, 100, "Number of nodes in the rb-tree");
+__param(int, perf_loops, 100000, "Number of iterations modifying the rb-tree");
+__param(int, check_loops, 100, "Number of iterations modifying and verifying the rb-tree");
struct test_node {
u32 key;
@@ -16,14 +23,14 @@ struct test_node {
u32 augmented;
};
-static struct rb_root root = RB_ROOT;
-static struct test_node nodes[NODES];
+static struct rb_root_cached root = RB_ROOT_CACHED;
+static struct test_node *nodes = NULL;
static struct rnd_state rnd;
-static void insert(struct test_node *node, struct rb_root *root)
+static void insert(struct test_node *node, struct rb_root_cached *root)
{
- struct rb_node **new = &root->rb_node, *parent = NULL;
+ struct rb_node **new = &root->rb_root.rb_node, *parent = NULL;
u32 key = node->key;
while (*new) {
@@ -35,14 +42,40 @@ static void insert(struct test_node *node, struct rb_root *root)
}
rb_link_node(&node->rb, parent, new);
- rb_insert_color(&node->rb, root);
+ rb_insert_color(&node->rb, &root->rb_root);
+}
+
+static void insert_cached(struct test_node *node, struct rb_root_cached *root)
+{
+ struct rb_node **new = &root->rb_root.rb_node, *parent = NULL;
+ u32 key = node->key;
+ bool leftmost = true;
+
+ while (*new) {
+ parent = *new;
+ if (key < rb_entry(parent, struct test_node, rb)->key)
+ new = &parent->rb_left;
+ else {
+ new = &parent->rb_right;
+ leftmost = false;
+ }
+ }
+
+ rb_link_node(&node->rb, parent, new);
+ rb_insert_color_cached(&node->rb, root, leftmost);
}
-static inline void erase(struct test_node *node, struct rb_root *root)
+static inline void erase(struct test_node *node, struct rb_root_cached *root)
{
- rb_erase(&node->rb, root);
+ rb_erase(&node->rb, &root->rb_root);
}
+static inline void erase_cached(struct test_node *node, struct rb_root_cached *root)
+{
+ rb_erase_cached(&node->rb, root);
+}
+
+
static inline u32 augment_recompute(struct test_node *node)
{
u32 max = node->val, child_augmented;
@@ -64,9 +97,10 @@ static inline u32 augment_recompute(struct test_node *node)
RB_DECLARE_CALLBACKS(static, augment_callbacks, struct test_node, rb,
u32, augmented, augment_recompute)
-static void insert_augmented(struct test_node *node, struct rb_root *root)
+static void insert_augmented(struct test_node *node,
+ struct rb_root_cached *root)
{
- struct rb_node **new = &root->rb_node, *rb_parent = NULL;
+ struct rb_node **new = &root->rb_root.rb_node, *rb_parent = NULL;
u32 key = node->key;
u32 val = node->val;
struct test_node *parent;
@@ -84,18 +118,53 @@ static void insert_augmented(struct test_node *node, struct rb_root *root)
node->augmented = val;
rb_link_node(&node->rb, rb_parent, new);
- rb_insert_augmented(&node->rb, root, &augment_callbacks);
+ rb_insert_augmented(&node->rb, &root->rb_root, &augment_callbacks);
+}
+
+static void insert_augmented_cached(struct test_node *node,
+ struct rb_root_cached *root)
+{
+ struct rb_node **new = &root->rb_root.rb_node, *rb_parent = NULL;
+ u32 key = node->key;
+ u32 val = node->val;
+ struct test_node *parent;
+ bool leftmost = true;
+
+ while (*new) {
+ rb_parent = *new;
+ parent = rb_entry(rb_parent, struct test_node, rb);
+ if (parent->augmented < val)
+ parent->augmented = val;
+ if (key < parent->key)
+ new = &parent->rb.rb_left;
+ else {
+ new = &parent->rb.rb_right;
+ leftmost = false;
+ }
+ }
+
+ node->augmented = val;
+ rb_link_node(&node->rb, rb_parent, new);
+ rb_insert_augmented_cached(&node->rb, root,
+ leftmost, &augment_callbacks);
+}
+
+
+static void erase_augmented(struct test_node *node, struct rb_root_cached *root)
+{
+ rb_erase_augmented(&node->rb, &root->rb_root, &augment_callbacks);
}
-static void erase_augmented(struct test_node *node, struct rb_root *root)
+static void erase_augmented_cached(struct test_node *node,
+ struct rb_root_cached *root)
{
- rb_erase_augmented(&node->rb, root, &augment_callbacks);
+ rb_erase_augmented_cached(&node->rb, root, &augment_callbacks);
}
static void init(void)
{
int i;
- for (i = 0; i < NODES; i++) {
+ for (i = 0; i < nnodes; i++) {
nodes[i].key = prandom_u32_state(&rnd);
nodes[i].val = prandom_u32_state(&rnd);
}
@@ -118,7 +187,7 @@ static void check_postorder_foreach(int nr_nodes)
{
struct test_node *cur, *n;
int count = 0;
- rbtree_postorder_for_each_entry_safe(cur, n, &root, rb)
+ rbtree_postorder_for_each_entry_safe(cur, n, &root.rb_root, rb)
count++;
WARN_ON_ONCE(count != nr_nodes);
@@ -128,7 +197,7 @@ static void check_postorder(int nr_nodes)
{
struct rb_node *rb;
int count = 0;
- for (rb = rb_first_postorder(&root); rb; rb = rb_next_postorder(rb))
+ for (rb = rb_first_postorder(&root.rb_root); rb; rb = rb_next_postorder(rb))
count++;
WARN_ON_ONCE(count != nr_nodes);
@@ -140,7 +209,7 @@ static void check(int nr_nodes)
int count = 0, blacks = 0;
u32 prev_key = 0;
- for (rb = rb_first(&root); rb; rb = rb_next(rb)) {
+ for (rb = rb_first(&root.rb_root); rb; rb = rb_next(rb)) {
struct test_node *node = rb_entry(rb, struct test_node, rb);
WARN_ON_ONCE(node->key < prev_key);
WARN_ON_ONCE(is_red(rb) &&
@@ -155,7 +224,7 @@ static void check(int nr_nodes)
}
WARN_ON_ONCE(count != nr_nodes);
- WARN_ON_ONCE(count < (1 << black_path_count(rb_last(&root))) - 1);
+ WARN_ON_ONCE(count < (1 << black_path_count(rb_last(&root.rb_root))) - 1);
check_postorder(nr_nodes);
check_postorder_foreach(nr_nodes);
@@ -166,7 +235,7 @@ static void check_augmented(int nr_nodes)
struct rb_node *rb;
check(nr_nodes);
- for (rb = rb_first(&root); rb; rb = rb_next(rb)) {
+ for (rb = rb_first(&root.rb_root); rb; rb = rb_next(rb)) {
struct test_node *node = rb_entry(rb, struct test_node, rb);
WARN_ON_ONCE(node->augmented != augment_recompute(node));
}
@@ -176,6 +245,11 @@ static int __init rbtree_test_init(void)
{
int i, j;
cycles_t time1, time2, time;
+ struct rb_node *node;
+
+ nodes = kmalloc(nnodes * sizeof(*nodes), GFP_KERNEL);
+ if (!nodes)
+ return -ENOMEM;
printk(KERN_ALERT "rbtree testing");
@@ -184,27 +258,88 @@ static int __init rbtree_test_init(void)
time1 = get_cycles();
- for (i = 0; i < PERF_LOOPS; i++) {
- for (j = 0; j < NODES; j++)
+ for (i = 0; i < perf_loops; i++) {
+ for (j = 0; j < nnodes; j++)
insert(nodes + j, &root);
- for (j = 0; j < NODES; j++)
+ for (j = 0; j < nnodes; j++)
erase(nodes + j, &root);
}
time2 = get_cycles();
time = time2 - time1;
- time = div_u64(time, PERF_LOOPS);
- printk(" -> %llu cycles\n", (unsigned long long)time);
+ time = div_u64(time, perf_loops);
+ printk(" -> test 1 (latency of nnodes insert+delete): %llu cycles\n",
+ (unsigned long long)time);
+
+ time1 = get_cycles();
+
+ for (i = 0; i < perf_loops; i++) {
+ for (j = 0; j < nnodes; j++)
+ insert_cached(nodes + j, &root);
+ for (j = 0; j < nnodes; j++)
+ erase_cached(nodes + j, &root);
+ }
+
+ time2 = get_cycles();
+ time = time2 - time1;
+
+ time = div_u64(time, perf_loops);
+ printk(" -> test 2 (latency of nnodes cached insert+delete): %llu cycles\n",
+ (unsigned long long)time);
+
+ for (i = 0; i < nnodes; i++)
+ insert(nodes + i, &root);
+
+ time1 = get_cycles();
+
+ for (i = 0; i < perf_loops; i++) {
+ for (node = rb_first(&root.rb_root); node; node = rb_next(node))
+ ;
+ }
+
+ time2 = get_cycles();
+ time = time2 - time1;
+
+ time = div_u64(time, perf_loops);
+ printk(" -> test 3 (latency of inorder traversal): %llu cycles\n",
+ (unsigned long long)time);
+
+ time1 = get_cycles();
+
+ for (i = 0; i < perf_loops; i++)
+ node = rb_first(&root.rb_root);
+
+ time2 = get_cycles();
+ time = time2 - time1;
+
+ time = div_u64(time, perf_loops);
+ printk(" -> test 4 (latency to fetch first node)\n");
+ printk(" non-cached: %llu cycles\n", (unsigned long long)time);
+
+ time1 = get_cycles();
+
+ for (i = 0; i < perf_loops; i++)
+ node = rb_first_cached(&root);
+
+ time2 = get_cycles();
+ time = time2 - time1;
+
+ time = div_u64(time, perf_loops);
+ printk(" cached: %llu cycles\n", (unsigned long long)time);
- for (i = 0; i < CHECK_LOOPS; i++) {
+ for (i = 0; i < nnodes; i++)
+ erase(nodes + i, &root);
+
+ /* run checks */
+ for (i = 0; i < check_loops; i++) {
init();
- for (j = 0; j < NODES; j++) {
+ for (j = 0; j < nnodes; j++) {
check(j);
insert(nodes + j, &root);
}
- for (j = 0; j < NODES; j++) {
- check(NODES - j);
+ for (j = 0; j < nnodes; j++) {
+ check(nnodes - j);
erase(nodes + j, &root);
}
check(0);
@@ -216,32 +351,49 @@ static int __init rbtree_test_init(void)
time1 = get_cycles();
- for (i = 0; i < PERF_LOOPS; i++) {
- for (j = 0; j < NODES; j++)
+ for (i = 0; i < perf_loops; i++) {
+ for (j = 0; j < nnodes; j++)
insert_augmented(nodes + j, &root);
- for (j = 0; j < NODES; j++)
+ for (j = 0; j < nnodes; j++)
erase_augmented(nodes + j, &root);
}
time2 = get_cycles();
time = time2 - time1;
- time = div_u64(time, PERF_LOOPS);
- printk(" -> %llu cycles\n", (unsigned long long)time);
+ time = div_u64(time, perf_loops);
+ printk(" -> test 1 (latency of nnodes insert+delete): %llu cycles\n", (unsigned long long)time);
+
+ time1 = get_cycles();
+
+ for (i = 0; i < perf_loops; i++) {
+ for (j = 0; j < nnodes; j++)
+ insert_augmented_cached(nodes + j, &root);
+ for (j = 0; j < nnodes; j++)
+ erase_augmented_cached(nodes + j, &root);
+ }
+
+ time2 = get_cycles();
+ time = time2 - time1;
+
+ time = div_u64(time, perf_loops);
+ printk(" -> test 2 (latency of nnodes cached insert+delete): %llu cycles\n", (unsigned long long)time);
- for (i = 0; i < CHECK_LOOPS; i++) {
+ for (i = 0; i < check_loops; i++) {
init();
- for (j = 0; j < NODES; j++) {
+ for (j = 0; j < nnodes; j++) {
check_augmented(j);
insert_augmented(nodes + j, &root);
}
- for (j = 0; j < NODES; j++) {
- check_augmented(NODES - j);
+ for (j = 0; j < nnodes; j++) {
+ check_augmented(nnodes - j);
erase_augmented(nodes + j, &root);
}
check_augmented(0);
}
+ kfree(nodes);
+
return -EAGAIN; /* Fail will directly unload the module */
}
diff --git a/lib/reciprocal_div.c b/lib/reciprocal_div.c
index 464152410c51..fcb4ce682c6f 100644
--- a/lib/reciprocal_div.c
+++ b/lib/reciprocal_div.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <asm/div64.h>
#include <linux/reciprocal_div.h>
diff --git a/lib/refcount.c b/lib/refcount.c
index 9f906783987e..0eb48353abe3 100644
--- a/lib/refcount.c
+++ b/lib/refcount.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Variant of atomic_t specialized for reference counts.
*
@@ -37,6 +38,8 @@
#include <linux/refcount.h>
#include <linux/bug.h>
+#ifdef CONFIG_REFCOUNT_FULL
+
/**
* refcount_add_not_zero - add a value to a refcount unless it is 0
* @i: the value to add to the refcount
@@ -225,6 +228,7 @@ void refcount_dec(refcount_t *r)
WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
}
EXPORT_SYMBOL(refcount_dec);
+#endif /* CONFIG_REFCOUNT_FULL */
/**
* refcount_dec_if_one - decrement a refcount if it is 1
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index d9e7274a04cd..ddd7dde87c3c 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -211,11 +211,10 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
int i;
size = sizeof(*tbl) + nbuckets * sizeof(tbl->buckets[0]);
- if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER) ||
- gfp != GFP_KERNEL)
+ if (gfp != GFP_KERNEL)
tbl = kzalloc(size, gfp | __GFP_NOWARN | __GFP_NORETRY);
- if (tbl == NULL && gfp == GFP_KERNEL)
- tbl = vzalloc(size);
+ else
+ tbl = kvzalloc(size, gfp);
size = nbuckets;
@@ -235,7 +234,7 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
INIT_LIST_HEAD(&tbl->walkers);
- get_random_bytes(&tbl->hash_rnd, sizeof(tbl->hash_rnd));
+ tbl->hash_rnd = get_random_u32();
for (i = 0; i < nbuckets; i++)
INIT_RHT_NULLS_HEAD(tbl->buckets[i], ht, i);
@@ -736,9 +735,9 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_exit);
* rhashtable_walk_start - Start a hash table walk
* @iter: Hash table iterator
*
- * Start a hash table walk. Note that we take the RCU lock in all
- * cases including when we return an error. So you must always call
- * rhashtable_walk_stop to clean up.
+ * Start a hash table walk at the current iterator position. Note that we take
+ * the RCU lock in all cases including when we return an error. So you must
+ * always call rhashtable_walk_stop to clean up.
*
* Returns zero if successful.
*
@@ -847,7 +846,8 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_next);
* rhashtable_walk_stop - Finish a hash table walk
* @iter: Hash table iterator
*
- * Finish a hash table walk.
+ * Finish a hash table walk. Does not reset the iterator to the start of the
+ * hash table.
*/
void rhashtable_walk_stop(struct rhashtable_iter *iter)
__releases(RCU)
diff --git a/lib/scatterlist.c b/lib/scatterlist.c
index c6cf82242d65..be7b4dd6b68d 100644
--- a/lib/scatterlist.c
+++ b/lib/scatterlist.c
@@ -751,3 +751,38 @@ size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
return sg_copy_buffer(sgl, nents, buf, buflen, skip, true);
}
EXPORT_SYMBOL(sg_pcopy_to_buffer);
+
+/**
+ * sg_zero_buffer - Zero-out a part of a SG list
+ * @sgl: The SG list
+ * @nents: Number of SG entries
+ * @buflen: The number of bytes to zero out
+ * @skip: Number of bytes to skip before zeroing
+ *
+ * Returns the number of bytes zeroed.
+ **/
+size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents,
+ size_t buflen, off_t skip)
+{
+ unsigned int offset = 0;
+ struct sg_mapping_iter miter;
+ unsigned int sg_flags = SG_MITER_ATOMIC | SG_MITER_TO_SG;
+
+ sg_miter_start(&miter, sgl, nents, sg_flags);
+
+ if (!sg_miter_skip(&miter, skip))
+ return false;
+
+ while (offset < buflen && sg_miter_next(&miter)) {
+ unsigned int len;
+
+ len = min(miter.length, buflen - offset);
+ memset(miter.addr, 0, len);
+
+ offset += len;
+ }
+
+ sg_miter_stop(&miter);
+ return offset;
+}
+EXPORT_SYMBOL(sg_zero_buffer);
diff --git a/lib/seq_buf.c b/lib/seq_buf.c
index cb18469e1f49..11f2ae0f9099 100644
--- a/lib/seq_buf.c
+++ b/lib/seq_buf.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* seq_buf.c
*
diff --git a/lib/sha1.c b/lib/sha1.c
index 5a56dfd7b99d..1d96d2c02b82 100644
--- a/lib/sha1.c
+++ b/lib/sha1.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* SHA1 routine optimized to do word accesses rather than byte accesses,
* and to avoid unnecessary copies into the context array.
diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c
index 690d75b132fa..835cc6df2776 100644
--- a/lib/smp_processor_id.c
+++ b/lib/smp_processor_id.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* lib/smp_processor_id.c
*
@@ -28,7 +29,7 @@ notrace static unsigned int check_preemption_disabled(const char *what1,
/*
* It is valid to assume CPU-locality during early bootup:
*/
- if (system_state != SYSTEM_RUNNING)
+ if (system_state < SYSTEM_SCHEDULING)
goto out;
/*
diff --git a/lib/sort.c b/lib/sort.c
index 975c6ef6fec7..d6b7a202b0b6 100644
--- a/lib/sort.c
+++ b/lib/sort.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* A fast, small, non-recursive O(nlog n) sort for the Linux kernel
*
diff --git a/lib/string.c b/lib/string.c
index 1c1fc9187b05..5e8d410a93df 100644
--- a/lib/string.c
+++ b/lib/string.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* linux/lib/string.c
*
@@ -723,6 +724,72 @@ void memzero_explicit(void *s, size_t count)
}
EXPORT_SYMBOL(memzero_explicit);
+#ifndef __HAVE_ARCH_MEMSET16
+/**
+ * memset16() - Fill a memory area with a uint16_t
+ * @s: Pointer to the start of the area.
+ * @v: The value to fill the area with
+ * @count: The number of values to store
+ *
+ * Differs from memset() in that it fills with a uint16_t instead
+ * of a byte. Remember that @count is the number of uint16_ts to
+ * store, not the number of bytes.
+ */
+void *memset16(uint16_t *s, uint16_t v, size_t count)
+{
+ uint16_t *xs = s;
+
+ while (count--)
+ *xs++ = v;
+ return s;
+}
+EXPORT_SYMBOL(memset16);
+#endif
+
+#ifndef __HAVE_ARCH_MEMSET32
+/**
+ * memset32() - Fill a memory area with a uint32_t
+ * @s: Pointer to the start of the area.
+ * @v: The value to fill the area with
+ * @count: The number of values to store
+ *
+ * Differs from memset() in that it fills with a uint32_t instead
+ * of a byte. Remember that @count is the number of uint32_ts to
+ * store, not the number of bytes.
+ */
+void *memset32(uint32_t *s, uint32_t v, size_t count)
+{
+ uint32_t *xs = s;
+
+ while (count--)
+ *xs++ = v;
+ return s;
+}
+EXPORT_SYMBOL(memset32);
+#endif
+
+#ifndef __HAVE_ARCH_MEMSET64
+/**
+ * memset64() - Fill a memory area with a uint64_t
+ * @s: Pointer to the start of the area.
+ * @v: The value to fill the area with
+ * @count: The number of values to store
+ *
+ * Differs from memset() in that it fills with a uint64_t instead
+ * of a byte. Remember that @count is the number of uint64_ts to
+ * store, not the number of bytes.
+ */
+void *memset64(uint64_t *s, uint64_t v, size_t count)
+{
+ uint64_t *xs = s;
+
+ while (count--)
+ *xs++ = v;
+ return s;
+}
+EXPORT_SYMBOL(memset64);
+#endif
+
#ifndef __HAVE_ARCH_MEMCPY
/**
* memcpy - Copy one area of memory to another
@@ -978,3 +1045,151 @@ char *strreplace(char *s, char old, char new)
return s;
}
EXPORT_SYMBOL(strreplace);
+
+void fortify_panic(const char *name)
+{
+ pr_emerg("detected buffer overflow in %s\n", name);
+ BUG();
+}
+EXPORT_SYMBOL(fortify_panic);
+
+#ifdef CONFIG_STRING_SELFTEST
+#include <linux/slab.h>
+#include <linux/module.h>
+
+static __init int memset16_selftest(void)
+{
+ unsigned i, j, k;
+ u16 v, *p;
+
+ p = kmalloc(256 * 2 * 2, GFP_KERNEL);
+ if (!p)
+ return -1;
+
+ for (i = 0; i < 256; i++) {
+ for (j = 0; j < 256; j++) {
+ memset(p, 0xa1, 256 * 2 * sizeof(v));
+ memset16(p + i, 0xb1b2, j);
+ for (k = 0; k < 512; k++) {
+ v = p[k];
+ if (k < i) {
+ if (v != 0xa1a1)
+ goto fail;
+ } else if (k < i + j) {
+ if (v != 0xb1b2)
+ goto fail;
+ } else {
+ if (v != 0xa1a1)
+ goto fail;
+ }
+ }
+ }
+ }
+
+fail:
+ kfree(p);
+ if (i < 256)
+ return (i << 24) | (j << 16) | k;
+ return 0;
+}
+
+static __init int memset32_selftest(void)
+{
+ unsigned i, j, k;
+ u32 v, *p;
+
+ p = kmalloc(256 * 2 * 4, GFP_KERNEL);
+ if (!p)
+ return -1;
+
+ for (i = 0; i < 256; i++) {
+ for (j = 0; j < 256; j++) {
+ memset(p, 0xa1, 256 * 2 * sizeof(v));
+ memset32(p + i, 0xb1b2b3b4, j);
+ for (k = 0; k < 512; k++) {
+ v = p[k];
+ if (k < i) {
+ if (v != 0xa1a1a1a1)
+ goto fail;
+ } else if (k < i + j) {
+ if (v != 0xb1b2b3b4)
+ goto fail;
+ } else {
+ if (v != 0xa1a1a1a1)
+ goto fail;
+ }
+ }
+ }
+ }
+
+fail:
+ kfree(p);
+ if (i < 256)
+ return (i << 24) | (j << 16) | k;
+ return 0;
+}
+
+static __init int memset64_selftest(void)
+{
+ unsigned i, j, k;
+ u64 v, *p;
+
+ p = kmalloc(256 * 2 * 8, GFP_KERNEL);
+ if (!p)
+ return -1;
+
+ for (i = 0; i < 256; i++) {
+ for (j = 0; j < 256; j++) {
+ memset(p, 0xa1, 256 * 2 * sizeof(v));
+ memset64(p + i, 0xb1b2b3b4b5b6b7b8ULL, j);
+ for (k = 0; k < 512; k++) {
+ v = p[k];
+ if (k < i) {
+ if (v != 0xa1a1a1a1a1a1a1a1ULL)
+ goto fail;
+ } else if (k < i + j) {
+ if (v != 0xb1b2b3b4b5b6b7b8ULL)
+ goto fail;
+ } else {
+ if (v != 0xa1a1a1a1a1a1a1a1ULL)
+ goto fail;
+ }
+ }
+ }
+ }
+
+fail:
+ kfree(p);
+ if (i < 256)
+ return (i << 24) | (j << 16) | k;
+ return 0;
+}
+
+static __init int string_selftest_init(void)
+{
+ int test, subtest;
+
+ test = 1;
+ subtest = memset16_selftest();
+ if (subtest)
+ goto fail;
+
+ test = 2;
+ subtest = memset32_selftest();
+ if (subtest)
+ goto fail;
+
+ test = 3;
+ subtest = memset64_selftest();
+ if (subtest)
+ goto fail;
+
+ pr_info("String selftests succeeded\n");
+ return 0;
+fail:
+ pr_crit("String selftest failure %d.%08x\n", test, subtest);
+ return 0;
+}
+
+module_init(string_selftest_init);
+#endif /* CONFIG_STRING_SELFTEST */
diff --git a/lib/string_helpers.c b/lib/string_helpers.c
index ecaac2c0526f..29c490e5d478 100644
--- a/lib/string_helpers.c
+++ b/lib/string_helpers.c
@@ -576,7 +576,7 @@ char *kstrdup_quotable_cmdline(struct task_struct *task, gfp_t gfp)
char *buffer, *quoted;
int i, res;
- buffer = kmalloc(PAGE_SIZE, GFP_TEMPORARY);
+ buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!buffer)
return NULL;
@@ -612,7 +612,7 @@ char *kstrdup_quotable_file(struct file *file, gfp_t gfp)
return kstrdup("<unknown>", gfp);
/* We add 11 spaces for ' (deleted)' to be appended */
- temp = kmalloc(PATH_MAX + 11, GFP_TEMPORARY);
+ temp = kmalloc(PATH_MAX + 11, GFP_KERNEL);
if (!temp)
return kstrdup("<no_memory>", gfp);
diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c
index 7e35fc450c5b..b53e1b5d80f4 100644
--- a/lib/strncpy_from_user.c
+++ b/lib/strncpy_from_user.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/compiler.h>
#include <linux/export.h>
#include <linux/kasan-checks.h>
diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c
index 8e105ed4df12..60d0bbda8f5e 100644
--- a/lib/strnlen_user.c
+++ b/lib/strnlen_user.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/uaccess.h>
@@ -121,37 +122,3 @@ long strnlen_user(const char __user *str, long count)
return 0;
}
EXPORT_SYMBOL(strnlen_user);
-
-/**
- * strlen_user: - Get the size of a user string INCLUDING final NUL.
- * @str: The string to measure.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * Get the size of a NUL-terminated string in user space.
- *
- * Returns the size of the string INCLUDING the terminating NUL.
- * On exception, returns 0.
- *
- * If there is a limit on the length of a valid string, you may wish to
- * consider using strnlen_user() instead.
- */
-long strlen_user(const char __user *str)
-{
- unsigned long max_addr, src_addr;
-
- max_addr = user_addr_max();
- src_addr = (unsigned long)str;
- if (likely(src_addr < max_addr)) {
- unsigned long max = max_addr - src_addr;
- long retval;
-
- user_access_begin();
- retval = do_strnlen_user(str, ~0ul, max);
- user_access_end();
- return retval;
- }
- return 0;
-}
-EXPORT_SYMBOL(strlen_user);
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index a8d74a733a38..cea19aaf303c 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -30,6 +30,7 @@
#include <linux/highmem.h>
#include <linux/gfp.h>
#include <linux/scatterlist.h>
+#include <linux/mem_encrypt.h>
#include <asm/io.h>
#include <asm/dma.h>
@@ -155,6 +156,15 @@ unsigned long swiotlb_size_or_default(void)
return size ? size : (IO_TLB_DEFAULT_SIZE);
}
+void __weak swiotlb_set_mem_attributes(void *vaddr, unsigned long size) { }
+
+/* For swiotlb, clear memory encryption mask from dma addresses */
+static dma_addr_t swiotlb_phys_to_dma(struct device *hwdev,
+ phys_addr_t address)
+{
+ return __sme_clr(phys_to_dma(hwdev, address));
+}
+
/* Note that this doesn't work with highmem page */
static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
volatile void *address)
@@ -183,6 +193,31 @@ void swiotlb_print_info(void)
bytes >> 20, vstart, vend - 1);
}
+/*
+ * Early SWIOTLB allocation may be too early to allow an architecture to
+ * perform the desired operations. This function allows the architecture to
+ * call SWIOTLB when the operations are possible. It needs to be called
+ * before the SWIOTLB memory is used.
+ */
+void __init swiotlb_update_mem_attributes(void)
+{
+ void *vaddr;
+ unsigned long bytes;
+
+ if (no_iotlb_memory || late_alloc)
+ return;
+
+ vaddr = phys_to_virt(io_tlb_start);
+ bytes = PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT);
+ swiotlb_set_mem_attributes(vaddr, bytes);
+ memset(vaddr, 0, bytes);
+
+ vaddr = phys_to_virt(io_tlb_overflow_buffer);
+ bytes = PAGE_ALIGN(io_tlb_overflow);
+ swiotlb_set_mem_attributes(vaddr, bytes);
+ memset(vaddr, 0, bytes);
+}
+
int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
{
void *v_overflow_buffer;
@@ -320,6 +355,7 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
io_tlb_start = virt_to_phys(tlb);
io_tlb_end = io_tlb_start + bytes;
+ swiotlb_set_mem_attributes(tlb, bytes);
memset(tlb, 0, bytes);
/*
@@ -330,6 +366,8 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
if (!v_overflow_buffer)
goto cleanup2;
+ swiotlb_set_mem_attributes(v_overflow_buffer, io_tlb_overflow);
+ memset(v_overflow_buffer, 0, io_tlb_overflow);
io_tlb_overflow_buffer = virt_to_phys(v_overflow_buffer);
/*
@@ -469,6 +507,10 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,
if (no_iotlb_memory)
panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
+ if (mem_encrypt_active())
+ pr_warn_once("%s is active and system is using DMA bounce buffers\n",
+ sme_active() ? "SME" : "SEV");
+
mask = dma_get_seg_boundary(hwdev);
tbl_dma_addr &= mask;
@@ -581,7 +623,7 @@ map_single(struct device *hwdev, phys_addr_t phys, size_t size,
return SWIOTLB_MAP_ERROR;
}
- start_dma_addr = phys_to_dma(hwdev, io_tlb_start);
+ start_dma_addr = swiotlb_phys_to_dma(hwdev, io_tlb_start);
return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size,
dir, attrs);
}
@@ -702,7 +744,7 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
goto err_warn;
ret = phys_to_virt(paddr);
- dev_addr = phys_to_dma(hwdev, paddr);
+ dev_addr = swiotlb_phys_to_dma(hwdev, paddr);
/* Confirm address can be DMA'd by device */
if (dev_addr + size - 1 > dma_mask) {
@@ -812,10 +854,10 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
map = map_single(dev, phys, size, dir, attrs);
if (map == SWIOTLB_MAP_ERROR) {
swiotlb_full(dev, size, dir, 1);
- return phys_to_dma(dev, io_tlb_overflow_buffer);
+ return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer);
}
- dev_addr = phys_to_dma(dev, map);
+ dev_addr = swiotlb_phys_to_dma(dev, map);
/* Ensure that the address returned is DMA'ble */
if (dma_capable(dev, dev_addr, size))
@@ -824,7 +866,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
attrs |= DMA_ATTR_SKIP_CPU_SYNC;
swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
- return phys_to_dma(dev, io_tlb_overflow_buffer);
+ return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer);
}
EXPORT_SYMBOL_GPL(swiotlb_map_page);
@@ -958,7 +1000,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
sg_dma_len(sgl) = 0;
return 0;
}
- sg->dma_address = phys_to_dma(hwdev, map);
+ sg->dma_address = swiotlb_phys_to_dma(hwdev, map);
} else
sg->dma_address = dev_addr;
sg_dma_len(sg) = sg->length;
@@ -1026,7 +1068,7 @@ EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
int
swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
{
- return (dma_addr == phys_to_dma(hwdev, io_tlb_overflow_buffer));
+ return (dma_addr == swiotlb_phys_to_dma(hwdev, io_tlb_overflow_buffer));
}
EXPORT_SYMBOL(swiotlb_dma_mapping_error);
@@ -1039,6 +1081,6 @@ EXPORT_SYMBOL(swiotlb_dma_mapping_error);
int
swiotlb_dma_supported(struct device *hwdev, u64 mask)
{
- return phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
+ return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
}
EXPORT_SYMBOL(swiotlb_dma_supported);
diff --git a/lib/syscall.c b/lib/syscall.c
index 2c6cd1b5c3ea..1a7077f20eae 100644
--- a/lib/syscall.c
+++ b/lib/syscall.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/ptrace.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c
index e2cbd43d193c..aa1f2669bdd5 100644
--- a/lib/test_bitmap.c
+++ b/lib/test_bitmap.c
@@ -165,6 +165,96 @@ static void __init test_zero_fill_copy(void)
expect_eq_pbl("128-1023", bmap2, 1024);
}
+#define PARSE_TIME 0x1
+
+struct test_bitmap_parselist{
+ const int errno;
+ const char *in;
+ const unsigned long *expected;
+ const int nbits;
+ const int flags;
+};
+
+static const unsigned long exp[] __initconst = {
+ BITMAP_FROM_U64(1),
+ BITMAP_FROM_U64(2),
+ BITMAP_FROM_U64(0x0000ffff),
+ BITMAP_FROM_U64(0xffff0000),
+ BITMAP_FROM_U64(0x55555555),
+ BITMAP_FROM_U64(0xaaaaaaaa),
+ BITMAP_FROM_U64(0x11111111),
+ BITMAP_FROM_U64(0x22222222),
+ BITMAP_FROM_U64(0xffffffff),
+ BITMAP_FROM_U64(0xfffffffe),
+ BITMAP_FROM_U64(0x3333333311111111ULL),
+ BITMAP_FROM_U64(0xffffffff77777777ULL)
+};
+
+static const unsigned long exp2[] __initconst = {
+ BITMAP_FROM_U64(0x3333333311111111ULL),
+ BITMAP_FROM_U64(0xffffffff77777777ULL)
+};
+
+static const struct test_bitmap_parselist parselist_tests[] __initconst = {
+#define step (sizeof(u64) / sizeof(unsigned long))
+
+ {0, "0", &exp[0], 8, 0},
+ {0, "1", &exp[1 * step], 8, 0},
+ {0, "0-15", &exp[2 * step], 32, 0},
+ {0, "16-31", &exp[3 * step], 32, 0},
+ {0, "0-31:1/2", &exp[4 * step], 32, 0},
+ {0, "1-31:1/2", &exp[5 * step], 32, 0},
+ {0, "0-31:1/4", &exp[6 * step], 32, 0},
+ {0, "1-31:1/4", &exp[7 * step], 32, 0},
+ {0, "0-31:4/4", &exp[8 * step], 32, 0},
+ {0, "1-31:4/4", &exp[9 * step], 32, 0},
+ {0, "0-31:1/4,32-63:2/4", &exp[10 * step], 64, 0},
+ {0, "0-31:3/4,32-63:4/4", &exp[11 * step], 64, 0},
+
+ {0, "0-31:1/4,32-63:2/4,64-95:3/4,96-127:4/4", exp2, 128, 0},
+
+ {0, "0-2047:128/256", NULL, 2048, PARSE_TIME},
+
+ {-EINVAL, "-1", NULL, 8, 0},
+ {-EINVAL, "-0", NULL, 8, 0},
+ {-EINVAL, "10-1", NULL, 8, 0},
+ {-EINVAL, "0-31:10/1", NULL, 8, 0},
+};
+
+static void __init test_bitmap_parselist(void)
+{
+ int i;
+ int err;
+ cycles_t cycles;
+ DECLARE_BITMAP(bmap, 2048);
+
+ for (i = 0; i < ARRAY_SIZE(parselist_tests); i++) {
+#define ptest parselist_tests[i]
+
+ cycles = get_cycles();
+ err = bitmap_parselist(ptest.in, bmap, ptest.nbits);
+ cycles = get_cycles() - cycles;
+
+ if (err != ptest.errno) {
+ pr_err("test %d: input is %s, errno is %d, expected %d\n",
+ i, ptest.in, err, ptest.errno);
+ continue;
+ }
+
+ if (!err && ptest.expected
+ && !__bitmap_equal(bmap, ptest.expected, ptest.nbits)) {
+ pr_err("test %d: input is %s, result is 0x%lx, expected 0x%lx\n",
+ i, ptest.in, bmap[0], *ptest.expected);
+ continue;
+ }
+
+ if (ptest.flags & PARSE_TIME)
+ pr_err("test %d: input is '%s' OK, Time: %llu\n",
+ i, ptest.in,
+ (unsigned long long)cycles);
+ }
+}
+
static void __init test_bitmap_u32_array_conversions(void)
{
DECLARE_BITMAP(bmap1, 1024);
@@ -333,10 +423,40 @@ static void __init test_bitmap_u32_array_conversions(void)
}
}
+static void noinline __init test_mem_optimisations(void)
+{
+ DECLARE_BITMAP(bmap1, 1024);
+ DECLARE_BITMAP(bmap2, 1024);
+ unsigned int start, nbits;
+
+ for (start = 0; start < 1024; start += 8) {
+ memset(bmap1, 0x5a, sizeof(bmap1));
+ memset(bmap2, 0x5a, sizeof(bmap2));
+ for (nbits = 0; nbits < 1024 - start; nbits += 8) {
+ bitmap_set(bmap1, start, nbits);
+ __bitmap_set(bmap2, start, nbits);
+ if (!bitmap_equal(bmap1, bmap2, 1024))
+ printk("set not equal %d %d\n", start, nbits);
+ if (!__bitmap_equal(bmap1, bmap2, 1024))
+ printk("set not __equal %d %d\n", start, nbits);
+
+ bitmap_clear(bmap1, start, nbits);
+ __bitmap_clear(bmap2, start, nbits);
+ if (!bitmap_equal(bmap1, bmap2, 1024))
+ printk("clear not equal %d %d\n", start, nbits);
+ if (!__bitmap_equal(bmap1, bmap2, 1024))
+ printk("clear not __equal %d %d\n", start,
+ nbits);
+ }
+ }
+}
+
static int __init test_bitmap_init(void)
{
test_zero_fill_copy();
test_bitmap_u32_array_conversions();
+ test_bitmap_parselist();
+ test_mem_optimisations();
if (failed_tests == 0)
pr_info("all %u tests passed\n", total_tests);
diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index be88cbaadde3..aa8812ae6776 100644
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -84,6 +84,7 @@ struct bpf_test {
} test[MAX_SUBTESTS];
int (*fill_helper)(struct bpf_test *self);
__u8 frag_data[MAX_DATA];
+ int stack_depth; /* for eBPF only, since tests don't call verifier */
};
/* Large test cases need separate allocation and fill handler. */
@@ -434,6 +435,30 @@ loop:
return 0;
}
+static int bpf_fill_jump_around_ld_abs(struct bpf_test *self)
+{
+ unsigned int len = BPF_MAXINSNS;
+ struct bpf_insn *insn;
+ int i = 0;
+
+ insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+ if (!insn)
+ return -ENOMEM;
+
+ insn[i++] = BPF_MOV64_REG(R6, R1);
+ insn[i++] = BPF_LD_ABS(BPF_B, 0);
+ insn[i] = BPF_JMP_IMM(BPF_JEQ, R0, 10, len - i - 2);
+ i++;
+ while (i < len - 1)
+ insn[i++] = BPF_LD_ABS(BPF_B, 1);
+ insn[i] = BPF_EXIT_INSN();
+
+ self->u.ptr.insns = insn;
+ self->u.ptr.len = len;
+
+ return 0;
+}
+
static int __bpf_fill_stxdw(struct bpf_test *self, int size)
{
unsigned int len = BPF_MAXINSNS;
@@ -455,6 +480,7 @@ static int __bpf_fill_stxdw(struct bpf_test *self, int size)
self->u.ptr.insns = insn;
self->u.ptr.len = len;
+ self->stack_depth = 40;
return 0;
}
@@ -926,6 +952,32 @@ static struct bpf_test tests[] = {
{ { 2, 0 }, { 3, 1 }, { 4, MAX_K } },
},
{
+ "JGE (jt 0), test 1",
+ .u.insns = {
+ BPF_STMT(BPF_LDX | BPF_LEN, 0),
+ BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2),
+ BPF_JUMP(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 1),
+ BPF_STMT(BPF_RET | BPF_K, 1),
+ BPF_STMT(BPF_RET | BPF_K, MAX_K)
+ },
+ CLASSIC,
+ { 4, 4, 4, 3, 3 },
+ { { 2, 0 }, { 3, 1 }, { 4, 1 } },
+ },
+ {
+ "JGE (jt 0), test 2",
+ .u.insns = {
+ BPF_STMT(BPF_LDX | BPF_LEN, 0),
+ BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2),
+ BPF_JUMP(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 1),
+ BPF_STMT(BPF_RET | BPF_K, 1),
+ BPF_STMT(BPF_RET | BPF_K, MAX_K)
+ },
+ CLASSIC,
+ { 4, 4, 5, 3, 3 },
+ { { 4, 1 }, { 5, 1 }, { 6, MAX_K } },
+ },
+ {
"JGE",
.u.insns = {
BPF_STMT(BPF_LDX | BPF_LEN, 0),
@@ -2317,7 +2369,8 @@ static struct bpf_test tests[] = {
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x06, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6},
- { { 38, 256 } }
+ { { 38, 256 } },
+ .stack_depth = 64,
},
/* BPF_ALU | BPF_MOV | BPF_X */
{
@@ -4169,6 +4222,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0xff } },
+ .stack_depth = 40,
},
{
"ST_MEM_B: Store/Load byte: max positive",
@@ -4181,6 +4235,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0x7f } },
+ .stack_depth = 40,
},
{
"STX_MEM_B: Store/Load byte: max negative",
@@ -4194,6 +4249,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0xff } },
+ .stack_depth = 40,
},
{
"ST_MEM_H: Store/Load half word: max negative",
@@ -4206,6 +4262,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0xffff } },
+ .stack_depth = 40,
},
{
"ST_MEM_H: Store/Load half word: max positive",
@@ -4218,6 +4275,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0x7fff } },
+ .stack_depth = 40,
},
{
"STX_MEM_H: Store/Load half word: max negative",
@@ -4231,6 +4289,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0xffff } },
+ .stack_depth = 40,
},
{
"ST_MEM_W: Store/Load word: max negative",
@@ -4243,6 +4302,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0xffffffff } },
+ .stack_depth = 40,
},
{
"ST_MEM_W: Store/Load word: max positive",
@@ -4255,6 +4315,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0x7fffffff } },
+ .stack_depth = 40,
},
{
"STX_MEM_W: Store/Load word: max negative",
@@ -4268,6 +4329,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0xffffffff } },
+ .stack_depth = 40,
},
{
"ST_MEM_DW: Store/Load double word: max negative",
@@ -4280,6 +4342,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0xffffffff } },
+ .stack_depth = 40,
},
{
"ST_MEM_DW: Store/Load double word: max negative 2",
@@ -4297,6 +4360,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0x1 } },
+ .stack_depth = 40,
},
{
"ST_MEM_DW: Store/Load double word: max positive",
@@ -4309,6 +4373,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0x7fffffff } },
+ .stack_depth = 40,
},
{
"STX_MEM_DW: Store/Load double word: max negative",
@@ -4322,6 +4387,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0xffffffff } },
+ .stack_depth = 40,
},
/* BPF_STX | BPF_XADD | BPF_W/DW */
{
@@ -4336,6 +4402,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0x22 } },
+ .stack_depth = 40,
},
{
"STX_XADD_W: Test side-effects, r10: 0x12 + 0x10 = 0x22",
@@ -4351,6 +4418,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0 } },
+ .stack_depth = 40,
},
{
"STX_XADD_W: Test side-effects, r0: 0x12 + 0x10 = 0x22",
@@ -4363,6 +4431,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0x12 } },
+ .stack_depth = 40,
},
{
"STX_XADD_W: X + 1 + 1 + 1 + ...",
@@ -4384,6 +4453,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0x22 } },
+ .stack_depth = 40,
},
{
"STX_XADD_DW: Test side-effects, r10: 0x12 + 0x10 = 0x22",
@@ -4399,6 +4469,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0 } },
+ .stack_depth = 40,
},
{
"STX_XADD_DW: Test side-effects, r0: 0x12 + 0x10 = 0x22",
@@ -4411,6 +4482,7 @@ static struct bpf_test tests[] = {
INTERNAL,
{ },
{ { 0, 0x12 } },
+ .stack_depth = 40,
},
{
"STX_XADD_DW: X + 1 + 1 + 1 + ...",
@@ -4446,6 +4518,35 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ /* BPF_JMP | BPF_JSLT | BPF_K */
+ {
+ "JMP_JSLT_K: Signed jump: if (-2 < -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 0xfffffffffffffffeLL),
+ BPF_JMP_IMM(BPF_JSLT, R1, -1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSLT_K: Signed jump: if (-1 < -1) return 0",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_LD_IMM64(R1, 0xffffffffffffffffLL),
+ BPF_JMP_IMM(BPF_JSLT, R1, -1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
/* BPF_JMP | BPF_JSGT | BPF_K */
{
"JMP_JSGT_K: Signed jump: if (-1 > -2) return 1",
@@ -4475,6 +4576,73 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ /* BPF_JMP | BPF_JSLE | BPF_K */
+ {
+ "JMP_JSLE_K: Signed jump: if (-2 <= -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 0xfffffffffffffffeLL),
+ BPF_JMP_IMM(BPF_JSLE, R1, -1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSLE_K: Signed jump: if (-1 <= -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 0xffffffffffffffffLL),
+ BPF_JMP_IMM(BPF_JSLE, R1, -1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSLE_K: Signed jump: value walk 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JSLE, R1, 0, 6),
+ BPF_ALU64_IMM(BPF_SUB, R1, 1),
+ BPF_JMP_IMM(BPF_JSLE, R1, 0, 4),
+ BPF_ALU64_IMM(BPF_SUB, R1, 1),
+ BPF_JMP_IMM(BPF_JSLE, R1, 0, 2),
+ BPF_ALU64_IMM(BPF_SUB, R1, 1),
+ BPF_JMP_IMM(BPF_JSLE, R1, 0, 1),
+ BPF_EXIT_INSN(), /* bad exit */
+ BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSLE_K: Signed jump: value walk 2",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JSLE, R1, 0, 4),
+ BPF_ALU64_IMM(BPF_SUB, R1, 2),
+ BPF_JMP_IMM(BPF_JSLE, R1, 0, 2),
+ BPF_ALU64_IMM(BPF_SUB, R1, 2),
+ BPF_JMP_IMM(BPF_JSLE, R1, 0, 1),
+ BPF_EXIT_INSN(), /* bad exit */
+ BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
/* BPF_JMP | BPF_JSGE | BPF_K */
{
"JMP_JSGE_K: Signed jump: if (-1 >= -2) return 1",
@@ -4571,6 +4739,35 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ /* BPF_JMP | BPF_JLT | BPF_K */
+ {
+ "JMP_JLT_K: if (2 < 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 2),
+ BPF_JMP_IMM(BPF_JLT, R1, 3, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JGT_K: Unsigned jump: if (1 < -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 1),
+ BPF_JMP_IMM(BPF_JLT, R1, -1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
/* BPF_JMP | BPF_JGE | BPF_K */
{
"JMP_JGE_K: if (3 >= 2) return 1",
@@ -4586,6 +4783,21 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ /* BPF_JMP | BPF_JLE | BPF_K */
+ {
+ "JMP_JLE_K: if (2 <= 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 2),
+ BPF_JMP_IMM(BPF_JLE, R1, 3, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
/* BPF_JMP | BPF_JGT | BPF_K jump backwards */
{
"JMP_JGT_K: if (3 > 2) return 1 (jump backwards)",
@@ -4616,6 +4828,36 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ /* BPF_JMP | BPF_JLT | BPF_K jump backwards */
+ {
+ "JMP_JGT_K: if (2 < 3) return 1 (jump backwards)",
+ .u.insns_int = {
+ BPF_JMP_IMM(BPF_JA, 0, 0, 2), /* goto start */
+ BPF_ALU32_IMM(BPF_MOV, R0, 1), /* out: */
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 0), /* start: */
+ BPF_LD_IMM64(R1, 2), /* note: this takes 2 insns */
+ BPF_JMP_IMM(BPF_JLT, R1, 3, -6), /* goto out */
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JLE_K: if (3 <= 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_JMP_IMM(BPF_JLE, R1, 3, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
/* BPF_JMP | BPF_JNE | BPF_K */
{
"JMP_JNE_K: if (3 != 2) return 1",
@@ -4706,6 +4948,37 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ /* BPF_JMP | BPF_JSLT | BPF_X */
+ {
+ "JMP_JSLT_X: Signed jump: if (-2 < -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, -1),
+ BPF_LD_IMM64(R2, -2),
+ BPF_JMP_REG(BPF_JSLT, R2, R1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSLT_X: Signed jump: if (-1 < -1) return 0",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_LD_IMM64(R1, -1),
+ BPF_LD_IMM64(R2, -1),
+ BPF_JMP_REG(BPF_JSLT, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
/* BPF_JMP | BPF_JSGE | BPF_X */
{
"JMP_JSGE_X: Signed jump: if (-1 >= -2) return 1",
@@ -4737,6 +5010,37 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ /* BPF_JMP | BPF_JSLE | BPF_X */
+ {
+ "JMP_JSLE_X: Signed jump: if (-2 <= -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, -1),
+ BPF_LD_IMM64(R2, -2),
+ BPF_JMP_REG(BPF_JSLE, R2, R1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JSLE_X: Signed jump: if (-1 <= -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, -1),
+ BPF_LD_IMM64(R2, -1),
+ BPF_JMP_REG(BPF_JSLE, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
/* BPF_JMP | BPF_JGT | BPF_X */
{
"JMP_JGT_X: if (3 > 2) return 1",
@@ -4768,6 +5072,37 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ /* BPF_JMP | BPF_JLT | BPF_X */
+ {
+ "JMP_JLT_X: if (2 < 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 2),
+ BPF_JMP_REG(BPF_JLT, R2, R1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JLT_X: Unsigned jump: if (1 < -1) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, -1),
+ BPF_LD_IMM64(R2, 1),
+ BPF_JMP_REG(BPF_JLT, R2, R1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
/* BPF_JMP | BPF_JGE | BPF_X */
{
"JMP_JGE_X: if (3 >= 2) return 1",
@@ -4799,6 +5134,37 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ /* BPF_JMP | BPF_JLE | BPF_X */
+ {
+ "JMP_JLE_X: if (2 <= 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 2),
+ BPF_JMP_REG(BPF_JLE, R2, R1, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
+ {
+ "JMP_JLE_X: if (3 <= 3) return 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 3),
+ BPF_JMP_REG(BPF_JLE, R1, R2, 1),
+ BPF_EXIT_INSN(),
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
{
/* Mainly testing JIT + imm64 here. */
"JMP_JGE_X: ldimm64 test 1",
@@ -4844,6 +5210,50 @@ static struct bpf_test tests[] = {
{ },
{ { 0, 1 } },
},
+ {
+ "JMP_JLE_X: ldimm64 test 1",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 2),
+ BPF_JMP_REG(BPF_JLE, R2, R1, 2),
+ BPF_LD_IMM64(R0, 0xffffffffffffffffULL),
+ BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xeeeeeeeeU } },
+ },
+ {
+ "JMP_JLE_X: ldimm64 test 2",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 0),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 2),
+ BPF_JMP_REG(BPF_JLE, R2, R1, 0),
+ BPF_LD_IMM64(R0, 0xffffffffffffffffULL),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 0xffffffffU } },
+ },
+ {
+ "JMP_JLE_X: ldimm64 test 3",
+ .u.insns_int = {
+ BPF_ALU32_IMM(BPF_MOV, R0, 1),
+ BPF_LD_IMM64(R1, 3),
+ BPF_LD_IMM64(R2, 2),
+ BPF_JMP_REG(BPF_JLE, R2, R1, 4),
+ BPF_LD_IMM64(R0, 0xffffffffffffffffULL),
+ BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL),
+ BPF_EXIT_INSN(),
+ },
+ INTERNAL,
+ { },
+ { { 0, 1 } },
+ },
/* BPF_JMP | BPF_JNE | BPF_X */
{
"JMP_JNE_X: if (3 != 2) return 1",
@@ -5022,6 +5432,14 @@ static struct bpf_test tests[] = {
{ { ETH_HLEN, 0xbef } },
.fill_helper = bpf_fill_ld_abs_vlan_push_pop,
},
+ {
+ "BPF_MAXINSNS: jump around ld_abs",
+ { },
+ INTERNAL,
+ { 10, 11 },
+ { { 2, 10 } },
+ .fill_helper = bpf_fill_jump_around_ld_abs,
+ },
/*
* LD_IND / LD_ABS on fragmented SKBs
*/
@@ -5663,7 +6081,7 @@ static struct sk_buff *populate_skb(char *buf, int size)
if (!skb)
return NULL;
- memcpy(__skb_put(skb, size), buf, size);
+ __skb_put_data(skb, buf, size);
/* Initialize a fake skb with test pattern. */
skb_reset_mac_header(skb);
@@ -5809,6 +6227,7 @@ static struct bpf_prog *generate_filter(int which, int *err)
/* Type doesn't really matter here as long as it's not unspec. */
fp->type = BPF_PROG_TYPE_SOCKET_FILTER;
memcpy(fp->insnsi, fptr, fp->len * sizeof(struct bpf_insn));
+ fp->aux->stack_depth = tests[which].stack_depth;
/* We cannot error here as we don't need type compatibility
* checks.
diff --git a/lib/test_debug_virtual.c b/lib/test_debug_virtual.c
new file mode 100644
index 000000000000..b9cdeecc19dc
--- /dev/null
+++ b/lib/test_debug_virtual.c
@@ -0,0 +1,49 @@
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/export.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/sizes.h>
+
+#include <asm/page.h>
+#ifdef CONFIG_MIPS
+#include <asm/bootinfo.h>
+#endif
+
+struct foo {
+ unsigned int bar;
+};
+
+struct foo *foo;
+
+static int __init test_debug_virtual_init(void)
+{
+ phys_addr_t pa;
+ void *va;
+
+ va = (void *)VMALLOC_START;
+ pa = virt_to_phys(va);
+
+ pr_info("PA: %pa for VA: 0x%lx\n", &pa, (unsigned long)va);
+
+ foo = kzalloc(sizeof(*foo), GFP_KERNEL);
+ if (!foo)
+ return -ENOMEM;
+
+ pa = virt_to_phys(foo);
+ va = foo;
+ pr_info("PA: %pa for VA: 0x%lx\n", &pa, (unsigned long)va);
+
+ return 0;
+}
+module_init(test_debug_virtual_init);
+
+static void __exit test_debug_virtual_exit(void)
+{
+ kfree(foo);
+}
+module_exit(test_debug_virtual_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Test module for CONFIG_DEBUG_VIRTUAL");
diff --git a/lib/test_firmware.c b/lib/test_firmware.c
index 09371b0a9baf..64a4c76cba2b 100644
--- a/lib/test_firmware.c
+++ b/lib/test_firmware.c
@@ -19,10 +19,85 @@
#include <linux/miscdevice.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
+#include <linux/delay.h>
+#include <linux/kthread.h>
+
+#define TEST_FIRMWARE_NAME "test-firmware.bin"
+#define TEST_FIRMWARE_NUM_REQS 4
static DEFINE_MUTEX(test_fw_mutex);
static const struct firmware *test_firmware;
+struct test_batched_req {
+ u8 idx;
+ int rc;
+ bool sent;
+ const struct firmware *fw;
+ const char *name;
+ struct completion completion;
+ struct task_struct *task;
+ struct device *dev;
+};
+
+/**
+ * test_config - represents configuration for the test for different triggers
+ *
+ * @name: the name of the firmware file to look for
+ * @sync_direct: when the sync trigger is used if this is true
+ * request_firmware_direct() will be used instead.
+ * @send_uevent: whether or not to send a uevent for async requests
+ * @num_requests: number of requests to try per test case. This is trigger
+ * specific.
+ * @reqs: stores all requests information
+ * @read_fw_idx: index of thread from which we want to read firmware results
+ * from through the read_fw trigger.
+ * @test_result: a test may use this to collect the result from the call
+ * of the request_firmware*() calls used in their tests. In order of
+ * priority we always keep first any setup error. If no setup errors were
+ * found then we move on to the first error encountered while running the
+ * API. Note that for async calls this typically will be a successful
+ * result (0) unless of course you've used bogus parameters, or the system
+ * is out of memory. In the async case the callback is expected to do a
+ * bit more homework to figure out what happened, unfortunately the only
+ * information passed today on error is the fact that no firmware was
+ * found so we can only assume -ENOENT on async calls if the firmware is
+ * NULL.
+ *
+ * Errors you can expect:
+ *
+ * API specific:
+ *
+ * 0: success for sync, for async it means request was sent
+ * -EINVAL: invalid parameters or request
+ * -ENOENT: files not found
+ *
+ * System environment:
+ *
+ * -ENOMEM: memory pressure on system
+ * -ENODEV: out of number of devices to test
+ * -EINVAL: an unexpected error has occurred
+ * @req_firmware: if @sync_direct is true this is set to
+ * request_firmware_direct(), otherwise request_firmware()
+ */
+struct test_config {
+ char *name;
+ bool sync_direct;
+ bool send_uevent;
+ u8 num_requests;
+ u8 read_fw_idx;
+
+ /*
+ * These below don't belong her but we'll move them once we create
+ * a struct fw_test_device and stuff the misc_dev under there later.
+ */
+ struct test_batched_req *reqs;
+ int test_result;
+ int (*req_firmware)(const struct firmware **fw, const char *name,
+ struct device *device);
+};
+
+struct test_config *test_fw_config;
+
static ssize_t test_fw_misc_read(struct file *f, char __user *buf,
size_t size, loff_t *offset)
{
@@ -42,6 +117,338 @@ static const struct file_operations test_fw_fops = {
.read = test_fw_misc_read,
};
+static void __test_release_all_firmware(void)
+{
+ struct test_batched_req *req;
+ u8 i;
+
+ if (!test_fw_config->reqs)
+ return;
+
+ for (i = 0; i < test_fw_config->num_requests; i++) {
+ req = &test_fw_config->reqs[i];
+ if (req->fw)
+ release_firmware(req->fw);
+ }
+
+ vfree(test_fw_config->reqs);
+ test_fw_config->reqs = NULL;
+}
+
+static void test_release_all_firmware(void)
+{
+ mutex_lock(&test_fw_mutex);
+ __test_release_all_firmware();
+ mutex_unlock(&test_fw_mutex);
+}
+
+
+static void __test_firmware_config_free(void)
+{
+ __test_release_all_firmware();
+ kfree_const(test_fw_config->name);
+ test_fw_config->name = NULL;
+}
+
+/*
+ * XXX: move to kstrncpy() once merged.
+ *
+ * Users should use kfree_const() when freeing these.
+ */
+static int __kstrncpy(char **dst, const char *name, size_t count, gfp_t gfp)
+{
+ *dst = kstrndup(name, count, gfp);
+ if (!*dst)
+ return -ENOSPC;
+ return count;
+}
+
+static int __test_firmware_config_init(void)
+{
+ int ret;
+
+ ret = __kstrncpy(&test_fw_config->name, TEST_FIRMWARE_NAME,
+ strlen(TEST_FIRMWARE_NAME), GFP_KERNEL);
+ if (ret < 0)
+ goto out;
+
+ test_fw_config->num_requests = TEST_FIRMWARE_NUM_REQS;
+ test_fw_config->send_uevent = true;
+ test_fw_config->sync_direct = false;
+ test_fw_config->req_firmware = request_firmware;
+ test_fw_config->test_result = 0;
+ test_fw_config->reqs = NULL;
+
+ return 0;
+
+out:
+ __test_firmware_config_free();
+ return ret;
+}
+
+static ssize_t reset_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+
+ mutex_lock(&test_fw_mutex);
+
+ __test_firmware_config_free();
+
+ ret = __test_firmware_config_init();
+ if (ret < 0) {
+ ret = -ENOMEM;
+ pr_err("could not alloc settings for config trigger: %d\n",
+ ret);
+ goto out;
+ }
+
+ pr_info("reset\n");
+ ret = count;
+
+out:
+ mutex_unlock(&test_fw_mutex);
+
+ return ret;
+}
+static DEVICE_ATTR_WO(reset);
+
+static ssize_t config_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ int len = 0;
+
+ mutex_lock(&test_fw_mutex);
+
+ len += snprintf(buf, PAGE_SIZE,
+ "Custom trigger configuration for: %s\n",
+ dev_name(dev));
+
+ if (test_fw_config->name)
+ len += snprintf(buf+len, PAGE_SIZE,
+ "name:\t%s\n",
+ test_fw_config->name);
+ else
+ len += snprintf(buf+len, PAGE_SIZE,
+ "name:\tEMTPY\n");
+
+ len += snprintf(buf+len, PAGE_SIZE,
+ "num_requests:\t%u\n", test_fw_config->num_requests);
+
+ len += snprintf(buf+len, PAGE_SIZE,
+ "send_uevent:\t\t%s\n",
+ test_fw_config->send_uevent ?
+ "FW_ACTION_HOTPLUG" :
+ "FW_ACTION_NOHOTPLUG");
+ len += snprintf(buf+len, PAGE_SIZE,
+ "sync_direct:\t\t%s\n",
+ test_fw_config->sync_direct ? "true" : "false");
+ len += snprintf(buf+len, PAGE_SIZE,
+ "read_fw_idx:\t%u\n", test_fw_config->read_fw_idx);
+
+ mutex_unlock(&test_fw_mutex);
+
+ return len;
+}
+static DEVICE_ATTR_RO(config);
+
+static ssize_t config_name_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+
+ mutex_lock(&test_fw_mutex);
+ kfree_const(test_fw_config->name);
+ ret = __kstrncpy(&test_fw_config->name, buf, count, GFP_KERNEL);
+ mutex_unlock(&test_fw_mutex);
+
+ return ret;
+}
+
+/*
+ * As per sysfs_kf_seq_show() the buf is max PAGE_SIZE.
+ */
+static ssize_t config_test_show_str(char *dst,
+ char *src)
+{
+ int len;
+
+ mutex_lock(&test_fw_mutex);
+ len = snprintf(dst, PAGE_SIZE, "%s\n", src);
+ mutex_unlock(&test_fw_mutex);
+
+ return len;
+}
+
+static int test_dev_config_update_bool(const char *buf, size_t size,
+ bool *cfg)
+{
+ int ret;
+
+ mutex_lock(&test_fw_mutex);
+ if (strtobool(buf, cfg) < 0)
+ ret = -EINVAL;
+ else
+ ret = size;
+ mutex_unlock(&test_fw_mutex);
+
+ return ret;
+}
+
+static ssize_t
+test_dev_config_show_bool(char *buf,
+ bool config)
+{
+ bool val;
+
+ mutex_lock(&test_fw_mutex);
+ val = config;
+ mutex_unlock(&test_fw_mutex);
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", val);
+}
+
+static ssize_t test_dev_config_show_int(char *buf, int cfg)
+{
+ int val;
+
+ mutex_lock(&test_fw_mutex);
+ val = cfg;
+ mutex_unlock(&test_fw_mutex);
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", val);
+}
+
+static int test_dev_config_update_u8(const char *buf, size_t size, u8 *cfg)
+{
+ int ret;
+ long new;
+
+ ret = kstrtol(buf, 10, &new);
+ if (ret)
+ return ret;
+
+ if (new > U8_MAX)
+ return -EINVAL;
+
+ mutex_lock(&test_fw_mutex);
+ *(u8 *)cfg = new;
+ mutex_unlock(&test_fw_mutex);
+
+ /* Always return full write size even if we didn't consume all */
+ return size;
+}
+
+static ssize_t test_dev_config_show_u8(char *buf, u8 cfg)
+{
+ u8 val;
+
+ mutex_lock(&test_fw_mutex);
+ val = cfg;
+ mutex_unlock(&test_fw_mutex);
+
+ return snprintf(buf, PAGE_SIZE, "%u\n", val);
+}
+
+static ssize_t config_name_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return config_test_show_str(buf, test_fw_config->name);
+}
+static DEVICE_ATTR(config_name, 0644, config_name_show, config_name_store);
+
+static ssize_t config_num_requests_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int rc;
+
+ mutex_lock(&test_fw_mutex);
+ if (test_fw_config->reqs) {
+ pr_err("Must call release_all_firmware prior to changing config\n");
+ rc = -EINVAL;
+ goto out;
+ }
+ mutex_unlock(&test_fw_mutex);
+
+ rc = test_dev_config_update_u8(buf, count,
+ &test_fw_config->num_requests);
+
+out:
+ return rc;
+}
+
+static ssize_t config_num_requests_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return test_dev_config_show_u8(buf, test_fw_config->num_requests);
+}
+static DEVICE_ATTR(config_num_requests, 0644, config_num_requests_show,
+ config_num_requests_store);
+
+static ssize_t config_sync_direct_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int rc = test_dev_config_update_bool(buf, count,
+ &test_fw_config->sync_direct);
+
+ if (rc == count)
+ test_fw_config->req_firmware = test_fw_config->sync_direct ?
+ request_firmware_direct :
+ request_firmware;
+ return rc;
+}
+
+static ssize_t config_sync_direct_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return test_dev_config_show_bool(buf, test_fw_config->sync_direct);
+}
+static DEVICE_ATTR(config_sync_direct, 0644, config_sync_direct_show,
+ config_sync_direct_store);
+
+static ssize_t config_send_uevent_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ return test_dev_config_update_bool(buf, count,
+ &test_fw_config->send_uevent);
+}
+
+static ssize_t config_send_uevent_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return test_dev_config_show_bool(buf, test_fw_config->send_uevent);
+}
+static DEVICE_ATTR(config_send_uevent, 0644, config_send_uevent_show,
+ config_send_uevent_store);
+
+static ssize_t config_read_fw_idx_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ return test_dev_config_update_u8(buf, count,
+ &test_fw_config->read_fw_idx);
+}
+
+static ssize_t config_read_fw_idx_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return test_dev_config_show_u8(buf, test_fw_config->read_fw_idx);
+}
+static DEVICE_ATTR(config_read_fw_idx, 0644, config_read_fw_idx_show,
+ config_read_fw_idx_store);
+
+
static ssize_t trigger_request_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
@@ -170,12 +577,301 @@ out:
}
static DEVICE_ATTR_WO(trigger_custom_fallback);
+static int test_fw_run_batch_request(void *data)
+{
+ struct test_batched_req *req = data;
+
+ if (!req) {
+ test_fw_config->test_result = -EINVAL;
+ return -EINVAL;
+ }
+
+ req->rc = test_fw_config->req_firmware(&req->fw, req->name, req->dev);
+ if (req->rc) {
+ pr_info("#%u: batched sync load failed: %d\n",
+ req->idx, req->rc);
+ if (!test_fw_config->test_result)
+ test_fw_config->test_result = req->rc;
+ } else if (req->fw) {
+ req->sent = true;
+ pr_info("#%u: batched sync loaded %zu\n",
+ req->idx, req->fw->size);
+ }
+ complete(&req->completion);
+
+ req->task = NULL;
+
+ return 0;
+}
+
+/*
+ * We use a kthread as otherwise the kernel serializes all our sync requests
+ * and we would not be able to mimic batched requests on a sync call. Batched
+ * requests on a sync call can for instance happen on a device driver when
+ * multiple cards are used and firmware loading happens outside of probe.
+ */
+static ssize_t trigger_batched_requests_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct test_batched_req *req;
+ int rc;
+ u8 i;
+
+ mutex_lock(&test_fw_mutex);
+
+ test_fw_config->reqs = vzalloc(sizeof(struct test_batched_req) *
+ test_fw_config->num_requests * 2);
+ if (!test_fw_config->reqs) {
+ rc = -ENOMEM;
+ goto out_unlock;
+ }
+
+ pr_info("batched sync firmware loading '%s' %u times\n",
+ test_fw_config->name, test_fw_config->num_requests);
+
+ for (i = 0; i < test_fw_config->num_requests; i++) {
+ req = &test_fw_config->reqs[i];
+ if (!req) {
+ WARN_ON(1);
+ rc = -ENOMEM;
+ goto out_bail;
+ }
+ req->fw = NULL;
+ req->idx = i;
+ req->name = test_fw_config->name;
+ req->dev = dev;
+ init_completion(&req->completion);
+ req->task = kthread_run(test_fw_run_batch_request, req,
+ "%s-%u", KBUILD_MODNAME, req->idx);
+ if (!req->task || IS_ERR(req->task)) {
+ pr_err("Setting up thread %u failed\n", req->idx);
+ req->task = NULL;
+ rc = -ENOMEM;
+ goto out_bail;
+ }
+ }
+
+ rc = count;
+
+ /*
+ * We require an explicit release to enable more time and delay of
+ * calling release_firmware() to improve our chances of forcing a
+ * batched request. If we instead called release_firmware() right away
+ * then we might miss on an opportunity of having a successful firmware
+ * request pass on the opportunity to be come a batched request.
+ */
+
+out_bail:
+ for (i = 0; i < test_fw_config->num_requests; i++) {
+ req = &test_fw_config->reqs[i];
+ if (req->task || req->sent)
+ wait_for_completion(&req->completion);
+ }
+
+ /* Override any worker error if we had a general setup error */
+ if (rc < 0)
+ test_fw_config->test_result = rc;
+
+out_unlock:
+ mutex_unlock(&test_fw_mutex);
+
+ return rc;
+}
+static DEVICE_ATTR_WO(trigger_batched_requests);
+
+/*
+ * We wait for each callback to return with the lock held, no need to lock here
+ */
+static void trigger_batched_cb(const struct firmware *fw, void *context)
+{
+ struct test_batched_req *req = context;
+
+ if (!req) {
+ test_fw_config->test_result = -EINVAL;
+ return;
+ }
+
+ /* forces *some* batched requests to queue up */
+ if (!req->idx)
+ ssleep(2);
+
+ req->fw = fw;
+
+ /*
+ * Unfortunately the firmware API gives us nothing other than a null FW
+ * if the firmware was not found on async requests. Best we can do is
+ * just assume -ENOENT. A better API would pass the actual return
+ * value to the callback.
+ */
+ if (!fw && !test_fw_config->test_result)
+ test_fw_config->test_result = -ENOENT;
+
+ complete(&req->completion);
+}
+
+static
+ssize_t trigger_batched_requests_async_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct test_batched_req *req;
+ bool send_uevent;
+ int rc;
+ u8 i;
+
+ mutex_lock(&test_fw_mutex);
+
+ test_fw_config->reqs = vzalloc(sizeof(struct test_batched_req) *
+ test_fw_config->num_requests * 2);
+ if (!test_fw_config->reqs) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ pr_info("batched loading '%s' custom fallback mechanism %u times\n",
+ test_fw_config->name, test_fw_config->num_requests);
+
+ send_uevent = test_fw_config->send_uevent ? FW_ACTION_HOTPLUG :
+ FW_ACTION_NOHOTPLUG;
+
+ for (i = 0; i < test_fw_config->num_requests; i++) {
+ req = &test_fw_config->reqs[i];
+ if (!req) {
+ WARN_ON(1);
+ goto out_bail;
+ }
+ req->name = test_fw_config->name;
+ req->fw = NULL;
+ req->idx = i;
+ init_completion(&req->completion);
+ rc = request_firmware_nowait(THIS_MODULE, send_uevent,
+ req->name,
+ dev, GFP_KERNEL, req,
+ trigger_batched_cb);
+ if (rc) {
+ pr_info("#%u: batched async load failed setup: %d\n",
+ i, rc);
+ req->rc = rc;
+ goto out_bail;
+ } else
+ req->sent = true;
+ }
+
+ rc = count;
+
+out_bail:
+
+ /*
+ * We require an explicit release to enable more time and delay of
+ * calling release_firmware() to improve our chances of forcing a
+ * batched request. If we instead called release_firmware() right away
+ * then we might miss on an opportunity of having a successful firmware
+ * request pass on the opportunity to be come a batched request.
+ */
+
+ for (i = 0; i < test_fw_config->num_requests; i++) {
+ req = &test_fw_config->reqs[i];
+ if (req->sent)
+ wait_for_completion(&req->completion);
+ }
+
+ /* Override any worker error if we had a general setup error */
+ if (rc < 0)
+ test_fw_config->test_result = rc;
+
+out:
+ mutex_unlock(&test_fw_mutex);
+
+ return rc;
+}
+static DEVICE_ATTR_WO(trigger_batched_requests_async);
+
+static ssize_t test_result_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return test_dev_config_show_int(buf, test_fw_config->test_result);
+}
+static DEVICE_ATTR_RO(test_result);
+
+static ssize_t release_all_firmware_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ test_release_all_firmware();
+ return count;
+}
+static DEVICE_ATTR_WO(release_all_firmware);
+
+static ssize_t read_firmware_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct test_batched_req *req;
+ u8 idx;
+ ssize_t rc = 0;
+
+ mutex_lock(&test_fw_mutex);
+
+ idx = test_fw_config->read_fw_idx;
+ if (idx >= test_fw_config->num_requests) {
+ rc = -ERANGE;
+ goto out;
+ }
+
+ if (!test_fw_config->reqs) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ req = &test_fw_config->reqs[idx];
+ if (!req->fw) {
+ pr_err("#%u: failed to async load firmware\n", idx);
+ rc = -ENOENT;
+ goto out;
+ }
+
+ pr_info("#%u: loaded %zu\n", idx, req->fw->size);
+
+ if (req->fw->size > PAGE_SIZE) {
+ pr_err("Testing interface must use PAGE_SIZE firmware for now\n");
+ rc = -EINVAL;
+ }
+ memcpy(buf, req->fw->data, req->fw->size);
+
+ rc = req->fw->size;
+out:
+ mutex_unlock(&test_fw_mutex);
+
+ return rc;
+}
+static DEVICE_ATTR_RO(read_firmware);
+
#define TEST_FW_DEV_ATTR(name) &dev_attr_##name.attr
static struct attribute *test_dev_attrs[] = {
+ TEST_FW_DEV_ATTR(reset),
+
+ TEST_FW_DEV_ATTR(config),
+ TEST_FW_DEV_ATTR(config_name),
+ TEST_FW_DEV_ATTR(config_num_requests),
+ TEST_FW_DEV_ATTR(config_sync_direct),
+ TEST_FW_DEV_ATTR(config_send_uevent),
+ TEST_FW_DEV_ATTR(config_read_fw_idx),
+
+ /* These don't use the config at all - they could be ported! */
TEST_FW_DEV_ATTR(trigger_request),
TEST_FW_DEV_ATTR(trigger_async_request),
TEST_FW_DEV_ATTR(trigger_custom_fallback),
+
+ /* These use the config and can use the test_result */
+ TEST_FW_DEV_ATTR(trigger_batched_requests),
+ TEST_FW_DEV_ATTR(trigger_batched_requests_async),
+
+ TEST_FW_DEV_ATTR(release_all_firmware),
+ TEST_FW_DEV_ATTR(test_result),
+ TEST_FW_DEV_ATTR(read_firmware),
NULL,
};
@@ -192,8 +888,17 @@ static int __init test_firmware_init(void)
{
int rc;
+ test_fw_config = kzalloc(sizeof(struct test_config), GFP_KERNEL);
+ if (!test_fw_config)
+ return -ENOMEM;
+
+ rc = __test_firmware_config_init();
+ if (rc)
+ return rc;
+
rc = misc_register(&test_fw_misc_device);
if (rc) {
+ kfree(test_fw_config);
pr_err("could not register misc device: %d\n", rc);
return rc;
}
@@ -207,8 +912,13 @@ module_init(test_firmware_init);
static void __exit test_firmware_exit(void)
{
+ mutex_lock(&test_fw_mutex);
release_firmware(test_firmware);
misc_deregister(&test_fw_misc_device);
+ __test_firmware_config_free();
+ kfree(test_fw_config);
+ mutex_unlock(&test_fw_mutex);
+
pr_warn("removed interface\n");
}
diff --git a/lib/test_kmod.c b/lib/test_kmod.c
new file mode 100644
index 000000000000..fba78d25e825
--- /dev/null
+++ b/lib/test_kmod.c
@@ -0,0 +1,1246 @@
+/*
+ * kmod stress test driver
+ *
+ * Copyright (C) 2017 Luis R. Rodriguez <mcgrof@kernel.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or at your option any
+ * later version; or, when distributed separately from the Linux kernel or
+ * when incorporated into other software packages, subject to the following
+ * license:
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of copyleft-next (version 0.3.1 or later) as published
+ * at http://copyleft-next.org/.
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+/*
+ * This driver provides an interface to trigger and test the kernel's
+ * module loader through a series of configurations and a few triggers.
+ * To test this driver use the following script as root:
+ *
+ * tools/testing/selftests/kmod/kmod.sh --help
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/kmod.h>
+#include <linux/printk.h>
+#include <linux/kthread.h>
+#include <linux/sched.h>
+#include <linux/fs.h>
+#include <linux/miscdevice.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/device.h>
+
+#define TEST_START_NUM_THREADS 50
+#define TEST_START_DRIVER "test_module"
+#define TEST_START_TEST_FS "xfs"
+#define TEST_START_TEST_CASE TEST_KMOD_DRIVER
+
+
+static bool force_init_test = false;
+module_param(force_init_test, bool_enable_only, 0644);
+MODULE_PARM_DESC(force_init_test,
+ "Force kicking a test immediately after driver loads");
+
+/*
+ * For device allocation / registration
+ */
+static DEFINE_MUTEX(reg_dev_mutex);
+static LIST_HEAD(reg_test_devs);
+
+/*
+ * num_test_devs actually represents the *next* ID of the next
+ * device we will allow to create.
+ */
+static int num_test_devs;
+
+/**
+ * enum kmod_test_case - linker table test case
+ *
+ * If you add a test case, please be sure to review if you need to se
+ * @need_mod_put for your tests case.
+ *
+ * @TEST_KMOD_DRIVER: stress tests request_module()
+ * @TEST_KMOD_FS_TYPE: stress tests get_fs_type()
+ */
+enum kmod_test_case {
+ __TEST_KMOD_INVALID = 0,
+
+ TEST_KMOD_DRIVER,
+ TEST_KMOD_FS_TYPE,
+
+ __TEST_KMOD_MAX,
+};
+
+struct test_config {
+ char *test_driver;
+ char *test_fs;
+ unsigned int num_threads;
+ enum kmod_test_case test_case;
+ int test_result;
+};
+
+struct kmod_test_device;
+
+/**
+ * kmod_test_device_info - thread info
+ *
+ * @ret_sync: return value if request_module() is used, sync request for
+ * @TEST_KMOD_DRIVER
+ * @fs_sync: return value of get_fs_type() for @TEST_KMOD_FS_TYPE
+ * @thread_idx: thread ID
+ * @test_dev: test device test is being performed under
+ * @need_mod_put: Some tests (get_fs_type() is one) requires putting the module
+ * (module_put(fs_sync->owner)) when done, otherwise you will not be able
+ * to unload the respective modules and re-test. We use this to keep
+ * accounting of when we need this and to help out in case we need to
+ * error out and deal with module_put() on error.
+ */
+struct kmod_test_device_info {
+ int ret_sync;
+ struct file_system_type *fs_sync;
+ struct task_struct *task_sync;
+ unsigned int thread_idx;
+ struct kmod_test_device *test_dev;
+ bool need_mod_put;
+};
+
+/**
+ * kmod_test_device - test device to help test kmod
+ *
+ * @dev_idx: unique ID for test device
+ * @config: configuration for the test
+ * @misc_dev: we use a misc device under the hood
+ * @dev: pointer to misc_dev's own struct device
+ * @config_mutex: protects configuration of test
+ * @trigger_mutex: the test trigger can only be fired once at a time
+ * @thread_lock: protects @done count, and the @info per each thread
+ * @done: number of threads which have completed or failed
+ * @test_is_oom: when we run out of memory, use this to halt moving forward
+ * @kthreads_done: completion used to signal when all work is done
+ * @list: needed to be part of the reg_test_devs
+ * @info: array of info for each thread
+ */
+struct kmod_test_device {
+ int dev_idx;
+ struct test_config config;
+ struct miscdevice misc_dev;
+ struct device *dev;
+ struct mutex config_mutex;
+ struct mutex trigger_mutex;
+ struct mutex thread_mutex;
+
+ unsigned int done;
+
+ bool test_is_oom;
+ struct completion kthreads_done;
+ struct list_head list;
+
+ struct kmod_test_device_info *info;
+};
+
+static const char *test_case_str(enum kmod_test_case test_case)
+{
+ switch (test_case) {
+ case TEST_KMOD_DRIVER:
+ return "TEST_KMOD_DRIVER";
+ case TEST_KMOD_FS_TYPE:
+ return "TEST_KMOD_FS_TYPE";
+ default:
+ return "invalid";
+ }
+}
+
+static struct miscdevice *dev_to_misc_dev(struct device *dev)
+{
+ return dev_get_drvdata(dev);
+}
+
+static struct kmod_test_device *misc_dev_to_test_dev(struct miscdevice *misc_dev)
+{
+ return container_of(misc_dev, struct kmod_test_device, misc_dev);
+}
+
+static struct kmod_test_device *dev_to_test_dev(struct device *dev)
+{
+ struct miscdevice *misc_dev;
+
+ misc_dev = dev_to_misc_dev(dev);
+
+ return misc_dev_to_test_dev(misc_dev);
+}
+
+/* Must run with thread_mutex held */
+static void kmod_test_done_check(struct kmod_test_device *test_dev,
+ unsigned int idx)
+{
+ struct test_config *config = &test_dev->config;
+
+ test_dev->done++;
+ dev_dbg(test_dev->dev, "Done thread count: %u\n", test_dev->done);
+
+ if (test_dev->done == config->num_threads) {
+ dev_info(test_dev->dev, "Done: %u threads have all run now\n",
+ test_dev->done);
+ dev_info(test_dev->dev, "Last thread to run: %u\n", idx);
+ complete(&test_dev->kthreads_done);
+ }
+}
+
+static void test_kmod_put_module(struct kmod_test_device_info *info)
+{
+ struct kmod_test_device *test_dev = info->test_dev;
+ struct test_config *config = &test_dev->config;
+
+ if (!info->need_mod_put)
+ return;
+
+ switch (config->test_case) {
+ case TEST_KMOD_DRIVER:
+ break;
+ case TEST_KMOD_FS_TYPE:
+ if (info && info->fs_sync && info->fs_sync->owner)
+ module_put(info->fs_sync->owner);
+ break;
+ default:
+ BUG();
+ }
+
+ info->need_mod_put = true;
+}
+
+static int run_request(void *data)
+{
+ struct kmod_test_device_info *info = data;
+ struct kmod_test_device *test_dev = info->test_dev;
+ struct test_config *config = &test_dev->config;
+
+ switch (config->test_case) {
+ case TEST_KMOD_DRIVER:
+ info->ret_sync = request_module("%s", config->test_driver);
+ break;
+ case TEST_KMOD_FS_TYPE:
+ info->fs_sync = get_fs_type(config->test_fs);
+ info->need_mod_put = true;
+ break;
+ default:
+ /* __trigger_config_run() already checked for test sanity */
+ BUG();
+ return -EINVAL;
+ }
+
+ dev_dbg(test_dev->dev, "Ran thread %u\n", info->thread_idx);
+
+ test_kmod_put_module(info);
+
+ mutex_lock(&test_dev->thread_mutex);
+ info->task_sync = NULL;
+ kmod_test_done_check(test_dev, info->thread_idx);
+ mutex_unlock(&test_dev->thread_mutex);
+
+ return 0;
+}
+
+static int tally_work_test(struct kmod_test_device_info *info)
+{
+ struct kmod_test_device *test_dev = info->test_dev;
+ struct test_config *config = &test_dev->config;
+ int err_ret = 0;
+
+ switch (config->test_case) {
+ case TEST_KMOD_DRIVER:
+ /*
+ * Only capture errors, if one is found that's
+ * enough, for now.
+ */
+ if (info->ret_sync != 0)
+ err_ret = info->ret_sync;
+ dev_info(test_dev->dev,
+ "Sync thread %d return status: %d\n",
+ info->thread_idx, info->ret_sync);
+ break;
+ case TEST_KMOD_FS_TYPE:
+ /* For now we make this simple */
+ if (!info->fs_sync)
+ err_ret = -EINVAL;
+ dev_info(test_dev->dev, "Sync thread %u fs: %s\n",
+ info->thread_idx, info->fs_sync ? config->test_fs :
+ "NULL");
+ break;
+ default:
+ BUG();
+ }
+
+ return err_ret;
+}
+
+/*
+ * XXX: add result option to display if all errors did not match.
+ * For now we just keep any error code if one was found.
+ *
+ * If this ran it means *all* tasks were created fine and we
+ * are now just collecting results.
+ *
+ * Only propagate errors, do not override with a subsequent sucess case.
+ */
+static void tally_up_work(struct kmod_test_device *test_dev)
+{
+ struct test_config *config = &test_dev->config;
+ struct kmod_test_device_info *info;
+ unsigned int idx;
+ int err_ret = 0;
+ int ret = 0;
+
+ mutex_lock(&test_dev->thread_mutex);
+
+ dev_info(test_dev->dev, "Results:\n");
+
+ for (idx=0; idx < config->num_threads; idx++) {
+ info = &test_dev->info[idx];
+ ret = tally_work_test(info);
+ if (ret)
+ err_ret = ret;
+ }
+
+ /*
+ * Note: request_module() returns 256 for a module not found even
+ * though modprobe itself returns 1.
+ */
+ config->test_result = err_ret;
+
+ mutex_unlock(&test_dev->thread_mutex);
+}
+
+static int try_one_request(struct kmod_test_device *test_dev, unsigned int idx)
+{
+ struct kmod_test_device_info *info = &test_dev->info[idx];
+ int fail_ret = -ENOMEM;
+
+ mutex_lock(&test_dev->thread_mutex);
+
+ info->thread_idx = idx;
+ info->test_dev = test_dev;
+ info->task_sync = kthread_run(run_request, info, "%s-%u",
+ KBUILD_MODNAME, idx);
+
+ if (!info->task_sync || IS_ERR(info->task_sync)) {
+ test_dev->test_is_oom = true;
+ dev_err(test_dev->dev, "Setting up thread %u failed\n", idx);
+ info->task_sync = NULL;
+ goto err_out;
+ } else
+ dev_dbg(test_dev->dev, "Kicked off thread %u\n", idx);
+
+ mutex_unlock(&test_dev->thread_mutex);
+
+ return 0;
+
+err_out:
+ info->ret_sync = fail_ret;
+ mutex_unlock(&test_dev->thread_mutex);
+
+ return fail_ret;
+}
+
+static void test_dev_kmod_stop_tests(struct kmod_test_device *test_dev)
+{
+ struct test_config *config = &test_dev->config;
+ struct kmod_test_device_info *info;
+ unsigned int i;
+
+ dev_info(test_dev->dev, "Ending request_module() tests\n");
+
+ mutex_lock(&test_dev->thread_mutex);
+
+ for (i=0; i < config->num_threads; i++) {
+ info = &test_dev->info[i];
+ if (info->task_sync && !IS_ERR(info->task_sync)) {
+ dev_info(test_dev->dev,
+ "Stopping still-running thread %i\n", i);
+ kthread_stop(info->task_sync);
+ }
+
+ /*
+ * info->task_sync is well protected, it can only be
+ * NULL or a pointer to a struct. If its NULL we either
+ * never ran, or we did and we completed the work. Completed
+ * tasks *always* put the module for us. This is a sanity
+ * check -- just in case.
+ */
+ if (info->task_sync && info->need_mod_put)
+ test_kmod_put_module(info);
+ }
+
+ mutex_unlock(&test_dev->thread_mutex);
+}
+
+/*
+ * Only wait *iff* we did not run into any errors during all of our thread
+ * set up. If run into any issues we stop threads and just bail out with
+ * an error to the trigger. This also means we don't need any tally work
+ * for any threads which fail.
+ */
+static int try_requests(struct kmod_test_device *test_dev)
+{
+ struct test_config *config = &test_dev->config;
+ unsigned int idx;
+ int ret;
+ bool any_error = false;
+
+ for (idx=0; idx < config->num_threads; idx++) {
+ if (test_dev->test_is_oom) {
+ any_error = true;
+ break;
+ }
+
+ ret = try_one_request(test_dev, idx);
+ if (ret) {
+ any_error = true;
+ break;
+ }
+ }
+
+ if (!any_error) {
+ test_dev->test_is_oom = false;
+ dev_info(test_dev->dev,
+ "No errors were found while initializing threads\n");
+ wait_for_completion(&test_dev->kthreads_done);
+ tally_up_work(test_dev);
+ } else {
+ test_dev->test_is_oom = true;
+ dev_info(test_dev->dev,
+ "At least one thread failed to start, stop all work\n");
+ test_dev_kmod_stop_tests(test_dev);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static int run_test_driver(struct kmod_test_device *test_dev)
+{
+ struct test_config *config = &test_dev->config;
+
+ dev_info(test_dev->dev, "Test case: %s (%u)\n",
+ test_case_str(config->test_case),
+ config->test_case);
+ dev_info(test_dev->dev, "Test driver to load: %s\n",
+ config->test_driver);
+ dev_info(test_dev->dev, "Number of threads to run: %u\n",
+ config->num_threads);
+ dev_info(test_dev->dev, "Thread IDs will range from 0 - %u\n",
+ config->num_threads - 1);
+
+ return try_requests(test_dev);
+}
+
+static int run_test_fs_type(struct kmod_test_device *test_dev)
+{
+ struct test_config *config = &test_dev->config;
+
+ dev_info(test_dev->dev, "Test case: %s (%u)\n",
+ test_case_str(config->test_case),
+ config->test_case);
+ dev_info(test_dev->dev, "Test filesystem to load: %s\n",
+ config->test_fs);
+ dev_info(test_dev->dev, "Number of threads to run: %u\n",
+ config->num_threads);
+ dev_info(test_dev->dev, "Thread IDs will range from 0 - %u\n",
+ config->num_threads - 1);
+
+ return try_requests(test_dev);
+}
+
+static ssize_t config_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+ int len = 0;
+
+ mutex_lock(&test_dev->config_mutex);
+
+ len += snprintf(buf, PAGE_SIZE,
+ "Custom trigger configuration for: %s\n",
+ dev_name(dev));
+
+ len += snprintf(buf+len, PAGE_SIZE - len,
+ "Number of threads:\t%u\n",
+ config->num_threads);
+
+ len += snprintf(buf+len, PAGE_SIZE - len,
+ "Test_case:\t%s (%u)\n",
+ test_case_str(config->test_case),
+ config->test_case);
+
+ if (config->test_driver)
+ len += snprintf(buf+len, PAGE_SIZE - len,
+ "driver:\t%s\n",
+ config->test_driver);
+ else
+ len += snprintf(buf+len, PAGE_SIZE - len,
+ "driver:\tEMPTY\n");
+
+ if (config->test_fs)
+ len += snprintf(buf+len, PAGE_SIZE - len,
+ "fs:\t%s\n",
+ config->test_fs);
+ else
+ len += snprintf(buf+len, PAGE_SIZE - len,
+ "fs:\tEMPTY\n");
+
+ mutex_unlock(&test_dev->config_mutex);
+
+ return len;
+}
+static DEVICE_ATTR_RO(config);
+
+/*
+ * This ensures we don't allow kicking threads through if our configuration
+ * is faulty.
+ */
+static int __trigger_config_run(struct kmod_test_device *test_dev)
+{
+ struct test_config *config = &test_dev->config;
+
+ test_dev->done = 0;
+
+ switch (config->test_case) {
+ case TEST_KMOD_DRIVER:
+ return run_test_driver(test_dev);
+ case TEST_KMOD_FS_TYPE:
+ return run_test_fs_type(test_dev);
+ default:
+ dev_warn(test_dev->dev,
+ "Invalid test case requested: %u\n",
+ config->test_case);
+ return -EINVAL;
+ }
+}
+
+static int trigger_config_run(struct kmod_test_device *test_dev)
+{
+ struct test_config *config = &test_dev->config;
+ int ret;
+
+ mutex_lock(&test_dev->trigger_mutex);
+ mutex_lock(&test_dev->config_mutex);
+
+ ret = __trigger_config_run(test_dev);
+ if (ret < 0)
+ goto out;
+ dev_info(test_dev->dev, "General test result: %d\n",
+ config->test_result);
+
+ /*
+ * We must return 0 after a trigger even unless something went
+ * wrong with the setup of the test. If the test setup went fine
+ * then userspace must just check the result of config->test_result.
+ * One issue with relying on the return from a call in the kernel
+ * is if the kernel returns a possitive value using this trigger
+ * will not return the value to userspace, it would be lost.
+ *
+ * By not relying on capturing the return value of tests we are using
+ * through the trigger it also us to run tests with set -e and only
+ * fail when something went wrong with the driver upon trigger
+ * requests.
+ */
+ ret = 0;
+
+out:
+ mutex_unlock(&test_dev->config_mutex);
+ mutex_unlock(&test_dev->trigger_mutex);
+
+ return ret;
+}
+
+static ssize_t
+trigger_config_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ int ret;
+
+ if (test_dev->test_is_oom)
+ return -ENOMEM;
+
+ /* For all intents and purposes we don't care what userspace
+ * sent this trigger, we care only that we were triggered.
+ * We treat the return value only for caputuring issues with
+ * the test setup. At this point all the test variables should
+ * have been allocated so typically this should never fail.
+ */
+ ret = trigger_config_run(test_dev);
+ if (unlikely(ret < 0))
+ goto out;
+
+ /*
+ * Note: any return > 0 will be treated as success
+ * and the error value will not be available to userspace.
+ * Do not rely on trying to send to userspace a test value
+ * return value as possitive return errors will be lost.
+ */
+ if (WARN_ON(ret > 0))
+ return -EINVAL;
+
+ ret = count;
+out:
+ return ret;
+}
+static DEVICE_ATTR_WO(trigger_config);
+
+/*
+ * XXX: move to kstrncpy() once merged.
+ *
+ * Users should use kfree_const() when freeing these.
+ */
+static int __kstrncpy(char **dst, const char *name, size_t count, gfp_t gfp)
+{
+ *dst = kstrndup(name, count, gfp);
+ if (!*dst)
+ return -ENOSPC;
+ return count;
+}
+
+static int config_copy_test_driver_name(struct test_config *config,
+ const char *name,
+ size_t count)
+{
+ return __kstrncpy(&config->test_driver, name, count, GFP_KERNEL);
+}
+
+
+static int config_copy_test_fs(struct test_config *config, const char *name,
+ size_t count)
+{
+ return __kstrncpy(&config->test_fs, name, count, GFP_KERNEL);
+}
+
+static void __kmod_config_free(struct test_config *config)
+{
+ if (!config)
+ return;
+
+ kfree_const(config->test_driver);
+ config->test_driver = NULL;
+
+ kfree_const(config->test_fs);
+ config->test_driver = NULL;
+}
+
+static void kmod_config_free(struct kmod_test_device *test_dev)
+{
+ struct test_config *config;
+
+ if (!test_dev)
+ return;
+
+ config = &test_dev->config;
+
+ mutex_lock(&test_dev->config_mutex);
+ __kmod_config_free(config);
+ mutex_unlock(&test_dev->config_mutex);
+}
+
+static ssize_t config_test_driver_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+ int copied;
+
+ mutex_lock(&test_dev->config_mutex);
+
+ kfree_const(config->test_driver);
+ config->test_driver = NULL;
+
+ copied = config_copy_test_driver_name(config, buf, count);
+ mutex_unlock(&test_dev->config_mutex);
+
+ return copied;
+}
+
+/*
+ * As per sysfs_kf_seq_show() the buf is max PAGE_SIZE.
+ */
+static ssize_t config_test_show_str(struct mutex *config_mutex,
+ char *dst,
+ char *src)
+{
+ int len;
+
+ mutex_lock(config_mutex);
+ len = snprintf(dst, PAGE_SIZE, "%s\n", src);
+ mutex_unlock(config_mutex);
+
+ return len;
+}
+
+static ssize_t config_test_driver_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+
+ return config_test_show_str(&test_dev->config_mutex, buf,
+ config->test_driver);
+}
+static DEVICE_ATTR(config_test_driver, 0644, config_test_driver_show,
+ config_test_driver_store);
+
+static ssize_t config_test_fs_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+ int copied;
+
+ mutex_lock(&test_dev->config_mutex);
+
+ kfree_const(config->test_fs);
+ config->test_fs = NULL;
+
+ copied = config_copy_test_fs(config, buf, count);
+ mutex_unlock(&test_dev->config_mutex);
+
+ return copied;
+}
+
+static ssize_t config_test_fs_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+
+ return config_test_show_str(&test_dev->config_mutex, buf,
+ config->test_fs);
+}
+static DEVICE_ATTR(config_test_fs, 0644, config_test_fs_show,
+ config_test_fs_store);
+
+static int trigger_config_run_type(struct kmod_test_device *test_dev,
+ enum kmod_test_case test_case,
+ const char *test_str)
+{
+ int copied = 0;
+ struct test_config *config = &test_dev->config;
+
+ mutex_lock(&test_dev->config_mutex);
+
+ switch (test_case) {
+ case TEST_KMOD_DRIVER:
+ kfree_const(config->test_driver);
+ config->test_driver = NULL;
+ copied = config_copy_test_driver_name(config, test_str,
+ strlen(test_str));
+ break;
+ case TEST_KMOD_FS_TYPE:
+ kfree_const(config->test_fs);
+ config->test_driver = NULL;
+ copied = config_copy_test_fs(config, test_str,
+ strlen(test_str));
+ break;
+ default:
+ mutex_unlock(&test_dev->config_mutex);
+ return -EINVAL;
+ }
+
+ config->test_case = test_case;
+
+ mutex_unlock(&test_dev->config_mutex);
+
+ if (copied <= 0 || copied != strlen(test_str)) {
+ test_dev->test_is_oom = true;
+ return -ENOMEM;
+ }
+
+ test_dev->test_is_oom = false;
+
+ return trigger_config_run(test_dev);
+}
+
+static void free_test_dev_info(struct kmod_test_device *test_dev)
+{
+ vfree(test_dev->info);
+ test_dev->info = NULL;
+}
+
+static int kmod_config_sync_info(struct kmod_test_device *test_dev)
+{
+ struct test_config *config = &test_dev->config;
+
+ free_test_dev_info(test_dev);
+ test_dev->info = vzalloc(config->num_threads *
+ sizeof(struct kmod_test_device_info));
+ if (!test_dev->info) {
+ dev_err(test_dev->dev, "Cannot alloc test_dev info\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+/*
+ * Old kernels may not have this, if you want to port this code to
+ * test it on older kernels.
+ */
+#ifdef get_kmod_umh_limit
+static unsigned int kmod_init_test_thread_limit(void)
+{
+ return get_kmod_umh_limit();
+}
+#else
+static unsigned int kmod_init_test_thread_limit(void)
+{
+ return TEST_START_NUM_THREADS;
+}
+#endif
+
+static int __kmod_config_init(struct kmod_test_device *test_dev)
+{
+ struct test_config *config = &test_dev->config;
+ int ret = -ENOMEM, copied;
+
+ __kmod_config_free(config);
+
+ copied = config_copy_test_driver_name(config, TEST_START_DRIVER,
+ strlen(TEST_START_DRIVER));
+ if (copied != strlen(TEST_START_DRIVER))
+ goto err_out;
+
+ copied = config_copy_test_fs(config, TEST_START_TEST_FS,
+ strlen(TEST_START_TEST_FS));
+ if (copied != strlen(TEST_START_TEST_FS))
+ goto err_out;
+
+ config->num_threads = kmod_init_test_thread_limit();
+ config->test_result = 0;
+ config->test_case = TEST_START_TEST_CASE;
+
+ ret = kmod_config_sync_info(test_dev);
+ if (ret)
+ goto err_out;
+
+ test_dev->test_is_oom = false;
+
+ return 0;
+
+err_out:
+ test_dev->test_is_oom = true;
+ WARN_ON(test_dev->test_is_oom);
+
+ __kmod_config_free(config);
+
+ return ret;
+}
+
+static ssize_t reset_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ int ret;
+
+ mutex_lock(&test_dev->trigger_mutex);
+ mutex_lock(&test_dev->config_mutex);
+
+ ret = __kmod_config_init(test_dev);
+ if (ret < 0) {
+ ret = -ENOMEM;
+ dev_err(dev, "could not alloc settings for config trigger: %d\n",
+ ret);
+ goto out;
+ }
+
+ dev_info(dev, "reset\n");
+ ret = count;
+
+out:
+ mutex_unlock(&test_dev->config_mutex);
+ mutex_unlock(&test_dev->trigger_mutex);
+
+ return ret;
+}
+static DEVICE_ATTR_WO(reset);
+
+static int test_dev_config_update_uint_sync(struct kmod_test_device *test_dev,
+ const char *buf, size_t size,
+ unsigned int *config,
+ int (*test_sync)(struct kmod_test_device *test_dev))
+{
+ int ret;
+ unsigned long new;
+ unsigned int old_val;
+
+ ret = kstrtoul(buf, 10, &new);
+ if (ret)
+ return ret;
+
+ if (new > UINT_MAX)
+ return -EINVAL;
+
+ mutex_lock(&test_dev->config_mutex);
+
+ old_val = *config;
+ *(unsigned int *)config = new;
+
+ ret = test_sync(test_dev);
+ if (ret) {
+ *(unsigned int *)config = old_val;
+
+ ret = test_sync(test_dev);
+ WARN_ON(ret);
+
+ mutex_unlock(&test_dev->config_mutex);
+ return -EINVAL;
+ }
+
+ mutex_unlock(&test_dev->config_mutex);
+ /* Always return full write size even if we didn't consume all */
+ return size;
+}
+
+static int test_dev_config_update_uint_range(struct kmod_test_device *test_dev,
+ const char *buf, size_t size,
+ unsigned int *config,
+ unsigned int min,
+ unsigned int max)
+{
+ int ret;
+ unsigned long new;
+
+ ret = kstrtoul(buf, 10, &new);
+ if (ret)
+ return ret;
+
+ if (new < min || new > max)
+ return -EINVAL;
+
+ mutex_lock(&test_dev->config_mutex);
+ *config = new;
+ mutex_unlock(&test_dev->config_mutex);
+
+ /* Always return full write size even if we didn't consume all */
+ return size;
+}
+
+static int test_dev_config_update_int(struct kmod_test_device *test_dev,
+ const char *buf, size_t size,
+ int *config)
+{
+ int ret;
+ long new;
+
+ ret = kstrtol(buf, 10, &new);
+ if (ret)
+ return ret;
+
+ if (new < INT_MIN || new > INT_MAX)
+ return -EINVAL;
+
+ mutex_lock(&test_dev->config_mutex);
+ *config = new;
+ mutex_unlock(&test_dev->config_mutex);
+ /* Always return full write size even if we didn't consume all */
+ return size;
+}
+
+static ssize_t test_dev_config_show_int(struct kmod_test_device *test_dev,
+ char *buf,
+ int config)
+{
+ int val;
+
+ mutex_lock(&test_dev->config_mutex);
+ val = config;
+ mutex_unlock(&test_dev->config_mutex);
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", val);
+}
+
+static ssize_t test_dev_config_show_uint(struct kmod_test_device *test_dev,
+ char *buf,
+ unsigned int config)
+{
+ unsigned int val;
+
+ mutex_lock(&test_dev->config_mutex);
+ val = config;
+ mutex_unlock(&test_dev->config_mutex);
+
+ return snprintf(buf, PAGE_SIZE, "%u\n", val);
+}
+
+static ssize_t test_result_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+
+ return test_dev_config_update_int(test_dev, buf, count,
+ &config->test_result);
+}
+
+static ssize_t config_num_threads_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+
+ return test_dev_config_update_uint_sync(test_dev, buf, count,
+ &config->num_threads,
+ kmod_config_sync_info);
+}
+
+static ssize_t config_num_threads_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+
+ return test_dev_config_show_int(test_dev, buf, config->num_threads);
+}
+static DEVICE_ATTR(config_num_threads, 0644, config_num_threads_show,
+ config_num_threads_store);
+
+static ssize_t config_test_case_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+
+ return test_dev_config_update_uint_range(test_dev, buf, count,
+ &config->test_case,
+ __TEST_KMOD_INVALID + 1,
+ __TEST_KMOD_MAX - 1);
+}
+
+static ssize_t config_test_case_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+
+ return test_dev_config_show_uint(test_dev, buf, config->test_case);
+}
+static DEVICE_ATTR(config_test_case, 0644, config_test_case_show,
+ config_test_case_store);
+
+static ssize_t test_result_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct kmod_test_device *test_dev = dev_to_test_dev(dev);
+ struct test_config *config = &test_dev->config;
+
+ return test_dev_config_show_int(test_dev, buf, config->test_result);
+}
+static DEVICE_ATTR(test_result, 0644, test_result_show, test_result_store);
+
+#define TEST_KMOD_DEV_ATTR(name) &dev_attr_##name.attr
+
+static struct attribute *test_dev_attrs[] = {
+ TEST_KMOD_DEV_ATTR(trigger_config),
+ TEST_KMOD_DEV_ATTR(config),
+ TEST_KMOD_DEV_ATTR(reset),
+
+ TEST_KMOD_DEV_ATTR(config_test_driver),
+ TEST_KMOD_DEV_ATTR(config_test_fs),
+ TEST_KMOD_DEV_ATTR(config_num_threads),
+ TEST_KMOD_DEV_ATTR(config_test_case),
+ TEST_KMOD_DEV_ATTR(test_result),
+
+ NULL,
+};
+
+ATTRIBUTE_GROUPS(test_dev);
+
+static int kmod_config_init(struct kmod_test_device *test_dev)
+{
+ int ret;
+
+ mutex_lock(&test_dev->config_mutex);
+ ret = __kmod_config_init(test_dev);
+ mutex_unlock(&test_dev->config_mutex);
+
+ return ret;
+}
+
+static struct kmod_test_device *alloc_test_dev_kmod(int idx)
+{
+ int ret;
+ struct kmod_test_device *test_dev;
+ struct miscdevice *misc_dev;
+
+ test_dev = vzalloc(sizeof(struct kmod_test_device));
+ if (!test_dev) {
+ pr_err("Cannot alloc test_dev\n");
+ goto err_out;
+ }
+
+ mutex_init(&test_dev->config_mutex);
+ mutex_init(&test_dev->trigger_mutex);
+ mutex_init(&test_dev->thread_mutex);
+
+ init_completion(&test_dev->kthreads_done);
+
+ ret = kmod_config_init(test_dev);
+ if (ret < 0) {
+ pr_err("Cannot alloc kmod_config_init()\n");
+ goto err_out_free;
+ }
+
+ test_dev->dev_idx = idx;
+ misc_dev = &test_dev->misc_dev;
+
+ misc_dev->minor = MISC_DYNAMIC_MINOR;
+ misc_dev->name = kasprintf(GFP_KERNEL, "test_kmod%d", idx);
+ if (!misc_dev->name) {
+ pr_err("Cannot alloc misc_dev->name\n");
+ goto err_out_free_config;
+ }
+ misc_dev->groups = test_dev_groups;
+
+ return test_dev;
+
+err_out_free_config:
+ free_test_dev_info(test_dev);
+ kmod_config_free(test_dev);
+err_out_free:
+ vfree(test_dev);
+ test_dev = NULL;
+err_out:
+ return NULL;
+}
+
+static void free_test_dev_kmod(struct kmod_test_device *test_dev)
+{
+ if (test_dev) {
+ kfree_const(test_dev->misc_dev.name);
+ test_dev->misc_dev.name = NULL;
+ free_test_dev_info(test_dev);
+ kmod_config_free(test_dev);
+ vfree(test_dev);
+ test_dev = NULL;
+ }
+}
+
+static struct kmod_test_device *register_test_dev_kmod(void)
+{
+ struct kmod_test_device *test_dev = NULL;
+ int ret;
+
+ mutex_lock(&reg_dev_mutex);
+
+ /* int should suffice for number of devices, test for wrap */
+ if (unlikely(num_test_devs + 1) < 0) {
+ pr_err("reached limit of number of test devices\n");
+ goto out;
+ }
+
+ test_dev = alloc_test_dev_kmod(num_test_devs);
+ if (!test_dev)
+ goto out;
+
+ ret = misc_register(&test_dev->misc_dev);
+ if (ret) {
+ pr_err("could not register misc device: %d\n", ret);
+ free_test_dev_kmod(test_dev);
+ goto out;
+ }
+
+ test_dev->dev = test_dev->misc_dev.this_device;
+ list_add_tail(&test_dev->list, &reg_test_devs);
+ dev_info(test_dev->dev, "interface ready\n");
+
+ num_test_devs++;
+
+out:
+ mutex_unlock(&reg_dev_mutex);
+
+ return test_dev;
+
+}
+
+static int __init test_kmod_init(void)
+{
+ struct kmod_test_device *test_dev;
+ int ret;
+
+ test_dev = register_test_dev_kmod();
+ if (!test_dev) {
+ pr_err("Cannot add first test kmod device\n");
+ return -ENODEV;
+ }
+
+ /*
+ * With some work we might be able to gracefully enable
+ * testing with this driver built-in, for now this seems
+ * rather risky. For those willing to try have at it,
+ * and enable the below. Good luck! If that works, try
+ * lowering the init level for more fun.
+ */
+ if (force_init_test) {
+ ret = trigger_config_run_type(test_dev,
+ TEST_KMOD_DRIVER, "tun");
+ if (WARN_ON(ret))
+ return ret;
+ ret = trigger_config_run_type(test_dev,
+ TEST_KMOD_FS_TYPE, "btrfs");
+ if (WARN_ON(ret))
+ return ret;
+ }
+
+ return 0;
+}
+late_initcall(test_kmod_init);
+
+static
+void unregister_test_dev_kmod(struct kmod_test_device *test_dev)
+{
+ mutex_lock(&test_dev->trigger_mutex);
+ mutex_lock(&test_dev->config_mutex);
+
+ test_dev_kmod_stop_tests(test_dev);
+
+ dev_info(test_dev->dev, "removing interface\n");
+ misc_deregister(&test_dev->misc_dev);
+ kfree(&test_dev->misc_dev.name);
+
+ mutex_unlock(&test_dev->config_mutex);
+ mutex_unlock(&test_dev->trigger_mutex);
+
+ free_test_dev_kmod(test_dev);
+}
+
+static void __exit test_kmod_exit(void)
+{
+ struct kmod_test_device *test_dev, *tmp;
+
+ mutex_lock(&reg_dev_mutex);
+ list_for_each_entry_safe(test_dev, tmp, &reg_test_devs, list) {
+ list_del(&test_dev->list);
+ unregister_test_dev_kmod(test_dev);
+ }
+ mutex_unlock(&reg_dev_mutex);
+}
+module_exit(test_kmod_exit);
+
+MODULE_AUTHOR("Luis R. Rodriguez <mcgrof@kernel.org>");
+MODULE_LICENSE("GPL");
diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c
index 64e899b63337..0ffca990a833 100644
--- a/lib/test_rhashtable.c
+++ b/lib/test_rhashtable.c
@@ -56,8 +56,13 @@ static bool enomem_retry = false;
module_param(enomem_retry, bool, 0);
MODULE_PARM_DESC(enomem_retry, "Retry insert even if -ENOMEM was returned (default: off)");
+struct test_obj_val {
+ int id;
+ int tid;
+};
+
struct test_obj {
- int value;
+ struct test_obj_val value;
struct rhash_head node;
};
@@ -72,7 +77,7 @@ static struct test_obj array[MAX_ENTRIES];
static struct rhashtable_params test_rht_params = {
.head_offset = offsetof(struct test_obj, node),
.key_offset = offsetof(struct test_obj, value),
- .key_len = sizeof(int),
+ .key_len = sizeof(struct test_obj_val),
.hashfn = jhash,
.nulls_base = (3U << RHT_BASE_SHIFT),
};
@@ -109,24 +114,26 @@ static int __init test_rht_lookup(struct rhashtable *ht)
for (i = 0; i < entries * 2; i++) {
struct test_obj *obj;
bool expected = !(i % 2);
- u32 key = i;
+ struct test_obj_val key = {
+ .id = i,
+ };
- if (array[i / 2].value == TEST_INSERT_FAIL)
+ if (array[i / 2].value.id == TEST_INSERT_FAIL)
expected = false;
obj = rhashtable_lookup_fast(ht, &key, test_rht_params);
if (expected && !obj) {
- pr_warn("Test failed: Could not find key %u\n", key);
+ pr_warn("Test failed: Could not find key %u\n", key.id);
return -ENOENT;
} else if (!expected && obj) {
pr_warn("Test failed: Unexpected entry found for key %u\n",
- key);
+ key.id);
return -EEXIST;
} else if (expected && obj) {
- if (obj->value != i) {
+ if (obj->value.id != i) {
pr_warn("Test failed: Lookup value mismatch %u!=%u\n",
- obj->value, i);
+ obj->value.id, i);
return -EINVAL;
}
}
@@ -195,7 +202,7 @@ static s64 __init test_rhashtable(struct rhashtable *ht)
for (i = 0; i < entries; i++) {
struct test_obj *obj = &array[i];
- obj->value = i * 2;
+ obj->value.id = i * 2;
err = insert_retry(ht, &obj->node, test_rht_params);
if (err > 0)
insert_retries += err;
@@ -216,9 +223,11 @@ static s64 __init test_rhashtable(struct rhashtable *ht)
pr_info(" Deleting %d keys\n", entries);
for (i = 0; i < entries; i++) {
- u32 key = i * 2;
+ struct test_obj_val key = {
+ .id = i * 2,
+ };
- if (array[i].value != TEST_INSERT_FAIL) {
+ if (array[i].value.id != TEST_INSERT_FAIL) {
obj = rhashtable_lookup_fast(ht, &key, test_rht_params);
BUG_ON(!obj);
@@ -242,18 +251,21 @@ static int thread_lookup_test(struct thread_data *tdata)
for (i = 0; i < entries; i++) {
struct test_obj *obj;
- int key = (tdata->id << 16) | i;
+ struct test_obj_val key = {
+ .id = i,
+ .tid = tdata->id,
+ };
obj = rhashtable_lookup_fast(&ht, &key, test_rht_params);
- if (obj && (tdata->objs[i].value == TEST_INSERT_FAIL)) {
- pr_err(" found unexpected object %d\n", key);
+ if (obj && (tdata->objs[i].value.id == TEST_INSERT_FAIL)) {
+ pr_err(" found unexpected object %d-%d\n", key.tid, key.id);
err++;
- } else if (!obj && (tdata->objs[i].value != TEST_INSERT_FAIL)) {
- pr_err(" object %d not found!\n", key);
+ } else if (!obj && (tdata->objs[i].value.id != TEST_INSERT_FAIL)) {
+ pr_err(" object %d-%d not found!\n", key.tid, key.id);
err++;
- } else if (obj && (obj->value != key)) {
- pr_err(" wrong object returned (got %d, expected %d)\n",
- obj->value, key);
+ } else if (obj && memcmp(&obj->value, &key, sizeof(key))) {
+ pr_err(" wrong object returned (got %d-%d, expected %d-%d)\n",
+ obj->value.tid, obj->value.id, key.tid, key.id);
err++;
}
@@ -272,7 +284,8 @@ static int threadfunc(void *data)
pr_err(" thread[%d]: down_interruptible failed\n", tdata->id);
for (i = 0; i < entries; i++) {
- tdata->objs[i].value = (tdata->id << 16) | i;
+ tdata->objs[i].value.id = i;
+ tdata->objs[i].value.tid = tdata->id;
err = insert_retry(&ht, &tdata->objs[i].node, test_rht_params);
if (err > 0) {
insert_retries += err;
@@ -295,7 +308,7 @@ static int threadfunc(void *data)
for (step = 10; step > 0; step--) {
for (i = 0; i < entries; i += step) {
- if (tdata->objs[i].value == TEST_INSERT_FAIL)
+ if (tdata->objs[i].value.id == TEST_INSERT_FAIL)
continue;
err = rhashtable_remove_fast(&ht, &tdata->objs[i].node,
test_rht_params);
@@ -304,7 +317,7 @@ static int threadfunc(void *data)
tdata->id);
goto out;
}
- tdata->objs[i].value = TEST_INSERT_FAIL;
+ tdata->objs[i].value.id = TEST_INSERT_FAIL;
cond_resched();
}
diff --git a/lib/test_sysctl.c b/lib/test_sysctl.c
new file mode 100644
index 000000000000..3dd801c1c85b
--- /dev/null
+++ b/lib/test_sysctl.c
@@ -0,0 +1,148 @@
+/*
+ * proc sysctl test driver
+ *
+ * Copyright (C) 2017 Luis R. Rodriguez <mcgrof@kernel.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or at your option any
+ * later version; or, when distributed separately from the Linux kernel or
+ * when incorporated into other software packages, subject to the following
+ * license:
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of copyleft-next (version 0.3.1 or later) as published
+ * at http://copyleft-next.org/.
+ */
+
+/*
+ * This module provides an interface to the the proc sysctl interfaces. This
+ * driver requires CONFIG_PROC_SYSCTL. It will not normally be loaded by the
+ * system unless explicitly requested by name. You can also build this driver
+ * into your kernel.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/printk.h>
+#include <linux/fs.h>
+#include <linux/miscdevice.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/async.h>
+#include <linux/delay.h>
+#include <linux/vmalloc.h>
+
+static int i_zero;
+static int i_one_hundred = 100;
+
+struct test_sysctl_data {
+ int int_0001;
+ int int_0002;
+ int int_0003[4];
+
+ unsigned int uint_0001;
+
+ char string_0001[65];
+};
+
+static struct test_sysctl_data test_data = {
+ .int_0001 = 60,
+ .int_0002 = 1,
+
+ .int_0003[0] = 0,
+ .int_0003[1] = 1,
+ .int_0003[2] = 2,
+ .int_0003[3] = 3,
+
+ .uint_0001 = 314,
+
+ .string_0001 = "(none)",
+};
+
+/* These are all under /proc/sys/debug/test_sysctl/ */
+static struct ctl_table test_table[] = {
+ {
+ .procname = "int_0001",
+ .data = &test_data.int_0001,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = &i_zero,
+ .extra2 = &i_one_hundred,
+ },
+ {
+ .procname = "int_0002",
+ .data = &test_data.int_0002,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ },
+ {
+ .procname = "int_0003",
+ .data = &test_data.int_0003,
+ .maxlen = sizeof(test_data.int_0003),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ },
+ {
+ .procname = "uint_0001",
+ .data = &test_data.uint_0001,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0644,
+ .proc_handler = proc_douintvec,
+ },
+ {
+ .procname = "string_0001",
+ .data = &test_data.string_0001,
+ .maxlen = sizeof(test_data.string_0001),
+ .mode = 0644,
+ .proc_handler = proc_dostring,
+ },
+ { }
+};
+
+static struct ctl_table test_sysctl_table[] = {
+ {
+ .procname = "test_sysctl",
+ .maxlen = 0,
+ .mode = 0555,
+ .child = test_table,
+ },
+ { }
+};
+
+static struct ctl_table test_sysctl_root_table[] = {
+ {
+ .procname = "debug",
+ .maxlen = 0,
+ .mode = 0555,
+ .child = test_sysctl_table,
+ },
+ { }
+};
+
+static struct ctl_table_header *test_sysctl_header;
+
+static int __init test_sysctl_init(void)
+{
+ test_sysctl_header = register_sysctl_table(test_sysctl_root_table);
+ if (!test_sysctl_header)
+ return -ENOMEM;
+ return 0;
+}
+late_initcall(test_sysctl_init);
+
+static void __exit test_sysctl_exit(void)
+{
+ if (test_sysctl_header)
+ unregister_sysctl_table(test_sysctl_header);
+}
+
+module_exit(test_sysctl_exit);
+
+MODULE_AUTHOR("Luis R. Rodriguez <mcgrof@kernel.org>");
+MODULE_LICENSE("GPL");
diff --git a/lib/test_uuid.c b/lib/test_uuid.c
index 547d3127a3cf..cd819c397dc7 100644
--- a/lib/test_uuid.c
+++ b/lib/test_uuid.c
@@ -11,25 +11,25 @@
struct test_uuid_data {
const char *uuid;
- uuid_le le;
- uuid_be be;
+ guid_t le;
+ uuid_t be;
};
static const struct test_uuid_data test_uuid_test_data[] = {
{
.uuid = "c33f4995-3701-450e-9fbf-206a2e98e576",
- .le = UUID_LE(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
- .be = UUID_BE(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
+ .le = GUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
+ .be = UUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
},
{
.uuid = "64b4371c-77c1-48f9-8221-29f054fc023b",
- .le = UUID_LE(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
- .be = UUID_BE(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
+ .le = GUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
+ .be = UUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
},
{
.uuid = "0cb4ddff-a545-4401-9d06-688af53e7f84",
- .le = UUID_LE(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
- .be = UUID_BE(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
+ .le = GUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
+ .be = UUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
},
};
@@ -61,28 +61,28 @@ static void __init test_uuid_failed(const char *prefix, bool wrong, bool be,
static void __init test_uuid_test(const struct test_uuid_data *data)
{
- uuid_le le;
- uuid_be be;
+ guid_t le;
+ uuid_t be;
char buf[48];
/* LE */
total_tests++;
- if (uuid_le_to_bin(data->uuid, &le))
+ if (guid_parse(data->uuid, &le))
test_uuid_failed("conversion", false, false, data->uuid, NULL);
total_tests++;
- if (uuid_le_cmp(data->le, le)) {
+ if (!guid_equal(&data->le, &le)) {
sprintf(buf, "%pUl", &le);
test_uuid_failed("cmp", false, false, data->uuid, buf);
}
/* BE */
total_tests++;
- if (uuid_be_to_bin(data->uuid, &be))
+ if (uuid_parse(data->uuid, &be))
test_uuid_failed("conversion", false, true, data->uuid, NULL);
total_tests++;
- if (uuid_be_cmp(data->be, be)) {
+ if (!uuid_equal(&data->be, &be)) {
sprintf(buf, "%pUb", &be);
test_uuid_failed("cmp", false, true, data->uuid, buf);
}
@@ -90,17 +90,17 @@ static void __init test_uuid_test(const struct test_uuid_data *data)
static void __init test_uuid_wrong(const char *data)
{
- uuid_le le;
- uuid_be be;
+ guid_t le;
+ uuid_t be;
/* LE */
total_tests++;
- if (!uuid_le_to_bin(data, &le))
+ if (!guid_parse(data, &le))
test_uuid_failed("negative", true, false, data, NULL);
/* BE */
total_tests++;
- if (!uuid_be_to_bin(data, &be))
+ if (!uuid_parse(data, &be))
test_uuid_failed("negative", true, true, data, NULL);
}
diff --git a/lib/ts_fsm.c b/lib/ts_fsm.c
index 5696a35184e4..69557c74ef9f 100644
--- a/lib/ts_fsm.c
+++ b/lib/ts_fsm.c
@@ -11,7 +11,7 @@
* ==========================================================================
*
* A finite state machine consists of n states (struct ts_fsm_token)
- * representing the pattern as a finite automation. The data is read
+ * representing the pattern as a finite automaton. The data is read
* sequentially on an octet basis. Every state token specifies the number
* of recurrences and the type of value accepted which can be either a
* specific character or ctype based set of characters. The available
diff --git a/lib/ts_kmp.c b/lib/ts_kmp.c
index 632f783e65f1..ffbe66cbb0ed 100644
--- a/lib/ts_kmp.c
+++ b/lib/ts_kmp.c
@@ -27,7 +27,7 @@
*
* [1] Cormen, Leiserson, Rivest, Stein
* Introdcution to Algorithms, 2nd Edition, MIT Press
- * [2] See finite automation theory
+ * [2] See finite automaton theory
*/
#include <linux/module.h>
diff --git a/lib/ubsan.h b/lib/ubsan.h
index b2d18d4a53f5..88f23557edbe 100644
--- a/lib/ubsan.h
+++ b/lib/ubsan.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LIB_UBSAN_H
#define _LIB_UBSAN_H
diff --git a/lib/ucs2_string.c b/lib/ucs2_string.c
index ae8d2491133c..d7e06b28de38 100644
--- a/lib/ucs2_string.c
+++ b/lib/ucs2_string.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/ucs2_string.h>
#include <linux/module.h>
diff --git a/lib/usercopy.c b/lib/usercopy.c
index 1b6010a3beb8..15e2e6fb060e 100644
--- a/lib/usercopy.c
+++ b/lib/usercopy.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
#include <linux/uaccess.h>
/* out-of-line parts */
@@ -6,8 +7,11 @@
unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n)
{
unsigned long res = n;
- if (likely(access_ok(VERIFY_READ, from, n)))
+ might_fault();
+ if (likely(access_ok(VERIFY_READ, from, n))) {
+ kasan_check_write(to, n);
res = raw_copy_from_user(to, from, n);
+ }
if (unlikely(res))
memset(to + (n - res), 0, res);
return res;
@@ -18,8 +22,11 @@ EXPORT_SYMBOL(_copy_from_user);
#ifndef INLINE_COPY_TO_USER
unsigned long _copy_to_user(void *to, const void __user *from, unsigned long n)
{
- if (likely(access_ok(VERIFY_WRITE, to, n)))
+ might_fault();
+ if (likely(access_ok(VERIFY_WRITE, to, n))) {
+ kasan_check_read(from, n);
n = raw_copy_to_user(to, from, n);
+ }
return n;
}
EXPORT_SYMBOL(_copy_to_user);
diff --git a/lib/uuid.c b/lib/uuid.c
index 37687af77ff8..680b9fb9ba09 100644
--- a/lib/uuid.c
+++ b/lib/uuid.c
@@ -21,10 +21,13 @@
#include <linux/uuid.h>
#include <linux/random.h>
-const u8 uuid_le_index[16] = {3,2,1,0,5,4,7,6,8,9,10,11,12,13,14,15};
-EXPORT_SYMBOL(uuid_le_index);
-const u8 uuid_be_index[16] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
-EXPORT_SYMBOL(uuid_be_index);
+const guid_t guid_null;
+EXPORT_SYMBOL(guid_null);
+const uuid_t uuid_null;
+EXPORT_SYMBOL(uuid_null);
+
+const u8 guid_index[16] = {3,2,1,0,5,4,7,6,8,9,10,11,12,13,14,15};
+const u8 uuid_index[16] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
/***************************************************************
* Random UUID interface
@@ -53,21 +56,21 @@ static void __uuid_gen_common(__u8 b[16])
b[8] = (b[8] & 0x3F) | 0x80;
}
-void uuid_le_gen(uuid_le *lu)
+void guid_gen(guid_t *lu)
{
__uuid_gen_common(lu->b);
/* version 4 : random generation */
lu->b[7] = (lu->b[7] & 0x0F) | 0x40;
}
-EXPORT_SYMBOL_GPL(uuid_le_gen);
+EXPORT_SYMBOL_GPL(guid_gen);
-void uuid_be_gen(uuid_be *bu)
+void uuid_gen(uuid_t *bu)
{
__uuid_gen_common(bu->b);
/* version 4 : random generation */
bu->b[6] = (bu->b[6] & 0x0F) | 0x40;
}
-EXPORT_SYMBOL_GPL(uuid_be_gen);
+EXPORT_SYMBOL_GPL(uuid_gen);
/**
* uuid_is_valid - checks if UUID string valid
@@ -97,7 +100,7 @@ bool uuid_is_valid(const char *uuid)
}
EXPORT_SYMBOL(uuid_is_valid);
-static int __uuid_to_bin(const char *uuid, __u8 b[16], const u8 ei[16])
+static int __uuid_parse(const char *uuid, __u8 b[16], const u8 ei[16])
{
static const u8 si[16] = {0,2,4,6,9,11,14,16,19,21,24,26,28,30,32,34};
unsigned int i;
@@ -115,14 +118,14 @@ static int __uuid_to_bin(const char *uuid, __u8 b[16], const u8 ei[16])
return 0;
}
-int uuid_le_to_bin(const char *uuid, uuid_le *u)
+int guid_parse(const char *uuid, guid_t *u)
{
- return __uuid_to_bin(uuid, u->b, uuid_le_index);
+ return __uuid_parse(uuid, u->b, guid_index);
}
-EXPORT_SYMBOL(uuid_le_to_bin);
+EXPORT_SYMBOL(guid_parse);
-int uuid_be_to_bin(const char *uuid, uuid_be *u)
+int uuid_parse(const char *uuid, uuid_t *u)
{
- return __uuid_to_bin(uuid, u->b, uuid_be_index);
+ return __uuid_parse(uuid, u->b, uuid_index);
}
-EXPORT_SYMBOL(uuid_be_to_bin);
+EXPORT_SYMBOL(uuid_parse);
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 2d41de3f98a1..1746bae94d41 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -31,6 +31,7 @@
#include <linux/dcache.h>
#include <linux/cred.h>
#include <linux/uuid.h>
+#include <linux/of.h>
#include <net/addrconf.h>
#ifdef CONFIG_BLOCK
#include <linux/blkdev.h>
@@ -619,8 +620,8 @@ char *dentry_name(char *buf, char *end, const struct dentry *d, struct printf_sp
rcu_read_lock();
for (i = 0; i < depth; i++, d = p) {
- p = ACCESS_ONCE(d->d_parent);
- array[i] = ACCESS_ONCE(d->d_name.name);
+ p = READ_ONCE(d->d_parent);
+ array[i] = READ_ONCE(d->d_name.name);
if (p == d) {
if (i)
array[i] = "";
@@ -1308,14 +1309,14 @@ char *uuid_string(char *buf, char *end, const u8 *addr,
char uuid[UUID_STRING_LEN + 1];
char *p = uuid;
int i;
- const u8 *index = uuid_be_index;
+ const u8 *index = uuid_index;
bool uc = false;
switch (*(++fmt)) {
case 'L':
uc = true; /* fall-through */
case 'l':
- index = uuid_le_index;
+ index = guid_index;
break;
case 'B':
uc = true;
@@ -1470,6 +1471,126 @@ char *flags_string(char *buf, char *end, void *flags_ptr, const char *fmt)
return format_flags(buf, end, flags, names);
}
+static const char *device_node_name_for_depth(const struct device_node *np, int depth)
+{
+ for ( ; np && depth; depth--)
+ np = np->parent;
+
+ return kbasename(np->full_name);
+}
+
+static noinline_for_stack
+char *device_node_gen_full_name(const struct device_node *np, char *buf, char *end)
+{
+ int depth;
+ const struct device_node *parent = np->parent;
+ static const struct printf_spec strspec = {
+ .field_width = -1,
+ .precision = -1,
+ };
+
+ /* special case for root node */
+ if (!parent)
+ return string(buf, end, "/", strspec);
+
+ for (depth = 0; parent->parent; depth++)
+ parent = parent->parent;
+
+ for ( ; depth >= 0; depth--) {
+ buf = string(buf, end, "/", strspec);
+ buf = string(buf, end, device_node_name_for_depth(np, depth),
+ strspec);
+ }
+ return buf;
+}
+
+static noinline_for_stack
+char *device_node_string(char *buf, char *end, struct device_node *dn,
+ struct printf_spec spec, const char *fmt)
+{
+ char tbuf[sizeof("xxxx") + 1];
+ const char *p;
+ int ret;
+ char *buf_start = buf;
+ struct property *prop;
+ bool has_mult, pass;
+ static const struct printf_spec num_spec = {
+ .flags = SMALL,
+ .field_width = -1,
+ .precision = -1,
+ .base = 10,
+ };
+
+ struct printf_spec str_spec = spec;
+ str_spec.field_width = -1;
+
+ if (!IS_ENABLED(CONFIG_OF))
+ return string(buf, end, "(!OF)", spec);
+
+ if ((unsigned long)dn < PAGE_SIZE)
+ return string(buf, end, "(null)", spec);
+
+ /* simple case without anything any more format specifiers */
+ fmt++;
+ if (fmt[0] == '\0' || strcspn(fmt,"fnpPFcC") > 0)
+ fmt = "f";
+
+ for (pass = false; strspn(fmt,"fnpPFcC"); fmt++, pass = true) {
+ if (pass) {
+ if (buf < end)
+ *buf = ':';
+ buf++;
+ }
+
+ switch (*fmt) {
+ case 'f': /* full_name */
+ buf = device_node_gen_full_name(dn, buf, end);
+ break;
+ case 'n': /* name */
+ buf = string(buf, end, dn->name, str_spec);
+ break;
+ case 'p': /* phandle */
+ buf = number(buf, end, (unsigned int)dn->phandle, num_spec);
+ break;
+ case 'P': /* path-spec */
+ p = kbasename(of_node_full_name(dn));
+ if (!p[1])
+ p = "/";
+ buf = string(buf, end, p, str_spec);
+ break;
+ case 'F': /* flags */
+ tbuf[0] = of_node_check_flag(dn, OF_DYNAMIC) ? 'D' : '-';
+ tbuf[1] = of_node_check_flag(dn, OF_DETACHED) ? 'd' : '-';
+ tbuf[2] = of_node_check_flag(dn, OF_POPULATED) ? 'P' : '-';
+ tbuf[3] = of_node_check_flag(dn, OF_POPULATED_BUS) ? 'B' : '-';
+ tbuf[4] = 0;
+ buf = string(buf, end, tbuf, str_spec);
+ break;
+ case 'c': /* major compatible string */
+ ret = of_property_read_string(dn, "compatible", &p);
+ if (!ret)
+ buf = string(buf, end, p, str_spec);
+ break;
+ case 'C': /* full compatible string */
+ has_mult = false;
+ of_property_for_each_string(dn, "compatible", prop, p) {
+ if (has_mult)
+ buf = string(buf, end, ",", str_spec);
+ buf = string(buf, end, "\"", str_spec);
+ buf = string(buf, end, p, str_spec);
+ buf = string(buf, end, "\"", str_spec);
+
+ has_mult = true;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ return widen_string(buf, buf - buf_start, end, spec);
+}
+
int kptr_restrict __read_mostly;
/*
@@ -1566,6 +1687,16 @@ int kptr_restrict __read_mostly;
* p page flags (see struct page) given as pointer to unsigned long
* g gfp flags (GFP_* and __GFP_*) given as pointer to gfp_t
* v vma flags (VM_*) given as pointer to unsigned long
+ * - 'O' For a kobject based struct. Must be one of the following:
+ * - 'OF[fnpPcCF]' For a device tree object
+ * Without any optional arguments prints the full_name
+ * f device node full_name
+ * n device node name
+ * p device node phandle
+ * P device node path spec (name + @unit)
+ * F device node flags
+ * c major compatible string
+ * C full compatible string
*
* ** Please update also Documentation/printk-formats.txt when making changes **
*
@@ -1721,6 +1852,11 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
case 'G':
return flags_string(buf, end, ptr, fmt);
+ case 'O':
+ switch (fmt[1]) {
+ case 'F':
+ return device_node_string(buf, end, ptr, spec, fmt + 1);
+ }
}
spec.flags |= SMALL;
if (spec.field_width == -1) {
diff --git a/lib/win_minmax.c b/lib/win_minmax.c
index c8420d404926..6bdc1cd15f76 100644
--- a/lib/win_minmax.c
+++ b/lib/win_minmax.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/**
* lib/minmax.c: windowed min/max tracker
*
diff --git a/lib/xxhash.c b/lib/xxhash.c
new file mode 100644
index 000000000000..aa61e2a3802f
--- /dev/null
+++ b/lib/xxhash.c
@@ -0,0 +1,500 @@
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at:
+ * - xxHash homepage: http://cyan4973.github.io/xxHash/
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+#include <asm/unaligned.h>
+#include <linux/errno.h>
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+#include <linux/xxhash.h>
+
+/*-*************************************
+ * Macros
+ **************************************/
+#define xxh_rotl32(x, r) ((x << r) | (x >> (32 - r)))
+#define xxh_rotl64(x, r) ((x << r) | (x >> (64 - r)))
+
+#ifdef __LITTLE_ENDIAN
+# define XXH_CPU_LITTLE_ENDIAN 1
+#else
+# define XXH_CPU_LITTLE_ENDIAN 0
+#endif
+
+/*-*************************************
+ * Constants
+ **************************************/
+static const uint32_t PRIME32_1 = 2654435761U;
+static const uint32_t PRIME32_2 = 2246822519U;
+static const uint32_t PRIME32_3 = 3266489917U;
+static const uint32_t PRIME32_4 = 668265263U;
+static const uint32_t PRIME32_5 = 374761393U;
+
+static const uint64_t PRIME64_1 = 11400714785074694791ULL;
+static const uint64_t PRIME64_2 = 14029467366897019727ULL;
+static const uint64_t PRIME64_3 = 1609587929392839161ULL;
+static const uint64_t PRIME64_4 = 9650029242287828579ULL;
+static const uint64_t PRIME64_5 = 2870177450012600261ULL;
+
+/*-**************************
+ * Utils
+ ***************************/
+void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src)
+{
+ memcpy(dst, src, sizeof(*dst));
+}
+EXPORT_SYMBOL(xxh32_copy_state);
+
+void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src)
+{
+ memcpy(dst, src, sizeof(*dst));
+}
+EXPORT_SYMBOL(xxh64_copy_state);
+
+/*-***************************
+ * Simple Hash Functions
+ ****************************/
+static uint32_t xxh32_round(uint32_t seed, const uint32_t input)
+{
+ seed += input * PRIME32_2;
+ seed = xxh_rotl32(seed, 13);
+ seed *= PRIME32_1;
+ return seed;
+}
+
+uint32_t xxh32(const void *input, const size_t len, const uint32_t seed)
+{
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t *b_end = p + len;
+ uint32_t h32;
+
+ if (len >= 16) {
+ const uint8_t *const limit = b_end - 16;
+ uint32_t v1 = seed + PRIME32_1 + PRIME32_2;
+ uint32_t v2 = seed + PRIME32_2;
+ uint32_t v3 = seed + 0;
+ uint32_t v4 = seed - PRIME32_1;
+
+ do {
+ v1 = xxh32_round(v1, get_unaligned_le32(p));
+ p += 4;
+ v2 = xxh32_round(v2, get_unaligned_le32(p));
+ p += 4;
+ v3 = xxh32_round(v3, get_unaligned_le32(p));
+ p += 4;
+ v4 = xxh32_round(v4, get_unaligned_le32(p));
+ p += 4;
+ } while (p <= limit);
+
+ h32 = xxh_rotl32(v1, 1) + xxh_rotl32(v2, 7) +
+ xxh_rotl32(v3, 12) + xxh_rotl32(v4, 18);
+ } else {
+ h32 = seed + PRIME32_5;
+ }
+
+ h32 += (uint32_t)len;
+
+ while (p + 4 <= b_end) {
+ h32 += get_unaligned_le32(p) * PRIME32_3;
+ h32 = xxh_rotl32(h32, 17) * PRIME32_4;
+ p += 4;
+ }
+
+ while (p < b_end) {
+ h32 += (*p) * PRIME32_5;
+ h32 = xxh_rotl32(h32, 11) * PRIME32_1;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+EXPORT_SYMBOL(xxh32);
+
+static uint64_t xxh64_round(uint64_t acc, const uint64_t input)
+{
+ acc += input * PRIME64_2;
+ acc = xxh_rotl64(acc, 31);
+ acc *= PRIME64_1;
+ return acc;
+}
+
+static uint64_t xxh64_merge_round(uint64_t acc, uint64_t val)
+{
+ val = xxh64_round(0, val);
+ acc ^= val;
+ acc = acc * PRIME64_1 + PRIME64_4;
+ return acc;
+}
+
+uint64_t xxh64(const void *input, const size_t len, const uint64_t seed)
+{
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t *const b_end = p + len;
+ uint64_t h64;
+
+ if (len >= 32) {
+ const uint8_t *const limit = b_end - 32;
+ uint64_t v1 = seed + PRIME64_1 + PRIME64_2;
+ uint64_t v2 = seed + PRIME64_2;
+ uint64_t v3 = seed + 0;
+ uint64_t v4 = seed - PRIME64_1;
+
+ do {
+ v1 = xxh64_round(v1, get_unaligned_le64(p));
+ p += 8;
+ v2 = xxh64_round(v2, get_unaligned_le64(p));
+ p += 8;
+ v3 = xxh64_round(v3, get_unaligned_le64(p));
+ p += 8;
+ v4 = xxh64_round(v4, get_unaligned_le64(p));
+ p += 8;
+ } while (p <= limit);
+
+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
+ h64 = xxh64_merge_round(h64, v1);
+ h64 = xxh64_merge_round(h64, v2);
+ h64 = xxh64_merge_round(h64, v3);
+ h64 = xxh64_merge_round(h64, v4);
+
+ } else {
+ h64 = seed + PRIME64_5;
+ }
+
+ h64 += (uint64_t)len;
+
+ while (p + 8 <= b_end) {
+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
+
+ h64 ^= k1;
+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
+ p += 8;
+ }
+
+ if (p + 4 <= b_end) {
+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ p += 4;
+ }
+
+ while (p < b_end) {
+ h64 ^= (*p) * PRIME64_5;
+ h64 = xxh_rotl64(h64, 11) * PRIME64_1;
+ p++;
+ }
+
+ h64 ^= h64 >> 33;
+ h64 *= PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= PRIME64_3;
+ h64 ^= h64 >> 32;
+
+ return h64;
+}
+EXPORT_SYMBOL(xxh64);
+
+/*-**************************************************
+ * Advanced Hash Functions
+ ***************************************************/
+void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed)
+{
+ /* use a local state for memcpy() to avoid strict-aliasing warnings */
+ struct xxh32_state state;
+
+ memset(&state, 0, sizeof(state));
+ state.v1 = seed + PRIME32_1 + PRIME32_2;
+ state.v2 = seed + PRIME32_2;
+ state.v3 = seed + 0;
+ state.v4 = seed - PRIME32_1;
+ memcpy(statePtr, &state, sizeof(state));
+}
+EXPORT_SYMBOL(xxh32_reset);
+
+void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed)
+{
+ /* use a local state for memcpy() to avoid strict-aliasing warnings */
+ struct xxh64_state state;
+
+ memset(&state, 0, sizeof(state));
+ state.v1 = seed + PRIME64_1 + PRIME64_2;
+ state.v2 = seed + PRIME64_2;
+ state.v3 = seed + 0;
+ state.v4 = seed - PRIME64_1;
+ memcpy(statePtr, &state, sizeof(state));
+}
+EXPORT_SYMBOL(xxh64_reset);
+
+int xxh32_update(struct xxh32_state *state, const void *input, const size_t len)
+{
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t *const b_end = p + len;
+
+ if (input == NULL)
+ return -EINVAL;
+
+ state->total_len_32 += (uint32_t)len;
+ state->large_len |= (len >= 16) | (state->total_len_32 >= 16);
+
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
+ memcpy((uint8_t *)(state->mem32) + state->memsize, input, len);
+ state->memsize += (uint32_t)len;
+ return 0;
+ }
+
+ if (state->memsize) { /* some data left from previous update */
+ const uint32_t *p32 = state->mem32;
+
+ memcpy((uint8_t *)(state->mem32) + state->memsize, input,
+ 16 - state->memsize);
+
+ state->v1 = xxh32_round(state->v1, get_unaligned_le32(p32));
+ p32++;
+ state->v2 = xxh32_round(state->v2, get_unaligned_le32(p32));
+ p32++;
+ state->v3 = xxh32_round(state->v3, get_unaligned_le32(p32));
+ p32++;
+ state->v4 = xxh32_round(state->v4, get_unaligned_le32(p32));
+ p32++;
+
+ p += 16-state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p <= b_end - 16) {
+ const uint8_t *const limit = b_end - 16;
+ uint32_t v1 = state->v1;
+ uint32_t v2 = state->v2;
+ uint32_t v3 = state->v3;
+ uint32_t v4 = state->v4;
+
+ do {
+ v1 = xxh32_round(v1, get_unaligned_le32(p));
+ p += 4;
+ v2 = xxh32_round(v2, get_unaligned_le32(p));
+ p += 4;
+ v3 = xxh32_round(v3, get_unaligned_le32(p));
+ p += 4;
+ v4 = xxh32_round(v4, get_unaligned_le32(p));
+ p += 4;
+ } while (p <= limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
+
+ if (p < b_end) {
+ memcpy(state->mem32, p, (size_t)(b_end-p));
+ state->memsize = (uint32_t)(b_end-p);
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(xxh32_update);
+
+uint32_t xxh32_digest(const struct xxh32_state *state)
+{
+ const uint8_t *p = (const uint8_t *)state->mem32;
+ const uint8_t *const b_end = (const uint8_t *)(state->mem32) +
+ state->memsize;
+ uint32_t h32;
+
+ if (state->large_len) {
+ h32 = xxh_rotl32(state->v1, 1) + xxh_rotl32(state->v2, 7) +
+ xxh_rotl32(state->v3, 12) + xxh_rotl32(state->v4, 18);
+ } else {
+ h32 = state->v3 /* == seed */ + PRIME32_5;
+ }
+
+ h32 += state->total_len_32;
+
+ while (p + 4 <= b_end) {
+ h32 += get_unaligned_le32(p) * PRIME32_3;
+ h32 = xxh_rotl32(h32, 17) * PRIME32_4;
+ p += 4;
+ }
+
+ while (p < b_end) {
+ h32 += (*p) * PRIME32_5;
+ h32 = xxh_rotl32(h32, 11) * PRIME32_1;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+EXPORT_SYMBOL(xxh32_digest);
+
+int xxh64_update(struct xxh64_state *state, const void *input, const size_t len)
+{
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t *const b_end = p + len;
+
+ if (input == NULL)
+ return -EINVAL;
+
+ state->total_len += len;
+
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
+ memcpy(((uint8_t *)state->mem64) + state->memsize, input, len);
+ state->memsize += (uint32_t)len;
+ return 0;
+ }
+
+ if (state->memsize) { /* tmp buffer is full */
+ uint64_t *p64 = state->mem64;
+
+ memcpy(((uint8_t *)p64) + state->memsize, input,
+ 32 - state->memsize);
+
+ state->v1 = xxh64_round(state->v1, get_unaligned_le64(p64));
+ p64++;
+ state->v2 = xxh64_round(state->v2, get_unaligned_le64(p64));
+ p64++;
+ state->v3 = xxh64_round(state->v3, get_unaligned_le64(p64));
+ p64++;
+ state->v4 = xxh64_round(state->v4, get_unaligned_le64(p64));
+
+ p += 32 - state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p + 32 <= b_end) {
+ const uint8_t *const limit = b_end - 32;
+ uint64_t v1 = state->v1;
+ uint64_t v2 = state->v2;
+ uint64_t v3 = state->v3;
+ uint64_t v4 = state->v4;
+
+ do {
+ v1 = xxh64_round(v1, get_unaligned_le64(p));
+ p += 8;
+ v2 = xxh64_round(v2, get_unaligned_le64(p));
+ p += 8;
+ v3 = xxh64_round(v3, get_unaligned_le64(p));
+ p += 8;
+ v4 = xxh64_round(v4, get_unaligned_le64(p));
+ p += 8;
+ } while (p <= limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
+
+ if (p < b_end) {
+ memcpy(state->mem64, p, (size_t)(b_end-p));
+ state->memsize = (uint32_t)(b_end - p);
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(xxh64_update);
+
+uint64_t xxh64_digest(const struct xxh64_state *state)
+{
+ const uint8_t *p = (const uint8_t *)state->mem64;
+ const uint8_t *const b_end = (const uint8_t *)state->mem64 +
+ state->memsize;
+ uint64_t h64;
+
+ if (state->total_len >= 32) {
+ const uint64_t v1 = state->v1;
+ const uint64_t v2 = state->v2;
+ const uint64_t v3 = state->v3;
+ const uint64_t v4 = state->v4;
+
+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
+ h64 = xxh64_merge_round(h64, v1);
+ h64 = xxh64_merge_round(h64, v2);
+ h64 = xxh64_merge_round(h64, v3);
+ h64 = xxh64_merge_round(h64, v4);
+ } else {
+ h64 = state->v3 + PRIME64_5;
+ }
+
+ h64 += (uint64_t)state->total_len;
+
+ while (p + 8 <= b_end) {
+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
+
+ h64 ^= k1;
+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
+ p += 8;
+ }
+
+ if (p + 4 <= b_end) {
+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ p += 4;
+ }
+
+ while (p < b_end) {
+ h64 ^= (*p) * PRIME64_5;
+ h64 = xxh_rotl64(h64, 11) * PRIME64_1;
+ p++;
+ }
+
+ h64 ^= h64 >> 33;
+ h64 *= PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= PRIME64_3;
+ h64 ^= h64 >> 32;
+
+ return h64;
+}
+EXPORT_SYMBOL(xxh64_digest);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("xxHash");
diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c
index ac809b1e64f7..bd1d182419d7 100644
--- a/lib/xz/xz_dec_stream.c
+++ b/lib/xz/xz_dec_stream.c
@@ -583,6 +583,8 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
if (ret != XZ_OK)
return ret;
+ /* Fall through */
+
case SEQ_BLOCK_START:
/* We need one byte of input to continue. */
if (b->in_pos == b->in_size)
@@ -606,6 +608,8 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
s->temp.pos = 0;
s->sequence = SEQ_BLOCK_HEADER;
+ /* Fall through */
+
case SEQ_BLOCK_HEADER:
if (!fill_temp(s, b))
return XZ_OK;
@@ -616,6 +620,8 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
s->sequence = SEQ_BLOCK_UNCOMPRESS;
+ /* Fall through */
+
case SEQ_BLOCK_UNCOMPRESS:
ret = dec_block(s, b);
if (ret != XZ_STREAM_END)
@@ -623,6 +629,8 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
s->sequence = SEQ_BLOCK_PADDING;
+ /* Fall through */
+
case SEQ_BLOCK_PADDING:
/*
* Size of Compressed Data + Block Padding
@@ -643,6 +651,8 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
s->sequence = SEQ_BLOCK_CHECK;
+ /* Fall through */
+
case SEQ_BLOCK_CHECK:
if (s->check_type == XZ_CHECK_CRC32) {
ret = crc32_validate(s, b);
@@ -665,6 +675,8 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
s->sequence = SEQ_INDEX_PADDING;
+ /* Fall through */
+
case SEQ_INDEX_PADDING:
while ((s->index.size + (b->in_pos - s->in_start))
& 3) {
@@ -687,6 +699,8 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
s->sequence = SEQ_INDEX_CRC32;
+ /* Fall through */
+
case SEQ_INDEX_CRC32:
ret = crc32_validate(s, b);
if (ret != XZ_STREAM_END)
@@ -695,6 +709,8 @@ static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
s->temp.size = STREAM_HEADER_SIZE;
s->sequence = SEQ_STREAM_FOOTER;
+ /* Fall through */
+
case SEQ_STREAM_FOOTER:
if (!fill_temp(s, b))
return XZ_OK;
diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile
new file mode 100644
index 000000000000..dd0a359c135b
--- /dev/null
+++ b/lib/zstd/Makefile
@@ -0,0 +1,18 @@
+obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o
+obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o
+
+ccflags-y += -O3
+
+# Object files unique to zstd_compress and zstd_decompress
+zstd_compress-y := fse_compress.o huf_compress.o compress.o
+zstd_decompress-y := huf_decompress.o decompress.o
+
+# These object files are shared between the modules.
+# Always add them to zstd_compress.
+# Unless both zstd_compress and zstd_decompress are built in
+# then also add them to zstd_decompress.
+zstd_compress-y += entropy_common.o fse_decompress.o zstd_common.o
+
+ifneq ($(CONFIG_ZSTD_COMPRESS)$(CONFIG_ZSTD_DECOMPRESS),yy)
+ zstd_decompress-y += entropy_common.o fse_decompress.o zstd_common.o
+endif
diff --git a/lib/zstd/bitstream.h b/lib/zstd/bitstream.h
new file mode 100644
index 000000000000..a826b99e1d63
--- /dev/null
+++ b/lib/zstd/bitstream.h
@@ -0,0 +1,374 @@
+/*
+ * bitstream
+ * Part of FSE library
+ * header file (to include)
+ * Copyright (C) 2013-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ */
+#ifndef BITSTREAM_H_MODULE
+#define BITSTREAM_H_MODULE
+
+/*
+* This API consists of small unitary functions, which must be inlined for best performance.
+* Since link-time-optimization is not available for all compilers,
+* these functions are defined into a .h to be included.
+*/
+
+/*-****************************************
+* Dependencies
+******************************************/
+#include "error_private.h" /* error codes and messages */
+#include "mem.h" /* unaligned access routines */
+
+/*=========================================
+* Target specific
+=========================================*/
+#define STREAM_ACCUMULATOR_MIN_32 25
+#define STREAM_ACCUMULATOR_MIN_64 57
+#define STREAM_ACCUMULATOR_MIN ((U32)(ZSTD_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
+
+/*-******************************************
+* bitStream encoding API (write forward)
+********************************************/
+/* bitStream can mix input from multiple sources.
+* A critical property of these streams is that they encode and decode in **reverse** direction.
+* So the first bit sequence you add will be the last to be read, like a LIFO stack.
+*/
+typedef struct {
+ size_t bitContainer;
+ int bitPos;
+ char *startPtr;
+ char *ptr;
+ char *endPtr;
+} BIT_CStream_t;
+
+ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *dstBuffer, size_t dstCapacity);
+ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits);
+ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC);
+ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC);
+
+/* Start with initCStream, providing the size of buffer to write into.
+* bitStream will never write outside of this buffer.
+* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
+*
+* bits are first added to a local register.
+* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
+* Writing data into memory is an explicit operation, performed by the flushBits function.
+* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
+* After a flushBits, a maximum of 7 bits might still be stored into local register.
+*
+* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
+*
+* Last operation is to close the bitStream.
+* The function returns the final size of CStream in bytes.
+* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
+*/
+
+/*-********************************************
+* bitStream decoding API (read backward)
+**********************************************/
+typedef struct {
+ size_t bitContainer;
+ unsigned bitsConsumed;
+ const char *ptr;
+ const char *start;
+} BIT_DStream_t;
+
+typedef enum {
+ BIT_DStream_unfinished = 0,
+ BIT_DStream_endOfBuffer = 1,
+ BIT_DStream_completed = 2,
+ BIT_DStream_overflow = 3
+} BIT_DStream_status; /* result of BIT_reloadDStream() */
+/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
+
+ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize);
+ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, unsigned nbBits);
+ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD);
+ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *bitD);
+
+/* Start by invoking BIT_initDStream().
+* A chunk of the bitStream is then stored into a local register.
+* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
+* You can then retrieve bitFields stored into the local register, **in reverse order**.
+* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
+* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
+* Otherwise, it can be less than that, so proceed accordingly.
+* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
+*/
+
+/*-****************************************
+* unsafe API
+******************************************/
+ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits);
+/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
+
+ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC);
+/* unsafe version; does not check buffer overflow */
+
+ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, unsigned nbBits);
+/* faster, but works only if nbBits >= 1 */
+
+/*-**************************************************************
+* Internal functions
+****************************************************************/
+ZSTD_STATIC unsigned BIT_highbit32(register U32 val) { return 31 - __builtin_clz(val); }
+
+/*===== Local Constants =====*/
+static const unsigned BIT_mask[] = {0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF,
+ 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
+ 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF}; /* up to 26 bits */
+
+/*-**************************************************************
+* bitStream encoding
+****************************************************************/
+/*! BIT_initCStream() :
+ * `dstCapacity` must be > sizeof(void*)
+ * @return : 0 if success,
+ otherwise an error code (can be tested using ERR_isError() ) */
+ZSTD_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *startPtr, size_t dstCapacity)
+{
+ bitC->bitContainer = 0;
+ bitC->bitPos = 0;
+ bitC->startPtr = (char *)startPtr;
+ bitC->ptr = bitC->startPtr;
+ bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
+ if (dstCapacity <= sizeof(bitC->ptr))
+ return ERROR(dstSize_tooSmall);
+ return 0;
+}
+
+/*! BIT_addBits() :
+ can add up to 26 bits into `bitC`.
+ Does not check for register overflow ! */
+ZSTD_STATIC void BIT_addBits(BIT_CStream_t *bitC, size_t value, unsigned nbBits)
+{
+ bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
+ bitC->bitPos += nbBits;
+}
+
+/*! BIT_addBitsFast() :
+ * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
+ZSTD_STATIC void BIT_addBitsFast(BIT_CStream_t *bitC, size_t value, unsigned nbBits)
+{
+ bitC->bitContainer |= value << bitC->bitPos;
+ bitC->bitPos += nbBits;
+}
+
+/*! BIT_flushBitsFast() :
+ * unsafe version; does not check buffer overflow */
+ZSTD_STATIC void BIT_flushBitsFast(BIT_CStream_t *bitC)
+{
+ size_t const nbBytes = bitC->bitPos >> 3;
+ ZSTD_writeLEST(bitC->ptr, bitC->bitContainer);
+ bitC->ptr += nbBytes;
+ bitC->bitPos &= 7;
+ bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
+}
+
+/*! BIT_flushBits() :
+ * safe version; check for buffer overflow, and prevents it.
+ * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
+ZSTD_STATIC void BIT_flushBits(BIT_CStream_t *bitC)
+{
+ size_t const nbBytes = bitC->bitPos >> 3;
+ ZSTD_writeLEST(bitC->ptr, bitC->bitContainer);
+ bitC->ptr += nbBytes;
+ if (bitC->ptr > bitC->endPtr)
+ bitC->ptr = bitC->endPtr;
+ bitC->bitPos &= 7;
+ bitC->bitContainer >>= nbBytes * 8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
+}
+
+/*! BIT_closeCStream() :
+ * @return : size of CStream, in bytes,
+ or 0 if it could not fit into dstBuffer */
+ZSTD_STATIC size_t BIT_closeCStream(BIT_CStream_t *bitC)
+{
+ BIT_addBitsFast(bitC, 1, 1); /* endMark */
+ BIT_flushBits(bitC);
+
+ if (bitC->ptr >= bitC->endPtr)
+ return 0; /* doesn't fit within authorized budget : cancel */
+
+ return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
+}
+
+/*-********************************************************
+* bitStream decoding
+**********************************************************/
+/*! BIT_initDStream() :
+* Initialize a BIT_DStream_t.
+* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
+* `srcSize` must be the *exact* size of the bitStream, in bytes.
+* @return : size of stream (== srcSize) or an errorCode if a problem is detected
+*/
+ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD, const void *srcBuffer, size_t srcSize)
+{
+ if (srcSize < 1) {
+ memset(bitD, 0, sizeof(*bitD));
+ return ERROR(srcSize_wrong);
+ }
+
+ if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
+ bitD->start = (const char *)srcBuffer;
+ bitD->ptr = (const char *)srcBuffer + srcSize - sizeof(bitD->bitContainer);
+ bitD->bitContainer = ZSTD_readLEST(bitD->ptr);
+ {
+ BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
+ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
+ if (lastByte == 0)
+ return ERROR(GENERIC); /* endMark not present */
+ }
+ } else {
+ bitD->start = (const char *)srcBuffer;
+ bitD->ptr = bitD->start;
+ bitD->bitContainer = *(const BYTE *)(bitD->start);
+ switch (srcSize) {
+ case 7: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16);
+ case 6: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24);
+ case 5: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32);
+ case 4: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[3]) << 24;
+ case 3: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[2]) << 16;
+ case 2: bitD->bitContainer += (size_t)(((const BYTE *)(srcBuffer))[1]) << 8;
+ default:;
+ }
+ {
+ BYTE const lastByte = ((const BYTE *)srcBuffer)[srcSize - 1];
+ bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+ if (lastByte == 0)
+ return ERROR(GENERIC); /* endMark not present */
+ }
+ bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8;
+ }
+
+ return srcSize;
+}
+
+ZSTD_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { return bitContainer >> start; }
+
+ZSTD_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { return (bitContainer >> start) & BIT_mask[nbBits]; }
+
+ZSTD_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { return bitContainer & BIT_mask[nbBits]; }
+
+/*! BIT_lookBits() :
+ * Provides next n bits from local register.
+ * local register is not modified.
+ * On 32-bits, maxNbBits==24.
+ * On 64-bits, maxNbBits==56.
+ * @return : value extracted
+ */
+ZSTD_STATIC size_t BIT_lookBits(const BIT_DStream_t *bitD, U32 nbBits)
+{
+ U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1;
+ return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask - nbBits) & bitMask);
+}
+
+/*! BIT_lookBitsFast() :
+* unsafe version; only works only if nbBits >= 1 */
+ZSTD_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t *bitD, U32 nbBits)
+{
+ U32 const bitMask = sizeof(bitD->bitContainer) * 8 - 1;
+ return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask + 1) - nbBits) & bitMask);
+}
+
+ZSTD_STATIC void BIT_skipBits(BIT_DStream_t *bitD, U32 nbBits) { bitD->bitsConsumed += nbBits; }
+
+/*! BIT_readBits() :
+ * Read (consume) next n bits from local register and update.
+ * Pay attention to not read more than nbBits contained into local register.
+ * @return : extracted value.
+ */
+ZSTD_STATIC size_t BIT_readBits(BIT_DStream_t *bitD, U32 nbBits)
+{
+ size_t const value = BIT_lookBits(bitD, nbBits);
+ BIT_skipBits(bitD, nbBits);
+ return value;
+}
+
+/*! BIT_readBitsFast() :
+* unsafe version; only works only if nbBits >= 1 */
+ZSTD_STATIC size_t BIT_readBitsFast(BIT_DStream_t *bitD, U32 nbBits)
+{
+ size_t const value = BIT_lookBitsFast(bitD, nbBits);
+ BIT_skipBits(bitD, nbBits);
+ return value;
+}
+
+/*! BIT_reloadDStream() :
+* Refill `bitD` from buffer previously set in BIT_initDStream() .
+* This function is safe, it guarantees it will not read beyond src buffer.
+* @return : status of `BIT_DStream_t` internal register.
+ if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
+ZSTD_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t *bitD)
+{
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer) * 8)) /* should not happen => corruption detected */
+ return BIT_DStream_overflow;
+
+ if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
+ bitD->ptr -= bitD->bitsConsumed >> 3;
+ bitD->bitsConsumed &= 7;
+ bitD->bitContainer = ZSTD_readLEST(bitD->ptr);
+ return BIT_DStream_unfinished;
+ }
+ if (bitD->ptr == bitD->start) {
+ if (bitD->bitsConsumed < sizeof(bitD->bitContainer) * 8)
+ return BIT_DStream_endOfBuffer;
+ return BIT_DStream_completed;
+ }
+ {
+ U32 nbBytes = bitD->bitsConsumed >> 3;
+ BIT_DStream_status result = BIT_DStream_unfinished;
+ if (bitD->ptr - nbBytes < bitD->start) {
+ nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
+ result = BIT_DStream_endOfBuffer;
+ }
+ bitD->ptr -= nbBytes;
+ bitD->bitsConsumed -= nbBytes * 8;
+ bitD->bitContainer = ZSTD_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
+ return result;
+ }
+}
+
+/*! BIT_endOfDStream() :
+* @return Tells if DStream has exactly reached its end (all bits consumed).
+*/
+ZSTD_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t *DStream)
+{
+ return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer) * 8));
+}
+
+#endif /* BITSTREAM_H_MODULE */
diff --git a/lib/zstd/compress.c b/lib/zstd/compress.c
new file mode 100644
index 000000000000..f9166cf4f7a9
--- /dev/null
+++ b/lib/zstd/compress.c
@@ -0,0 +1,3484 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
+ * An additional grant of patent rights can be found in the PATENTS file in the
+ * same directory.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ */
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "fse.h"
+#include "huf.h"
+#include "mem.h"
+#include "zstd_internal.h" /* includes zstd.h */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h> /* memset */
+
+/*-*************************************
+* Constants
+***************************************/
+static const U32 g_searchStrength = 8; /* control skip over incompressible data */
+#define HASH_READ_SIZE 8
+typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
+
+/*-*************************************
+* Helper functions
+***************************************/
+size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
+
+/*-*************************************
+* Sequence storage
+***************************************/
+static void ZSTD_resetSeqStore(seqStore_t *ssPtr)
+{
+ ssPtr->lit = ssPtr->litStart;
+ ssPtr->sequences = ssPtr->sequencesStart;
+ ssPtr->longLengthID = 0;
+}
+
+/*-*************************************
+* Context memory management
+***************************************/
+struct ZSTD_CCtx_s {
+ const BYTE *nextSrc; /* next block here to continue on curr prefix */
+ const BYTE *base; /* All regular indexes relative to this position */
+ const BYTE *dictBase; /* extDict indexes relative to this position */
+ U32 dictLimit; /* below that point, need extDict */
+ U32 lowLimit; /* below that point, no more data */
+ U32 nextToUpdate; /* index from which to continue dictionary update */
+ U32 nextToUpdate3; /* index from which to continue dictionary update */
+ U32 hashLog3; /* dispatch table : larger == faster, more memory */
+ U32 loadedDictEnd; /* index of end of dictionary */
+ U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
+ U32 forceRawDict; /* Force loading dictionary in "content-only" mode (no header analysis) */
+ ZSTD_compressionStage_e stage;
+ U32 rep[ZSTD_REP_NUM];
+ U32 repToConfirm[ZSTD_REP_NUM];
+ U32 dictID;
+ ZSTD_parameters params;
+ void *workSpace;
+ size_t workSpaceSize;
+ size_t blockSize;
+ U64 frameContentSize;
+ struct xxh64_state xxhState;
+ ZSTD_customMem customMem;
+
+ seqStore_t seqStore; /* sequences storage ptrs */
+ U32 *hashTable;
+ U32 *hashTable3;
+ U32 *chainTable;
+ HUF_CElt *hufTable;
+ U32 flagStaticTables;
+ HUF_repeat flagStaticHufTable;
+ FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
+ FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
+ FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
+ unsigned tmpCounters[HUF_COMPRESS_WORKSPACE_SIZE_U32];
+};
+
+size_t ZSTD_CCtxWorkspaceBound(ZSTD_compressionParameters cParams)
+{
+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
+ U32 const divider = (cParams.searchLength == 3) ? 3 : 4;
+ size_t const maxNbSeq = blockSize / divider;
+ size_t const tokenSpace = blockSize + 11 * maxNbSeq;
+ size_t const chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog);
+ size_t const hSize = ((size_t)1) << cParams.hashLog;
+ U32 const hashLog3 = (cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog);
+ size_t const h3Size = ((size_t)1) << hashLog3;
+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
+ size_t const optSpace =
+ ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
+ size_t const workspaceSize = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
+ (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
+
+ return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_CCtx)) + ZSTD_ALIGN(workspaceSize);
+}
+
+static ZSTD_CCtx *ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
+{
+ ZSTD_CCtx *cctx;
+ if (!customMem.customAlloc || !customMem.customFree)
+ return NULL;
+ cctx = (ZSTD_CCtx *)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
+ if (!cctx)
+ return NULL;
+ memset(cctx, 0, sizeof(ZSTD_CCtx));
+ cctx->customMem = customMem;
+ return cctx;
+}
+
+ZSTD_CCtx *ZSTD_initCCtx(void *workspace, size_t workspaceSize)
+{
+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
+ ZSTD_CCtx *cctx = ZSTD_createCCtx_advanced(stackMem);
+ if (cctx) {
+ cctx->workSpace = ZSTD_stackAllocAll(cctx->customMem.opaque, &cctx->workSpaceSize);
+ }
+ return cctx;
+}
+
+size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx)
+{
+ if (cctx == NULL)
+ return 0; /* support free on NULL */
+ ZSTD_free(cctx->workSpace, cctx->customMem);
+ ZSTD_free(cctx, cctx->customMem);
+ return 0; /* reserved as a potential error code in the future */
+}
+
+const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx) /* hidden interface */ { return &(ctx->seqStore); }
+
+static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx *cctx) { return cctx->params; }
+
+/** ZSTD_checkParams() :
+ ensure param values remain within authorized range.
+ @return : 0, or an error code if one value is beyond authorized range */
+size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
+{
+#define CLAMPCHECK(val, min, max) \
+ { \
+ if ((val < min) | (val > max)) \
+ return ERROR(compressionParameter_unsupported); \
+ }
+ CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+ CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
+ CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
+ CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
+ CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+ CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
+ if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2)
+ return ERROR(compressionParameter_unsupported);
+ return 0;
+}
+
+/** ZSTD_cycleLog() :
+ * condition for correct operation : hashLog > 1 */
+static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
+{
+ U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
+ return hashLog - btScale;
+}
+
+/** ZSTD_adjustCParams() :
+ optimize `cPar` for a given input (`srcSize` and `dictSize`).
+ mostly downsizing to reduce memory consumption and initialization.
+ Both `srcSize` and `dictSize` are optional (use 0 if unknown),
+ but if both are 0, no optimization can be done.
+ Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
+ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
+{
+ if (srcSize + dictSize == 0)
+ return cPar; /* no size information available : no adjustment */
+
+ /* resize params, to use less memory when necessary */
+ {
+ U32 const minSrcSize = (srcSize == 0) ? 500 : 0;
+ U64 const rSize = srcSize + dictSize + minSrcSize;
+ if (rSize < ((U64)1 << ZSTD_WINDOWLOG_MAX)) {
+ U32 const srcLog = MAX(ZSTD_HASHLOG_MIN, ZSTD_highbit32((U32)(rSize)-1) + 1);
+ if (cPar.windowLog > srcLog)
+ cPar.windowLog = srcLog;
+ }
+ }
+ if (cPar.hashLog > cPar.windowLog)
+ cPar.hashLog = cPar.windowLog;
+ {
+ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
+ if (cycleLog > cPar.windowLog)
+ cPar.chainLog -= (cycleLog - cPar.windowLog);
+ }
+
+ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
+ cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
+
+ return cPar;
+}
+
+static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2)
+{
+ return (param1.cParams.hashLog == param2.cParams.hashLog) & (param1.cParams.chainLog == param2.cParams.chainLog) &
+ (param1.cParams.strategy == param2.cParams.strategy) & ((param1.cParams.searchLength == 3) == (param2.cParams.searchLength == 3));
+}
+
+/*! ZSTD_continueCCtx() :
+ reuse CCtx without reset (note : requires no dictionary) */
+static size_t ZSTD_continueCCtx(ZSTD_CCtx *cctx, ZSTD_parameters params, U64 frameContentSize)
+{
+ U32 const end = (U32)(cctx->nextSrc - cctx->base);
+ cctx->params = params;
+ cctx->frameContentSize = frameContentSize;
+ cctx->lowLimit = end;
+ cctx->dictLimit = end;
+ cctx->nextToUpdate = end + 1;
+ cctx->stage = ZSTDcs_init;
+ cctx->dictID = 0;
+ cctx->loadedDictEnd = 0;
+ {
+ int i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ cctx->rep[i] = repStartValue[i];
+ }
+ cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */
+ xxh64_reset(&cctx->xxhState, 0);
+ return 0;
+}
+
+typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
+
+/*! ZSTD_resetCCtx_advanced() :
+ note : `params` must be validated */
+static size_t ZSTD_resetCCtx_advanced(ZSTD_CCtx *zc, ZSTD_parameters params, U64 frameContentSize, ZSTD_compResetPolicy_e const crp)
+{
+ if (crp == ZSTDcrp_continue)
+ if (ZSTD_equivalentParams(params, zc->params)) {
+ zc->flagStaticTables = 0;
+ zc->flagStaticHufTable = HUF_repeat_none;
+ return ZSTD_continueCCtx(zc, params, frameContentSize);
+ }
+
+ {
+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
+ U32 const divider = (params.cParams.searchLength == 3) ? 3 : 4;
+ size_t const maxNbSeq = blockSize / divider;
+ size_t const tokenSpace = blockSize + 11 * maxNbSeq;
+ size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
+ size_t const hSize = ((size_t)1) << params.cParams.hashLog;
+ U32 const hashLog3 = (params.cParams.searchLength > 3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
+ size_t const h3Size = ((size_t)1) << hashLog3;
+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
+ void *ptr;
+
+ /* Check if workSpace is large enough, alloc a new one if needed */
+ {
+ size_t const optSpace = ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) +
+ (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
+ size_t const neededSpace = tableSpace + (256 * sizeof(U32)) /* huffTable */ + tokenSpace +
+ (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
+ if (zc->workSpaceSize < neededSpace) {
+ ZSTD_free(zc->workSpace, zc->customMem);
+ zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
+ if (zc->workSpace == NULL)
+ return ERROR(memory_allocation);
+ zc->workSpaceSize = neededSpace;
+ }
+ }
+
+ if (crp != ZSTDcrp_noMemset)
+ memset(zc->workSpace, 0, tableSpace); /* reset tables only */
+ xxh64_reset(&zc->xxhState, 0);
+ zc->hashLog3 = hashLog3;
+ zc->hashTable = (U32 *)(zc->workSpace);
+ zc->chainTable = zc->hashTable + hSize;
+ zc->hashTable3 = zc->chainTable + chainSize;
+ ptr = zc->hashTable3 + h3Size;
+ zc->hufTable = (HUF_CElt *)ptr;
+ zc->flagStaticTables = 0;
+ zc->flagStaticHufTable = HUF_repeat_none;
+ ptr = ((U32 *)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
+
+ zc->nextToUpdate = 1;
+ zc->nextSrc = NULL;
+ zc->base = NULL;
+ zc->dictBase = NULL;
+ zc->dictLimit = 0;
+ zc->lowLimit = 0;
+ zc->params = params;
+ zc->blockSize = blockSize;
+ zc->frameContentSize = frameContentSize;
+ {
+ int i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ zc->rep[i] = repStartValue[i];
+ }
+
+ if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
+ zc->seqStore.litFreq = (U32 *)ptr;
+ zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1 << Litbits);
+ zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL + 1);
+ zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML + 1);
+ ptr = zc->seqStore.offCodeFreq + (MaxOff + 1);
+ zc->seqStore.matchTable = (ZSTD_match_t *)ptr;
+ ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM + 1;
+ zc->seqStore.priceTable = (ZSTD_optimal_t *)ptr;
+ ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM + 1;
+ zc->seqStore.litLengthSum = 0;
+ }
+ zc->seqStore.sequencesStart = (seqDef *)ptr;
+ ptr = zc->seqStore.sequencesStart + maxNbSeq;
+ zc->seqStore.llCode = (BYTE *)ptr;
+ zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq;
+ zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq;
+ zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq;
+
+ zc->stage = ZSTDcs_init;
+ zc->dictID = 0;
+ zc->loadedDictEnd = 0;
+
+ return 0;
+ }
+}
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx)
+{
+ int i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ cctx->rep[i] = 0;
+}
+
+/*! ZSTD_copyCCtx() :
+* Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
+* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
+* @return : 0, or an error code */
+size_t ZSTD_copyCCtx(ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize)
+{
+ if (srcCCtx->stage != ZSTDcs_init)
+ return ERROR(stage_wrong);
+
+ memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
+ {
+ ZSTD_parameters params = srcCCtx->params;
+ params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+ ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
+ }
+
+ /* copy tables */
+ {
+ size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
+ size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
+ size_t const h3Size = (size_t)1 << srcCCtx->hashLog3;
+ size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
+ memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
+ }
+
+ /* copy dictionary offsets */
+ dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
+ dstCCtx->nextToUpdate3 = srcCCtx->nextToUpdate3;
+ dstCCtx->nextSrc = srcCCtx->nextSrc;
+ dstCCtx->base = srcCCtx->base;
+ dstCCtx->dictBase = srcCCtx->dictBase;
+ dstCCtx->dictLimit = srcCCtx->dictLimit;
+ dstCCtx->lowLimit = srcCCtx->lowLimit;
+ dstCCtx->loadedDictEnd = srcCCtx->loadedDictEnd;
+ dstCCtx->dictID = srcCCtx->dictID;
+
+ /* copy entropy tables */
+ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
+ dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable;
+ if (srcCCtx->flagStaticTables) {
+ memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable));
+ memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable));
+ memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
+ }
+ if (srcCCtx->flagStaticHufTable) {
+ memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256 * 4);
+ }
+
+ return 0;
+}
+
+/*! ZSTD_reduceTable() :
+* reduce table indexes by `reducerValue` */
+static void ZSTD_reduceTable(U32 *const table, U32 const size, U32 const reducerValue)
+{
+ U32 u;
+ for (u = 0; u < size; u++) {
+ if (table[u] < reducerValue)
+ table[u] = 0;
+ else
+ table[u] -= reducerValue;
+ }
+}
+
+/*! ZSTD_reduceIndex() :
+* rescale all indexes to avoid future overflow (indexes are U32) */
+static void ZSTD_reduceIndex(ZSTD_CCtx *zc, const U32 reducerValue)
+{
+ {
+ U32 const hSize = 1 << zc->params.cParams.hashLog;
+ ZSTD_reduceTable(zc->hashTable, hSize, reducerValue);
+ }
+
+ {
+ U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
+ ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue);
+ }
+
+ {
+ U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
+ ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue);
+ }
+}
+
+/*-*******************************************************
+* Block entropic compression
+*********************************************************/
+
+/* See doc/zstd_compression_format.md for detailed format description */
+
+size_t ZSTD_noCompressBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ if (srcSize + ZSTD_blockHeaderSize > dstCapacity)
+ return ERROR(dstSize_tooSmall);
+ memcpy((BYTE *)dst + ZSTD_blockHeaderSize, src, srcSize);
+ ZSTD_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw);
+ return ZSTD_blockHeaderSize + srcSize;
+}
+
+static size_t ZSTD_noCompressLiterals(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ BYTE *const ostart = (BYTE * const)dst;
+ U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095);
+
+ if (srcSize + flSize > dstCapacity)
+ return ERROR(dstSize_tooSmall);
+
+ switch (flSize) {
+ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_basic + (srcSize << 3)); break;
+ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_basic + (1 << 2) + (srcSize << 4))); break;
+ default: /*note : should not be necessary : flSize is within {1,2,3} */
+ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_basic + (3 << 2) + (srcSize << 4))); break;
+ }
+
+ memcpy(ostart + flSize, src, srcSize);
+ return srcSize + flSize;
+}
+
+static size_t ZSTD_compressRleLiteralsBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ BYTE *const ostart = (BYTE * const)dst;
+ U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095);
+
+ (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */
+
+ switch (flSize) {
+ case 1: /* 2 - 1 - 5 */ ostart[0] = (BYTE)((U32)set_rle + (srcSize << 3)); break;
+ case 2: /* 2 - 2 - 12 */ ZSTD_writeLE16(ostart, (U16)((U32)set_rle + (1 << 2) + (srcSize << 4))); break;
+ default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */
+ case 3: /* 2 - 2 - 20 */ ZSTD_writeLE32(ostart, (U32)((U32)set_rle + (3 << 2) + (srcSize << 4))); break;
+ }
+
+ ostart[flSize] = *(const BYTE *)src;
+ return flSize + 1;
+}
+
+static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
+
+static size_t ZSTD_compressLiterals(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ size_t const minGain = ZSTD_minGain(srcSize);
+ size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
+ BYTE *const ostart = (BYTE *)dst;
+ U32 singleStream = srcSize < 256;
+ symbolEncodingType_e hType = set_compressed;
+ size_t cLitSize;
+
+/* small ? don't even attempt compression (speed opt) */
+#define LITERAL_NOENTROPY 63
+ {
+ size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
+ if (srcSize <= minLitSize)
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+ }
+
+ if (dstCapacity < lhSize + 1)
+ return ERROR(dstSize_tooSmall); /* not enough space for compression */
+ {
+ HUF_repeat repeat = zc->flagStaticHufTable;
+ int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
+ if (repeat == HUF_repeat_valid && lhSize == 3)
+ singleStream = 1;
+ cLitSize = singleStream ? HUF_compress1X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
+ sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
+ : HUF_compress4X_repeat(ostart + lhSize, dstCapacity - lhSize, src, srcSize, 255, 11, zc->tmpCounters,
+ sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
+ if (repeat != HUF_repeat_none) {
+ hType = set_repeat;
+ } /* reused the existing table */
+ else {
+ zc->flagStaticHufTable = HUF_repeat_check;
+ } /* now have a table to reuse */
+ }
+
+ if ((cLitSize == 0) | (cLitSize >= srcSize - minGain)) {
+ zc->flagStaticHufTable = HUF_repeat_none;
+ return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+ }
+ if (cLitSize == 1) {
+ zc->flagStaticHufTable = HUF_repeat_none;
+ return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
+ }
+
+ /* Build header */
+ switch (lhSize) {
+ case 3: /* 2 - 2 - 10 - 10 */
+ {
+ U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 14);
+ ZSTD_writeLE24(ostart, lhc);
+ break;
+ }
+ case 4: /* 2 - 2 - 14 - 14 */
+ {
+ U32 const lhc = hType + (2 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 18);
+ ZSTD_writeLE32(ostart, lhc);
+ break;
+ }
+ default: /* should not be necessary, lhSize is only {3,4,5} */
+ case 5: /* 2 - 2 - 18 - 18 */
+ {
+ U32 const lhc = hType + (3 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 22);
+ ZSTD_writeLE32(ostart, lhc);
+ ostart[4] = (BYTE)(cLitSize >> 10);
+ break;
+ }
+ }
+ return lhSize + cLitSize;
+}
+
+static const BYTE LL_Code[64] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18,
+ 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23,
+ 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24};
+
+static const BYTE ML_Code[128] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
+ 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38,
+ 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+ 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 42,
+ 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42};
+
+void ZSTD_seqToCodes(const seqStore_t *seqStorePtr)
+{
+ BYTE const LL_deltaCode = 19;
+ BYTE const ML_deltaCode = 36;
+ const seqDef *const sequences = seqStorePtr->sequencesStart;
+ BYTE *const llCodeTable = seqStorePtr->llCode;
+ BYTE *const ofCodeTable = seqStorePtr->ofCode;
+ BYTE *const mlCodeTable = seqStorePtr->mlCode;
+ U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ U32 u;
+ for (u = 0; u < nbSeq; u++) {
+ U32 const llv = sequences[u].litLength;
+ U32 const mlv = sequences[u].matchLength;
+ llCodeTable[u] = (llv > 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv];
+ ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset);
+ mlCodeTable[u] = (mlv > 127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv];
+ }
+ if (seqStorePtr->longLengthID == 1)
+ llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
+ if (seqStorePtr->longLengthID == 2)
+ mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
+}
+
+ZSTD_STATIC size_t ZSTD_compressSequences_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity)
+{
+ const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
+ const seqStore_t *seqStorePtr = &(zc->seqStore);
+ FSE_CTable *CTable_LitLength = zc->litlengthCTable;
+ FSE_CTable *CTable_OffsetBits = zc->offcodeCTable;
+ FSE_CTable *CTable_MatchLength = zc->matchlengthCTable;
+ U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
+ const seqDef *const sequences = seqStorePtr->sequencesStart;
+ const BYTE *const ofCodeTable = seqStorePtr->ofCode;
+ const BYTE *const llCodeTable = seqStorePtr->llCode;
+ const BYTE *const mlCodeTable = seqStorePtr->mlCode;
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *const oend = ostart + dstCapacity;
+ BYTE *op = ostart;
+ size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
+ BYTE *seqHead;
+
+ U32 *count;
+ S16 *norm;
+ U32 *workspace;
+ size_t workspaceSize = sizeof(zc->tmpCounters);
+ {
+ size_t spaceUsed32 = 0;
+ count = (U32 *)zc->tmpCounters + spaceUsed32;
+ spaceUsed32 += MaxSeq + 1;
+ norm = (S16 *)((U32 *)zc->tmpCounters + spaceUsed32);
+ spaceUsed32 += ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2;
+
+ workspace = (U32 *)zc->tmpCounters + spaceUsed32;
+ workspaceSize -= (spaceUsed32 << 2);
+ }
+
+ /* Compress literals */
+ {
+ const BYTE *const literals = seqStorePtr->litStart;
+ size_t const litSize = seqStorePtr->lit - literals;
+ size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize);
+ if (ZSTD_isError(cSize))
+ return cSize;
+ op += cSize;
+ }
+
+ /* Sequences Header */
+ if ((oend - op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */)
+ return ERROR(dstSize_tooSmall);
+ if (nbSeq < 0x7F)
+ *op++ = (BYTE)nbSeq;
+ else if (nbSeq < LONGNBSEQ)
+ op[0] = (BYTE)((nbSeq >> 8) + 0x80), op[1] = (BYTE)nbSeq, op += 2;
+ else
+ op[0] = 0xFF, ZSTD_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3;
+ if (nbSeq == 0)
+ return op - ostart;
+
+ /* seqHead : flags for FSE encoding type */
+ seqHead = op++;
+
+#define MIN_SEQ_FOR_DYNAMIC_FSE 64
+#define MAX_SEQ_FOR_STATIC_FSE 1000
+
+ /* convert length/distances into codes */
+ ZSTD_seqToCodes(seqStorePtr);
+
+ /* CTable for Literal Lengths */
+ {
+ U32 max = MaxLL;
+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace);
+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
+ *op++ = llCodeTable[0];
+ FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
+ LLtype = set_rle;
+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+ LLtype = set_repeat;
+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog - 1)))) {
+ FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, workspace, workspaceSize);
+ LLtype = set_basic;
+ } else {
+ size_t nbSeq_1 = nbSeq;
+ const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
+ if (count[llCodeTable[nbSeq - 1]] > 1) {
+ count[llCodeTable[nbSeq - 1]]--;
+ nbSeq_1--;
+ }
+ FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
+ {
+ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
+ if (FSE_isError(NCountSize))
+ return NCountSize;
+ op += NCountSize;
+ }
+ FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, workspace, workspaceSize);
+ LLtype = set_compressed;
+ }
+ }
+
+ /* CTable for Offsets */
+ {
+ U32 max = MaxOff;
+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace);
+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
+ *op++ = ofCodeTable[0];
+ FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
+ Offtype = set_rle;
+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+ Offtype = set_repeat;
+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog - 1)))) {
+ FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, workspace, workspaceSize);
+ Offtype = set_basic;
+ } else {
+ size_t nbSeq_1 = nbSeq;
+ const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
+ if (count[ofCodeTable[nbSeq - 1]] > 1) {
+ count[ofCodeTable[nbSeq - 1]]--;
+ nbSeq_1--;
+ }
+ FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
+ {
+ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
+ if (FSE_isError(NCountSize))
+ return NCountSize;
+ op += NCountSize;
+ }
+ FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, workspace, workspaceSize);
+ Offtype = set_compressed;
+ }
+ }
+
+ /* CTable for MatchLengths */
+ {
+ U32 max = MaxML;
+ size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace);
+ if ((mostFrequent == nbSeq) && (nbSeq > 2)) {
+ *op++ = *mlCodeTable;
+ FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
+ MLtype = set_rle;
+ } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
+ MLtype = set_repeat;
+ } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog - 1)))) {
+ FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, workspace, workspaceSize);
+ MLtype = set_basic;
+ } else {
+ size_t nbSeq_1 = nbSeq;
+ const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
+ if (count[mlCodeTable[nbSeq - 1]] > 1) {
+ count[mlCodeTable[nbSeq - 1]]--;
+ nbSeq_1--;
+ }
+ FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
+ {
+ size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */
+ if (FSE_isError(NCountSize))
+ return NCountSize;
+ op += NCountSize;
+ }
+ FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, workspace, workspaceSize);
+ MLtype = set_compressed;
+ }
+ }
+
+ *seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2));
+ zc->flagStaticTables = 0;
+
+ /* Encoding Sequences */
+ {
+ BIT_CStream_t blockStream;
+ FSE_CState_t stateMatchLength;
+ FSE_CState_t stateOffsetBits;
+ FSE_CState_t stateLitLength;
+
+ CHECK_E(BIT_initCStream(&blockStream, op, oend - op), dstSize_tooSmall); /* not enough space remaining */
+
+ /* first symbols */
+ FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq - 1]);
+ FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]);
+ FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]);
+ BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]);
+ if (ZSTD_32bits())
+ BIT_flushBits(&blockStream);
+ BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]);
+ if (ZSTD_32bits())
+ BIT_flushBits(&blockStream);
+ if (longOffsets) {
+ U32 const ofBits = ofCodeTable[nbSeq - 1];
+ int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
+ if (extraBits) {
+ BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits);
+ BIT_flushBits(&blockStream);
+ }
+ BIT_addBits(&blockStream, sequences[nbSeq - 1].offset >> extraBits, ofBits - extraBits);
+ } else {
+ BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]);
+ }
+ BIT_flushBits(&blockStream);
+
+ {
+ size_t n;
+ for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */
+ BYTE const llCode = llCodeTable[n];
+ BYTE const ofCode = ofCodeTable[n];
+ BYTE const mlCode = mlCodeTable[n];
+ U32 const llBits = LL_bits[llCode];
+ U32 const ofBits = ofCode; /* 32b*/ /* 64b*/
+ U32 const mlBits = ML_bits[mlCode];
+ /* (7)*/ /* (7)*/
+ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
+ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
+ if (ZSTD_32bits())
+ BIT_flushBits(&blockStream); /* (7)*/
+ FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */
+ if (ZSTD_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
+ BIT_flushBits(&blockStream); /* (7)*/
+ BIT_addBits(&blockStream, sequences[n].litLength, llBits);
+ if (ZSTD_32bits() && ((llBits + mlBits) > 24))
+ BIT_flushBits(&blockStream);
+ BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
+ if (ZSTD_32bits())
+ BIT_flushBits(&blockStream); /* (7)*/
+ if (longOffsets) {
+ int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1);
+ if (extraBits) {
+ BIT_addBits(&blockStream, sequences[n].offset, extraBits);
+ BIT_flushBits(&blockStream); /* (7)*/
+ }
+ BIT_addBits(&blockStream, sequences[n].offset >> extraBits, ofBits - extraBits); /* 31 */
+ } else {
+ BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */
+ }
+ BIT_flushBits(&blockStream); /* (7)*/
+ }
+ }
+
+ FSE_flushCState(&blockStream, &stateMatchLength);
+ FSE_flushCState(&blockStream, &stateOffsetBits);
+ FSE_flushCState(&blockStream, &stateLitLength);
+
+ {
+ size_t const streamSize = BIT_closeCStream(&blockStream);
+ if (streamSize == 0)
+ return ERROR(dstSize_tooSmall); /* not enough space */
+ op += streamSize;
+ }
+ }
+ return op - ostart;
+}
+
+ZSTD_STATIC size_t ZSTD_compressSequences(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, size_t srcSize)
+{
+ size_t const cSize = ZSTD_compressSequences_internal(zc, dst, dstCapacity);
+ size_t const minGain = ZSTD_minGain(srcSize);
+ size_t const maxCSize = srcSize - minGain;
+ /* If the srcSize <= dstCapacity, then there is enough space to write a
+ * raw uncompressed block. Since we ran out of space, the block must not
+ * be compressible, so fall back to a raw uncompressed block.
+ */
+ int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity;
+ int i;
+
+ if (ZSTD_isError(cSize) && !uncompressibleError)
+ return cSize;
+ if (cSize >= maxCSize || uncompressibleError) {
+ zc->flagStaticHufTable = HUF_repeat_none;
+ return 0;
+ }
+ /* confirm repcodes */
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ zc->rep[i] = zc->repToConfirm[i];
+ return cSize;
+}
+
+/*! ZSTD_storeSeq() :
+ Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
+ `offsetCode` : distance to match, or 0 == repCode.
+ `matchCode` : matchLength - MINMATCH
+*/
+ZSTD_STATIC void ZSTD_storeSeq(seqStore_t *seqStorePtr, size_t litLength, const void *literals, U32 offsetCode, size_t matchCode)
+{
+ /* copy Literals */
+ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
+ seqStorePtr->lit += litLength;
+
+ /* literal Length */
+ if (litLength > 0xFFFF) {
+ seqStorePtr->longLengthID = 1;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].litLength = (U16)litLength;
+
+ /* match offset */
+ seqStorePtr->sequences[0].offset = offsetCode + 1;
+
+ /* match Length */
+ if (matchCode > 0xFFFF) {
+ seqStorePtr->longLengthID = 2;
+ seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+ }
+ seqStorePtr->sequences[0].matchLength = (U16)matchCode;
+
+ seqStorePtr->sequences++;
+}
+
+/*-*************************************
+* Match length counter
+***************************************/
+static unsigned ZSTD_NbCommonBytes(register size_t val)
+{
+ if (ZSTD_isLittleEndian()) {
+ if (ZSTD_64bits()) {
+ return (__builtin_ctzll((U64)val) >> 3);
+ } else { /* 32 bits */
+ return (__builtin_ctz((U32)val) >> 3);
+ }
+ } else { /* Big Endian CPU */
+ if (ZSTD_64bits()) {
+ return (__builtin_clzll(val) >> 3);
+ } else { /* 32 bits */
+ return (__builtin_clz((U32)val) >> 3);
+ }
+ }
+}
+
+static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit)
+{
+ const BYTE *const pStart = pIn;
+ const BYTE *const pInLoopLimit = pInLimit - (sizeof(size_t) - 1);
+
+ while (pIn < pInLoopLimit) {
+ size_t const diff = ZSTD_readST(pMatch) ^ ZSTD_readST(pIn);
+ if (!diff) {
+ pIn += sizeof(size_t);
+ pMatch += sizeof(size_t);
+ continue;
+ }
+ pIn += ZSTD_NbCommonBytes(diff);
+ return (size_t)(pIn - pStart);
+ }
+ if (ZSTD_64bits())
+ if ((pIn < (pInLimit - 3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) {
+ pIn += 4;
+ pMatch += 4;
+ }
+ if ((pIn < (pInLimit - 1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) {
+ pIn += 2;
+ pMatch += 2;
+ }
+ if ((pIn < pInLimit) && (*pMatch == *pIn))
+ pIn++;
+ return (size_t)(pIn - pStart);
+}
+
+/** ZSTD_count_2segments() :
+* can count match length with `ip` & `match` in 2 different segments.
+* convention : on reaching mEnd, match count continue starting from iStart
+*/
+static size_t ZSTD_count_2segments(const BYTE *ip, const BYTE *match, const BYTE *iEnd, const BYTE *mEnd, const BYTE *iStart)
+{
+ const BYTE *const vEnd = MIN(ip + (mEnd - match), iEnd);
+ size_t const matchLength = ZSTD_count(ip, match, vEnd);
+ if (match + matchLength != mEnd)
+ return matchLength;
+ return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd);
+}
+
+/*-*************************************
+* Hashes
+***************************************/
+static const U32 prime3bytes = 506832829U;
+static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32 - 24)) * prime3bytes) >> (32 - h); }
+ZSTD_STATIC size_t ZSTD_hash3Ptr(const void *ptr, U32 h) { return ZSTD_hash3(ZSTD_readLE32(ptr), h); } /* only in zstd_opt.h */
+
+static const U32 prime4bytes = 2654435761U;
+static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32 - h); }
+static size_t ZSTD_hash4Ptr(const void *ptr, U32 h) { return ZSTD_hash4(ZSTD_read32(ptr), h); }
+
+static const U64 prime5bytes = 889523592379ULL;
+static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64 - 40)) * prime5bytes) >> (64 - h)); }
+static size_t ZSTD_hash5Ptr(const void *p, U32 h) { return ZSTD_hash5(ZSTD_readLE64(p), h); }
+
+static const U64 prime6bytes = 227718039650203ULL;
+static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64 - 48)) * prime6bytes) >> (64 - h)); }
+static size_t ZSTD_hash6Ptr(const void *p, U32 h) { return ZSTD_hash6(ZSTD_readLE64(p), h); }
+
+static const U64 prime7bytes = 58295818150454627ULL;
+static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64 - 56)) * prime7bytes) >> (64 - h)); }
+static size_t ZSTD_hash7Ptr(const void *p, U32 h) { return ZSTD_hash7(ZSTD_readLE64(p), h); }
+
+static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
+static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u)*prime8bytes) >> (64 - h)); }
+static size_t ZSTD_hash8Ptr(const void *p, U32 h) { return ZSTD_hash8(ZSTD_readLE64(p), h); }
+
+static size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls)
+{
+ switch (mls) {
+ // case 3: return ZSTD_hash3Ptr(p, hBits);
+ default:
+ case 4: return ZSTD_hash4Ptr(p, hBits);
+ case 5: return ZSTD_hash5Ptr(p, hBits);
+ case 6: return ZSTD_hash6Ptr(p, hBits);
+ case 7: return ZSTD_hash7Ptr(p, hBits);
+ case 8: return ZSTD_hash8Ptr(p, hBits);
+ }
+}
+
+/*-*************************************
+* Fast Scan
+***************************************/
+static void ZSTD_fillHashTable(ZSTD_CCtx *zc, const void *end, const U32 mls)
+{
+ U32 *const hashTable = zc->hashTable;
+ U32 const hBits = zc->params.cParams.hashLog;
+ const BYTE *const base = zc->base;
+ const BYTE *ip = base + zc->nextToUpdate;
+ const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE;
+ const size_t fastHashFillStep = 3;
+
+ while (ip <= iend) {
+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
+ ip += fastHashFillStep;
+ }
+}
+
+FORCE_INLINE
+void ZSTD_compressBlock_fast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
+{
+ U32 *const hashTable = cctx->hashTable;
+ U32 const hBits = cctx->params.cParams.hashLog;
+ seqStore_t *seqStorePtr = &(cctx->seqStore);
+ const BYTE *const base = cctx->base;
+ const BYTE *const istart = (const BYTE *)src;
+ const BYTE *ip = istart;
+ const BYTE *anchor = istart;
+ const U32 lowestIndex = cctx->dictLimit;
+ const BYTE *const lowest = base + lowestIndex;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *const ilimit = iend - HASH_READ_SIZE;
+ U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1];
+ U32 offsetSaved = 0;
+
+ /* init */
+ ip += (ip == lowest);
+ {
+ U32 const maxRep = (U32)(ip - lowest);
+ if (offset_2 > maxRep)
+ offsetSaved = offset_2, offset_2 = 0;
+ if (offset_1 > maxRep)
+ offsetSaved = offset_1, offset_1 = 0;
+ }
+
+ /* Main Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
+ size_t mLength;
+ size_t const h = ZSTD_hashPtr(ip, hBits, mls);
+ U32 const curr = (U32)(ip - base);
+ U32 const matchIndex = hashTable[h];
+ const BYTE *match = base + matchIndex;
+ hashTable[h] = curr; /* update hash table */
+
+ if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) {
+ mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4;
+ ip++;
+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
+ } else {
+ U32 offset;
+ if ((matchIndex <= lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) {
+ ip += ((ip - anchor) >> g_searchStrength) + 1;
+ continue;
+ }
+ mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
+ offset = (U32)(ip - match);
+ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
+ ip--;
+ match--;
+ mLength++;
+ } /* catch up */
+ offset_2 = offset_1;
+ offset_1 = offset;
+
+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
+ }
+
+ /* match found */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Fill Table */
+ hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2; /* here because curr+2 could be > iend-8 */
+ hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base);
+ /* check immediate repcode */
+ while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
+ /* store sequence */
+ size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
+ {
+ U32 const tmpOff = offset_2;
+ offset_2 = offset_1;
+ offset_1 = tmpOff;
+ } /* swap offset_2 <=> offset_1 */
+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base);
+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
+ ip += rLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ }
+ }
+ }
+
+ /* save reps for next block */
+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
+
+ /* Last Literals */
+ {
+ size_t const lastLLSize = iend - anchor;
+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+ }
+}
+
+static void ZSTD_compressBlock_fast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+ const U32 mls = ctx->params.cParams.searchLength;
+ switch (mls) {
+ default: /* includes case 3 */
+ case 4: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return;
+ case 5: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return;
+ case 6: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return;
+ case 7: ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return;
+ }
+}
+
+static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
+{
+ U32 *hashTable = ctx->hashTable;
+ const U32 hBits = ctx->params.cParams.hashLog;
+ seqStore_t *seqStorePtr = &(ctx->seqStore);
+ const BYTE *const base = ctx->base;
+ const BYTE *const dictBase = ctx->dictBase;
+ const BYTE *const istart = (const BYTE *)src;
+ const BYTE *ip = istart;
+ const BYTE *anchor = istart;
+ const U32 lowestIndex = ctx->lowLimit;
+ const BYTE *const dictStart = dictBase + lowestIndex;
+ const U32 dictLimit = ctx->dictLimit;
+ const BYTE *const lowPrefixPtr = base + dictLimit;
+ const BYTE *const dictEnd = dictBase + dictLimit;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *const ilimit = iend - 8;
+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
+
+ /* Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
+ const size_t h = ZSTD_hashPtr(ip, hBits, mls);
+ const U32 matchIndex = hashTable[h];
+ const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base;
+ const BYTE *match = matchBase + matchIndex;
+ const U32 curr = (U32)(ip - base);
+ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
+ const BYTE *repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE *repMatch = repBase + repIndex;
+ size_t mLength;
+ hashTable[h] = curr; /* update hash table */
+
+ if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
+ (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
+ const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
+ mLength = ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
+ ip++;
+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
+ } else {
+ if ((matchIndex < lowestIndex) || (ZSTD_read32(match) != ZSTD_read32(ip))) {
+ ip += ((ip - anchor) >> g_searchStrength) + 1;
+ continue;
+ }
+ {
+ const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
+ const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
+ U32 offset;
+ mLength = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32;
+ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
+ ip--;
+ match--;
+ mLength++;
+ } /* catch up */
+ offset = curr - matchIndex;
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
+ }
+ }
+
+ /* found a match : store it */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Fill Table */
+ hashTable[ZSTD_hashPtr(base + curr + 2, hBits, mls)] = curr + 2;
+ hashTable[ZSTD_hashPtr(ip - 2, hBits, mls)] = (U32)(ip - 2 - base);
+ /* check immediate repcode */
+ while (ip <= ilimit) {
+ U32 const curr2 = (U32)(ip - base);
+ U32 const repIndex2 = curr2 - offset_2;
+ const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
+ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
+ && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
+ const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
+ size_t repLength2 =
+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
+ U32 tmpOffset = offset_2;
+ offset_2 = offset_1;
+ offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
+ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = curr2;
+ ip += repLength2;
+ anchor = ip;
+ continue;
+ }
+ break;
+ }
+ }
+ }
+
+ /* save reps for next block */
+ ctx->repToConfirm[0] = offset_1;
+ ctx->repToConfirm[1] = offset_2;
+
+ /* Last Literals */
+ {
+ size_t const lastLLSize = iend - anchor;
+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+ }
+}
+
+static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+ U32 const mls = ctx->params.cParams.searchLength;
+ switch (mls) {
+ default: /* includes case 3 */
+ case 4: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return;
+ case 5: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return;
+ case 6: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return;
+ case 7: ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return;
+ }
+}
+
+/*-*************************************
+* Double Fast
+***************************************/
+static void ZSTD_fillDoubleHashTable(ZSTD_CCtx *cctx, const void *end, const U32 mls)
+{
+ U32 *const hashLarge = cctx->hashTable;
+ U32 const hBitsL = cctx->params.cParams.hashLog;
+ U32 *const hashSmall = cctx->chainTable;
+ U32 const hBitsS = cctx->params.cParams.chainLog;
+ const BYTE *const base = cctx->base;
+ const BYTE *ip = base + cctx->nextToUpdate;
+ const BYTE *const iend = ((const BYTE *)end) - HASH_READ_SIZE;
+ const size_t fastHashFillStep = 3;
+
+ while (ip <= iend) {
+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base);
+ hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base);
+ ip += fastHashFillStep;
+ }
+}
+
+FORCE_INLINE
+void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx *cctx, const void *src, size_t srcSize, const U32 mls)
+{
+ U32 *const hashLong = cctx->hashTable;
+ const U32 hBitsL = cctx->params.cParams.hashLog;
+ U32 *const hashSmall = cctx->chainTable;
+ const U32 hBitsS = cctx->params.cParams.chainLog;
+ seqStore_t *seqStorePtr = &(cctx->seqStore);
+ const BYTE *const base = cctx->base;
+ const BYTE *const istart = (const BYTE *)src;
+ const BYTE *ip = istart;
+ const BYTE *anchor = istart;
+ const U32 lowestIndex = cctx->dictLimit;
+ const BYTE *const lowest = base + lowestIndex;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *const ilimit = iend - HASH_READ_SIZE;
+ U32 offset_1 = cctx->rep[0], offset_2 = cctx->rep[1];
+ U32 offsetSaved = 0;
+
+ /* init */
+ ip += (ip == lowest);
+ {
+ U32 const maxRep = (U32)(ip - lowest);
+ if (offset_2 > maxRep)
+ offsetSaved = offset_2, offset_2 = 0;
+ if (offset_1 > maxRep)
+ offsetSaved = offset_1, offset_1 = 0;
+ }
+
+ /* Main Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
+ size_t mLength;
+ size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
+ size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
+ U32 const curr = (U32)(ip - base);
+ U32 const matchIndexL = hashLong[h2];
+ U32 const matchIndexS = hashSmall[h];
+ const BYTE *matchLong = base + matchIndexL;
+ const BYTE *match = base + matchIndexS;
+ hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
+
+ if ((offset_1 > 0) & (ZSTD_read32(ip + 1 - offset_1) == ZSTD_read32(ip + 1))) { /* note : by construction, offset_1 <= curr */
+ mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4;
+ ip++;
+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
+ } else {
+ U32 offset;
+ if ((matchIndexL > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) {
+ mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8;
+ offset = (U32)(ip - matchLong);
+ while (((ip > anchor) & (matchLong > lowest)) && (ip[-1] == matchLong[-1])) {
+ ip--;
+ matchLong--;
+ mLength++;
+ } /* catch up */
+ } else if ((matchIndexS > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) {
+ size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8);
+ U32 const matchIndex3 = hashLong[h3];
+ const BYTE *match3 = base + matchIndex3;
+ hashLong[h3] = curr + 1;
+ if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) {
+ mLength = ZSTD_count(ip + 9, match3 + 8, iend) + 8;
+ ip++;
+ offset = (U32)(ip - match3);
+ while (((ip > anchor) & (match3 > lowest)) && (ip[-1] == match3[-1])) {
+ ip--;
+ match3--;
+ mLength++;
+ } /* catch up */
+ } else {
+ mLength = ZSTD_count(ip + 4, match + 4, iend) + 4;
+ offset = (U32)(ip - match);
+ while (((ip > anchor) & (match > lowest)) && (ip[-1] == match[-1])) {
+ ip--;
+ match--;
+ mLength++;
+ } /* catch up */
+ }
+ } else {
+ ip += ((ip - anchor) >> g_searchStrength) + 1;
+ continue;
+ }
+
+ offset_2 = offset_1;
+ offset_1 = offset;
+
+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
+ }
+
+ /* match found */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Fill Table */
+ hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] =
+ curr + 2; /* here because curr+2 could be > iend-8 */
+ hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base);
+
+ /* check immediate repcode */
+ while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
+ /* store sequence */
+ size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4;
+ {
+ U32 const tmpOff = offset_2;
+ offset_2 = offset_1;
+ offset_1 = tmpOff;
+ } /* swap offset_2 <=> offset_1 */
+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base);
+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base);
+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength - MINMATCH);
+ ip += rLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ }
+ }
+ }
+
+ /* save reps for next block */
+ cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
+ cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
+
+ /* Last Literals */
+ {
+ size_t const lastLLSize = iend - anchor;
+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+ }
+}
+
+static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+ const U32 mls = ctx->params.cParams.searchLength;
+ switch (mls) {
+ default: /* includes case 3 */
+ case 4: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return;
+ case 5: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return;
+ case 6: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return;
+ case 7: ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return;
+ }
+}
+
+static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 mls)
+{
+ U32 *const hashLong = ctx->hashTable;
+ U32 const hBitsL = ctx->params.cParams.hashLog;
+ U32 *const hashSmall = ctx->chainTable;
+ U32 const hBitsS = ctx->params.cParams.chainLog;
+ seqStore_t *seqStorePtr = &(ctx->seqStore);
+ const BYTE *const base = ctx->base;
+ const BYTE *const dictBase = ctx->dictBase;
+ const BYTE *const istart = (const BYTE *)src;
+ const BYTE *ip = istart;
+ const BYTE *anchor = istart;
+ const U32 lowestIndex = ctx->lowLimit;
+ const BYTE *const dictStart = dictBase + lowestIndex;
+ const U32 dictLimit = ctx->dictLimit;
+ const BYTE *const lowPrefixPtr = base + dictLimit;
+ const BYTE *const dictEnd = dictBase + dictLimit;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *const ilimit = iend - 8;
+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
+
+ /* Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
+ const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
+ const U32 matchIndex = hashSmall[hSmall];
+ const BYTE *matchBase = matchIndex < dictLimit ? dictBase : base;
+ const BYTE *match = matchBase + matchIndex;
+
+ const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8);
+ const U32 matchLongIndex = hashLong[hLong];
+ const BYTE *matchLongBase = matchLongIndex < dictLimit ? dictBase : base;
+ const BYTE *matchLong = matchLongBase + matchLongIndex;
+
+ const U32 curr = (U32)(ip - base);
+ const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
+ const BYTE *repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE *repMatch = repBase + repIndex;
+ size_t mLength;
+ hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
+
+ if ((((U32)((dictLimit - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) &&
+ (ZSTD_read32(repMatch) == ZSTD_read32(ip + 1))) {
+ const BYTE *repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
+ mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, lowPrefixPtr) + 4;
+ ip++;
+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, 0, mLength - MINMATCH);
+ } else {
+ if ((matchLongIndex > lowestIndex) && (ZSTD_read64(matchLong) == ZSTD_read64(ip))) {
+ const BYTE *matchEnd = matchLongIndex < dictLimit ? dictEnd : iend;
+ const BYTE *lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr;
+ U32 offset;
+ mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, lowPrefixPtr) + 8;
+ offset = curr - matchLongIndex;
+ while (((ip > anchor) & (matchLong > lowMatchPtr)) && (ip[-1] == matchLong[-1])) {
+ ip--;
+ matchLong--;
+ mLength++;
+ } /* catch up */
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
+
+ } else if ((matchIndex > lowestIndex) && (ZSTD_read32(match) == ZSTD_read32(ip))) {
+ size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8);
+ U32 const matchIndex3 = hashLong[h3];
+ const BYTE *const match3Base = matchIndex3 < dictLimit ? dictBase : base;
+ const BYTE *match3 = match3Base + matchIndex3;
+ U32 offset;
+ hashLong[h3] = curr + 1;
+ if ((matchIndex3 > lowestIndex) && (ZSTD_read64(match3) == ZSTD_read64(ip + 1))) {
+ const BYTE *matchEnd = matchIndex3 < dictLimit ? dictEnd : iend;
+ const BYTE *lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr;
+ mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, lowPrefixPtr) + 8;
+ ip++;
+ offset = curr + 1 - matchIndex3;
+ while (((ip > anchor) & (match3 > lowMatchPtr)) && (ip[-1] == match3[-1])) {
+ ip--;
+ match3--;
+ mLength++;
+ } /* catch up */
+ } else {
+ const BYTE *matchEnd = matchIndex < dictLimit ? dictEnd : iend;
+ const BYTE *lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
+ mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, lowPrefixPtr) + 4;
+ offset = curr - matchIndex;
+ while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) {
+ ip--;
+ match--;
+ mLength++;
+ } /* catch up */
+ }
+ offset_2 = offset_1;
+ offset_1 = offset;
+ ZSTD_storeSeq(seqStorePtr, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH);
+
+ } else {
+ ip += ((ip - anchor) >> g_searchStrength) + 1;
+ continue;
+ }
+ }
+
+ /* found a match : store it */
+ ip += mLength;
+ anchor = ip;
+
+ if (ip <= ilimit) {
+ /* Fill Table */
+ hashSmall[ZSTD_hashPtr(base + curr + 2, hBitsS, mls)] = curr + 2;
+ hashLong[ZSTD_hashPtr(base + curr + 2, hBitsL, 8)] = curr + 2;
+ hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base);
+ hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = (U32)(ip - 2 - base);
+ /* check immediate repcode */
+ while (ip <= ilimit) {
+ U32 const curr2 = (U32)(ip - base);
+ U32 const repIndex2 = curr2 - offset_2;
+ const BYTE *repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
+ if ((((U32)((dictLimit - 1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
+ && (ZSTD_read32(repMatch2) == ZSTD_read32(ip))) {
+ const BYTE *const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
+ size_t const repLength2 =
+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch2 + EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
+ U32 tmpOffset = offset_2;
+ offset_2 = offset_1;
+ offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2 - MINMATCH);
+ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = curr2;
+ hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = curr2;
+ ip += repLength2;
+ anchor = ip;
+ continue;
+ }
+ break;
+ }
+ }
+ }
+
+ /* save reps for next block */
+ ctx->repToConfirm[0] = offset_1;
+ ctx->repToConfirm[1] = offset_2;
+
+ /* Last Literals */
+ {
+ size_t const lastLLSize = iend - anchor;
+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+ }
+}
+
+static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+ U32 const mls = ctx->params.cParams.searchLength;
+ switch (mls) {
+ default: /* includes case 3 */
+ case 4: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return;
+ case 5: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return;
+ case 6: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return;
+ case 7: ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return;
+ }
+}
+
+/*-*************************************
+* Binary Tree search
+***************************************/
+/** ZSTD_insertBt1() : add one or multiple positions to tree.
+* ip : assumed <= iend-8 .
+* @return : nb of positions added */
+static U32 ZSTD_insertBt1(ZSTD_CCtx *zc, const BYTE *const ip, const U32 mls, const BYTE *const iend, U32 nbCompares, U32 extDict)
+{
+ U32 *const hashTable = zc->hashTable;
+ U32 const hashLog = zc->params.cParams.hashLog;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32 *const bt = zc->chainTable;
+ U32 const btLog = zc->params.cParams.chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ U32 matchIndex = hashTable[h];
+ size_t commonLengthSmaller = 0, commonLengthLarger = 0;
+ const BYTE *const base = zc->base;
+ const BYTE *const dictBase = zc->dictBase;
+ const U32 dictLimit = zc->dictLimit;
+ const BYTE *const dictEnd = dictBase + dictLimit;
+ const BYTE *const prefixStart = base + dictLimit;
+ const BYTE *match;
+ const U32 curr = (U32)(ip - base);
+ const U32 btLow = btMask >= curr ? 0 : curr - btMask;
+ U32 *smallerPtr = bt + 2 * (curr & btMask);
+ U32 *largerPtr = smallerPtr + 1;
+ U32 dummy32; /* to be nullified at the end */
+ U32 const windowLow = zc->lowLimit;
+ U32 matchEndIdx = curr + 8;
+ size_t bestLength = 8;
+
+ hashTable[h] = curr; /* Update Hash Table */
+
+ while (nbCompares-- && (matchIndex > windowLow)) {
+ U32 *const nextPtr = bt + 2 * (matchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+
+ if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
+ match = base + matchIndex;
+ if (match[matchLength] == ip[matchLength])
+ matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1;
+ } else {
+ match = dictBase + matchIndex;
+ matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart);
+ if (matchIndex + matchLength >= dictLimit)
+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
+ }
+
+ if (matchLength > bestLength) {
+ bestLength = matchLength;
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ }
+
+ if (ip + matchLength == iend) /* equal : no way to know if inf or sup */
+ break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
+
+ if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */
+ /* match is smaller than curr */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) {
+ smallerPtr = &dummy32;
+ break;
+ } /* beyond tree size, stop the search */
+ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
+ } else {
+ /* match is larger than curr */
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) {
+ largerPtr = &dummy32;
+ break;
+ } /* beyond tree size, stop the search */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ }
+ }
+
+ *smallerPtr = *largerPtr = 0;
+ if (bestLength > 384)
+ return MIN(192, (U32)(bestLength - 384)); /* speed optimization */
+ if (matchEndIdx > curr + 8)
+ return matchEndIdx - curr - 8;
+ return 1;
+}
+
+static size_t ZSTD_insertBtAndFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, size_t *offsetPtr, U32 nbCompares, const U32 mls,
+ U32 extDict)
+{
+ U32 *const hashTable = zc->hashTable;
+ U32 const hashLog = zc->params.cParams.hashLog;
+ size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32 *const bt = zc->chainTable;
+ U32 const btLog = zc->params.cParams.chainLog - 1;
+ U32 const btMask = (1 << btLog) - 1;
+ U32 matchIndex = hashTable[h];
+ size_t commonLengthSmaller = 0, commonLengthLarger = 0;
+ const BYTE *const base = zc->base;
+ const BYTE *const dictBase = zc->dictBase;
+ const U32 dictLimit = zc->dictLimit;
+ const BYTE *const dictEnd = dictBase + dictLimit;
+ const BYTE *const prefixStart = base + dictLimit;
+ const U32 curr = (U32)(ip - base);
+ const U32 btLow = btMask >= curr ? 0 : curr - btMask;
+ const U32 windowLow = zc->lowLimit;
+ U32 *smallerPtr = bt + 2 * (curr & btMask);
+ U32 *largerPtr = bt + 2 * (curr & btMask) + 1;
+ U32 matchEndIdx = curr + 8;
+ U32 dummy32; /* to be nullified at the end */
+ size_t bestLength = 0;
+
+ hashTable[h] = curr; /* Update Hash Table */
+
+ while (nbCompares-- && (matchIndex > windowLow)) {
+ U32 *const nextPtr = bt + 2 * (matchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ const BYTE *match;
+
+ if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
+ match = base + matchIndex;
+ if (match[matchLength] == ip[matchLength])
+ matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iend) + 1;
+ } else {
+ match = dictBase + matchIndex;
+ matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart);
+ if (matchIndex + matchLength >= dictLimit)
+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
+ }
+
+ if (matchLength > bestLength) {
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ if ((4 * (int)(matchLength - bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)offsetPtr[0] + 1)))
+ bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex;
+ if (ip + matchLength == iend) /* equal : no way to know if inf or sup */
+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
+ }
+
+ if (match[matchLength] < ip[matchLength]) {
+ /* match is smaller than curr */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) {
+ smallerPtr = &dummy32;
+ break;
+ } /* beyond tree size, stop the search */
+ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
+ } else {
+ /* match is larger than curr */
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) {
+ largerPtr = &dummy32;
+ break;
+ } /* beyond tree size, stop the search */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ }
+ }
+
+ *smallerPtr = *largerPtr = 0;
+
+ zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1;
+ return bestLength;
+}
+
+static void ZSTD_updateTree(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
+{
+ const BYTE *const base = zc->base;
+ const U32 target = (U32)(ip - base);
+ U32 idx = zc->nextToUpdate;
+
+ while (idx < target)
+ idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 0);
+}
+
+/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
+static size_t ZSTD_BtFindBestMatch(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls)
+{
+ if (ip < zc->base + zc->nextToUpdate)
+ return 0; /* skipped area */
+ ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
+ return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0);
+}
+
+static size_t ZSTD_BtFindBestMatch_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
+ const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 matchLengthSearch)
+{
+ switch (matchLengthSearch) {
+ default: /* includes case 3 */
+ case 4: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
+ case 5: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
+ case 7:
+ case 6: return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
+ }
+}
+
+static void ZSTD_updateTree_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iend, const U32 nbCompares, const U32 mls)
+{
+ const BYTE *const base = zc->base;
+ const U32 target = (U32)(ip - base);
+ U32 idx = zc->nextToUpdate;
+
+ while (idx < target)
+ idx += ZSTD_insertBt1(zc, base + idx, mls, iend, nbCompares, 1);
+}
+
+/** Tree updater, providing best match */
+static size_t ZSTD_BtFindBestMatch_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
+ const U32 mls)
+{
+ if (ip < zc->base + zc->nextToUpdate)
+ return 0; /* skipped area */
+ ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
+ return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1);
+}
+
+static size_t ZSTD_BtFindBestMatch_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
+ const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
+ const U32 matchLengthSearch)
+{
+ switch (matchLengthSearch) {
+ default: /* includes case 3 */
+ case 4: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
+ case 5: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
+ case 7:
+ case 6: return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
+ }
+}
+
+/* *********************************
+* Hash Chain
+***********************************/
+#define NEXT_IN_CHAIN(d, mask) chainTable[(d)&mask]
+
+/* Update chains up to ip (excluded)
+ Assumption : always within prefix (i.e. not within extDict) */
+FORCE_INLINE
+U32 ZSTD_insertAndFindFirstIndex(ZSTD_CCtx *zc, const BYTE *ip, U32 mls)
+{
+ U32 *const hashTable = zc->hashTable;
+ const U32 hashLog = zc->params.cParams.hashLog;
+ U32 *const chainTable = zc->chainTable;
+ const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1;
+ const BYTE *const base = zc->base;
+ const U32 target = (U32)(ip - base);
+ U32 idx = zc->nextToUpdate;
+
+ while (idx < target) { /* catch up */
+ size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls);
+ NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
+ hashTable[h] = idx;
+ idx++;
+ }
+
+ zc->nextToUpdate = target;
+ return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
+}
+
+/* inlining is important to hardwire a hot branch (template emulation) */
+FORCE_INLINE
+size_t ZSTD_HcFindBestMatch_generic(ZSTD_CCtx *zc, /* Index table will be updated */
+ const BYTE *const ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts, const U32 mls,
+ const U32 extDict)
+{
+ U32 *const chainTable = zc->chainTable;
+ const U32 chainSize = (1 << zc->params.cParams.chainLog);
+ const U32 chainMask = chainSize - 1;
+ const BYTE *const base = zc->base;
+ const BYTE *const dictBase = zc->dictBase;
+ const U32 dictLimit = zc->dictLimit;
+ const BYTE *const prefixStart = base + dictLimit;
+ const BYTE *const dictEnd = dictBase + dictLimit;
+ const U32 lowLimit = zc->lowLimit;
+ const U32 curr = (U32)(ip - base);
+ const U32 minChain = curr > chainSize ? curr - chainSize : 0;
+ int nbAttempts = maxNbAttempts;
+ size_t ml = EQUAL_READ32 - 1;
+
+ /* HC4 match finder */
+ U32 matchIndex = ZSTD_insertAndFindFirstIndex(zc, ip, mls);
+
+ for (; (matchIndex > lowLimit) & (nbAttempts > 0); nbAttempts--) {
+ const BYTE *match;
+ size_t currMl = 0;
+ if ((!extDict) || matchIndex >= dictLimit) {
+ match = base + matchIndex;
+ if (match[ml] == ip[ml]) /* potentially better */
+ currMl = ZSTD_count(ip, match, iLimit);
+ } else {
+ match = dictBase + matchIndex;
+ if (ZSTD_read32(match) == ZSTD_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
+ currMl = ZSTD_count_2segments(ip + EQUAL_READ32, match + EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32;
+ }
+
+ /* save best solution */
+ if (currMl > ml) {
+ ml = currMl;
+ *offsetPtr = curr - matchIndex + ZSTD_REP_MOVE;
+ if (ip + currMl == iLimit)
+ break; /* best possible, and avoid read overflow*/
+ }
+
+ if (matchIndex <= minChain)
+ break;
+ matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
+ }
+
+ return ml;
+}
+
+FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
+ const U32 matchLengthSearch)
+{
+ switch (matchLengthSearch) {
+ default: /* includes case 3 */
+ case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0);
+ case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0);
+ case 7:
+ case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0);
+ }
+}
+
+FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS(ZSTD_CCtx *zc, const BYTE *ip, const BYTE *const iLimit, size_t *offsetPtr, const U32 maxNbAttempts,
+ const U32 matchLengthSearch)
+{
+ switch (matchLengthSearch) {
+ default: /* includes case 3 */
+ case 4: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1);
+ case 5: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1);
+ case 7:
+ case 6: return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1);
+ }
+}
+
+/* *******************************
+* Common parser - lazy strategy
+*********************************/
+FORCE_INLINE
+void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
+{
+ seqStore_t *seqStorePtr = &(ctx->seqStore);
+ const BYTE *const istart = (const BYTE *)src;
+ const BYTE *ip = istart;
+ const BYTE *anchor = istart;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *const ilimit = iend - 8;
+ const BYTE *const base = ctx->base + ctx->dictLimit;
+
+ U32 const maxSearches = 1 << ctx->params.cParams.searchLog;
+ U32 const mls = ctx->params.cParams.searchLength;
+
+ typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
+ searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset = 0;
+
+ /* init */
+ ip += (ip == base);
+ ctx->nextToUpdate3 = ctx->nextToUpdate;
+ {
+ U32 const maxRep = (U32)(ip - base);
+ if (offset_2 > maxRep)
+ savedOffset = offset_2, offset_2 = 0;
+ if (offset_1 > maxRep)
+ savedOffset = offset_1, offset_1 = 0;
+ }
+
+ /* Match Loop */
+ while (ip < ilimit) {
+ size_t matchLength = 0;
+ size_t offset = 0;
+ const BYTE *start = ip + 1;
+
+ /* check repCode */
+ if ((offset_1 > 0) & (ZSTD_read32(ip + 1) == ZSTD_read32(ip + 1 - offset_1))) {
+ /* repcode : we take it */
+ matchLength = ZSTD_count(ip + 1 + EQUAL_READ32, ip + 1 + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
+ if (depth == 0)
+ goto _storeSequence;
+ }
+
+ /* first search (depth 0) */
+ {
+ size_t offsetFound = 99999999;
+ size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
+ if (ml2 > matchLength)
+ matchLength = ml2, start = ip, offset = offsetFound;
+ }
+
+ if (matchLength < EQUAL_READ32) {
+ ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
+ continue;
+ }
+
+ /* let's try to find a better solution */
+ if (depth >= 1)
+ while (ip < ilimit) {
+ ip++;
+ if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) {
+ size_t const mlRep = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
+ int const gain2 = (int)(mlRep * 3);
+ int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1);
+ if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
+ matchLength = mlRep, offset = 0, start = ip;
+ }
+ {
+ size_t offset2 = 99999999;
+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
+ matchLength = ml2, offset = offset2, start = ip;
+ continue; /* search a better one */
+ }
+ }
+
+ /* let's find an even better one */
+ if ((depth == 2) && (ip < ilimit)) {
+ ip++;
+ if ((offset) && ((offset_1 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_1)))) {
+ size_t const ml2 = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_1, iend) + EQUAL_READ32;
+ int const gain2 = (int)(ml2 * 4);
+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1);
+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
+ matchLength = ml2, offset = 0, start = ip;
+ }
+ {
+ size_t offset2 = 99999999;
+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
+ matchLength = ml2, offset = offset2, start = ip;
+ continue;
+ }
+ }
+ }
+ break; /* nothing found : store previous solution */
+ }
+
+ /* NOTE:
+ * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior.
+ * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which
+ * overflows the pointer, which is undefined behavior.
+ */
+ /* catch up */
+ if (offset) {
+ while ((start > anchor) && (start > base + offset - ZSTD_REP_MOVE) &&
+ (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1])) /* only search for offset within prefix */
+ {
+ start--;
+ matchLength++;
+ }
+ offset_2 = offset_1;
+ offset_1 = (U32)(offset - ZSTD_REP_MOVE);
+ }
+
+ /* store sequence */
+_storeSequence:
+ {
+ size_t const litLength = start - anchor;
+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
+ anchor = ip = start + matchLength;
+ }
+
+ /* check immediate repcode */
+ while ((ip <= ilimit) && ((offset_2 > 0) & (ZSTD_read32(ip) == ZSTD_read32(ip - offset_2)))) {
+ /* store sequence */
+ matchLength = ZSTD_count(ip + EQUAL_READ32, ip + EQUAL_READ32 - offset_2, iend) + EQUAL_READ32;
+ offset = offset_2;
+ offset_2 = offset_1;
+ offset_1 = (U32)offset; /* swap repcodes */
+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
+ ip += matchLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ }
+ }
+
+ /* Save reps for next block */
+ ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
+ ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
+
+ /* Last Literals */
+ {
+ size_t const lastLLSize = iend - anchor;
+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+ }
+}
+
+static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); }
+
+static void ZSTD_compressBlock_lazy2(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); }
+
+static void ZSTD_compressBlock_lazy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); }
+
+static void ZSTD_compressBlock_greedy(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); }
+
+FORCE_INLINE
+void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const U32 searchMethod, const U32 depth)
+{
+ seqStore_t *seqStorePtr = &(ctx->seqStore);
+ const BYTE *const istart = (const BYTE *)src;
+ const BYTE *ip = istart;
+ const BYTE *anchor = istart;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *const ilimit = iend - 8;
+ const BYTE *const base = ctx->base;
+ const U32 dictLimit = ctx->dictLimit;
+ const U32 lowestIndex = ctx->lowLimit;
+ const BYTE *const prefixStart = base + dictLimit;
+ const BYTE *const dictBase = ctx->dictBase;
+ const BYTE *const dictEnd = dictBase + dictLimit;
+ const BYTE *const dictStart = dictBase + ctx->lowLimit;
+
+ const U32 maxSearches = 1 << ctx->params.cParams.searchLog;
+ const U32 mls = ctx->params.cParams.searchLength;
+
+ typedef size_t (*searchMax_f)(ZSTD_CCtx * zc, const BYTE *ip, const BYTE *iLimit, size_t *offsetPtr, U32 maxNbAttempts, U32 matchLengthSearch);
+ searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS;
+
+ U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1];
+
+ /* init */
+ ctx->nextToUpdate3 = ctx->nextToUpdate;
+ ip += (ip == prefixStart);
+
+ /* Match Loop */
+ while (ip < ilimit) {
+ size_t matchLength = 0;
+ size_t offset = 0;
+ const BYTE *start = ip + 1;
+ U32 curr = (U32)(ip - base);
+
+ /* check repCode */
+ {
+ const U32 repIndex = (U32)(curr + 1 - offset_1);
+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE *const repMatch = repBase + repIndex;
+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
+ if (ZSTD_read32(ip + 1) == ZSTD_read32(repMatch)) {
+ /* repcode detected we should take it */
+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ matchLength =
+ ZSTD_count_2segments(ip + 1 + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
+ if (depth == 0)
+ goto _storeSequence;
+ }
+ }
+
+ /* first search (depth 0) */
+ {
+ size_t offsetFound = 99999999;
+ size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls);
+ if (ml2 > matchLength)
+ matchLength = ml2, start = ip, offset = offsetFound;
+ }
+
+ if (matchLength < EQUAL_READ32) {
+ ip += ((ip - anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
+ continue;
+ }
+
+ /* let's try to find a better solution */
+ if (depth >= 1)
+ while (ip < ilimit) {
+ ip++;
+ curr++;
+ /* check repCode */
+ if (offset) {
+ const U32 repIndex = (U32)(curr - offset_1);
+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE *const repMatch = repBase + repIndex;
+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
+ if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
+ /* repcode detected */
+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ size_t const repLength =
+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) +
+ EQUAL_READ32;
+ int const gain2 = (int)(repLength * 3);
+ int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1);
+ if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
+ matchLength = repLength, offset = 0, start = ip;
+ }
+ }
+
+ /* search match, depth 1 */
+ {
+ size_t offset2 = 99999999;
+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4);
+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
+ matchLength = ml2, offset = offset2, start = ip;
+ continue; /* search a better one */
+ }
+ }
+
+ /* let's find an even better one */
+ if ((depth == 2) && (ip < ilimit)) {
+ ip++;
+ curr++;
+ /* check repCode */
+ if (offset) {
+ const U32 repIndex = (U32)(curr - offset_1);
+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE *const repMatch = repBase + repIndex;
+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
+ if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
+ /* repcode detected */
+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ size_t repLength = ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend,
+ repEnd, prefixStart) +
+ EQUAL_READ32;
+ int gain2 = (int)(repLength * 4);
+ int gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1);
+ if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
+ matchLength = repLength, offset = 0, start = ip;
+ }
+ }
+
+ /* search match, depth 2 */
+ {
+ size_t offset2 = 99999999;
+ size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
+ int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */
+ int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7);
+ if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
+ matchLength = ml2, offset = offset2, start = ip;
+ continue;
+ }
+ }
+ }
+ break; /* nothing found : store previous solution */
+ }
+
+ /* catch up */
+ if (offset) {
+ U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE));
+ const BYTE *match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
+ const BYTE *const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
+ while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) {
+ start--;
+ match--;
+ matchLength++;
+ } /* catch up */
+ offset_2 = offset_1;
+ offset_1 = (U32)(offset - ZSTD_REP_MOVE);
+ }
+
+ /* store sequence */
+ _storeSequence : {
+ size_t const litLength = start - anchor;
+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength - MINMATCH);
+ anchor = ip = start + matchLength;
+ }
+
+ /* check immediate repcode */
+ while (ip <= ilimit) {
+ const U32 repIndex = (U32)((ip - base) - offset_2);
+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE *const repMatch = repBase + repIndex;
+ if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
+ if (ZSTD_read32(ip) == ZSTD_read32(repMatch)) {
+ /* repcode detected we should take it */
+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ matchLength =
+ ZSTD_count_2segments(ip + EQUAL_READ32, repMatch + EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
+ offset = offset_2;
+ offset_2 = offset_1;
+ offset_1 = (U32)offset; /* swap offset history */
+ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength - MINMATCH);
+ ip += matchLength;
+ anchor = ip;
+ continue; /* faster when present ... (?) */
+ }
+ break;
+ }
+ }
+
+ /* Save reps for next block */
+ ctx->repToConfirm[0] = offset_1;
+ ctx->repToConfirm[1] = offset_2;
+
+ /* Last Literals */
+ {
+ size_t const lastLLSize = iend - anchor;
+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+ }
+}
+
+void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize) { ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); }
+
+static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+ ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1);
+}
+
+static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+ ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2);
+}
+
+static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+ ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2);
+}
+
+/* The optimal parser */
+#include "zstd_opt.h"
+
+static void ZSTD_compressBlock_btopt(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
+#else
+ (void)ctx;
+ (void)src;
+ (void)srcSize;
+ return;
+#endif
+}
+
+static void ZSTD_compressBlock_btopt2(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+ ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
+#else
+ (void)ctx;
+ (void)src;
+ (void)srcSize;
+ return;
+#endif
+}
+
+static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
+#else
+ (void)ctx;
+ (void)src;
+ (void)srcSize;
+ return;
+#endif
+}
+
+static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx *ctx, const void *src, size_t srcSize)
+{
+#ifdef ZSTD_OPT_H_91842398743
+ ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
+#else
+ (void)ctx;
+ (void)src;
+ (void)srcSize;
+ return;
+#endif
+}
+
+typedef void (*ZSTD_blockCompressor)(ZSTD_CCtx *ctx, const void *src, size_t srcSize);
+
+static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
+{
+ static const ZSTD_blockCompressor blockCompressor[2][8] = {
+ {ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2,
+ ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2},
+ {ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,
+ ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict}};
+
+ return blockCompressor[extDict][(U32)strat];
+}
+
+static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
+ const BYTE *const base = zc->base;
+ const BYTE *const istart = (const BYTE *)src;
+ const U32 curr = (U32)(istart - base);
+ if (srcSize < MIN_CBLOCK_SIZE + ZSTD_blockHeaderSize + 1)
+ return 0; /* don't even attempt compression below a certain srcSize */
+ ZSTD_resetSeqStore(&(zc->seqStore));
+ if (curr > zc->nextToUpdate + 384)
+ zc->nextToUpdate = curr - MIN(192, (U32)(curr - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */
+ blockCompressor(zc, src, srcSize);
+ return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize);
+}
+
+/*! ZSTD_compress_generic() :
+* Compress a chunk of data into one or multiple blocks.
+* All blocks will be terminated, all input will be consumed.
+* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
+* Frame is supposed already started (header already produced)
+* @return : compressed size, or an error code
+*/
+static size_t ZSTD_compress_generic(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastFrameChunk)
+{
+ size_t blockSize = cctx->blockSize;
+ size_t remaining = srcSize;
+ const BYTE *ip = (const BYTE *)src;
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *op = ostart;
+ U32 const maxDist = 1 << cctx->params.cParams.windowLog;
+
+ if (cctx->params.fParams.checksumFlag && srcSize)
+ xxh64_update(&cctx->xxhState, src, srcSize);
+
+ while (remaining) {
+ U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
+ size_t cSize;
+
+ if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
+ return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
+ if (remaining < blockSize)
+ blockSize = remaining;
+
+ /* preemptive overflow correction */
+ if (cctx->lowLimit > (3U << 29)) {
+ U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
+ U32 const curr = (U32)(ip - cctx->base);
+ U32 const newCurr = (curr & cycleMask) + (1 << cctx->params.cParams.windowLog);
+ U32 const correction = curr - newCurr;
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30);
+ ZSTD_reduceIndex(cctx, correction);
+ cctx->base += correction;
+ cctx->dictBase += correction;
+ cctx->lowLimit -= correction;
+ cctx->dictLimit -= correction;
+ if (cctx->nextToUpdate < correction)
+ cctx->nextToUpdate = 0;
+ else
+ cctx->nextToUpdate -= correction;
+ }
+
+ if ((U32)(ip + blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
+ /* enforce maxDist */
+ U32 const newLowLimit = (U32)(ip + blockSize - cctx->base) - maxDist;
+ if (cctx->lowLimit < newLowLimit)
+ cctx->lowLimit = newLowLimit;
+ if (cctx->dictLimit < cctx->lowLimit)
+ cctx->dictLimit = cctx->lowLimit;
+ }
+
+ cSize = ZSTD_compressBlock_internal(cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize);
+ if (ZSTD_isError(cSize))
+ return cSize;
+
+ if (cSize == 0) { /* block is not compressible */
+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(blockSize << 3);
+ if (blockSize + ZSTD_blockHeaderSize > dstCapacity)
+ return ERROR(dstSize_tooSmall);
+ ZSTD_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */
+ memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
+ cSize = ZSTD_blockHeaderSize + blockSize;
+ } else {
+ U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed) << 1) + (U32)(cSize << 3);
+ ZSTD_writeLE24(op, cBlockHeader24);
+ cSize += ZSTD_blockHeaderSize;
+ }
+
+ remaining -= blockSize;
+ dstCapacity -= cSize;
+ ip += blockSize;
+ op += cSize;
+ }
+
+ if (lastFrameChunk && (op > ostart))
+ cctx->stage = ZSTDcs_ending;
+ return op - ostart;
+}
+
+static size_t ZSTD_writeFrameHeader(void *dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
+{
+ BYTE *const op = (BYTE *)dst;
+ U32 const dictIDSizeCode = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */
+ U32 const checksumFlag = params.fParams.checksumFlag > 0;
+ U32 const windowSize = 1U << params.cParams.windowLog;
+ U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
+ BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
+ U32 const fcsCode =
+ params.fParams.contentSizeFlag ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) : 0; /* 0-3 */
+ BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6));
+ size_t pos;
+
+ if (dstCapacity < ZSTD_frameHeaderSize_max)
+ return ERROR(dstSize_tooSmall);
+
+ ZSTD_writeLE32(dst, ZSTD_MAGICNUMBER);
+ op[4] = frameHeaderDecriptionByte;
+ pos = 5;
+ if (!singleSegment)
+ op[pos++] = windowLogByte;
+ switch (dictIDSizeCode) {
+ default: /* impossible */
+ case 0: break;
+ case 1:
+ op[pos] = (BYTE)(dictID);
+ pos++;
+ break;
+ case 2:
+ ZSTD_writeLE16(op + pos, (U16)dictID);
+ pos += 2;
+ break;
+ case 3:
+ ZSTD_writeLE32(op + pos, dictID);
+ pos += 4;
+ break;
+ }
+ switch (fcsCode) {
+ default: /* impossible */
+ case 0:
+ if (singleSegment)
+ op[pos++] = (BYTE)(pledgedSrcSize);
+ break;
+ case 1:
+ ZSTD_writeLE16(op + pos, (U16)(pledgedSrcSize - 256));
+ pos += 2;
+ break;
+ case 2:
+ ZSTD_writeLE32(op + pos, (U32)(pledgedSrcSize));
+ pos += 4;
+ break;
+ case 3:
+ ZSTD_writeLE64(op + pos, (U64)(pledgedSrcSize));
+ pos += 8;
+ break;
+ }
+ return pos;
+}
+
+static size_t ZSTD_compressContinue_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame, U32 lastFrameChunk)
+{
+ const BYTE *const ip = (const BYTE *)src;
+ size_t fhSize = 0;
+
+ if (cctx->stage == ZSTDcs_created)
+ return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
+
+ if (frame && (cctx->stage == ZSTDcs_init)) {
+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID);
+ if (ZSTD_isError(fhSize))
+ return fhSize;
+ dstCapacity -= fhSize;
+ dst = (char *)dst + fhSize;
+ cctx->stage = ZSTDcs_ongoing;
+ }
+
+ /* Check if blocks follow each other */
+ if (src != cctx->nextSrc) {
+ /* not contiguous */
+ ptrdiff_t const delta = cctx->nextSrc - ip;
+ cctx->lowLimit = cctx->dictLimit;
+ cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base);
+ cctx->dictBase = cctx->base;
+ cctx->base -= delta;
+ cctx->nextToUpdate = cctx->dictLimit;
+ if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE)
+ cctx->lowLimit = cctx->dictLimit; /* too small extDict */
+ }
+
+ /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
+ if ((ip + srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
+ ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase;
+ U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx;
+ cctx->lowLimit = lowLimitMax;
+ }
+
+ cctx->nextSrc = ip + srcSize;
+
+ if (srcSize) {
+ size_t const cSize = frame ? ZSTD_compress_generic(cctx, dst, dstCapacity, src, srcSize, lastFrameChunk)
+ : ZSTD_compressBlock_internal(cctx, dst, dstCapacity, src, srcSize);
+ if (ZSTD_isError(cSize))
+ return cSize;
+ return cSize + fhSize;
+ } else
+ return fhSize;
+}
+
+size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0);
+}
+
+size_t ZSTD_getBlockSizeMax(ZSTD_CCtx *cctx) { return MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); }
+
+size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
+ if (srcSize > blockSizeMax)
+ return ERROR(srcSize_wrong);
+ return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0);
+}
+
+/*! ZSTD_loadDictionaryContent() :
+ * @return : 0, or an error code
+ */
+static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx *zc, const void *src, size_t srcSize)
+{
+ const BYTE *const ip = (const BYTE *)src;
+ const BYTE *const iend = ip + srcSize;
+
+ /* input becomes curr prefix */
+ zc->lowLimit = zc->dictLimit;
+ zc->dictLimit = (U32)(zc->nextSrc - zc->base);
+ zc->dictBase = zc->base;
+ zc->base += ip - zc->nextSrc;
+ zc->nextToUpdate = zc->dictLimit;
+ zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base);
+
+ zc->nextSrc = iend;
+ if (srcSize <= HASH_READ_SIZE)
+ return 0;
+
+ switch (zc->params.cParams.strategy) {
+ case ZSTD_fast: ZSTD_fillHashTable(zc, iend, zc->params.cParams.searchLength); break;
+
+ case ZSTD_dfast: ZSTD_fillDoubleHashTable(zc, iend, zc->params.cParams.searchLength); break;
+
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ if (srcSize >= HASH_READ_SIZE)
+ ZSTD_insertAndFindFirstIndex(zc, iend - HASH_READ_SIZE, zc->params.cParams.searchLength);
+ break;
+
+ case ZSTD_btlazy2:
+ case ZSTD_btopt:
+ case ZSTD_btopt2:
+ if (srcSize >= HASH_READ_SIZE)
+ ZSTD_updateTree(zc, iend - HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength);
+ break;
+
+ default:
+ return ERROR(GENERIC); /* strategy doesn't exist; impossible */
+ }
+
+ zc->nextToUpdate = (U32)(iend - zc->base);
+ return 0;
+}
+
+/* Dictionaries that assign zero probability to symbols that show up causes problems
+ when FSE encoding. Refuse dictionaries that assign zero probability to symbols
+ that we may encounter during compression.
+ NOTE: This behavior is not standard and could be improved in the future. */
+static size_t ZSTD_checkDictNCount(short *normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue)
+{
+ U32 s;
+ if (dictMaxSymbolValue < maxSymbolValue)
+ return ERROR(dictionary_corrupted);
+ for (s = 0; s <= maxSymbolValue; ++s) {
+ if (normalizedCounter[s] == 0)
+ return ERROR(dictionary_corrupted);
+ }
+ return 0;
+}
+
+/* Dictionary format :
+ * See :
+ * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format
+ */
+/*! ZSTD_loadZstdDictionary() :
+ * @return : 0, or an error code
+ * assumptions : magic number supposed already checked
+ * dictSize supposed > 8
+ */
+static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
+{
+ const BYTE *dictPtr = (const BYTE *)dict;
+ const BYTE *const dictEnd = dictPtr + dictSize;
+ short offcodeNCount[MaxOff + 1];
+ unsigned offcodeMaxValue = MaxOff;
+
+ dictPtr += 4; /* skip magic number */
+ cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : ZSTD_readLE32(dictPtr);
+ dictPtr += 4;
+
+ {
+ size_t const hufHeaderSize = HUF_readCTable_wksp(cctx->hufTable, 255, dictPtr, dictEnd - dictPtr, cctx->tmpCounters, sizeof(cctx->tmpCounters));
+ if (HUF_isError(hufHeaderSize))
+ return ERROR(dictionary_corrupted);
+ dictPtr += hufHeaderSize;
+ }
+
+ {
+ unsigned offcodeLog;
+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
+ if (FSE_isError(offcodeHeaderSize))
+ return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSELog)
+ return ERROR(dictionary_corrupted);
+ /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
+ CHECK_E(FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)),
+ dictionary_corrupted);
+ dictPtr += offcodeHeaderSize;
+ }
+
+ {
+ short matchlengthNCount[MaxML + 1];
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
+ if (FSE_isError(matchlengthHeaderSize))
+ return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSELog)
+ return ERROR(dictionary_corrupted);
+ /* Every match length code must have non-zero probability */
+ CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
+ CHECK_E(
+ FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)),
+ dictionary_corrupted);
+ dictPtr += matchlengthHeaderSize;
+ }
+
+ {
+ short litlengthNCount[MaxLL + 1];
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
+ if (FSE_isError(litlengthHeaderSize))
+ return ERROR(dictionary_corrupted);
+ if (litlengthLog > LLFSELog)
+ return ERROR(dictionary_corrupted);
+ /* Every literal length code must have non-zero probability */
+ CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
+ CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->tmpCounters, sizeof(cctx->tmpCounters)),
+ dictionary_corrupted);
+ dictPtr += litlengthHeaderSize;
+ }
+
+ if (dictPtr + 12 > dictEnd)
+ return ERROR(dictionary_corrupted);
+ cctx->rep[0] = ZSTD_readLE32(dictPtr + 0);
+ cctx->rep[1] = ZSTD_readLE32(dictPtr + 4);
+ cctx->rep[2] = ZSTD_readLE32(dictPtr + 8);
+ dictPtr += 12;
+
+ {
+ size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
+ U32 offcodeMax = MaxOff;
+ if (dictContentSize <= ((U32)-1) - 128 KB) {
+ U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */
+ offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */
+ }
+ /* All offset values <= dictContentSize + 128 KB must be representable */
+ CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
+ /* All repCodes must be <= dictContentSize and != 0*/
+ {
+ U32 u;
+ for (u = 0; u < 3; u++) {
+ if (cctx->rep[u] == 0)
+ return ERROR(dictionary_corrupted);
+ if (cctx->rep[u] > dictContentSize)
+ return ERROR(dictionary_corrupted);
+ }
+ }
+
+ cctx->flagStaticTables = 1;
+ cctx->flagStaticHufTable = HUF_repeat_valid;
+ return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize);
+ }
+}
+
+/** ZSTD_compress_insertDictionary() :
+* @return : 0, or an error code */
+static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
+{
+ if ((dict == NULL) || (dictSize <= 8))
+ return 0;
+
+ /* dict as pure content */
+ if ((ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict))
+ return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
+
+ /* dict as zstd dictionary */
+ return ZSTD_loadZstdDictionary(cctx, dict, dictSize);
+}
+
+/*! ZSTD_compressBegin_internal() :
+* @return : 0, or an error code */
+static size_t ZSTD_compressBegin_internal(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize)
+{
+ ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
+ CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp));
+ return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
+}
+
+/*! ZSTD_compressBegin_advanced() :
+* @return : 0, or an error code */
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+ /* compression parameters verification and optimization */
+ CHECK_F(ZSTD_checkCParams(params.cParams));
+ return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
+}
+
+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
+ return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
+}
+
+size_t ZSTD_compressBegin(ZSTD_CCtx *cctx, int compressionLevel) { return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); }
+
+/*! ZSTD_writeEpilogue() :
+* Ends a frame.
+* @return : nb of bytes written into dst (or an error code) */
+static size_t ZSTD_writeEpilogue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity)
+{
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *op = ostart;
+ size_t fhSize = 0;
+
+ if (cctx->stage == ZSTDcs_created)
+ return ERROR(stage_wrong); /* init missing */
+
+ /* special case : empty frame */
+ if (cctx->stage == ZSTDcs_init) {
+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
+ if (ZSTD_isError(fhSize))
+ return fhSize;
+ dstCapacity -= fhSize;
+ op += fhSize;
+ cctx->stage = ZSTDcs_ongoing;
+ }
+
+ if (cctx->stage != ZSTDcs_ending) {
+ /* write one last empty block, make it the "last" block */
+ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0;
+ if (dstCapacity < 4)
+ return ERROR(dstSize_tooSmall);
+ ZSTD_writeLE32(op, cBlockHeader24);
+ op += ZSTD_blockHeaderSize;
+ dstCapacity -= ZSTD_blockHeaderSize;
+ }
+
+ if (cctx->params.fParams.checksumFlag) {
+ U32 const checksum = (U32)xxh64_digest(&cctx->xxhState);
+ if (dstCapacity < 4)
+ return ERROR(dstSize_tooSmall);
+ ZSTD_writeLE32(op, checksum);
+ op += 4;
+ }
+
+ cctx->stage = ZSTDcs_created; /* return to "created but no init" status */
+ return op - ostart;
+}
+
+size_t ZSTD_compressEnd(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ size_t endResult;
+ size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1);
+ if (ZSTD_isError(cSize))
+ return cSize;
+ endResult = ZSTD_writeEpilogue(cctx, (char *)dst + cSize, dstCapacity - cSize);
+ if (ZSTD_isError(endResult))
+ return endResult;
+ return cSize + endResult;
+}
+
+static size_t ZSTD_compress_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
+ ZSTD_parameters params)
+{
+ CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize));
+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
+}
+
+size_t ZSTD_compress_usingDict(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
+ ZSTD_parameters params)
+{
+ return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
+}
+
+size_t ZSTD_compressCCtx(ZSTD_CCtx *ctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, ZSTD_parameters params)
+{
+ return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, NULL, 0, params);
+}
+
+/* ===== Dictionary API ===== */
+
+struct ZSTD_CDict_s {
+ void *dictBuffer;
+ const void *dictContent;
+ size_t dictContentSize;
+ ZSTD_CCtx *refContext;
+}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
+
+size_t ZSTD_CDictWorkspaceBound(ZSTD_compressionParameters cParams) { return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CDict)); }
+
+static ZSTD_CDict *ZSTD_createCDict_advanced(const void *dictBuffer, size_t dictSize, unsigned byReference, ZSTD_parameters params, ZSTD_customMem customMem)
+{
+ if (!customMem.customAlloc || !customMem.customFree)
+ return NULL;
+
+ {
+ ZSTD_CDict *const cdict = (ZSTD_CDict *)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
+ ZSTD_CCtx *const cctx = ZSTD_createCCtx_advanced(customMem);
+
+ if (!cdict || !cctx) {
+ ZSTD_free(cdict, customMem);
+ ZSTD_freeCCtx(cctx);
+ return NULL;
+ }
+
+ if ((byReference) || (!dictBuffer) || (!dictSize)) {
+ cdict->dictBuffer = NULL;
+ cdict->dictContent = dictBuffer;
+ } else {
+ void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
+ if (!internalBuffer) {
+ ZSTD_free(cctx, customMem);
+ ZSTD_free(cdict, customMem);
+ return NULL;
+ }
+ memcpy(internalBuffer, dictBuffer, dictSize);
+ cdict->dictBuffer = internalBuffer;
+ cdict->dictContent = internalBuffer;
+ }
+
+ {
+ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
+ if (ZSTD_isError(errorCode)) {
+ ZSTD_free(cdict->dictBuffer, customMem);
+ ZSTD_free(cdict, customMem);
+ ZSTD_freeCCtx(cctx);
+ return NULL;
+ }
+ }
+
+ cdict->refContext = cctx;
+ cdict->dictContentSize = dictSize;
+ return cdict;
+ }
+}
+
+ZSTD_CDict *ZSTD_initCDict(const void *dict, size_t dictSize, ZSTD_parameters params, void *workspace, size_t workspaceSize)
+{
+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
+ return ZSTD_createCDict_advanced(dict, dictSize, 1, params, stackMem);
+}
+
+size_t ZSTD_freeCDict(ZSTD_CDict *cdict)
+{
+ if (cdict == NULL)
+ return 0; /* support free on NULL */
+ {
+ ZSTD_customMem const cMem = cdict->refContext->customMem;
+ ZSTD_freeCCtx(cdict->refContext);
+ ZSTD_free(cdict->dictBuffer, cMem);
+ ZSTD_free(cdict, cMem);
+ return 0;
+ }
+}
+
+static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict *cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); }
+
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize)
+{
+ if (cdict->dictContentSize)
+ CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
+ else {
+ ZSTD_parameters params = cdict->refContext->params;
+ params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+ CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize));
+ }
+ return 0;
+}
+
+/*! ZSTD_compress_usingCDict() :
+* Compression using a digested Dictionary.
+* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
+* Note that compression level is decided during dictionary creation */
+size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict)
+{
+ CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));
+
+ if (cdict->refContext->params.fParams.contentSizeFlag == 1) {
+ cctx->params.fParams.contentSizeFlag = 1;
+ cctx->frameContentSize = srcSize;
+ } else {
+ cctx->params.fParams.contentSizeFlag = 0;
+ }
+
+ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
+}
+
+/* ******************************************************************
+* Streaming
+********************************************************************/
+
+typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
+
+struct ZSTD_CStream_s {
+ ZSTD_CCtx *cctx;
+ ZSTD_CDict *cdictLocal;
+ const ZSTD_CDict *cdict;
+ char *inBuff;
+ size_t inBuffSize;
+ size_t inToCompress;
+ size_t inBuffPos;
+ size_t inBuffTarget;
+ size_t blockSize;
+ char *outBuff;
+ size_t outBuffSize;
+ size_t outBuffContentSize;
+ size_t outBuffFlushedSize;
+ ZSTD_cStreamStage stage;
+ U32 checksum;
+ U32 frameEnded;
+ U64 pledgedSrcSize;
+ U64 inputProcessed;
+ ZSTD_parameters params;
+ ZSTD_customMem customMem;
+}; /* typedef'd to ZSTD_CStream within "zstd.h" */
+
+size_t ZSTD_CStreamWorkspaceBound(ZSTD_compressionParameters cParams)
+{
+ size_t const inBuffSize = (size_t)1 << cParams.windowLog;
+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, inBuffSize);
+ size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
+
+ return ZSTD_CCtxWorkspaceBound(cParams) + ZSTD_ALIGN(sizeof(ZSTD_CStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize);
+}
+
+ZSTD_CStream *ZSTD_createCStream_advanced(ZSTD_customMem customMem)
+{
+ ZSTD_CStream *zcs;
+
+ if (!customMem.customAlloc || !customMem.customFree)
+ return NULL;
+
+ zcs = (ZSTD_CStream *)ZSTD_malloc(sizeof(ZSTD_CStream), customMem);
+ if (zcs == NULL)
+ return NULL;
+ memset(zcs, 0, sizeof(ZSTD_CStream));
+ memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
+ zcs->cctx = ZSTD_createCCtx_advanced(customMem);
+ if (zcs->cctx == NULL) {
+ ZSTD_freeCStream(zcs);
+ return NULL;
+ }
+ return zcs;
+}
+
+size_t ZSTD_freeCStream(ZSTD_CStream *zcs)
+{
+ if (zcs == NULL)
+ return 0; /* support free on NULL */
+ {
+ ZSTD_customMem const cMem = zcs->customMem;
+ ZSTD_freeCCtx(zcs->cctx);
+ zcs->cctx = NULL;
+ ZSTD_freeCDict(zcs->cdictLocal);
+ zcs->cdictLocal = NULL;
+ ZSTD_free(zcs->inBuff, cMem);
+ zcs->inBuff = NULL;
+ ZSTD_free(zcs->outBuff, cMem);
+ zcs->outBuff = NULL;
+ ZSTD_free(zcs, cMem);
+ return 0;
+ }
+}
+
+/*====== Initialization ======*/
+
+size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
+size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */; }
+
+static size_t ZSTD_resetCStream_internal(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize)
+{
+ if (zcs->inBuffSize == 0)
+ return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
+
+ if (zcs->cdict)
+ CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
+ else
+ CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
+
+ zcs->inToCompress = 0;
+ zcs->inBuffPos = 0;
+ zcs->inBuffTarget = zcs->blockSize;
+ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
+ zcs->stage = zcss_load;
+ zcs->frameEnded = 0;
+ zcs->pledgedSrcSize = pledgedSrcSize;
+ zcs->inputProcessed = 0;
+ return 0; /* ready to go */
+}
+
+size_t ZSTD_resetCStream(ZSTD_CStream *zcs, unsigned long long pledgedSrcSize)
+{
+
+ zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+
+ return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
+}
+
+static size_t ZSTD_initCStream_advanced(ZSTD_CStream *zcs, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
+{
+ /* allocate buffers */
+ {
+ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
+ if (zcs->inBuffSize < neededInBuffSize) {
+ zcs->inBuffSize = neededInBuffSize;
+ ZSTD_free(zcs->inBuff, zcs->customMem);
+ zcs->inBuff = (char *)ZSTD_malloc(neededInBuffSize, zcs->customMem);
+ if (zcs->inBuff == NULL)
+ return ERROR(memory_allocation);
+ }
+ zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
+ }
+ if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize) + 1) {
+ zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize) + 1;
+ ZSTD_free(zcs->outBuff, zcs->customMem);
+ zcs->outBuff = (char *)ZSTD_malloc(zcs->outBuffSize, zcs->customMem);
+ if (zcs->outBuff == NULL)
+ return ERROR(memory_allocation);
+ }
+
+ if (dict && dictSize >= 8) {
+ ZSTD_freeCDict(zcs->cdictLocal);
+ zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
+ if (zcs->cdictLocal == NULL)
+ return ERROR(memory_allocation);
+ zcs->cdict = zcs->cdictLocal;
+ } else
+ zcs->cdict = NULL;
+
+ zcs->checksum = params.fParams.checksumFlag > 0;
+ zcs->params = params;
+
+ return ZSTD_resetCStream_internal(zcs, pledgedSrcSize);
+}
+
+ZSTD_CStream *ZSTD_initCStream(ZSTD_parameters params, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize)
+{
+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
+ ZSTD_CStream *const zcs = ZSTD_createCStream_advanced(stackMem);
+ if (zcs) {
+ size_t const code = ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
+ if (ZSTD_isError(code)) {
+ return NULL;
+ }
+ }
+ return zcs;
+}
+
+ZSTD_CStream *ZSTD_initCStream_usingCDict(const ZSTD_CDict *cdict, unsigned long long pledgedSrcSize, void *workspace, size_t workspaceSize)
+{
+ ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
+ ZSTD_CStream *const zcs = ZSTD_initCStream(params, pledgedSrcSize, workspace, workspaceSize);
+ if (zcs) {
+ zcs->cdict = cdict;
+ if (ZSTD_isError(ZSTD_resetCStream_internal(zcs, pledgedSrcSize))) {
+ return NULL;
+ }
+ }
+ return zcs;
+}
+
+/*====== Compression ======*/
+
+typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e;
+
+ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ size_t const length = MIN(dstCapacity, srcSize);
+ memcpy(dst, src, length);
+ return length;
+}
+
+static size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, void *dst, size_t *dstCapacityPtr, const void *src, size_t *srcSizePtr, ZSTD_flush_e const flush)
+{
+ U32 someMoreWork = 1;
+ const char *const istart = (const char *)src;
+ const char *const iend = istart + *srcSizePtr;
+ const char *ip = istart;
+ char *const ostart = (char *)dst;
+ char *const oend = ostart + *dstCapacityPtr;
+ char *op = ostart;
+
+ while (someMoreWork) {
+ switch (zcs->stage) {
+ case zcss_init:
+ return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
+
+ case zcss_load:
+ /* complete inBuffer */
+ {
+ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
+ size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip);
+ zcs->inBuffPos += loaded;
+ ip += loaded;
+ if ((zcs->inBuffPos == zcs->inToCompress) || (!flush && (toLoad != loaded))) {
+ someMoreWork = 0;
+ break; /* not enough input to get a full block : stop there, wait for more */
+ }
+ }
+ /* compress curr block (note : this stage cannot be stopped in the middle) */
+ {
+ void *cDst;
+ size_t cSize;
+ size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
+ size_t oSize = oend - op;
+ if (oSize >= ZSTD_compressBound(iSize))
+ cDst = op; /* compress directly into output buffer (avoid flush stage) */
+ else
+ cDst = zcs->outBuff, oSize = zcs->outBuffSize;
+ cSize = (flush == zsf_end) ? ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize)
+ : ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
+ if (ZSTD_isError(cSize))
+ return cSize;
+ if (flush == zsf_end)
+ zcs->frameEnded = 1;
+ /* prepare next block */
+ zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
+ if (zcs->inBuffTarget > zcs->inBuffSize)
+ zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */
+ zcs->inToCompress = zcs->inBuffPos;
+ if (cDst == op) {
+ op += cSize;
+ break;
+ } /* no need to flush */
+ zcs->outBuffContentSize = cSize;
+ zcs->outBuffFlushedSize = 0;
+ zcs->stage = zcss_flush; /* pass-through to flush stage */
+ }
+
+ case zcss_flush: {
+ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
+ size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
+ op += flushed;
+ zcs->outBuffFlushedSize += flushed;
+ if (toFlush != flushed) {
+ someMoreWork = 0;
+ break;
+ } /* dst too small to store flushed data : stop there */
+ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
+ zcs->stage = zcss_load;
+ break;
+ }
+
+ case zcss_final:
+ someMoreWork = 0; /* do nothing */
+ break;
+
+ default:
+ return ERROR(GENERIC); /* impossible */
+ }
+ }
+
+ *srcSizePtr = ip - istart;
+ *dstCapacityPtr = op - ostart;
+ zcs->inputProcessed += *srcSizePtr;
+ if (zcs->frameEnded)
+ return 0;
+ {
+ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
+ if (hintInSize == 0)
+ hintInSize = zcs->blockSize;
+ return hintInSize;
+ }
+}
+
+size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
+{
+ size_t sizeRead = input->size - input->pos;
+ size_t sizeWritten = output->size - output->pos;
+ size_t const result =
+ ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, (const char *)(input->src) + input->pos, &sizeRead, zsf_gather);
+ input->pos += sizeRead;
+ output->pos += sizeWritten;
+ return result;
+}
+
+/*====== Finalize ======*/
+
+/*! ZSTD_flushStream() :
+* @return : amount of data remaining to flush */
+size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
+{
+ size_t srcSize = 0;
+ size_t sizeWritten = output->size - output->pos;
+ size_t const result = ZSTD_compressStream_generic(zcs, (char *)(output->dst) + output->pos, &sizeWritten, &srcSize,
+ &srcSize, /* use a valid src address instead of NULL */
+ zsf_flush);
+ output->pos += sizeWritten;
+ if (ZSTD_isError(result))
+ return result;
+ return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
+}
+
+size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
+{
+ BYTE *const ostart = (BYTE *)(output->dst) + output->pos;
+ BYTE *const oend = (BYTE *)(output->dst) + output->size;
+ BYTE *op = ostart;
+
+ if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
+ return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */
+
+ if (zcs->stage != zcss_final) {
+ /* flush whatever remains */
+ size_t srcSize = 0;
+ size_t sizeWritten = output->size - output->pos;
+ size_t const notEnded =
+ ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */
+ size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
+ op += sizeWritten;
+ if (remainingToFlush) {
+ output->pos += sizeWritten;
+ return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4);
+ }
+ /* create epilogue */
+ zcs->stage = zcss_final;
+ zcs->outBuffContentSize = !notEnded ? 0 : ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL,
+ 0); /* write epilogue, including final empty block, into outBuff */
+ }
+
+ /* flush epilogue */
+ {
+ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
+ size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
+ op += flushed;
+ zcs->outBuffFlushedSize += flushed;
+ output->pos += op - ostart;
+ if (toFlush == flushed)
+ zcs->stage = zcss_init; /* end reached */
+ return toFlush - flushed;
+ }
+}
+
+/*-===== Pre-defined compression levels =====-*/
+
+#define ZSTD_DEFAULT_CLEVEL 1
+#define ZSTD_MAX_CLEVEL 22
+int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
+
+static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = {
+ {
+ /* "default" */
+ /* W, C, H, S, L, TL, strat */
+ {18, 12, 12, 1, 7, 16, ZSTD_fast}, /* level 0 - never used */
+ {19, 13, 14, 1, 7, 16, ZSTD_fast}, /* level 1 */
+ {19, 15, 16, 1, 6, 16, ZSTD_fast}, /* level 2 */
+ {20, 16, 17, 1, 5, 16, ZSTD_dfast}, /* level 3.*/
+ {20, 18, 18, 1, 5, 16, ZSTD_dfast}, /* level 4.*/
+ {20, 15, 18, 3, 5, 16, ZSTD_greedy}, /* level 5 */
+ {21, 16, 19, 2, 5, 16, ZSTD_lazy}, /* level 6 */
+ {21, 17, 20, 3, 5, 16, ZSTD_lazy}, /* level 7 */
+ {21, 18, 20, 3, 5, 16, ZSTD_lazy2}, /* level 8 */
+ {21, 20, 20, 3, 5, 16, ZSTD_lazy2}, /* level 9 */
+ {21, 19, 21, 4, 5, 16, ZSTD_lazy2}, /* level 10 */
+ {22, 20, 22, 4, 5, 16, ZSTD_lazy2}, /* level 11 */
+ {22, 20, 22, 5, 5, 16, ZSTD_lazy2}, /* level 12 */
+ {22, 21, 22, 5, 5, 16, ZSTD_lazy2}, /* level 13 */
+ {22, 21, 22, 6, 5, 16, ZSTD_lazy2}, /* level 14 */
+ {22, 21, 21, 5, 5, 16, ZSTD_btlazy2}, /* level 15 */
+ {23, 22, 22, 5, 5, 16, ZSTD_btlazy2}, /* level 16 */
+ {23, 21, 22, 4, 5, 24, ZSTD_btopt}, /* level 17 */
+ {23, 23, 22, 6, 5, 32, ZSTD_btopt}, /* level 18 */
+ {23, 23, 22, 6, 3, 48, ZSTD_btopt}, /* level 19 */
+ {25, 25, 23, 7, 3, 64, ZSTD_btopt2}, /* level 20 */
+ {26, 26, 23, 7, 3, 256, ZSTD_btopt2}, /* level 21 */
+ {27, 27, 25, 9, 3, 512, ZSTD_btopt2}, /* level 22 */
+ },
+ {
+ /* for srcSize <= 256 KB */
+ /* W, C, H, S, L, T, strat */
+ {0, 0, 0, 0, 0, 0, ZSTD_fast}, /* level 0 - not used */
+ {18, 13, 14, 1, 6, 8, ZSTD_fast}, /* level 1 */
+ {18, 14, 13, 1, 5, 8, ZSTD_dfast}, /* level 2 */
+ {18, 16, 15, 1, 5, 8, ZSTD_dfast}, /* level 3 */
+ {18, 15, 17, 1, 5, 8, ZSTD_greedy}, /* level 4.*/
+ {18, 16, 17, 4, 5, 8, ZSTD_greedy}, /* level 5.*/
+ {18, 16, 17, 3, 5, 8, ZSTD_lazy}, /* level 6.*/
+ {18, 17, 17, 4, 4, 8, ZSTD_lazy}, /* level 7 */
+ {18, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */
+ {18, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */
+ {18, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */
+ {18, 18, 17, 6, 4, 8, ZSTD_lazy2}, /* level 11.*/
+ {18, 18, 17, 7, 4, 8, ZSTD_lazy2}, /* level 12.*/
+ {18, 19, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13 */
+ {18, 18, 18, 4, 4, 16, ZSTD_btopt}, /* level 14.*/
+ {18, 18, 18, 4, 3, 16, ZSTD_btopt}, /* level 15.*/
+ {18, 19, 18, 6, 3, 32, ZSTD_btopt}, /* level 16.*/
+ {18, 19, 18, 8, 3, 64, ZSTD_btopt}, /* level 17.*/
+ {18, 19, 18, 9, 3, 128, ZSTD_btopt}, /* level 18.*/
+ {18, 19, 18, 10, 3, 256, ZSTD_btopt}, /* level 19.*/
+ {18, 19, 18, 11, 3, 512, ZSTD_btopt2}, /* level 20.*/
+ {18, 19, 18, 12, 3, 512, ZSTD_btopt2}, /* level 21.*/
+ {18, 19, 18, 13, 3, 512, ZSTD_btopt2}, /* level 22.*/
+ },
+ {
+ /* for srcSize <= 128 KB */
+ /* W, C, H, S, L, T, strat */
+ {17, 12, 12, 1, 7, 8, ZSTD_fast}, /* level 0 - not used */
+ {17, 12, 13, 1, 6, 8, ZSTD_fast}, /* level 1 */
+ {17, 13, 16, 1, 5, 8, ZSTD_fast}, /* level 2 */
+ {17, 16, 16, 2, 5, 8, ZSTD_dfast}, /* level 3 */
+ {17, 13, 15, 3, 4, 8, ZSTD_greedy}, /* level 4 */
+ {17, 15, 17, 4, 4, 8, ZSTD_greedy}, /* level 5 */
+ {17, 16, 17, 3, 4, 8, ZSTD_lazy}, /* level 6 */
+ {17, 15, 17, 4, 4, 8, ZSTD_lazy2}, /* level 7 */
+ {17, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */
+ {17, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */
+ {17, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */
+ {17, 17, 17, 7, 4, 8, ZSTD_lazy2}, /* level 11 */
+ {17, 17, 17, 8, 4, 8, ZSTD_lazy2}, /* level 12 */
+ {17, 18, 17, 6, 4, 8, ZSTD_btlazy2}, /* level 13.*/
+ {17, 17, 17, 7, 3, 8, ZSTD_btopt}, /* level 14.*/
+ {17, 17, 17, 7, 3, 16, ZSTD_btopt}, /* level 15.*/
+ {17, 18, 17, 7, 3, 32, ZSTD_btopt}, /* level 16.*/
+ {17, 18, 17, 7, 3, 64, ZSTD_btopt}, /* level 17.*/
+ {17, 18, 17, 7, 3, 256, ZSTD_btopt}, /* level 18.*/
+ {17, 18, 17, 8, 3, 256, ZSTD_btopt}, /* level 19.*/
+ {17, 18, 17, 9, 3, 256, ZSTD_btopt2}, /* level 20.*/
+ {17, 18, 17, 10, 3, 256, ZSTD_btopt2}, /* level 21.*/
+ {17, 18, 17, 11, 3, 512, ZSTD_btopt2}, /* level 22.*/
+ },
+ {
+ /* for srcSize <= 16 KB */
+ /* W, C, H, S, L, T, strat */
+ {14, 12, 12, 1, 7, 6, ZSTD_fast}, /* level 0 - not used */
+ {14, 14, 14, 1, 6, 6, ZSTD_fast}, /* level 1 */
+ {14, 14, 14, 1, 4, 6, ZSTD_fast}, /* level 2 */
+ {14, 14, 14, 1, 4, 6, ZSTD_dfast}, /* level 3.*/
+ {14, 14, 14, 4, 4, 6, ZSTD_greedy}, /* level 4.*/
+ {14, 14, 14, 3, 4, 6, ZSTD_lazy}, /* level 5.*/
+ {14, 14, 14, 4, 4, 6, ZSTD_lazy2}, /* level 6 */
+ {14, 14, 14, 5, 4, 6, ZSTD_lazy2}, /* level 7 */
+ {14, 14, 14, 6, 4, 6, ZSTD_lazy2}, /* level 8.*/
+ {14, 15, 14, 6, 4, 6, ZSTD_btlazy2}, /* level 9.*/
+ {14, 15, 14, 3, 3, 6, ZSTD_btopt}, /* level 10.*/
+ {14, 15, 14, 6, 3, 8, ZSTD_btopt}, /* level 11.*/
+ {14, 15, 14, 6, 3, 16, ZSTD_btopt}, /* level 12.*/
+ {14, 15, 14, 6, 3, 24, ZSTD_btopt}, /* level 13.*/
+ {14, 15, 15, 6, 3, 48, ZSTD_btopt}, /* level 14.*/
+ {14, 15, 15, 6, 3, 64, ZSTD_btopt}, /* level 15.*/
+ {14, 15, 15, 6, 3, 96, ZSTD_btopt}, /* level 16.*/
+ {14, 15, 15, 6, 3, 128, ZSTD_btopt}, /* level 17.*/
+ {14, 15, 15, 6, 3, 256, ZSTD_btopt}, /* level 18.*/
+ {14, 15, 15, 7, 3, 256, ZSTD_btopt}, /* level 19.*/
+ {14, 15, 15, 8, 3, 256, ZSTD_btopt2}, /* level 20.*/
+ {14, 15, 15, 9, 3, 256, ZSTD_btopt2}, /* level 21.*/
+ {14, 15, 15, 10, 3, 256, ZSTD_btopt2}, /* level 22.*/
+ },
+};
+
+/*! ZSTD_getCParams() :
+* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
+* Size values are optional, provide 0 if not known or unused */
+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
+{
+ ZSTD_compressionParameters cp;
+ size_t const addedSize = srcSize ? 0 : 500;
+ U64 const rSize = srcSize + dictSize ? srcSize + dictSize + addedSize : (U64)-1;
+ U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
+ if (compressionLevel <= 0)
+ compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */
+ if (compressionLevel > ZSTD_MAX_CLEVEL)
+ compressionLevel = ZSTD_MAX_CLEVEL;
+ cp = ZSTD_defaultCParameters[tableID][compressionLevel];
+ if (ZSTD_32bits()) { /* auto-correction, for 32-bits mode */
+ if (cp.windowLog > ZSTD_WINDOWLOG_MAX)
+ cp.windowLog = ZSTD_WINDOWLOG_MAX;
+ if (cp.chainLog > ZSTD_CHAINLOG_MAX)
+ cp.chainLog = ZSTD_CHAINLOG_MAX;
+ if (cp.hashLog > ZSTD_HASHLOG_MAX)
+ cp.hashLog = ZSTD_HASHLOG_MAX;
+ }
+ cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
+ return cp;
+}
+
+/*! ZSTD_getParams() :
+* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`).
+* All fields of `ZSTD_frameParameters` are set to default (0) */
+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
+{
+ ZSTD_parameters params;
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
+ memset(&params, 0, sizeof(params));
+ params.cParams = cParams;
+ return params;
+}
+
+EXPORT_SYMBOL(ZSTD_maxCLevel);
+EXPORT_SYMBOL(ZSTD_compressBound);
+
+EXPORT_SYMBOL(ZSTD_CCtxWorkspaceBound);
+EXPORT_SYMBOL(ZSTD_initCCtx);
+EXPORT_SYMBOL(ZSTD_compressCCtx);
+EXPORT_SYMBOL(ZSTD_compress_usingDict);
+
+EXPORT_SYMBOL(ZSTD_CDictWorkspaceBound);
+EXPORT_SYMBOL(ZSTD_initCDict);
+EXPORT_SYMBOL(ZSTD_compress_usingCDict);
+
+EXPORT_SYMBOL(ZSTD_CStreamWorkspaceBound);
+EXPORT_SYMBOL(ZSTD_initCStream);
+EXPORT_SYMBOL(ZSTD_initCStream_usingCDict);
+EXPORT_SYMBOL(ZSTD_resetCStream);
+EXPORT_SYMBOL(ZSTD_compressStream);
+EXPORT_SYMBOL(ZSTD_flushStream);
+EXPORT_SYMBOL(ZSTD_endStream);
+EXPORT_SYMBOL(ZSTD_CStreamInSize);
+EXPORT_SYMBOL(ZSTD_CStreamOutSize);
+
+EXPORT_SYMBOL(ZSTD_getCParams);
+EXPORT_SYMBOL(ZSTD_getParams);
+EXPORT_SYMBOL(ZSTD_checkCParams);
+EXPORT_SYMBOL(ZSTD_adjustCParams);
+
+EXPORT_SYMBOL(ZSTD_compressBegin);
+EXPORT_SYMBOL(ZSTD_compressBegin_usingDict);
+EXPORT_SYMBOL(ZSTD_compressBegin_advanced);
+EXPORT_SYMBOL(ZSTD_copyCCtx);
+EXPORT_SYMBOL(ZSTD_compressBegin_usingCDict);
+EXPORT_SYMBOL(ZSTD_compressContinue);
+EXPORT_SYMBOL(ZSTD_compressEnd);
+
+EXPORT_SYMBOL(ZSTD_getBlockSizeMax);
+EXPORT_SYMBOL(ZSTD_compressBlock);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Zstd Compressor");
diff --git a/lib/zstd/decompress.c b/lib/zstd/decompress.c
new file mode 100644
index 000000000000..b17846725ca0
--- /dev/null
+++ b/lib/zstd/decompress.c
@@ -0,0 +1,2528 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
+ * An additional grant of patent rights can be found in the PATENTS file in the
+ * same directory.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ */
+
+/* ***************************************************************
+* Tuning parameters
+*****************************************************************/
+/*!
+* MAXWINDOWSIZE_DEFAULT :
+* maximum window size accepted by DStream, by default.
+* Frames requiring more memory will be rejected.
+*/
+#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
+#define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
+#endif
+
+/*-*******************************************************
+* Dependencies
+*********************************************************/
+#include "fse.h"
+#include "huf.h"
+#include "mem.h" /* low level memory routines */
+#include "zstd_internal.h"
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h> /* memcpy, memmove, memset */
+
+#define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
+
+/*-*************************************
+* Macros
+***************************************/
+#define ZSTD_isError ERR_isError /* for inlining */
+#define FSE_isError ERR_isError
+#define HUF_isError ERR_isError
+
+/*_*******************************************************
+* Memory operations
+**********************************************************/
+static void ZSTD_copy4(void *dst, const void *src) { memcpy(dst, src, 4); }
+
+/*-*************************************************************
+* Context management
+***************************************************************/
+typedef enum {
+ ZSTDds_getFrameHeaderSize,
+ ZSTDds_decodeFrameHeader,
+ ZSTDds_decodeBlockHeader,
+ ZSTDds_decompressBlock,
+ ZSTDds_decompressLastBlock,
+ ZSTDds_checkChecksum,
+ ZSTDds_decodeSkippableHeader,
+ ZSTDds_skipFrame
+} ZSTD_dStage;
+
+typedef struct {
+ FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
+ FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+ FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+ HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
+ U64 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32 / 2];
+ U32 rep[ZSTD_REP_NUM];
+} ZSTD_entropyTables_t;
+
+struct ZSTD_DCtx_s {
+ const FSE_DTable *LLTptr;
+ const FSE_DTable *MLTptr;
+ const FSE_DTable *OFTptr;
+ const HUF_DTable *HUFptr;
+ ZSTD_entropyTables_t entropy;
+ const void *previousDstEnd; /* detect continuity */
+ const void *base; /* start of curr segment */
+ const void *vBase; /* virtual start of previous segment if it was just before curr one */
+ const void *dictEnd; /* end of previous segment */
+ size_t expected;
+ ZSTD_frameParams fParams;
+ blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
+ ZSTD_dStage stage;
+ U32 litEntropy;
+ U32 fseEntropy;
+ struct xxh64_state xxhState;
+ size_t headerSize;
+ U32 dictID;
+ const BYTE *litPtr;
+ ZSTD_customMem customMem;
+ size_t litSize;
+ size_t rleSize;
+ BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
+
+size_t ZSTD_DCtxWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DCtx)); }
+
+size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx)
+{
+ dctx->expected = ZSTD_frameHeaderSize_prefix;
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ dctx->previousDstEnd = NULL;
+ dctx->base = NULL;
+ dctx->vBase = NULL;
+ dctx->dictEnd = NULL;
+ dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
+ dctx->litEntropy = dctx->fseEntropy = 0;
+ dctx->dictID = 0;
+ ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
+ memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
+ dctx->LLTptr = dctx->entropy.LLTable;
+ dctx->MLTptr = dctx->entropy.MLTable;
+ dctx->OFTptr = dctx->entropy.OFTable;
+ dctx->HUFptr = dctx->entropy.hufTable;
+ return 0;
+}
+
+ZSTD_DCtx *ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
+{
+ ZSTD_DCtx *dctx;
+
+ if (!customMem.customAlloc || !customMem.customFree)
+ return NULL;
+
+ dctx = (ZSTD_DCtx *)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
+ if (!dctx)
+ return NULL;
+ memcpy(&dctx->customMem, &customMem, sizeof(customMem));
+ ZSTD_decompressBegin(dctx);
+ return dctx;
+}
+
+ZSTD_DCtx *ZSTD_initDCtx(void *workspace, size_t workspaceSize)
+{
+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
+ return ZSTD_createDCtx_advanced(stackMem);
+}
+
+size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx)
+{
+ if (dctx == NULL)
+ return 0; /* support free on NULL */
+ ZSTD_free(dctx, dctx->customMem);
+ return 0; /* reserved as a potential error code in the future */
+}
+
+void ZSTD_copyDCtx(ZSTD_DCtx *dstDCtx, const ZSTD_DCtx *srcDCtx)
+{
+ size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
+ memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
+}
+
+static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict);
+
+/*-*************************************************************
+* Decompression section
+***************************************************************/
+
+/*! ZSTD_isFrame() :
+ * Tells if the content of `buffer` starts with a valid Frame Identifier.
+ * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ * Note 3 : Skippable Frame Identifiers are considered valid. */
+unsigned ZSTD_isFrame(const void *buffer, size_t size)
+{
+ if (size < 4)
+ return 0;
+ {
+ U32 const magic = ZSTD_readLE32(buffer);
+ if (magic == ZSTD_MAGICNUMBER)
+ return 1;
+ if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START)
+ return 1;
+ }
+ return 0;
+}
+
+/** ZSTD_frameHeaderSize() :
+* srcSize must be >= ZSTD_frameHeaderSize_prefix.
+* @return : size of the Frame Header */
+static size_t ZSTD_frameHeaderSize(const void *src, size_t srcSize)
+{
+ if (srcSize < ZSTD_frameHeaderSize_prefix)
+ return ERROR(srcSize_wrong);
+ {
+ BYTE const fhd = ((const BYTE *)src)[4];
+ U32 const dictID = fhd & 3;
+ U32 const singleSegment = (fhd >> 5) & 1;
+ U32 const fcsId = fhd >> 6;
+ return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + (singleSegment && !fcsId);
+ }
+}
+
+/** ZSTD_getFrameParams() :
+* decode Frame Header, or require larger `srcSize`.
+* @return : 0, `fparamsPtr` is correctly filled,
+* >0, `srcSize` is too small, result is expected `srcSize`,
+* or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_getFrameParams(ZSTD_frameParams *fparamsPtr, const void *src, size_t srcSize)
+{
+ const BYTE *ip = (const BYTE *)src;
+
+ if (srcSize < ZSTD_frameHeaderSize_prefix)
+ return ZSTD_frameHeaderSize_prefix;
+ if (ZSTD_readLE32(src) != ZSTD_MAGICNUMBER) {
+ if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ if (srcSize < ZSTD_skippableHeaderSize)
+ return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
+ memset(fparamsPtr, 0, sizeof(*fparamsPtr));
+ fparamsPtr->frameContentSize = ZSTD_readLE32((const char *)src + 4);
+ fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
+ return 0;
+ }
+ return ERROR(prefix_unknown);
+ }
+
+ /* ensure there is enough `srcSize` to fully read/decode frame header */
+ {
+ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
+ if (srcSize < fhsize)
+ return fhsize;
+ }
+
+ {
+ BYTE const fhdByte = ip[4];
+ size_t pos = 5;
+ U32 const dictIDSizeCode = fhdByte & 3;
+ U32 const checksumFlag = (fhdByte >> 2) & 1;
+ U32 const singleSegment = (fhdByte >> 5) & 1;
+ U32 const fcsID = fhdByte >> 6;
+ U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
+ U32 windowSize = 0;
+ U32 dictID = 0;
+ U64 frameContentSize = 0;
+ if ((fhdByte & 0x08) != 0)
+ return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
+ if (!singleSegment) {
+ BYTE const wlByte = ip[pos++];
+ U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
+ if (windowLog > ZSTD_WINDOWLOG_MAX)
+ return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
+ windowSize = (1U << windowLog);
+ windowSize += (windowSize >> 3) * (wlByte & 7);
+ }
+
+ switch (dictIDSizeCode) {
+ default: /* impossible */
+ case 0: break;
+ case 1:
+ dictID = ip[pos];
+ pos++;
+ break;
+ case 2:
+ dictID = ZSTD_readLE16(ip + pos);
+ pos += 2;
+ break;
+ case 3:
+ dictID = ZSTD_readLE32(ip + pos);
+ pos += 4;
+ break;
+ }
+ switch (fcsID) {
+ default: /* impossible */
+ case 0:
+ if (singleSegment)
+ frameContentSize = ip[pos];
+ break;
+ case 1: frameContentSize = ZSTD_readLE16(ip + pos) + 256; break;
+ case 2: frameContentSize = ZSTD_readLE32(ip + pos); break;
+ case 3: frameContentSize = ZSTD_readLE64(ip + pos); break;
+ }
+ if (!windowSize)
+ windowSize = (U32)frameContentSize;
+ if (windowSize > windowSizeMax)
+ return ERROR(frameParameter_windowTooLarge);
+ fparamsPtr->frameContentSize = frameContentSize;
+ fparamsPtr->windowSize = windowSize;
+ fparamsPtr->dictID = dictID;
+ fparamsPtr->checksumFlag = checksumFlag;
+ }
+ return 0;
+}
+
+/** ZSTD_getFrameContentSize() :
+* compatible with legacy mode
+* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
+* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
+unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
+{
+ {
+ ZSTD_frameParams fParams;
+ if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0)
+ return ZSTD_CONTENTSIZE_ERROR;
+ if (fParams.windowSize == 0) {
+ /* Either skippable or empty frame, size == 0 either way */
+ return 0;
+ } else if (fParams.frameContentSize != 0) {
+ return fParams.frameContentSize;
+ } else {
+ return ZSTD_CONTENTSIZE_UNKNOWN;
+ }
+ }
+}
+
+/** ZSTD_findDecompressedSize() :
+ * compatible with legacy mode
+ * `srcSize` must be the exact length of some number of ZSTD compressed and/or
+ * skippable frames
+ * @return : decompressed size of the frames contained */
+unsigned long long ZSTD_findDecompressedSize(const void *src, size_t srcSize)
+{
+ {
+ unsigned long long totalDstSize = 0;
+ while (srcSize >= ZSTD_frameHeaderSize_prefix) {
+ const U32 magicNumber = ZSTD_readLE32(src);
+
+ if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ size_t skippableSize;
+ if (srcSize < ZSTD_skippableHeaderSize)
+ return ERROR(srcSize_wrong);
+ skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
+ if (srcSize < skippableSize) {
+ return ZSTD_CONTENTSIZE_ERROR;
+ }
+
+ src = (const BYTE *)src + skippableSize;
+ srcSize -= skippableSize;
+ continue;
+ }
+
+ {
+ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+ if (ret >= ZSTD_CONTENTSIZE_ERROR)
+ return ret;
+
+ /* check for overflow */
+ if (totalDstSize + ret < totalDstSize)
+ return ZSTD_CONTENTSIZE_ERROR;
+ totalDstSize += ret;
+ }
+ {
+ size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
+ if (ZSTD_isError(frameSrcSize)) {
+ return ZSTD_CONTENTSIZE_ERROR;
+ }
+
+ src = (const BYTE *)src + frameSrcSize;
+ srcSize -= frameSrcSize;
+ }
+ }
+
+ if (srcSize) {
+ return ZSTD_CONTENTSIZE_ERROR;
+ }
+
+ return totalDstSize;
+ }
+}
+
+/** ZSTD_decodeFrameHeader() :
+* `headerSize` must be the size provided by ZSTD_frameHeaderSize().
+* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
+static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx *dctx, const void *src, size_t headerSize)
+{
+ size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize);
+ if (ZSTD_isError(result))
+ return result; /* invalid header */
+ if (result > 0)
+ return ERROR(srcSize_wrong); /* headerSize too small */
+ if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID))
+ return ERROR(dictionary_wrong);
+ if (dctx->fParams.checksumFlag)
+ xxh64_reset(&dctx->xxhState, 0);
+ return 0;
+}
+
+typedef struct {
+ blockType_e blockType;
+ U32 lastBlock;
+ U32 origSize;
+} blockProperties_t;
+
+/*! ZSTD_getcBlockSize() :
+* Provides the size of compressed block from block header `src` */
+size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t *bpPtr)
+{
+ if (srcSize < ZSTD_blockHeaderSize)
+ return ERROR(srcSize_wrong);
+ {
+ U32 const cBlockHeader = ZSTD_readLE24(src);
+ U32 const cSize = cBlockHeader >> 3;
+ bpPtr->lastBlock = cBlockHeader & 1;
+ bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
+ bpPtr->origSize = cSize; /* only useful for RLE */
+ if (bpPtr->blockType == bt_rle)
+ return 1;
+ if (bpPtr->blockType == bt_reserved)
+ return ERROR(corruption_detected);
+ return cSize;
+ }
+}
+
+static size_t ZSTD_copyRawBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ if (srcSize > dstCapacity)
+ return ERROR(dstSize_tooSmall);
+ memcpy(dst, src, srcSize);
+ return srcSize;
+}
+
+static size_t ZSTD_setRleBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize, size_t regenSize)
+{
+ if (srcSize != 1)
+ return ERROR(srcSize_wrong);
+ if (regenSize > dstCapacity)
+ return ERROR(dstSize_tooSmall);
+ memset(dst, *(const BYTE *)src, regenSize);
+ return regenSize;
+}
+
+/*! ZSTD_decodeLiteralsBlock() :
+ @return : nb of bytes read from src (< srcSize ) */
+size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
+{
+ if (srcSize < MIN_CBLOCK_SIZE)
+ return ERROR(corruption_detected);
+
+ {
+ const BYTE *const istart = (const BYTE *)src;
+ symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
+
+ switch (litEncType) {
+ case set_repeat:
+ if (dctx->litEntropy == 0)
+ return ERROR(dictionary_corrupted);
+ /* fall-through */
+ case set_compressed:
+ if (srcSize < 5)
+ return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
+ {
+ size_t lhSize, litSize, litCSize;
+ U32 singleStream = 0;
+ U32 const lhlCode = (istart[0] >> 2) & 3;
+ U32 const lhc = ZSTD_readLE32(istart);
+ switch (lhlCode) {
+ case 0:
+ case 1:
+ default: /* note : default is impossible, since lhlCode into [0..3] */
+ /* 2 - 2 - 10 - 10 */
+ singleStream = !lhlCode;
+ lhSize = 3;
+ litSize = (lhc >> 4) & 0x3FF;
+ litCSize = (lhc >> 14) & 0x3FF;
+ break;
+ case 2:
+ /* 2 - 2 - 14 - 14 */
+ lhSize = 4;
+ litSize = (lhc >> 4) & 0x3FFF;
+ litCSize = lhc >> 18;
+ break;
+ case 3:
+ /* 2 - 2 - 18 - 18 */
+ lhSize = 5;
+ litSize = (lhc >> 4) & 0x3FFFF;
+ litCSize = (lhc >> 22) + (istart[4] << 10);
+ break;
+ }
+ if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
+ return ERROR(corruption_detected);
+ if (litCSize + lhSize > srcSize)
+ return ERROR(corruption_detected);
+
+ if (HUF_isError(
+ (litEncType == set_repeat)
+ ? (singleStream ? HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr)
+ : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr))
+ : (singleStream
+ ? HUF_decompress1X2_DCtx_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize,
+ dctx->entropy.workspace, sizeof(dctx->entropy.workspace))
+ : HUF_decompress4X_hufOnly_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart + lhSize, litCSize,
+ dctx->entropy.workspace, sizeof(dctx->entropy.workspace)))))
+ return ERROR(corruption_detected);
+
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litSize = litSize;
+ dctx->litEntropy = 1;
+ if (litEncType == set_compressed)
+ dctx->HUFptr = dctx->entropy.hufTable;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
+ return litCSize + lhSize;
+ }
+
+ case set_basic: {
+ size_t litSize, lhSize;
+ U32 const lhlCode = ((istart[0]) >> 2) & 3;
+ switch (lhlCode) {
+ case 0:
+ case 2:
+ default: /* note : default is impossible, since lhlCode into [0..3] */
+ lhSize = 1;
+ litSize = istart[0] >> 3;
+ break;
+ case 1:
+ lhSize = 2;
+ litSize = ZSTD_readLE16(istart) >> 4;
+ break;
+ case 3:
+ lhSize = 3;
+ litSize = ZSTD_readLE24(istart) >> 4;
+ break;
+ }
+
+ if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
+ if (litSize + lhSize > srcSize)
+ return ERROR(corruption_detected);
+ memcpy(dctx->litBuffer, istart + lhSize, litSize);
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litSize = litSize;
+ memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
+ return lhSize + litSize;
+ }
+ /* direct reference into compressed stream */
+ dctx->litPtr = istart + lhSize;
+ dctx->litSize = litSize;
+ return lhSize + litSize;
+ }
+
+ case set_rle: {
+ U32 const lhlCode = ((istart[0]) >> 2) & 3;
+ size_t litSize, lhSize;
+ switch (lhlCode) {
+ case 0:
+ case 2:
+ default: /* note : default is impossible, since lhlCode into [0..3] */
+ lhSize = 1;
+ litSize = istart[0] >> 3;
+ break;
+ case 1:
+ lhSize = 2;
+ litSize = ZSTD_readLE16(istart) >> 4;
+ break;
+ case 3:
+ lhSize = 3;
+ litSize = ZSTD_readLE24(istart) >> 4;
+ if (srcSize < 4)
+ return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
+ break;
+ }
+ if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX)
+ return ERROR(corruption_detected);
+ memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
+ dctx->litPtr = dctx->litBuffer;
+ dctx->litSize = litSize;
+ return lhSize + 1;
+ }
+ default:
+ return ERROR(corruption_detected); /* impossible */
+ }
+ }
+}
+
+typedef union {
+ FSE_decode_t realData;
+ U32 alignedBy4;
+} FSE_decode_t4;
+
+static const FSE_decode_t4 LL_defaultDTable[(1 << LL_DEFAULTNORMLOG) + 1] = {
+ {{LL_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
+ {{0, 0, 4}}, /* 0 : base, symbol, bits */
+ {{16, 0, 4}},
+ {{32, 1, 5}},
+ {{0, 3, 5}},
+ {{0, 4, 5}},
+ {{0, 6, 5}},
+ {{0, 7, 5}},
+ {{0, 9, 5}},
+ {{0, 10, 5}},
+ {{0, 12, 5}},
+ {{0, 14, 6}},
+ {{0, 16, 5}},
+ {{0, 18, 5}},
+ {{0, 19, 5}},
+ {{0, 21, 5}},
+ {{0, 22, 5}},
+ {{0, 24, 5}},
+ {{32, 25, 5}},
+ {{0, 26, 5}},
+ {{0, 27, 6}},
+ {{0, 29, 6}},
+ {{0, 31, 6}},
+ {{32, 0, 4}},
+ {{0, 1, 4}},
+ {{0, 2, 5}},
+ {{32, 4, 5}},
+ {{0, 5, 5}},
+ {{32, 7, 5}},
+ {{0, 8, 5}},
+ {{32, 10, 5}},
+ {{0, 11, 5}},
+ {{0, 13, 6}},
+ {{32, 16, 5}},
+ {{0, 17, 5}},
+ {{32, 19, 5}},
+ {{0, 20, 5}},
+ {{32, 22, 5}},
+ {{0, 23, 5}},
+ {{0, 25, 4}},
+ {{16, 25, 4}},
+ {{32, 26, 5}},
+ {{0, 28, 6}},
+ {{0, 30, 6}},
+ {{48, 0, 4}},
+ {{16, 1, 4}},
+ {{32, 2, 5}},
+ {{32, 3, 5}},
+ {{32, 5, 5}},
+ {{32, 6, 5}},
+ {{32, 8, 5}},
+ {{32, 9, 5}},
+ {{32, 11, 5}},
+ {{32, 12, 5}},
+ {{0, 15, 6}},
+ {{32, 17, 5}},
+ {{32, 18, 5}},
+ {{32, 20, 5}},
+ {{32, 21, 5}},
+ {{32, 23, 5}},
+ {{32, 24, 5}},
+ {{0, 35, 6}},
+ {{0, 34, 6}},
+ {{0, 33, 6}},
+ {{0, 32, 6}},
+}; /* LL_defaultDTable */
+
+static const FSE_decode_t4 ML_defaultDTable[(1 << ML_DEFAULTNORMLOG) + 1] = {
+ {{ML_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
+ {{0, 0, 6}}, /* 0 : base, symbol, bits */
+ {{0, 1, 4}},
+ {{32, 2, 5}},
+ {{0, 3, 5}},
+ {{0, 5, 5}},
+ {{0, 6, 5}},
+ {{0, 8, 5}},
+ {{0, 10, 6}},
+ {{0, 13, 6}},
+ {{0, 16, 6}},
+ {{0, 19, 6}},
+ {{0, 22, 6}},
+ {{0, 25, 6}},
+ {{0, 28, 6}},
+ {{0, 31, 6}},
+ {{0, 33, 6}},
+ {{0, 35, 6}},
+ {{0, 37, 6}},
+ {{0, 39, 6}},
+ {{0, 41, 6}},
+ {{0, 43, 6}},
+ {{0, 45, 6}},
+ {{16, 1, 4}},
+ {{0, 2, 4}},
+ {{32, 3, 5}},
+ {{0, 4, 5}},
+ {{32, 6, 5}},
+ {{0, 7, 5}},
+ {{0, 9, 6}},
+ {{0, 12, 6}},
+ {{0, 15, 6}},
+ {{0, 18, 6}},
+ {{0, 21, 6}},
+ {{0, 24, 6}},
+ {{0, 27, 6}},
+ {{0, 30, 6}},
+ {{0, 32, 6}},
+ {{0, 34, 6}},
+ {{0, 36, 6}},
+ {{0, 38, 6}},
+ {{0, 40, 6}},
+ {{0, 42, 6}},
+ {{0, 44, 6}},
+ {{32, 1, 4}},
+ {{48, 1, 4}},
+ {{16, 2, 4}},
+ {{32, 4, 5}},
+ {{32, 5, 5}},
+ {{32, 7, 5}},
+ {{32, 8, 5}},
+ {{0, 11, 6}},
+ {{0, 14, 6}},
+ {{0, 17, 6}},
+ {{0, 20, 6}},
+ {{0, 23, 6}},
+ {{0, 26, 6}},
+ {{0, 29, 6}},
+ {{0, 52, 6}},
+ {{0, 51, 6}},
+ {{0, 50, 6}},
+ {{0, 49, 6}},
+ {{0, 48, 6}},
+ {{0, 47, 6}},
+ {{0, 46, 6}},
+}; /* ML_defaultDTable */
+
+static const FSE_decode_t4 OF_defaultDTable[(1 << OF_DEFAULTNORMLOG) + 1] = {
+ {{OF_DEFAULTNORMLOG, 1, 1}}, /* header : tableLog, fastMode, fastMode */
+ {{0, 0, 5}}, /* 0 : base, symbol, bits */
+ {{0, 6, 4}},
+ {{0, 9, 5}},
+ {{0, 15, 5}},
+ {{0, 21, 5}},
+ {{0, 3, 5}},
+ {{0, 7, 4}},
+ {{0, 12, 5}},
+ {{0, 18, 5}},
+ {{0, 23, 5}},
+ {{0, 5, 5}},
+ {{0, 8, 4}},
+ {{0, 14, 5}},
+ {{0, 20, 5}},
+ {{0, 2, 5}},
+ {{16, 7, 4}},
+ {{0, 11, 5}},
+ {{0, 17, 5}},
+ {{0, 22, 5}},
+ {{0, 4, 5}},
+ {{16, 8, 4}},
+ {{0, 13, 5}},
+ {{0, 19, 5}},
+ {{0, 1, 5}},
+ {{16, 6, 4}},
+ {{0, 10, 5}},
+ {{0, 16, 5}},
+ {{0, 28, 5}},
+ {{0, 27, 5}},
+ {{0, 26, 5}},
+ {{0, 25, 5}},
+ {{0, 24, 5}},
+}; /* OF_defaultDTable */
+
+/*! ZSTD_buildSeqTable() :
+ @return : nb bytes read from src,
+ or an error code if it fails, testable with ZSTD_isError()
+*/
+static size_t ZSTD_buildSeqTable(FSE_DTable *DTableSpace, const FSE_DTable **DTablePtr, symbolEncodingType_e type, U32 max, U32 maxLog, const void *src,
+ size_t srcSize, const FSE_decode_t4 *defaultTable, U32 flagRepeatTable, void *workspace, size_t workspaceSize)
+{
+ const void *const tmpPtr = defaultTable; /* bypass strict aliasing */
+ switch (type) {
+ case set_rle:
+ if (!srcSize)
+ return ERROR(srcSize_wrong);
+ if ((*(const BYTE *)src) > max)
+ return ERROR(corruption_detected);
+ FSE_buildDTable_rle(DTableSpace, *(const BYTE *)src);
+ *DTablePtr = DTableSpace;
+ return 1;
+ case set_basic: *DTablePtr = (const FSE_DTable *)tmpPtr; return 0;
+ case set_repeat:
+ if (!flagRepeatTable)
+ return ERROR(corruption_detected);
+ return 0;
+ default: /* impossible */
+ case set_compressed: {
+ U32 tableLog;
+ S16 *norm = (S16 *)workspace;
+ size_t const spaceUsed32 = ALIGN(sizeof(S16) * (MaxSeq + 1), sizeof(U32)) >> 2;
+
+ if ((spaceUsed32 << 2) > workspaceSize)
+ return ERROR(GENERIC);
+ workspace = (U32 *)workspace + spaceUsed32;
+ workspaceSize -= (spaceUsed32 << 2);
+ {
+ size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
+ if (FSE_isError(headerSize))
+ return ERROR(corruption_detected);
+ if (tableLog > maxLog)
+ return ERROR(corruption_detected);
+ FSE_buildDTable_wksp(DTableSpace, norm, max, tableLog, workspace, workspaceSize);
+ *DTablePtr = DTableSpace;
+ return headerSize;
+ }
+ }
+ }
+}
+
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx *dctx, int *nbSeqPtr, const void *src, size_t srcSize)
+{
+ const BYTE *const istart = (const BYTE *const)src;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *ip = istart;
+
+ /* check */
+ if (srcSize < MIN_SEQUENCES_SIZE)
+ return ERROR(srcSize_wrong);
+
+ /* SeqHead */
+ {
+ int nbSeq = *ip++;
+ if (!nbSeq) {
+ *nbSeqPtr = 0;
+ return 1;
+ }
+ if (nbSeq > 0x7F) {
+ if (nbSeq == 0xFF) {
+ if (ip + 2 > iend)
+ return ERROR(srcSize_wrong);
+ nbSeq = ZSTD_readLE16(ip) + LONGNBSEQ, ip += 2;
+ } else {
+ if (ip >= iend)
+ return ERROR(srcSize_wrong);
+ nbSeq = ((nbSeq - 0x80) << 8) + *ip++;
+ }
+ }
+ *nbSeqPtr = nbSeq;
+ }
+
+ /* FSE table descriptors */
+ if (ip + 4 > iend)
+ return ERROR(srcSize_wrong); /* minimum possible size */
+ {
+ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
+ symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
+ symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
+ ip++;
+
+ /* Build DTables */
+ {
+ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, LLtype, MaxLL, LLFSELog, ip, iend - ip,
+ LL_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace));
+ if (ZSTD_isError(llhSize))
+ return ERROR(corruption_detected);
+ ip += llhSize;
+ }
+ {
+ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, OFtype, MaxOff, OffFSELog, ip, iend - ip,
+ OF_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace));
+ if (ZSTD_isError(ofhSize))
+ return ERROR(corruption_detected);
+ ip += ofhSize;
+ }
+ {
+ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, MLtype, MaxML, MLFSELog, ip, iend - ip,
+ ML_defaultDTable, dctx->fseEntropy, dctx->entropy.workspace, sizeof(dctx->entropy.workspace));
+ if (ZSTD_isError(mlhSize))
+ return ERROR(corruption_detected);
+ ip += mlhSize;
+ }
+ }
+
+ return ip - istart;
+}
+
+typedef struct {
+ size_t litLength;
+ size_t matchLength;
+ size_t offset;
+ const BYTE *match;
+} seq_t;
+
+typedef struct {
+ BIT_DStream_t DStream;
+ FSE_DState_t stateLL;
+ FSE_DState_t stateOffb;
+ FSE_DState_t stateML;
+ size_t prevOffset[ZSTD_REP_NUM];
+ const BYTE *base;
+ size_t pos;
+ uPtrDiff gotoDict;
+} seqState_t;
+
+FORCE_NOINLINE
+size_t ZSTD_execSequenceLast7(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
+ const BYTE *const vBase, const BYTE *const dictEnd)
+{
+ BYTE *const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
+ const BYTE *const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE *match = oLitEnd - sequence.offset;
+
+ /* check */
+ if (oMatchEnd > oend)
+ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+ if (iLitEnd > litLimit)
+ return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (oLitEnd <= oend_w)
+ return ERROR(GENERIC); /* Precondition */
+
+ /* copy literals */
+ if (op < oend_w) {
+ ZSTD_wildcopy(op, *litPtr, oend_w - op);
+ *litPtr += oend_w - op;
+ op = oend_w;
+ }
+ while (op < oLitEnd)
+ *op++ = *(*litPtr)++;
+
+ /* copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - base)) {
+ /* offset beyond prefix */
+ if (sequence.offset > (size_t)(oLitEnd - vBase))
+ return ERROR(corruption_detected);
+ match = dictEnd - (base - match);
+ if (match + sequence.matchLength <= dictEnd) {
+ memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currPrefixSegment */
+ {
+ size_t const length1 = dictEnd - match;
+ memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = base;
+ }
+ }
+ while (op < oMatchEnd)
+ *op++ = *match++;
+ return sequenceLength;
+}
+
+static seq_t ZSTD_decodeSequence(seqState_t *seqState)
+{
+ seq_t seq;
+
+ U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
+ U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
+ U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */
+
+ U32 const llBits = LL_bits[llCode];
+ U32 const mlBits = ML_bits[mlCode];
+ U32 const ofBits = ofCode;
+ U32 const totalBits = llBits + mlBits + ofBits;
+
+ static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
+ 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
+
+ static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
+ 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
+
+ static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
+ 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
+ 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
+
+ /* sequence */
+ {
+ size_t offset;
+ if (!ofCode)
+ offset = 0;
+ else {
+ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
+ if (ZSTD_32bits())
+ BIT_reloadDStream(&seqState->DStream);
+ }
+
+ if (ofCode <= 1) {
+ offset += (llCode == 0);
+ if (offset) {
+ size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
+ if (offset != 1)
+ seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset = temp;
+ } else {
+ offset = seqState->prevOffset[0];
+ }
+ } else {
+ seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset;
+ }
+ seq.offset = offset;
+ }
+
+ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
+ if (ZSTD_32bits() && (mlBits + llBits > 24))
+ BIT_reloadDStream(&seqState->DStream);
+
+ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
+ if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
+ BIT_reloadDStream(&seqState->DStream);
+
+ /* ANS state update */
+ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
+ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
+ if (ZSTD_32bits())
+ BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
+ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
+
+ seq.match = NULL;
+
+ return seq;
+}
+
+FORCE_INLINE
+size_t ZSTD_execSequence(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
+ const BYTE *const vBase, const BYTE *const dictEnd)
+{
+ BYTE *const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
+ const BYTE *const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE *match = oLitEnd - sequence.offset;
+
+ /* check */
+ if (oMatchEnd > oend)
+ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+ if (iLitEnd > litLimit)
+ return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (oLitEnd > oend_w)
+ return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
+
+ /* copy Literals */
+ ZSTD_copy8(op, *litPtr);
+ if (sequence.litLength > 8)
+ ZSTD_wildcopy(op + 8, (*litPtr) + 8,
+ sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+ op = oLitEnd;
+ *litPtr = iLitEnd; /* update for next sequence */
+
+ /* copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - base)) {
+ /* offset beyond prefix */
+ if (sequence.offset > (size_t)(oLitEnd - vBase))
+ return ERROR(corruption_detected);
+ match = dictEnd + (match - base);
+ if (match + sequence.matchLength <= dictEnd) {
+ memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currPrefixSegment */
+ {
+ size_t const length1 = dictEnd - match;
+ memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = base;
+ if (op > oend_w || sequence.matchLength < MINMATCH) {
+ U32 i;
+ for (i = 0; i < sequence.matchLength; ++i)
+ op[i] = match[i];
+ return sequenceLength;
+ }
+ }
+ }
+ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
+
+ /* match within prefix */
+ if (sequence.offset < 8) {
+ /* close range match, overlap */
+ static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */
+ int const sub2 = dec64table[sequence.offset];
+ op[0] = match[0];
+ op[1] = match[1];
+ op[2] = match[2];
+ op[3] = match[3];
+ match += dec32table[sequence.offset];
+ ZSTD_copy4(op + 4, match);
+ match -= sub2;
+ } else {
+ ZSTD_copy8(op, match);
+ }
+ op += 8;
+ match += 8;
+
+ if (oMatchEnd > oend - (16 - MINMATCH)) {
+ if (op < oend_w) {
+ ZSTD_wildcopy(op, match, oend_w - op);
+ match += oend_w - op;
+ op = oend_w;
+ }
+ while (op < oMatchEnd)
+ *op++ = *match++;
+ } else {
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
+ }
+ return sequenceLength;
+}
+
+static size_t ZSTD_decompressSequences(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
+{
+ const BYTE *ip = (const BYTE *)seqStart;
+ const BYTE *const iend = ip + seqSize;
+ BYTE *const ostart = (BYTE * const)dst;
+ BYTE *const oend = ostart + maxDstSize;
+ BYTE *op = ostart;
+ const BYTE *litPtr = dctx->litPtr;
+ const BYTE *const litEnd = litPtr + dctx->litSize;
+ const BYTE *const base = (const BYTE *)(dctx->base);
+ const BYTE *const vBase = (const BYTE *)(dctx->vBase);
+ const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd);
+ int nbSeq;
+
+ /* Build Decoding Tables */
+ {
+ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
+ if (ZSTD_isError(seqHSize))
+ return seqHSize;
+ ip += seqHSize;
+ }
+
+ /* Regen sequences */
+ if (nbSeq) {
+ seqState_t seqState;
+ dctx->fseEntropy = 1;
+ {
+ U32 i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ seqState.prevOffset[i] = dctx->entropy.rep[i];
+ }
+ CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected);
+ FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+
+ for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) {
+ nbSeq--;
+ {
+ seq_t const sequence = ZSTD_decodeSequence(&seqState);
+ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
+ if (ZSTD_isError(oneSeqSize))
+ return oneSeqSize;
+ op += oneSeqSize;
+ }
+ }
+
+ /* check if reached exact end */
+ if (nbSeq)
+ return ERROR(corruption_detected);
+ /* save reps for next block */
+ {
+ U32 i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
+ }
+ }
+
+ /* last literal segment */
+ {
+ size_t const lastLLSize = litEnd - litPtr;
+ if (lastLLSize > (size_t)(oend - op))
+ return ERROR(dstSize_tooSmall);
+ memcpy(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+
+ return op - ostart;
+}
+
+FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t *seqState, int const longOffsets)
+{
+ seq_t seq;
+
+ U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
+ U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
+ U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */
+
+ U32 const llBits = LL_bits[llCode];
+ U32 const mlBits = ML_bits[mlCode];
+ U32 const ofBits = ofCode;
+ U32 const totalBits = llBits + mlBits + ofBits;
+
+ static const U32 LL_base[MaxLL + 1] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18,
+ 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000};
+
+ static const U32 ML_base[MaxML + 1] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41,
+ 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003};
+
+ static const U32 OF_base[MaxOff + 1] = {0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD,
+ 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD,
+ 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD};
+
+ /* sequence */
+ {
+ size_t offset;
+ if (!ofCode)
+ offset = 0;
+ else {
+ if (longOffsets) {
+ int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
+ offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
+ if (ZSTD_32bits() || extraBits)
+ BIT_reloadDStream(&seqState->DStream);
+ if (extraBits)
+ offset += BIT_readBitsFast(&seqState->DStream, extraBits);
+ } else {
+ offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
+ if (ZSTD_32bits())
+ BIT_reloadDStream(&seqState->DStream);
+ }
+ }
+
+ if (ofCode <= 1) {
+ offset += (llCode == 0);
+ if (offset) {
+ size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
+ if (offset != 1)
+ seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset = temp;
+ } else {
+ offset = seqState->prevOffset[0];
+ }
+ } else {
+ seqState->prevOffset[2] = seqState->prevOffset[1];
+ seqState->prevOffset[1] = seqState->prevOffset[0];
+ seqState->prevOffset[0] = offset;
+ }
+ seq.offset = offset;
+ }
+
+ seq.matchLength = ML_base[mlCode] + ((mlCode > 31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
+ if (ZSTD_32bits() && (mlBits + llBits > 24))
+ BIT_reloadDStream(&seqState->DStream);
+
+ seq.litLength = LL_base[llCode] + ((llCode > 15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
+ if (ZSTD_32bits() || (totalBits > 64 - 7 - (LLFSELog + MLFSELog + OffFSELog)))
+ BIT_reloadDStream(&seqState->DStream);
+
+ {
+ size_t const pos = seqState->pos + seq.litLength;
+ seq.match = seqState->base + pos - seq.offset; /* single memory segment */
+ if (seq.offset > pos)
+ seq.match += seqState->gotoDict; /* separate memory segment */
+ seqState->pos = pos + seq.matchLength;
+ }
+
+ /* ANS state update */
+ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
+ FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
+ if (ZSTD_32bits())
+ BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
+ FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
+
+ return seq;
+}
+
+static seq_t ZSTD_decodeSequenceLong(seqState_t *seqState, unsigned const windowSize)
+{
+ if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) {
+ return ZSTD_decodeSequenceLong_generic(seqState, 1);
+ } else {
+ return ZSTD_decodeSequenceLong_generic(seqState, 0);
+ }
+}
+
+FORCE_INLINE
+size_t ZSTD_execSequenceLong(BYTE *op, BYTE *const oend, seq_t sequence, const BYTE **litPtr, const BYTE *const litLimit, const BYTE *const base,
+ const BYTE *const vBase, const BYTE *const dictEnd)
+{
+ BYTE *const oLitEnd = op + sequence.litLength;
+ size_t const sequenceLength = sequence.litLength + sequence.matchLength;
+ BYTE *const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
+ BYTE *const oend_w = oend - WILDCOPY_OVERLENGTH;
+ const BYTE *const iLitEnd = *litPtr + sequence.litLength;
+ const BYTE *match = sequence.match;
+
+ /* check */
+ if (oMatchEnd > oend)
+ return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
+ if (iLitEnd > litLimit)
+ return ERROR(corruption_detected); /* over-read beyond lit buffer */
+ if (oLitEnd > oend_w)
+ return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
+
+ /* copy Literals */
+ ZSTD_copy8(op, *litPtr);
+ if (sequence.litLength > 8)
+ ZSTD_wildcopy(op + 8, (*litPtr) + 8,
+ sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+ op = oLitEnd;
+ *litPtr = iLitEnd; /* update for next sequence */
+
+ /* copy Match */
+ if (sequence.offset > (size_t)(oLitEnd - base)) {
+ /* offset beyond prefix */
+ if (sequence.offset > (size_t)(oLitEnd - vBase))
+ return ERROR(corruption_detected);
+ if (match + sequence.matchLength <= dictEnd) {
+ memmove(oLitEnd, match, sequence.matchLength);
+ return sequenceLength;
+ }
+ /* span extDict & currPrefixSegment */
+ {
+ size_t const length1 = dictEnd - match;
+ memmove(oLitEnd, match, length1);
+ op = oLitEnd + length1;
+ sequence.matchLength -= length1;
+ match = base;
+ if (op > oend_w || sequence.matchLength < MINMATCH) {
+ U32 i;
+ for (i = 0; i < sequence.matchLength; ++i)
+ op[i] = match[i];
+ return sequenceLength;
+ }
+ }
+ }
+ /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
+
+ /* match within prefix */
+ if (sequence.offset < 8) {
+ /* close range match, overlap */
+ static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
+ static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */
+ int const sub2 = dec64table[sequence.offset];
+ op[0] = match[0];
+ op[1] = match[1];
+ op[2] = match[2];
+ op[3] = match[3];
+ match += dec32table[sequence.offset];
+ ZSTD_copy4(op + 4, match);
+ match -= sub2;
+ } else {
+ ZSTD_copy8(op, match);
+ }
+ op += 8;
+ match += 8;
+
+ if (oMatchEnd > oend - (16 - MINMATCH)) {
+ if (op < oend_w) {
+ ZSTD_wildcopy(op, match, oend_w - op);
+ match += oend_w - op;
+ op = oend_w;
+ }
+ while (op < oMatchEnd)
+ *op++ = *match++;
+ } else {
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */
+ }
+ return sequenceLength;
+}
+
+static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, size_t seqSize)
+{
+ const BYTE *ip = (const BYTE *)seqStart;
+ const BYTE *const iend = ip + seqSize;
+ BYTE *const ostart = (BYTE * const)dst;
+ BYTE *const oend = ostart + maxDstSize;
+ BYTE *op = ostart;
+ const BYTE *litPtr = dctx->litPtr;
+ const BYTE *const litEnd = litPtr + dctx->litSize;
+ const BYTE *const base = (const BYTE *)(dctx->base);
+ const BYTE *const vBase = (const BYTE *)(dctx->vBase);
+ const BYTE *const dictEnd = (const BYTE *)(dctx->dictEnd);
+ unsigned const windowSize = dctx->fParams.windowSize;
+ int nbSeq;
+
+ /* Build Decoding Tables */
+ {
+ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
+ if (ZSTD_isError(seqHSize))
+ return seqHSize;
+ ip += seqHSize;
+ }
+
+ /* Regen sequences */
+ if (nbSeq) {
+#define STORED_SEQS 4
+#define STOSEQ_MASK (STORED_SEQS - 1)
+#define ADVANCED_SEQS 4
+ seq_t *sequences = (seq_t *)dctx->entropy.workspace;
+ int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
+ seqState_t seqState;
+ int seqNb;
+ ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.workspace) >= sizeof(seq_t) * STORED_SEQS);
+ dctx->fseEntropy = 1;
+ {
+ U32 i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ seqState.prevOffset[i] = dctx->entropy.rep[i];
+ }
+ seqState.base = base;
+ seqState.pos = (size_t)(op - base);
+ seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
+ CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected);
+ FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
+ FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
+ FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+
+ /* prepare in advance */
+ for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb < seqAdvance; seqNb++) {
+ sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize);
+ }
+ if (seqNb < seqAdvance)
+ return ERROR(corruption_detected);
+
+ /* decode and decompress */
+ for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb < nbSeq; seqNb++) {
+ seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize);
+ size_t const oneSeqSize =
+ ZSTD_execSequenceLong(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
+ if (ZSTD_isError(oneSeqSize))
+ return oneSeqSize;
+ ZSTD_PREFETCH(sequence.match);
+ sequences[seqNb & STOSEQ_MASK] = sequence;
+ op += oneSeqSize;
+ }
+ if (seqNb < nbSeq)
+ return ERROR(corruption_detected);
+
+ /* finish queue */
+ seqNb -= seqAdvance;
+ for (; seqNb < nbSeq; seqNb++) {
+ size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
+ if (ZSTD_isError(oneSeqSize))
+ return oneSeqSize;
+ op += oneSeqSize;
+ }
+
+ /* save reps for next block */
+ {
+ U32 i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]);
+ }
+ }
+
+ /* last literal segment */
+ {
+ size_t const lastLLSize = litEnd - litPtr;
+ if (lastLLSize > (size_t)(oend - op))
+ return ERROR(dstSize_tooSmall);
+ memcpy(op, litPtr, lastLLSize);
+ op += lastLLSize;
+ }
+
+ return op - ostart;
+}
+
+static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{ /* blockType == blockCompressed */
+ const BYTE *ip = (const BYTE *)src;
+
+ if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX)
+ return ERROR(srcSize_wrong);
+
+ /* Decode literals section */
+ {
+ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
+ if (ZSTD_isError(litCSize))
+ return litCSize;
+ ip += litCSize;
+ srcSize -= litCSize;
+ }
+ if (sizeof(size_t) > 4) /* do not enable prefetching on 32-bits x86, as it's performance detrimental */
+ /* likely because of register pressure */
+ /* if that's the correct cause, then 32-bits ARM should be affected differently */
+ /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */
+ if (dctx->fParams.windowSize > (1 << 23))
+ return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
+ return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
+}
+
+static void ZSTD_checkContinuity(ZSTD_DCtx *dctx, const void *dst)
+{
+ if (dst != dctx->previousDstEnd) { /* not contiguous */
+ dctx->dictEnd = dctx->previousDstEnd;
+ dctx->vBase = (const char *)dst - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base));
+ dctx->base = dst;
+ dctx->previousDstEnd = dst;
+ }
+}
+
+size_t ZSTD_decompressBlock(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ size_t dSize;
+ ZSTD_checkContinuity(dctx, dst);
+ dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
+ dctx->previousDstEnd = (char *)dst + dSize;
+ return dSize;
+}
+
+/** ZSTD_insertBlock() :
+ insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
+size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void *blockStart, size_t blockSize)
+{
+ ZSTD_checkContinuity(dctx, blockStart);
+ dctx->previousDstEnd = (const char *)blockStart + blockSize;
+ return blockSize;
+}
+
+size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t length)
+{
+ if (length > dstCapacity)
+ return ERROR(dstSize_tooSmall);
+ memset(dst, byte, length);
+ return length;
+}
+
+/** ZSTD_findFrameCompressedSize() :
+ * compatible with legacy mode
+ * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
+ * `srcSize` must be at least as large as the frame contained
+ * @return : the compressed size of the frame starting at `src` */
+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
+{
+ if (srcSize >= ZSTD_skippableHeaderSize && (ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ return ZSTD_skippableHeaderSize + ZSTD_readLE32((const BYTE *)src + 4);
+ } else {
+ const BYTE *ip = (const BYTE *)src;
+ const BYTE *const ipstart = ip;
+ size_t remainingSize = srcSize;
+ ZSTD_frameParams fParams;
+
+ size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
+ if (ZSTD_isError(headerSize))
+ return headerSize;
+
+ /* Frame Header */
+ {
+ size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
+ if (ZSTD_isError(ret))
+ return ret;
+ if (ret > 0)
+ return ERROR(srcSize_wrong);
+ }
+
+ ip += headerSize;
+ remainingSize -= headerSize;
+
+ /* Loop on each block */
+ while (1) {
+ blockProperties_t blockProperties;
+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
+ if (ZSTD_isError(cBlockSize))
+ return cBlockSize;
+
+ if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
+ return ERROR(srcSize_wrong);
+
+ ip += ZSTD_blockHeaderSize + cBlockSize;
+ remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
+
+ if (blockProperties.lastBlock)
+ break;
+ }
+
+ if (fParams.checksumFlag) { /* Frame content checksum */
+ if (remainingSize < 4)
+ return ERROR(srcSize_wrong);
+ ip += 4;
+ remainingSize -= 4;
+ }
+
+ return ip - ipstart;
+ }
+}
+
+/*! ZSTD_decompressFrame() :
+* @dctx must be properly initialized */
+static size_t ZSTD_decompressFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void **srcPtr, size_t *srcSizePtr)
+{
+ const BYTE *ip = (const BYTE *)(*srcPtr);
+ BYTE *const ostart = (BYTE * const)dst;
+ BYTE *const oend = ostart + dstCapacity;
+ BYTE *op = ostart;
+ size_t remainingSize = *srcSizePtr;
+
+ /* check */
+ if (remainingSize < ZSTD_frameHeaderSize_min + ZSTD_blockHeaderSize)
+ return ERROR(srcSize_wrong);
+
+ /* Frame Header */
+ {
+ size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix);
+ if (ZSTD_isError(frameHeaderSize))
+ return frameHeaderSize;
+ if (remainingSize < frameHeaderSize + ZSTD_blockHeaderSize)
+ return ERROR(srcSize_wrong);
+ CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize));
+ ip += frameHeaderSize;
+ remainingSize -= frameHeaderSize;
+ }
+
+ /* Loop on each block */
+ while (1) {
+ size_t decodedSize;
+ blockProperties_t blockProperties;
+ size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
+ if (ZSTD_isError(cBlockSize))
+ return cBlockSize;
+
+ ip += ZSTD_blockHeaderSize;
+ remainingSize -= ZSTD_blockHeaderSize;
+ if (cBlockSize > remainingSize)
+ return ERROR(srcSize_wrong);
+
+ switch (blockProperties.blockType) {
+ case bt_compressed: decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend - op, ip, cBlockSize); break;
+ case bt_raw: decodedSize = ZSTD_copyRawBlock(op, oend - op, ip, cBlockSize); break;
+ case bt_rle: decodedSize = ZSTD_generateNxBytes(op, oend - op, *ip, blockProperties.origSize); break;
+ case bt_reserved:
+ default: return ERROR(corruption_detected);
+ }
+
+ if (ZSTD_isError(decodedSize))
+ return decodedSize;
+ if (dctx->fParams.checksumFlag)
+ xxh64_update(&dctx->xxhState, op, decodedSize);
+ op += decodedSize;
+ ip += cBlockSize;
+ remainingSize -= cBlockSize;
+ if (blockProperties.lastBlock)
+ break;
+ }
+
+ if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
+ U32 const checkCalc = (U32)xxh64_digest(&dctx->xxhState);
+ U32 checkRead;
+ if (remainingSize < 4)
+ return ERROR(checksum_wrong);
+ checkRead = ZSTD_readLE32(ip);
+ if (checkRead != checkCalc)
+ return ERROR(checksum_wrong);
+ ip += 4;
+ remainingSize -= 4;
+ }
+
+ /* Allow caller to get size read */
+ *srcPtr = ip;
+ *srcSizePtr = remainingSize;
+ return op - ostart;
+}
+
+static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict);
+static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict);
+
+static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize,
+ const ZSTD_DDict *ddict)
+{
+ void *const dststart = dst;
+
+ if (ddict) {
+ if (dict) {
+ /* programmer error, these two cases should be mutually exclusive */
+ return ERROR(GENERIC);
+ }
+
+ dict = ZSTD_DDictDictContent(ddict);
+ dictSize = ZSTD_DDictDictSize(ddict);
+ }
+
+ while (srcSize >= ZSTD_frameHeaderSize_prefix) {
+ U32 magicNumber;
+
+ magicNumber = ZSTD_readLE32(src);
+ if (magicNumber != ZSTD_MAGICNUMBER) {
+ if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+ size_t skippableSize;
+ if (srcSize < ZSTD_skippableHeaderSize)
+ return ERROR(srcSize_wrong);
+ skippableSize = ZSTD_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize;
+ if (srcSize < skippableSize) {
+ return ERROR(srcSize_wrong);
+ }
+
+ src = (const BYTE *)src + skippableSize;
+ srcSize -= skippableSize;
+ continue;
+ } else {
+ return ERROR(prefix_unknown);
+ }
+ }
+
+ if (ddict) {
+ /* we were called from ZSTD_decompress_usingDDict */
+ ZSTD_refDDict(dctx, ddict);
+ } else {
+ /* this will initialize correctly with no dict if dict == NULL, so
+ * use this in all cases but ddict */
+ CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
+ }
+ ZSTD_checkContinuity(dctx, dst);
+
+ {
+ const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize);
+ if (ZSTD_isError(res))
+ return res;
+ /* don't need to bounds check this, ZSTD_decompressFrame will have
+ * already */
+ dst = (BYTE *)dst + res;
+ dstCapacity -= res;
+ }
+ }
+
+ if (srcSize)
+ return ERROR(srcSize_wrong); /* input not entirely consumed */
+
+ return (BYTE *)dst - (BYTE *)dststart;
+}
+
+size_t ZSTD_decompress_usingDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize)
+{
+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
+}
+
+size_t ZSTD_decompressDCtx(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
+}
+
+/*-**************************************
+* Advanced Streaming Decompression API
+* Bufferless and synchronous
+****************************************/
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx *dctx) { return dctx->expected; }
+
+ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx *dctx)
+{
+ switch (dctx->stage) {
+ default: /* should not happen */
+ case ZSTDds_getFrameHeaderSize:
+ case ZSTDds_decodeFrameHeader: return ZSTDnit_frameHeader;
+ case ZSTDds_decodeBlockHeader: return ZSTDnit_blockHeader;
+ case ZSTDds_decompressBlock: return ZSTDnit_block;
+ case ZSTDds_decompressLastBlock: return ZSTDnit_lastBlock;
+ case ZSTDds_checkChecksum: return ZSTDnit_checksum;
+ case ZSTDds_decodeSkippableHeader:
+ case ZSTDds_skipFrame: return ZSTDnit_skippableFrame;
+ }
+}
+
+int ZSTD_isSkipFrame(ZSTD_DCtx *dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */
+
+/** ZSTD_decompressContinue() :
+* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
+* or an error code, which can be tested using ZSTD_isError() */
+size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ /* Sanity check */
+ if (srcSize != dctx->expected)
+ return ERROR(srcSize_wrong);
+ if (dstCapacity)
+ ZSTD_checkContinuity(dctx, dst);
+
+ switch (dctx->stage) {
+ case ZSTDds_getFrameHeaderSize:
+ if (srcSize != ZSTD_frameHeaderSize_prefix)
+ return ERROR(srcSize_wrong); /* impossible */
+ if ((ZSTD_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
+ dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */
+ dctx->stage = ZSTDds_decodeSkippableHeader;
+ return 0;
+ }
+ dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
+ if (ZSTD_isError(dctx->headerSize))
+ return dctx->headerSize;
+ memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
+ if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) {
+ dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix;
+ dctx->stage = ZSTDds_decodeFrameHeader;
+ return 0;
+ }
+ dctx->expected = 0; /* not necessary to copy more */
+
+ case ZSTDds_decodeFrameHeader:
+ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
+ CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
+ dctx->expected = ZSTD_blockHeaderSize;
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ return 0;
+
+ case ZSTDds_decodeBlockHeader: {
+ blockProperties_t bp;
+ size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
+ if (ZSTD_isError(cBlockSize))
+ return cBlockSize;
+ dctx->expected = cBlockSize;
+ dctx->bType = bp.blockType;
+ dctx->rleSize = bp.origSize;
+ if (cBlockSize) {
+ dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
+ return 0;
+ }
+ /* empty block */
+ if (bp.lastBlock) {
+ if (dctx->fParams.checksumFlag) {
+ dctx->expected = 4;
+ dctx->stage = ZSTDds_checkChecksum;
+ } else {
+ dctx->expected = 0; /* end of frame */
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ }
+ } else {
+ dctx->expected = 3; /* go directly to next header */
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ }
+ return 0;
+ }
+ case ZSTDds_decompressLastBlock:
+ case ZSTDds_decompressBlock: {
+ size_t rSize;
+ switch (dctx->bType) {
+ case bt_compressed: rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break;
+ case bt_raw: rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break;
+ case bt_rle: rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); break;
+ case bt_reserved: /* should never happen */
+ default: return ERROR(corruption_detected);
+ }
+ if (ZSTD_isError(rSize))
+ return rSize;
+ if (dctx->fParams.checksumFlag)
+ xxh64_update(&dctx->xxhState, dst, rSize);
+
+ if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
+ if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
+ dctx->expected = 4;
+ dctx->stage = ZSTDds_checkChecksum;
+ } else {
+ dctx->expected = 0; /* ends here */
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ }
+ } else {
+ dctx->stage = ZSTDds_decodeBlockHeader;
+ dctx->expected = ZSTD_blockHeaderSize;
+ dctx->previousDstEnd = (char *)dst + rSize;
+ }
+ return rSize;
+ }
+ case ZSTDds_checkChecksum: {
+ U32 const h32 = (U32)xxh64_digest(&dctx->xxhState);
+ U32 const check32 = ZSTD_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */
+ if (check32 != h32)
+ return ERROR(checksum_wrong);
+ dctx->expected = 0;
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ return 0;
+ }
+ case ZSTDds_decodeSkippableHeader: {
+ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
+ dctx->expected = ZSTD_readLE32(dctx->headerBuffer + 4);
+ dctx->stage = ZSTDds_skipFrame;
+ return 0;
+ }
+ case ZSTDds_skipFrame: {
+ dctx->expected = 0;
+ dctx->stage = ZSTDds_getFrameHeaderSize;
+ return 0;
+ }
+ default:
+ return ERROR(GENERIC); /* impossible */
+ }
+}
+
+static size_t ZSTD_refDictContent(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
+{
+ dctx->dictEnd = dctx->previousDstEnd;
+ dctx->vBase = (const char *)dict - ((const char *)(dctx->previousDstEnd) - (const char *)(dctx->base));
+ dctx->base = dict;
+ dctx->previousDstEnd = (const char *)dict + dictSize;
+ return 0;
+}
+
+/* ZSTD_loadEntropy() :
+ * dict : must point at beginning of a valid zstd dictionary
+ * @return : size of entropy tables read */
+static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t *entropy, const void *const dict, size_t const dictSize)
+{
+ const BYTE *dictPtr = (const BYTE *)dict;
+ const BYTE *const dictEnd = dictPtr + dictSize;
+
+ if (dictSize <= 8)
+ return ERROR(dictionary_corrupted);
+ dictPtr += 8; /* skip header = magic + dictID */
+
+ {
+ size_t const hSize = HUF_readDTableX4_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, entropy->workspace, sizeof(entropy->workspace));
+ if (HUF_isError(hSize))
+ return ERROR(dictionary_corrupted);
+ dictPtr += hSize;
+ }
+
+ {
+ short offcodeNCount[MaxOff + 1];
+ U32 offcodeMaxValue = MaxOff, offcodeLog;
+ size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr);
+ if (FSE_isError(offcodeHeaderSize))
+ return ERROR(dictionary_corrupted);
+ if (offcodeLog > OffFSELog)
+ return ERROR(dictionary_corrupted);
+ CHECK_E(FSE_buildDTable_wksp(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted);
+ dictPtr += offcodeHeaderSize;
+ }
+
+ {
+ short matchlengthNCount[MaxML + 1];
+ unsigned matchlengthMaxValue = MaxML, matchlengthLog;
+ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr);
+ if (FSE_isError(matchlengthHeaderSize))
+ return ERROR(dictionary_corrupted);
+ if (matchlengthLog > MLFSELog)
+ return ERROR(dictionary_corrupted);
+ CHECK_E(FSE_buildDTable_wksp(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted);
+ dictPtr += matchlengthHeaderSize;
+ }
+
+ {
+ short litlengthNCount[MaxLL + 1];
+ unsigned litlengthMaxValue = MaxLL, litlengthLog;
+ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr);
+ if (FSE_isError(litlengthHeaderSize))
+ return ERROR(dictionary_corrupted);
+ if (litlengthLog > LLFSELog)
+ return ERROR(dictionary_corrupted);
+ CHECK_E(FSE_buildDTable_wksp(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog, entropy->workspace, sizeof(entropy->workspace)), dictionary_corrupted);
+ dictPtr += litlengthHeaderSize;
+ }
+
+ if (dictPtr + 12 > dictEnd)
+ return ERROR(dictionary_corrupted);
+ {
+ int i;
+ size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12));
+ for (i = 0; i < 3; i++) {
+ U32 const rep = ZSTD_readLE32(dictPtr);
+ dictPtr += 4;
+ if (rep == 0 || rep >= dictContentSize)
+ return ERROR(dictionary_corrupted);
+ entropy->rep[i] = rep;
+ }
+ }
+
+ return dictPtr - (const BYTE *)dict;
+}
+
+static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
+{
+ if (dictSize < 8)
+ return ZSTD_refDictContent(dctx, dict, dictSize);
+ {
+ U32 const magic = ZSTD_readLE32(dict);
+ if (magic != ZSTD_DICT_MAGIC) {
+ return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
+ }
+ }
+ dctx->dictID = ZSTD_readLE32((const char *)dict + 4);
+
+ /* load entropy tables */
+ {
+ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
+ if (ZSTD_isError(eSize))
+ return ERROR(dictionary_corrupted);
+ dict = (const char *)dict + eSize;
+ dictSize -= eSize;
+ }
+ dctx->litEntropy = dctx->fseEntropy = 1;
+
+ /* reference dictionary content */
+ return ZSTD_refDictContent(dctx, dict, dictSize);
+}
+
+size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx *dctx, const void *dict, size_t dictSize)
+{
+ CHECK_F(ZSTD_decompressBegin(dctx));
+ if (dict && dictSize)
+ CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
+ return 0;
+}
+
+/* ====== ZSTD_DDict ====== */
+
+struct ZSTD_DDict_s {
+ void *dictBuffer;
+ const void *dictContent;
+ size_t dictSize;
+ ZSTD_entropyTables_t entropy;
+ U32 dictID;
+ U32 entropyPresent;
+ ZSTD_customMem cMem;
+}; /* typedef'd to ZSTD_DDict within "zstd.h" */
+
+size_t ZSTD_DDictWorkspaceBound(void) { return ZSTD_ALIGN(sizeof(ZSTD_stack)) + ZSTD_ALIGN(sizeof(ZSTD_DDict)); }
+
+static const void *ZSTD_DDictDictContent(const ZSTD_DDict *ddict) { return ddict->dictContent; }
+
+static size_t ZSTD_DDictDictSize(const ZSTD_DDict *ddict) { return ddict->dictSize; }
+
+static void ZSTD_refDDict(ZSTD_DCtx *dstDCtx, const ZSTD_DDict *ddict)
+{
+ ZSTD_decompressBegin(dstDCtx); /* init */
+ if (ddict) { /* support refDDict on NULL */
+ dstDCtx->dictID = ddict->dictID;
+ dstDCtx->base = ddict->dictContent;
+ dstDCtx->vBase = ddict->dictContent;
+ dstDCtx->dictEnd = (const BYTE *)ddict->dictContent + ddict->dictSize;
+ dstDCtx->previousDstEnd = dstDCtx->dictEnd;
+ if (ddict->entropyPresent) {
+ dstDCtx->litEntropy = 1;
+ dstDCtx->fseEntropy = 1;
+ dstDCtx->LLTptr = ddict->entropy.LLTable;
+ dstDCtx->MLTptr = ddict->entropy.MLTable;
+ dstDCtx->OFTptr = ddict->entropy.OFTable;
+ dstDCtx->HUFptr = ddict->entropy.hufTable;
+ dstDCtx->entropy.rep[0] = ddict->entropy.rep[0];
+ dstDCtx->entropy.rep[1] = ddict->entropy.rep[1];
+ dstDCtx->entropy.rep[2] = ddict->entropy.rep[2];
+ } else {
+ dstDCtx->litEntropy = 0;
+ dstDCtx->fseEntropy = 0;
+ }
+ }
+}
+
+static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict *ddict)
+{
+ ddict->dictID = 0;
+ ddict->entropyPresent = 0;
+ if (ddict->dictSize < 8)
+ return 0;
+ {
+ U32 const magic = ZSTD_readLE32(ddict->dictContent);
+ if (magic != ZSTD_DICT_MAGIC)
+ return 0; /* pure content mode */
+ }
+ ddict->dictID = ZSTD_readLE32((const char *)ddict->dictContent + 4);
+
+ /* load entropy tables */
+ CHECK_E(ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted);
+ ddict->entropyPresent = 1;
+ return 0;
+}
+
+static ZSTD_DDict *ZSTD_createDDict_advanced(const void *dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
+{
+ if (!customMem.customAlloc || !customMem.customFree)
+ return NULL;
+
+ {
+ ZSTD_DDict *const ddict = (ZSTD_DDict *)ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
+ if (!ddict)
+ return NULL;
+ ddict->cMem = customMem;
+
+ if ((byReference) || (!dict) || (!dictSize)) {
+ ddict->dictBuffer = NULL;
+ ddict->dictContent = dict;
+ } else {
+ void *const internalBuffer = ZSTD_malloc(dictSize, customMem);
+ if (!internalBuffer) {
+ ZSTD_freeDDict(ddict);
+ return NULL;
+ }
+ memcpy(internalBuffer, dict, dictSize);
+ ddict->dictBuffer = internalBuffer;
+ ddict->dictContent = internalBuffer;
+ }
+ ddict->dictSize = dictSize;
+ ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
+ /* parse dictionary content */
+ {
+ size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
+ if (ZSTD_isError(errorCode)) {
+ ZSTD_freeDDict(ddict);
+ return NULL;
+ }
+ }
+
+ return ddict;
+ }
+}
+
+/*! ZSTD_initDDict() :
+* Create a digested dictionary, to start decompression without startup delay.
+* `dict` content is copied inside DDict.
+* Consequently, `dict` can be released after `ZSTD_DDict` creation */
+ZSTD_DDict *ZSTD_initDDict(const void *dict, size_t dictSize, void *workspace, size_t workspaceSize)
+{
+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
+ return ZSTD_createDDict_advanced(dict, dictSize, 1, stackMem);
+}
+
+size_t ZSTD_freeDDict(ZSTD_DDict *ddict)
+{
+ if (ddict == NULL)
+ return 0; /* support free on NULL */
+ {
+ ZSTD_customMem const cMem = ddict->cMem;
+ ZSTD_free(ddict->dictBuffer, cMem);
+ ZSTD_free(ddict, cMem);
+ return 0;
+ }
+}
+
+/*! ZSTD_getDictID_fromDict() :
+ * Provides the dictID stored within dictionary.
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
+ * It can still be loaded, but as a content-only dictionary. */
+unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize)
+{
+ if (dictSize < 8)
+ return 0;
+ if (ZSTD_readLE32(dict) != ZSTD_DICT_MAGIC)
+ return 0;
+ return ZSTD_readLE32((const char *)dict + 4);
+}
+
+/*! ZSTD_getDictID_fromDDict() :
+ * Provides the dictID of the dictionary loaded into `ddict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict)
+{
+ if (ddict == NULL)
+ return 0;
+ return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
+}
+
+/*! ZSTD_getDictID_fromFrame() :
+ * Provides the dictID required to decompressed the frame stored within `src`.
+ * If @return == 0, the dictID could not be decoded.
+ * This could for one of the following reasons :
+ * - The frame does not require a dictionary to be decoded (most common case).
+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ * Note : this use case also happens when using a non-conformant dictionary.
+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ * - This is not a Zstandard frame.
+ * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
+unsigned ZSTD_getDictID_fromFrame(const void *src, size_t srcSize)
+{
+ ZSTD_frameParams zfp = {0, 0, 0, 0};
+ size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
+ if (ZSTD_isError(hError))
+ return 0;
+ return zfp.dictID;
+}
+
+/*! ZSTD_decompress_usingDDict() :
+* Decompression using a pre-digested Dictionary
+* Use dictionary without significant overhead. */
+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx *dctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_DDict *ddict)
+{
+ /* pass content and size in case legacy frames are encountered */
+ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, NULL, 0, ddict);
+}
+
+/*=====================================
+* Streaming decompression
+*====================================*/
+
+typedef enum { zdss_init, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
+
+/* *** Resource management *** */
+struct ZSTD_DStream_s {
+ ZSTD_DCtx *dctx;
+ ZSTD_DDict *ddictLocal;
+ const ZSTD_DDict *ddict;
+ ZSTD_frameParams fParams;
+ ZSTD_dStreamStage stage;
+ char *inBuff;
+ size_t inBuffSize;
+ size_t inPos;
+ size_t maxWindowSize;
+ char *outBuff;
+ size_t outBuffSize;
+ size_t outStart;
+ size_t outEnd;
+ size_t blockSize;
+ BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */
+ size_t lhSize;
+ ZSTD_customMem customMem;
+ void *legacyContext;
+ U32 previousLegacyVersion;
+ U32 legacyVersion;
+ U32 hostageByte;
+}; /* typedef'd to ZSTD_DStream within "zstd.h" */
+
+size_t ZSTD_DStreamWorkspaceBound(size_t maxWindowSize)
+{
+ size_t const blockSize = MIN(maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+ size_t const inBuffSize = blockSize;
+ size_t const outBuffSize = maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
+ return ZSTD_DCtxWorkspaceBound() + ZSTD_ALIGN(sizeof(ZSTD_DStream)) + ZSTD_ALIGN(inBuffSize) + ZSTD_ALIGN(outBuffSize);
+}
+
+static ZSTD_DStream *ZSTD_createDStream_advanced(ZSTD_customMem customMem)
+{
+ ZSTD_DStream *zds;
+
+ if (!customMem.customAlloc || !customMem.customFree)
+ return NULL;
+
+ zds = (ZSTD_DStream *)ZSTD_malloc(sizeof(ZSTD_DStream), customMem);
+ if (zds == NULL)
+ return NULL;
+ memset(zds, 0, sizeof(ZSTD_DStream));
+ memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
+ zds->dctx = ZSTD_createDCtx_advanced(customMem);
+ if (zds->dctx == NULL) {
+ ZSTD_freeDStream(zds);
+ return NULL;
+ }
+ zds->stage = zdss_init;
+ zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
+ return zds;
+}
+
+ZSTD_DStream *ZSTD_initDStream(size_t maxWindowSize, void *workspace, size_t workspaceSize)
+{
+ ZSTD_customMem const stackMem = ZSTD_initStack(workspace, workspaceSize);
+ ZSTD_DStream *zds = ZSTD_createDStream_advanced(stackMem);
+ if (!zds) {
+ return NULL;
+ }
+
+ zds->maxWindowSize = maxWindowSize;
+ zds->stage = zdss_loadHeader;
+ zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
+ ZSTD_freeDDict(zds->ddictLocal);
+ zds->ddictLocal = NULL;
+ zds->ddict = zds->ddictLocal;
+ zds->legacyVersion = 0;
+ zds->hostageByte = 0;
+
+ {
+ size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+ size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
+
+ zds->inBuff = (char *)ZSTD_malloc(blockSize, zds->customMem);
+ zds->inBuffSize = blockSize;
+ zds->outBuff = (char *)ZSTD_malloc(neededOutSize, zds->customMem);
+ zds->outBuffSize = neededOutSize;
+ if (zds->inBuff == NULL || zds->outBuff == NULL) {
+ ZSTD_freeDStream(zds);
+ return NULL;
+ }
+ }
+ return zds;
+}
+
+ZSTD_DStream *ZSTD_initDStream_usingDDict(size_t maxWindowSize, const ZSTD_DDict *ddict, void *workspace, size_t workspaceSize)
+{
+ ZSTD_DStream *zds = ZSTD_initDStream(maxWindowSize, workspace, workspaceSize);
+ if (zds) {
+ zds->ddict = ddict;
+ }
+ return zds;
+}
+
+size_t ZSTD_freeDStream(ZSTD_DStream *zds)
+{
+ if (zds == NULL)
+ return 0; /* support free on null */
+ {
+ ZSTD_customMem const cMem = zds->customMem;
+ ZSTD_freeDCtx(zds->dctx);
+ zds->dctx = NULL;
+ ZSTD_freeDDict(zds->ddictLocal);
+ zds->ddictLocal = NULL;
+ ZSTD_free(zds->inBuff, cMem);
+ zds->inBuff = NULL;
+ ZSTD_free(zds->outBuff, cMem);
+ zds->outBuff = NULL;
+ ZSTD_free(zds, cMem);
+ return 0;
+ }
+}
+
+/* *** Initialization *** */
+
+size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
+size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
+
+size_t ZSTD_resetDStream(ZSTD_DStream *zds)
+{
+ zds->stage = zdss_loadHeader;
+ zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
+ zds->legacyVersion = 0;
+ zds->hostageByte = 0;
+ return ZSTD_frameHeaderSize_prefix;
+}
+
+/* ***** Decompression ***** */
+
+ZSTD_STATIC size_t ZSTD_limitCopy(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
+{
+ size_t const length = MIN(dstCapacity, srcSize);
+ memcpy(dst, src, length);
+ return length;
+}
+
+size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
+{
+ const char *const istart = (const char *)(input->src) + input->pos;
+ const char *const iend = (const char *)(input->src) + input->size;
+ const char *ip = istart;
+ char *const ostart = (char *)(output->dst) + output->pos;
+ char *const oend = (char *)(output->dst) + output->size;
+ char *op = ostart;
+ U32 someMoreWork = 1;
+
+ while (someMoreWork) {
+ switch (zds->stage) {
+ case zdss_init:
+ ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
+ /* fall-through */
+
+ case zdss_loadHeader: {
+ size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
+ if (ZSTD_isError(hSize))
+ return hSize;
+ if (hSize != 0) { /* need more input */
+ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
+ if (toLoad > (size_t)(iend - ip)) { /* not enough input to load full header */
+ memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip);
+ zds->lhSize += iend - ip;
+ input->pos = input->size;
+ return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) +
+ ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
+ }
+ memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad);
+ zds->lhSize = hSize;
+ ip += toLoad;
+ break;
+ }
+
+ /* check for single-pass mode opportunity */
+ if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
+ && (U64)(size_t)(oend - op) >= zds->fParams.frameContentSize) {
+ size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart);
+ if (cSize <= (size_t)(iend - istart)) {
+ size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend - op, istart, cSize, zds->ddict);
+ if (ZSTD_isError(decompressedSize))
+ return decompressedSize;
+ ip = istart + cSize;
+ op += decompressedSize;
+ zds->dctx->expected = 0;
+ zds->stage = zdss_init;
+ someMoreWork = 0;
+ break;
+ }
+ }
+
+ /* Consume header */
+ ZSTD_refDDict(zds->dctx, zds->ddict);
+ {
+ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
+ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
+ {
+ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
+ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer + h1Size, h2Size));
+ }
+ }
+
+ zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
+ if (zds->fParams.windowSize > zds->maxWindowSize)
+ return ERROR(frameParameter_windowTooLarge);
+
+ /* Buffers are preallocated, but double check */
+ {
+ size_t const blockSize = MIN(zds->maxWindowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+ size_t const neededOutSize = zds->maxWindowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
+ if (zds->inBuffSize < blockSize) {
+ return ERROR(GENERIC);
+ }
+ if (zds->outBuffSize < neededOutSize) {
+ return ERROR(GENERIC);
+ }
+ zds->blockSize = blockSize;
+ }
+ zds->stage = zdss_read;
+ }
+ /* pass-through */
+
+ case zdss_read: {
+ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
+ if (neededInSize == 0) { /* end of frame */
+ zds->stage = zdss_init;
+ someMoreWork = 0;
+ break;
+ }
+ if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */
+ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
+ size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart,
+ (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize);
+ if (ZSTD_isError(decodedSize))
+ return decodedSize;
+ ip += neededInSize;
+ if (!decodedSize && !isSkipFrame)
+ break; /* this was just a header */
+ zds->outEnd = zds->outStart + decodedSize;
+ zds->stage = zdss_flush;
+ break;
+ }
+ if (ip == iend) {
+ someMoreWork = 0;
+ break;
+ } /* no more input */
+ zds->stage = zdss_load;
+ /* pass-through */
+ }
+
+ case zdss_load: {
+ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
+ size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
+ size_t loadedSize;
+ if (toLoad > zds->inBuffSize - zds->inPos)
+ return ERROR(corruption_detected); /* should never happen */
+ loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip);
+ ip += loadedSize;
+ zds->inPos += loadedSize;
+ if (loadedSize < toLoad) {
+ someMoreWork = 0;
+ break;
+ } /* not enough input, wait for more */
+
+ /* decode loaded input */
+ {
+ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
+ size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
+ zds->inBuff, neededInSize);
+ if (ZSTD_isError(decodedSize))
+ return decodedSize;
+ zds->inPos = 0; /* input is consumed */
+ if (!decodedSize && !isSkipFrame) {
+ zds->stage = zdss_read;
+ break;
+ } /* this was just a header */
+ zds->outEnd = zds->outStart + decodedSize;
+ zds->stage = zdss_flush;
+ /* pass-through */
+ }
+ }
+
+ case zdss_flush: {
+ size_t const toFlushSize = zds->outEnd - zds->outStart;
+ size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize);
+ op += flushedSize;
+ zds->outStart += flushedSize;
+ if (flushedSize == toFlushSize) { /* flush completed */
+ zds->stage = zdss_read;
+ if (zds->outStart + zds->blockSize > zds->outBuffSize)
+ zds->outStart = zds->outEnd = 0;
+ break;
+ }
+ /* cannot complete flush */
+ someMoreWork = 0;
+ break;
+ }
+ default:
+ return ERROR(GENERIC); /* impossible */
+ }
+ }
+
+ /* result */
+ input->pos += (size_t)(ip - istart);
+ output->pos += (size_t)(op - ostart);
+ {
+ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
+ if (!nextSrcSizeHint) { /* frame fully decoded */
+ if (zds->outEnd == zds->outStart) { /* output fully flushed */
+ if (zds->hostageByte) {
+ if (input->pos >= input->size) {
+ zds->stage = zdss_read;
+ return 1;
+ } /* can't release hostage (not present) */
+ input->pos++; /* release hostage */
+ }
+ return 0;
+ }
+ if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
+ input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
+ zds->hostageByte = 1;
+ }
+ return 1;
+ }
+ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */
+ if (zds->inPos > nextSrcSizeHint)
+ return ERROR(GENERIC); /* should never happen */
+ nextSrcSizeHint -= zds->inPos; /* already loaded*/
+ return nextSrcSizeHint;
+ }
+}
+
+EXPORT_SYMBOL(ZSTD_DCtxWorkspaceBound);
+EXPORT_SYMBOL(ZSTD_initDCtx);
+EXPORT_SYMBOL(ZSTD_decompressDCtx);
+EXPORT_SYMBOL(ZSTD_decompress_usingDict);
+
+EXPORT_SYMBOL(ZSTD_DDictWorkspaceBound);
+EXPORT_SYMBOL(ZSTD_initDDict);
+EXPORT_SYMBOL(ZSTD_decompress_usingDDict);
+
+EXPORT_SYMBOL(ZSTD_DStreamWorkspaceBound);
+EXPORT_SYMBOL(ZSTD_initDStream);
+EXPORT_SYMBOL(ZSTD_initDStream_usingDDict);
+EXPORT_SYMBOL(ZSTD_resetDStream);
+EXPORT_SYMBOL(ZSTD_decompressStream);
+EXPORT_SYMBOL(ZSTD_DStreamInSize);
+EXPORT_SYMBOL(ZSTD_DStreamOutSize);
+
+EXPORT_SYMBOL(ZSTD_findFrameCompressedSize);
+EXPORT_SYMBOL(ZSTD_getFrameContentSize);
+EXPORT_SYMBOL(ZSTD_findDecompressedSize);
+
+EXPORT_SYMBOL(ZSTD_isFrame);
+EXPORT_SYMBOL(ZSTD_getDictID_fromDict);
+EXPORT_SYMBOL(ZSTD_getDictID_fromDDict);
+EXPORT_SYMBOL(ZSTD_getDictID_fromFrame);
+
+EXPORT_SYMBOL(ZSTD_getFrameParams);
+EXPORT_SYMBOL(ZSTD_decompressBegin);
+EXPORT_SYMBOL(ZSTD_decompressBegin_usingDict);
+EXPORT_SYMBOL(ZSTD_copyDCtx);
+EXPORT_SYMBOL(ZSTD_nextSrcSizeToDecompress);
+EXPORT_SYMBOL(ZSTD_decompressContinue);
+EXPORT_SYMBOL(ZSTD_nextInputType);
+
+EXPORT_SYMBOL(ZSTD_decompressBlock);
+EXPORT_SYMBOL(ZSTD_insertBlock);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Zstd Decompressor");
diff --git a/lib/zstd/entropy_common.c b/lib/zstd/entropy_common.c
new file mode 100644
index 000000000000..2b0a643c32c4
--- /dev/null
+++ b/lib/zstd/entropy_common.c
@@ -0,0 +1,243 @@
+/*
+ * Common functions of New Generation Entropy library
+ * Copyright (C) 2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ */
+
+/* *************************************
+* Dependencies
+***************************************/
+#include "error_private.h" /* ERR_*, ERROR */
+#include "fse.h"
+#include "huf.h"
+#include "mem.h"
+
+/*=== Version ===*/
+unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
+
+/*=== Error Management ===*/
+unsigned FSE_isError(size_t code) { return ERR_isError(code); }
+
+unsigned HUF_isError(size_t code) { return ERR_isError(code); }
+
+/*-**************************************************************
+* FSE NCount encoding-decoding
+****************************************************************/
+size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSVPtr, unsigned *tableLogPtr, const void *headerBuffer, size_t hbSize)
+{
+ const BYTE *const istart = (const BYTE *)headerBuffer;
+ const BYTE *const iend = istart + hbSize;
+ const BYTE *ip = istart;
+ int nbBits;
+ int remaining;
+ int threshold;
+ U32 bitStream;
+ int bitCount;
+ unsigned charnum = 0;
+ int previous0 = 0;
+
+ if (hbSize < 4)
+ return ERROR(srcSize_wrong);
+ bitStream = ZSTD_readLE32(ip);
+ nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
+ if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX)
+ return ERROR(tableLog_tooLarge);
+ bitStream >>= 4;
+ bitCount = 4;
+ *tableLogPtr = nbBits;
+ remaining = (1 << nbBits) + 1;
+ threshold = 1 << nbBits;
+ nbBits++;
+
+ while ((remaining > 1) & (charnum <= *maxSVPtr)) {
+ if (previous0) {
+ unsigned n0 = charnum;
+ while ((bitStream & 0xFFFF) == 0xFFFF) {
+ n0 += 24;
+ if (ip < iend - 5) {
+ ip += 2;
+ bitStream = ZSTD_readLE32(ip) >> bitCount;
+ } else {
+ bitStream >>= 16;
+ bitCount += 16;
+ }
+ }
+ while ((bitStream & 3) == 3) {
+ n0 += 3;
+ bitStream >>= 2;
+ bitCount += 2;
+ }
+ n0 += bitStream & 3;
+ bitCount += 2;
+ if (n0 > *maxSVPtr)
+ return ERROR(maxSymbolValue_tooSmall);
+ while (charnum < n0)
+ normalizedCounter[charnum++] = 0;
+ if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) {
+ ip += bitCount >> 3;
+ bitCount &= 7;
+ bitStream = ZSTD_readLE32(ip) >> bitCount;
+ } else {
+ bitStream >>= 2;
+ }
+ }
+ {
+ int const max = (2 * threshold - 1) - remaining;
+ int count;
+
+ if ((bitStream & (threshold - 1)) < (U32)max) {
+ count = bitStream & (threshold - 1);
+ bitCount += nbBits - 1;
+ } else {
+ count = bitStream & (2 * threshold - 1);
+ if (count >= threshold)
+ count -= max;
+ bitCount += nbBits;
+ }
+
+ count--; /* extra accuracy */
+ remaining -= count < 0 ? -count : count; /* -1 means +1 */
+ normalizedCounter[charnum++] = (short)count;
+ previous0 = !count;
+ while (remaining < threshold) {
+ nbBits--;
+ threshold >>= 1;
+ }
+
+ if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) {
+ ip += bitCount >> 3;
+ bitCount &= 7;
+ } else {
+ bitCount -= (int)(8 * (iend - 4 - ip));
+ ip = iend - 4;
+ }
+ bitStream = ZSTD_readLE32(ip) >> (bitCount & 31);
+ }
+ } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
+ if (remaining != 1)
+ return ERROR(corruption_detected);
+ if (bitCount > 32)
+ return ERROR(corruption_detected);
+ *maxSVPtr = charnum - 1;
+
+ ip += (bitCount + 7) >> 3;
+ return ip - istart;
+}
+
+/*! HUF_readStats() :
+ Read compact Huffman tree, saved by HUF_writeCTable().
+ `huffWeight` is destination buffer.
+ `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
+ @return : size read from `src` , or an error Code .
+ Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
+*/
+size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
+{
+ U32 weightTotal;
+ const BYTE *ip = (const BYTE *)src;
+ size_t iSize;
+ size_t oSize;
+
+ if (!srcSize)
+ return ERROR(srcSize_wrong);
+ iSize = ip[0];
+ /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
+
+ if (iSize >= 128) { /* special header */
+ oSize = iSize - 127;
+ iSize = ((oSize + 1) / 2);
+ if (iSize + 1 > srcSize)
+ return ERROR(srcSize_wrong);
+ if (oSize >= hwSize)
+ return ERROR(corruption_detected);
+ ip += 1;
+ {
+ U32 n;
+ for (n = 0; n < oSize; n += 2) {
+ huffWeight[n] = ip[n / 2] >> 4;
+ huffWeight[n + 1] = ip[n / 2] & 15;
+ }
+ }
+ } else { /* header compressed with FSE (normal case) */
+ if (iSize + 1 > srcSize)
+ return ERROR(srcSize_wrong);
+ oSize = FSE_decompress_wksp(huffWeight, hwSize - 1, ip + 1, iSize, 6, workspace, workspaceSize); /* max (hwSize-1) values decoded, as last one is implied */
+ if (FSE_isError(oSize))
+ return oSize;
+ }
+
+ /* collect weight stats */
+ memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
+ weightTotal = 0;
+ {
+ U32 n;
+ for (n = 0; n < oSize; n++) {
+ if (huffWeight[n] >= HUF_TABLELOG_MAX)
+ return ERROR(corruption_detected);
+ rankStats[huffWeight[n]]++;
+ weightTotal += (1 << huffWeight[n]) >> 1;
+ }
+ }
+ if (weightTotal == 0)
+ return ERROR(corruption_detected);
+
+ /* get last non-null symbol weight (implied, total must be 2^n) */
+ {
+ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
+ if (tableLog > HUF_TABLELOG_MAX)
+ return ERROR(corruption_detected);
+ *tableLogPtr = tableLog;
+ /* determine last weight */
+ {
+ U32 const total = 1 << tableLog;
+ U32 const rest = total - weightTotal;
+ U32 const verif = 1 << BIT_highbit32(rest);
+ U32 const lastWeight = BIT_highbit32(rest) + 1;
+ if (verif != rest)
+ return ERROR(corruption_detected); /* last value must be a clean power of 2 */
+ huffWeight[oSize] = (BYTE)lastWeight;
+ rankStats[lastWeight]++;
+ }
+ }
+
+ /* check tree construction validity */
+ if ((rankStats[1] < 2) || (rankStats[1] & 1))
+ return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
+
+ /* results */
+ *nbSymbolsPtr = (U32)(oSize + 1);
+ return iSize + 1;
+}
diff --git a/lib/zstd/error_private.h b/lib/zstd/error_private.h
new file mode 100644
index 000000000000..1a60b31f706c
--- /dev/null
+++ b/lib/zstd/error_private.h
@@ -0,0 +1,53 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
+ * An additional grant of patent rights can be found in the PATENTS file in the
+ * same directory.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ */
+
+/* Note : this module is expected to remain private, do not expose it */
+
+#ifndef ERROR_H_MODULE
+#define ERROR_H_MODULE
+
+/* ****************************************
+* Dependencies
+******************************************/
+#include <linux/types.h> /* size_t */
+#include <linux/zstd.h> /* enum list */
+
+/* ****************************************
+* Compiler-specific
+******************************************/
+#define ERR_STATIC static __attribute__((unused))
+
+/*-****************************************
+* Customization (error_public.h)
+******************************************/
+typedef ZSTD_ErrorCode ERR_enum;
+#define PREFIX(name) ZSTD_error_##name
+
+/*-****************************************
+* Error codes handling
+******************************************/
+#define ERROR(name) ((size_t)-PREFIX(name))
+
+ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
+
+ERR_STATIC ERR_enum ERR_getErrorCode(size_t code)
+{
+ if (!ERR_isError(code))
+ return (ERR_enum)0;
+ return (ERR_enum)(0 - code);
+}
+
+#endif /* ERROR_H_MODULE */
diff --git a/lib/zstd/fse.h b/lib/zstd/fse.h
new file mode 100644
index 000000000000..7460ab04b191
--- /dev/null
+++ b/lib/zstd/fse.h
@@ -0,0 +1,575 @@
+/*
+ * FSE : Finite State Entropy codec
+ * Public Prototypes declaration
+ * Copyright (C) 2013-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ */
+#ifndef FSE_H
+#define FSE_H
+
+/*-*****************************************
+* Dependencies
+******************************************/
+#include <linux/types.h> /* size_t, ptrdiff_t */
+
+/*-*****************************************
+* FSE_PUBLIC_API : control library symbols visibility
+******************************************/
+#define FSE_PUBLIC_API
+
+/*------ Version ------*/
+#define FSE_VERSION_MAJOR 0
+#define FSE_VERSION_MINOR 9
+#define FSE_VERSION_RELEASE 0
+
+#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
+#define FSE_QUOTE(str) #str
+#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
+#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
+
+#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR * 100 * 100 + FSE_VERSION_MINOR * 100 + FSE_VERSION_RELEASE)
+FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
+
+/*-*****************************************
+* Tool functions
+******************************************/
+FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
+
+/* Error Management */
+FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
+
+/*-*****************************************
+* FSE detailed API
+******************************************/
+/*!
+FSE_compress() does the following:
+1. count symbol occurrence from source[] into table count[]
+2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
+3. save normalized counters to memory buffer using writeNCount()
+4. build encoding table 'CTable' from normalized counters
+5. encode the data stream using encoding table 'CTable'
+
+FSE_decompress() does the following:
+1. read normalized counters with readNCount()
+2. build decoding table 'DTable' from normalized counters
+3. decode the data stream using decoding table 'DTable'
+
+The following API allows targeting specific sub-functions for advanced tasks.
+For example, it's possible to compress several blocks using the same 'CTable',
+or to save and provide normalized distribution using external method.
+*/
+
+/* *** COMPRESSION *** */
+/*! FSE_optimalTableLog():
+ dynamically downsize 'tableLog' when conditions are met.
+ It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
+ @return : recommended tableLog (necessarily <= 'maxTableLog') */
+FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
+
+/*! FSE_normalizeCount():
+ normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
+ 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
+ @return : tableLog,
+ or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue);
+
+/*! FSE_NCountWriteBound():
+ Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
+ Typically useful for allocation purpose. */
+FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
+
+/*! FSE_writeNCount():
+ Compactly save 'normalizedCounter' into 'buffer'.
+ @return : size of the compressed table,
+ or an errorCode, which can be tested using FSE_isError(). */
+FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
+
+/*! Constructor and Destructor of FSE_CTable.
+ Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
+typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
+
+/*! FSE_compress_usingCTable():
+ Compress `src` using `ct` into `dst` which must be already allocated.
+ @return : size of compressed data (<= `dstCapacity`),
+ or 0 if compressed data could not fit into `dst`,
+ or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_compress_usingCTable(void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct);
+
+/*!
+Tutorial :
+----------
+The first step is to count all symbols. FSE_count() does this job very fast.
+Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
+'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
+maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
+FSE_count() will return the number of occurrence of the most frequent symbol.
+This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
+
+The next step is to normalize the frequencies.
+FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
+It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
+You can use 'tableLog'==0 to mean "use default tableLog value".
+If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
+which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
+
+The result of FSE_normalizeCount() will be saved into a table,
+called 'normalizedCounter', which is a table of signed short.
+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
+The return value is tableLog if everything proceeded as expected.
+It is 0 if there is a single symbol within distribution.
+If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
+
+'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
+'buffer' must be already allocated.
+For guaranteed success, buffer size must be at least FSE_headerBound().
+The result of the function is the number of bytes written into 'buffer'.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
+
+'normalizedCounter' can then be used to create the compression table 'CTable'.
+The space required by 'CTable' must be already allocated, using FSE_createCTable().
+You can then use FSE_buildCTable() to fill 'CTable'.
+If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
+
+'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
+Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
+The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
+If it returns '0', compressed data could not fit into 'dst'.
+If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
+*/
+
+/* *** DECOMPRESSION *** */
+
+/*! FSE_readNCount():
+ Read compactly saved 'normalizedCounter' from 'rBuffer'.
+ @return : size read from 'rBuffer',
+ or an errorCode, which can be tested using FSE_isError().
+ maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
+FSE_PUBLIC_API size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize);
+
+/*! Constructor and Destructor of FSE_DTable.
+ Note that its size depends on 'tableLog' */
+typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
+
+/*! FSE_buildDTable():
+ Builds 'dt', which must be already allocated, using FSE_createDTable().
+ return : 0, or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize);
+
+/*! FSE_decompress_usingDTable():
+ Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
+ into `dst` which must be already allocated.
+ @return : size of regenerated data (necessarily <= `dstCapacity`),
+ or an errorCode, which can be tested using FSE_isError() */
+FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt);
+
+/*!
+Tutorial :
+----------
+(Note : these functions only decompress FSE-compressed blocks.
+ If block is uncompressed, use memcpy() instead
+ If block is a single repeated byte, use memset() instead )
+
+The first step is to obtain the normalized frequencies of symbols.
+This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
+'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
+In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
+or size the table to handle worst case situations (typically 256).
+FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
+The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
+Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
+If there is an error, the function will return an error code, which can be tested using FSE_isError().
+
+The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
+This is performed by the function FSE_buildDTable().
+The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
+If there is an error, the function will return an error code, which can be tested using FSE_isError().
+
+`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
+`cSrcSize` must be strictly correct, otherwise decompression will fail.
+FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
+If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
+*/
+
+/* *** Dependency *** */
+#include "bitstream.h"
+
+/* *****************************************
+* Static allocation
+*******************************************/
+/* FSE buffer bounds */
+#define FSE_NCOUNTBOUND 512
+#define FSE_BLOCKBOUND(size) (size + (size >> 7))
+#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
+
+/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
+#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2))
+#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog))
+
+/* *****************************************
+* FSE advanced API
+*******************************************/
+/* FSE_count_wksp() :
+ * Same as FSE_count(), but using an externally provided scratch buffer.
+ * `workSpace` size must be table of >= `1024` unsigned
+ */
+size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace);
+
+/* FSE_countFast_wksp() :
+ * Same as FSE_countFast(), but using an externally provided scratch buffer.
+ * `workSpace` must be a table of minimum `1024` unsigned
+ */
+size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize, unsigned *workSpace);
+
+/*! FSE_count_simple
+ * Same as FSE_countFast(), but does not use any additional memory (not even on stack).
+ * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
+*/
+size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize);
+
+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
+/**< same as FSE_optimalTableLog(), which used `minus==2` */
+
+size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits);
+/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
+
+size_t FSE_buildCTable_rle(FSE_CTable *ct, unsigned char symbolValue);
+/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
+
+/* FSE_buildCTable_wksp() :
+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
+ * `wkspSize` must be >= `(1<<tableLog)`.
+ */
+size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, size_t wkspSize);
+
+size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits);
+/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
+
+size_t FSE_buildDTable_rle(FSE_DTable *dt, unsigned char symbolValue);
+/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
+
+size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize);
+/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
+
+/* *****************************************
+* FSE symbol compression API
+*******************************************/
+/*!
+ This API consists of small unitary functions, which highly benefit from being inlined.
+ Hence their body are included in next section.
+*/
+typedef struct {
+ ptrdiff_t value;
+ const void *stateTable;
+ const void *symbolTT;
+ unsigned stateLog;
+} FSE_CState_t;
+
+static void FSE_initCState(FSE_CState_t *CStatePtr, const FSE_CTable *ct);
+
+static void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *CStatePtr, unsigned symbol);
+
+static void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *CStatePtr);
+
+/**<
+These functions are inner components of FSE_compress_usingCTable().
+They allow the creation of custom streams, mixing multiple tables and bit sources.
+
+A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
+So the first symbol you will encode is the last you will decode, like a LIFO stack.
+
+You will need a few variables to track your CStream. They are :
+
+FSE_CTable ct; // Provided by FSE_buildCTable()
+BIT_CStream_t bitStream; // bitStream tracking structure
+FSE_CState_t state; // State tracking structure (can have several)
+
+
+The first thing to do is to init bitStream and state.
+ size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
+ FSE_initCState(&state, ct);
+
+Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
+You can then encode your input data, byte after byte.
+FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
+Remember decoding will be done in reverse direction.
+ FSE_encodeByte(&bitStream, &state, symbol);
+
+At any time, you can also add any bit sequence.
+Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
+ BIT_addBits(&bitStream, bitField, nbBits);
+
+The above methods don't commit data to memory, they just store it into local register, for speed.
+Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
+Writing data to memory is a manual operation, performed by the flushBits function.
+ BIT_flushBits(&bitStream);
+
+Your last FSE encoding operation shall be to flush your last state value(s).
+ FSE_flushState(&bitStream, &state);
+
+Finally, you must close the bitStream.
+The function returns the size of CStream in bytes.
+If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
+If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
+ size_t size = BIT_closeCStream(&bitStream);
+*/
+
+/* *****************************************
+* FSE symbol decompression API
+*******************************************/
+typedef struct {
+ size_t state;
+ const void *table; /* precise table may vary, depending on U16 */
+} FSE_DState_t;
+
+static void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt);
+
+static unsigned char FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
+
+static unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr);
+
+/**<
+Let's now decompose FSE_decompress_usingDTable() into its unitary components.
+You will decode FSE-encoded symbols from the bitStream,
+and also any other bitFields you put in, **in reverse order**.
+
+You will need a few variables to track your bitStream. They are :
+
+BIT_DStream_t DStream; // Stream context
+FSE_DState_t DState; // State context. Multiple ones are possible
+FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
+
+The first thing to do is to init the bitStream.
+ errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
+
+You should then retrieve your initial state(s)
+(in reverse flushing order if you have several ones) :
+ errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
+
+You can then decode your data, symbol after symbol.
+For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
+Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
+ unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
+
+You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
+Note : maximum allowed nbBits is 25, for 32-bits compatibility
+ size_t bitField = BIT_readBits(&DStream, nbBits);
+
+All above operations only read from local register (which size depends on size_t).
+Refueling the register from memory is manually performed by the reload method.
+ endSignal = FSE_reloadDStream(&DStream);
+
+BIT_reloadDStream() result tells if there is still some more data to read from DStream.
+BIT_DStream_unfinished : there is still some data left into the DStream.
+BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
+BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
+BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
+
+When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
+to properly detect the exact end of stream.
+After each decoded symbol, check if DStream is fully consumed using this simple test :
+ BIT_reloadDStream(&DStream) >= BIT_DStream_completed
+
+When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
+Checking if DStream has reached its end is performed by :
+ BIT_endOfDStream(&DStream);
+Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
+ FSE_endOfDState(&DState);
+*/
+
+/* *****************************************
+* FSE unsafe API
+*******************************************/
+static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD);
+/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
+
+/* *****************************************
+* Implementation of inlined functions
+*******************************************/
+typedef struct {
+ int deltaFindState;
+ U32 deltaNbBits;
+} FSE_symbolCompressionTransform; /* total 8 bytes */
+
+ZSTD_STATIC void FSE_initCState(FSE_CState_t *statePtr, const FSE_CTable *ct)
+{
+ const void *ptr = ct;
+ const U16 *u16ptr = (const U16 *)ptr;
+ const U32 tableLog = ZSTD_read16(ptr);
+ statePtr->value = (ptrdiff_t)1 << tableLog;
+ statePtr->stateTable = u16ptr + 2;
+ statePtr->symbolTT = ((const U32 *)ct + 1 + (tableLog ? (1 << (tableLog - 1)) : 1));
+ statePtr->stateLog = tableLog;
+}
+
+/*! FSE_initCState2() :
+* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
+* uses the smallest state value possible, saving the cost of this symbol */
+ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol)
+{
+ FSE_initCState(statePtr, ct);
+ {
+ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol];
+ const U16 *stateTable = (const U16 *)(statePtr->stateTable);
+ U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16);
+ statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
+ statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
+ }
+}
+
+ZSTD_STATIC void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *statePtr, U32 symbol)
+{
+ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol];
+ const U16 *const stateTable = (const U16 *)(statePtr->stateTable);
+ U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
+ BIT_addBits(bitC, statePtr->value, nbBitsOut);
+ statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
+}
+
+ZSTD_STATIC void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *statePtr)
+{
+ BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
+ BIT_flushBits(bitC);
+}
+
+/* ====== Decompression ====== */
+
+typedef struct {
+ U16 tableLog;
+ U16 fastMode;
+} FSE_DTableHeader; /* sizeof U32 */
+
+typedef struct {
+ unsigned short newState;
+ unsigned char symbol;
+ unsigned char nbBits;
+} FSE_decode_t; /* size == U32 */
+
+ZSTD_STATIC void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt)
+{
+ const void *ptr = dt;
+ const FSE_DTableHeader *const DTableH = (const FSE_DTableHeader *)ptr;
+ DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
+ BIT_reloadDStream(bitD);
+ DStatePtr->table = dt + 1;
+}
+
+ZSTD_STATIC BYTE FSE_peekSymbol(const FSE_DState_t *DStatePtr)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
+ return DInfo.symbol;
+}
+
+ZSTD_STATIC void FSE_updateState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
+ U32 const nbBits = DInfo.nbBits;
+ size_t const lowBits = BIT_readBits(bitD, nbBits);
+ DStatePtr->state = DInfo.newState + lowBits;
+}
+
+ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
+ U32 const nbBits = DInfo.nbBits;
+ BYTE const symbol = DInfo.symbol;
+ size_t const lowBits = BIT_readBits(bitD, nbBits);
+
+ DStatePtr->state = DInfo.newState + lowBits;
+ return symbol;
+}
+
+/*! FSE_decodeSymbolFast() :
+ unsafe, only works if no symbol has a probability > 50% */
+ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD)
+{
+ FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state];
+ U32 const nbBits = DInfo.nbBits;
+ BYTE const symbol = DInfo.symbol;
+ size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
+
+ DStatePtr->state = DInfo.newState + lowBits;
+ return symbol;
+}
+
+ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; }
+
+/* **************************************************************
+* Tuning parameters
+****************************************************************/
+/*!MEMORY_USAGE :
+* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+* Increasing memory usage improves compression ratio
+* Reduced memory usage can improve speed, due to cache effect
+* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
+#ifndef FSE_MAX_MEMORY_USAGE
+#define FSE_MAX_MEMORY_USAGE 14
+#endif
+#ifndef FSE_DEFAULT_MEMORY_USAGE
+#define FSE_DEFAULT_MEMORY_USAGE 13
+#endif
+
+/*!FSE_MAX_SYMBOL_VALUE :
+* Maximum symbol value authorized.
+* Required for proper stack allocation */
+#ifndef FSE_MAX_SYMBOL_VALUE
+#define FSE_MAX_SYMBOL_VALUE 255
+#endif
+
+/* **************************************************************
+* template functions type & suffix
+****************************************************************/
+#define FSE_FUNCTION_TYPE BYTE
+#define FSE_FUNCTION_EXTENSION
+#define FSE_DECODE_TYPE FSE_decode_t
+
+/* ***************************************************************
+* Constants
+*****************************************************************/
+#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE - 2)
+#define FSE_MAX_TABLESIZE (1U << FSE_MAX_TABLELOG)
+#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE - 1)
+#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE - 2)
+#define FSE_MIN_TABLELOG 5
+
+#define FSE_TABLELOG_ABSOLUTE_MAX 15
+#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
+#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
+#endif
+
+#define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3)
+
+#endif /* FSE_H */
diff --git a/lib/zstd/fse_compress.c b/lib/zstd/fse_compress.c
new file mode 100644
index 000000000000..ef3d1741d532
--- /dev/null
+++ b/lib/zstd/fse_compress.c
@@ -0,0 +1,795 @@
+/*
+ * FSE : Finite State Entropy encoder
+ * Copyright (C) 2013-2015, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ */
+
+/* **************************************************************
+* Compiler specifics
+****************************************************************/
+#define FORCE_INLINE static __always_inline
+
+/* **************************************************************
+* Includes
+****************************************************************/
+#include "bitstream.h"
+#include "fse.h"
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/math64.h>
+#include <linux/string.h> /* memcpy, memset */
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define FSE_STATIC_ASSERT(c) \
+ { \
+ enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
+ } /* use only *after* variable declarations */
+
+/* **************************************************************
+* Templates
+****************************************************************/
+/*
+ designed to be included
+ for type-specific functions (template emulation in C)
+ Objective is to write these functions only once, for improved maintenance
+*/
+
+/* safety checks */
+#ifndef FSE_FUNCTION_EXTENSION
+#error "FSE_FUNCTION_EXTENSION must be defined"
+#endif
+#ifndef FSE_FUNCTION_TYPE
+#error "FSE_FUNCTION_TYPE must be defined"
+#endif
+
+/* Function names */
+#define FSE_CAT(X, Y) X##Y
+#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
+#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
+
+/* Function templates */
+
+/* FSE_buildCTable_wksp() :
+ * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
+ * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
+ * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
+ */
+size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize)
+{
+ U32 const tableSize = 1 << tableLog;
+ U32 const tableMask = tableSize - 1;
+ void *const ptr = ct;
+ U16 *const tableU16 = ((U16 *)ptr) + 2;
+ void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableLog ? tableSize >> 1 : 1);
+ FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT);
+ U32 const step = FSE_TABLESTEP(tableSize);
+ U32 highThreshold = tableSize - 1;
+
+ U32 *cumul;
+ FSE_FUNCTION_TYPE *tableSymbol;
+ size_t spaceUsed32 = 0;
+
+ cumul = (U32 *)workspace + spaceUsed32;
+ spaceUsed32 += FSE_MAX_SYMBOL_VALUE + 2;
+ tableSymbol = (FSE_FUNCTION_TYPE *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += ALIGN(sizeof(FSE_FUNCTION_TYPE) * ((size_t)1 << tableLog), sizeof(U32)) >> 2;
+
+ if ((spaceUsed32 << 2) > workspaceSize)
+ return ERROR(tableLog_tooLarge);
+ workspace = (U32 *)workspace + spaceUsed32;
+ workspaceSize -= (spaceUsed32 << 2);
+
+ /* CTable header */
+ tableU16[-2] = (U16)tableLog;
+ tableU16[-1] = (U16)maxSymbolValue;
+
+ /* For explanations on how to distribute symbol values over the table :
+ * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
+
+ /* symbol start positions */
+ {
+ U32 u;
+ cumul[0] = 0;
+ for (u = 1; u <= maxSymbolValue + 1; u++) {
+ if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */
+ cumul[u] = cumul[u - 1] + 1;
+ tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1);
+ } else {
+ cumul[u] = cumul[u - 1] + normalizedCounter[u - 1];
+ }
+ }
+ cumul[maxSymbolValue + 1] = tableSize + 1;
+ }
+
+ /* Spread symbols */
+ {
+ U32 position = 0;
+ U32 symbol;
+ for (symbol = 0; symbol <= maxSymbolValue; symbol++) {
+ int nbOccurences;
+ for (nbOccurences = 0; nbOccurences < normalizedCounter[symbol]; nbOccurences++) {
+ tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
+ position = (position + step) & tableMask;
+ while (position > highThreshold)
+ position = (position + step) & tableMask; /* Low proba area */
+ }
+ }
+
+ if (position != 0)
+ return ERROR(GENERIC); /* Must have gone through all positions */
+ }
+
+ /* Build table */
+ {
+ U32 u;
+ for (u = 0; u < tableSize; u++) {
+ FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
+ tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */
+ }
+ }
+
+ /* Build Symbol Transformation Table */
+ {
+ unsigned total = 0;
+ unsigned s;
+ for (s = 0; s <= maxSymbolValue; s++) {
+ switch (normalizedCounter[s]) {
+ case 0: break;
+
+ case -1:
+ case 1:
+ symbolTT[s].deltaNbBits = (tableLog << 16) - (1 << tableLog);
+ symbolTT[s].deltaFindState = total - 1;
+ total++;
+ break;
+ default: {
+ U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1);
+ U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
+ symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
+ symbolTT[s].deltaFindState = total - normalizedCounter[s];
+ total += normalizedCounter[s];
+ }
+ }
+ }
+ }
+
+ return 0;
+}
+
+/*-**************************************************************
+* FSE NCount encoding-decoding
+****************************************************************/
+size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
+{
+ size_t const maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3;
+ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
+}
+
+static size_t FSE_writeNCount_generic(void *header, size_t headerBufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
+ unsigned writeIsSafe)
+{
+ BYTE *const ostart = (BYTE *)header;
+ BYTE *out = ostart;
+ BYTE *const oend = ostart + headerBufferSize;
+ int nbBits;
+ const int tableSize = 1 << tableLog;
+ int remaining;
+ int threshold;
+ U32 bitStream;
+ int bitCount;
+ unsigned charnum = 0;
+ int previous0 = 0;
+
+ bitStream = 0;
+ bitCount = 0;
+ /* Table Size */
+ bitStream += (tableLog - FSE_MIN_TABLELOG) << bitCount;
+ bitCount += 4;
+
+ /* Init */
+ remaining = tableSize + 1; /* +1 for extra accuracy */
+ threshold = tableSize;
+ nbBits = tableLog + 1;
+
+ while (remaining > 1) { /* stops at 1 */
+ if (previous0) {
+ unsigned start = charnum;
+ while (!normalizedCounter[charnum])
+ charnum++;
+ while (charnum >= start + 24) {
+ start += 24;
+ bitStream += 0xFFFFU << bitCount;
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream >> 8);
+ out += 2;
+ bitStream >>= 16;
+ }
+ while (charnum >= start + 3) {
+ start += 3;
+ bitStream += 3 << bitCount;
+ bitCount += 2;
+ }
+ bitStream += (charnum - start) << bitCount;
+ bitCount += 2;
+ if (bitCount > 16) {
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream >> 8);
+ out += 2;
+ bitStream >>= 16;
+ bitCount -= 16;
+ }
+ }
+ {
+ int count = normalizedCounter[charnum++];
+ int const max = (2 * threshold - 1) - remaining;
+ remaining -= count < 0 ? -count : count;
+ count++; /* +1 for extra accuracy */
+ if (count >= threshold)
+ count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
+ bitStream += count << bitCount;
+ bitCount += nbBits;
+ bitCount -= (count < max);
+ previous0 = (count == 1);
+ if (remaining < 1)
+ return ERROR(GENERIC);
+ while (remaining < threshold)
+ nbBits--, threshold >>= 1;
+ }
+ if (bitCount > 16) {
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream >> 8);
+ out += 2;
+ bitStream >>= 16;
+ bitCount -= 16;
+ }
+ }
+
+ /* flush remaining bitStream */
+ if ((!writeIsSafe) && (out > oend - 2))
+ return ERROR(dstSize_tooSmall); /* Buffer overflow */
+ out[0] = (BYTE)bitStream;
+ out[1] = (BYTE)(bitStream >> 8);
+ out += (bitCount + 7) / 8;
+
+ if (charnum > maxSymbolValue + 1)
+ return ERROR(GENERIC);
+
+ return (out - ostart);
+}
+
+size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
+{
+ if (tableLog > FSE_MAX_TABLELOG)
+ return ERROR(tableLog_tooLarge); /* Unsupported */
+ if (tableLog < FSE_MIN_TABLELOG)
+ return ERROR(GENERIC); /* Unsupported */
+
+ if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
+
+ return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
+}
+
+/*-**************************************************************
+* Counting histogram
+****************************************************************/
+/*! FSE_count_simple
+ This function counts byte values within `src`, and store the histogram into table `count`.
+ It doesn't use any additional memory.
+ But this function is unsafe : it doesn't check that all values within `src` can fit into `count`.
+ For this reason, prefer using a table `count` with 256 elements.
+ @return : count of most numerous element
+*/
+size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize)
+{
+ const BYTE *ip = (const BYTE *)src;
+ const BYTE *const end = ip + srcSize;
+ unsigned maxSymbolValue = *maxSymbolValuePtr;
+ unsigned max = 0;
+
+ memset(count, 0, (maxSymbolValue + 1) * sizeof(*count));
+ if (srcSize == 0) {
+ *maxSymbolValuePtr = 0;
+ return 0;
+ }
+
+ while (ip < end)
+ count[*ip++]++;
+
+ while (!count[maxSymbolValue])
+ maxSymbolValue--;
+ *maxSymbolValuePtr = maxSymbolValue;
+
+ {
+ U32 s;
+ for (s = 0; s <= maxSymbolValue; s++)
+ if (count[s] > max)
+ max = count[s];
+ }
+
+ return (size_t)max;
+}
+
+/* FSE_count_parallel_wksp() :
+ * Same as FSE_count_parallel(), but using an externally provided scratch buffer.
+ * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */
+static size_t FSE_count_parallel_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned checkMax,
+ unsigned *const workSpace)
+{
+ const BYTE *ip = (const BYTE *)source;
+ const BYTE *const iend = ip + sourceSize;
+ unsigned maxSymbolValue = *maxSymbolValuePtr;
+ unsigned max = 0;
+ U32 *const Counting1 = workSpace;
+ U32 *const Counting2 = Counting1 + 256;
+ U32 *const Counting3 = Counting2 + 256;
+ U32 *const Counting4 = Counting3 + 256;
+
+ memset(Counting1, 0, 4 * 256 * sizeof(unsigned));
+
+ /* safety checks */
+ if (!sourceSize) {
+ memset(count, 0, maxSymbolValue + 1);
+ *maxSymbolValuePtr = 0;
+ return 0;
+ }
+ if (!maxSymbolValue)
+ maxSymbolValue = 255; /* 0 == default */
+
+ /* by stripes of 16 bytes */
+ {
+ U32 cached = ZSTD_read32(ip);
+ ip += 4;
+ while (ip < iend - 15) {
+ U32 c = cached;
+ cached = ZSTD_read32(ip);
+ ip += 4;
+ Counting1[(BYTE)c]++;
+ Counting2[(BYTE)(c >> 8)]++;
+ Counting3[(BYTE)(c >> 16)]++;
+ Counting4[c >> 24]++;
+ c = cached;
+ cached = ZSTD_read32(ip);
+ ip += 4;
+ Counting1[(BYTE)c]++;
+ Counting2[(BYTE)(c >> 8)]++;
+ Counting3[(BYTE)(c >> 16)]++;
+ Counting4[c >> 24]++;
+ c = cached;
+ cached = ZSTD_read32(ip);
+ ip += 4;
+ Counting1[(BYTE)c]++;
+ Counting2[(BYTE)(c >> 8)]++;
+ Counting3[(BYTE)(c >> 16)]++;
+ Counting4[c >> 24]++;
+ c = cached;
+ cached = ZSTD_read32(ip);
+ ip += 4;
+ Counting1[(BYTE)c]++;
+ Counting2[(BYTE)(c >> 8)]++;
+ Counting3[(BYTE)(c >> 16)]++;
+ Counting4[c >> 24]++;
+ }
+ ip -= 4;
+ }
+
+ /* finish last symbols */
+ while (ip < iend)
+ Counting1[*ip++]++;
+
+ if (checkMax) { /* verify stats will fit into destination table */
+ U32 s;
+ for (s = 255; s > maxSymbolValue; s--) {
+ Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
+ if (Counting1[s])
+ return ERROR(maxSymbolValue_tooSmall);
+ }
+ }
+
+ {
+ U32 s;
+ for (s = 0; s <= maxSymbolValue; s++) {
+ count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
+ if (count[s] > max)
+ max = count[s];
+ }
+ }
+
+ while (!count[maxSymbolValue])
+ maxSymbolValue--;
+ *maxSymbolValuePtr = maxSymbolValue;
+ return (size_t)max;
+}
+
+/* FSE_countFast_wksp() :
+ * Same as FSE_countFast(), but using an externally provided scratch buffer.
+ * `workSpace` size must be table of >= `1024` unsigned */
+size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
+{
+ if (sourceSize < 1500)
+ return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize);
+ return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
+}
+
+/* FSE_count_wksp() :
+ * Same as FSE_count(), but using an externally provided scratch buffer.
+ * `workSpace` size must be table of >= `1024` unsigned */
+size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace)
+{
+ if (*maxSymbolValuePtr < 255)
+ return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
+ *maxSymbolValuePtr = 255;
+ return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
+}
+
+/*-**************************************************************
+* FSE Compression Code
+****************************************************************/
+/*! FSE_sizeof_CTable() :
+ FSE_CTable is a variable size structure which contains :
+ `U16 tableLog;`
+ `U16 maxSymbolValue;`
+ `U16 nextStateNumber[1 << tableLog];` // This size is variable
+ `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable
+Allocation is manual (C standard does not support variable-size structures).
+*/
+size_t FSE_sizeof_CTable(unsigned maxSymbolValue, unsigned tableLog)
+{
+ if (tableLog > FSE_MAX_TABLELOG)
+ return ERROR(tableLog_tooLarge);
+ return FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32);
+}
+
+/* provides the minimum logSize to safely represent a distribution */
+static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
+{
+ U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1;
+ U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
+ U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
+ return minBits;
+}
+
+unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
+{
+ U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
+ U32 tableLog = maxTableLog;
+ U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
+ if (tableLog == 0)
+ tableLog = FSE_DEFAULT_TABLELOG;
+ if (maxBitsSrc < tableLog)
+ tableLog = maxBitsSrc; /* Accuracy can be reduced */
+ if (minBits > tableLog)
+ tableLog = minBits; /* Need a minimum to safely represent all symbol values */
+ if (tableLog < FSE_MIN_TABLELOG)
+ tableLog = FSE_MIN_TABLELOG;
+ if (tableLog > FSE_MAX_TABLELOG)
+ tableLog = FSE_MAX_TABLELOG;
+ return tableLog;
+}
+
+unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
+{
+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
+}
+
+/* Secondary normalization method.
+ To be used when primary method fails. */
+
+static size_t FSE_normalizeM2(short *norm, U32 tableLog, const unsigned *count, size_t total, U32 maxSymbolValue)
+{
+ short const NOT_YET_ASSIGNED = -2;
+ U32 s;
+ U32 distributed = 0;
+ U32 ToDistribute;
+
+ /* Init */
+ U32 const lowThreshold = (U32)(total >> tableLog);
+ U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
+
+ for (s = 0; s <= maxSymbolValue; s++) {
+ if (count[s] == 0) {
+ norm[s] = 0;
+ continue;
+ }
+ if (count[s] <= lowThreshold) {
+ norm[s] = -1;
+ distributed++;
+ total -= count[s];
+ continue;
+ }
+ if (count[s] <= lowOne) {
+ norm[s] = 1;
+ distributed++;
+ total -= count[s];
+ continue;
+ }
+
+ norm[s] = NOT_YET_ASSIGNED;
+ }
+ ToDistribute = (1 << tableLog) - distributed;
+
+ if ((total / ToDistribute) > lowOne) {
+ /* risk of rounding to zero */
+ lowOne = (U32)((total * 3) / (ToDistribute * 2));
+ for (s = 0; s <= maxSymbolValue; s++) {
+ if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
+ norm[s] = 1;
+ distributed++;
+ total -= count[s];
+ continue;
+ }
+ }
+ ToDistribute = (1 << tableLog) - distributed;
+ }
+
+ if (distributed == maxSymbolValue + 1) {
+ /* all values are pretty poor;
+ probably incompressible data (should have already been detected);
+ find max, then give all remaining points to max */
+ U32 maxV = 0, maxC = 0;
+ for (s = 0; s <= maxSymbolValue; s++)
+ if (count[s] > maxC)
+ maxV = s, maxC = count[s];
+ norm[maxV] += (short)ToDistribute;
+ return 0;
+ }
+
+ if (total == 0) {
+ /* all of the symbols were low enough for the lowOne or lowThreshold */
+ for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1))
+ if (norm[s] > 0)
+ ToDistribute--, norm[s]++;
+ return 0;
+ }
+
+ {
+ U64 const vStepLog = 62 - tableLog;
+ U64 const mid = (1ULL << (vStepLog - 1)) - 1;
+ U64 const rStep = div_u64((((U64)1 << vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
+ U64 tmpTotal = mid;
+ for (s = 0; s <= maxSymbolValue; s++) {
+ if (norm[s] == NOT_YET_ASSIGNED) {
+ U64 const end = tmpTotal + (count[s] * rStep);
+ U32 const sStart = (U32)(tmpTotal >> vStepLog);
+ U32 const sEnd = (U32)(end >> vStepLog);
+ U32 const weight = sEnd - sStart;
+ if (weight < 1)
+ return ERROR(GENERIC);
+ norm[s] = (short)weight;
+ tmpTotal = end;
+ }
+ }
+ }
+
+ return 0;
+}
+
+size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t total, unsigned maxSymbolValue)
+{
+ /* Sanity checks */
+ if (tableLog == 0)
+ tableLog = FSE_DEFAULT_TABLELOG;
+ if (tableLog < FSE_MIN_TABLELOG)
+ return ERROR(GENERIC); /* Unsupported size */
+ if (tableLog > FSE_MAX_TABLELOG)
+ return ERROR(tableLog_tooLarge); /* Unsupported size */
+ if (tableLog < FSE_minTableLog(total, maxSymbolValue))
+ return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
+
+ {
+ U32 const rtbTable[] = {0, 473195, 504333, 520860, 550000, 700000, 750000, 830000};
+ U64 const scale = 62 - tableLog;
+ U64 const step = div_u64((U64)1 << 62, (U32)total); /* <== here, one division ! */
+ U64 const vStep = 1ULL << (scale - 20);
+ int stillToDistribute = 1 << tableLog;
+ unsigned s;
+ unsigned largest = 0;
+ short largestP = 0;
+ U32 lowThreshold = (U32)(total >> tableLog);
+
+ for (s = 0; s <= maxSymbolValue; s++) {
+ if (count[s] == total)
+ return 0; /* rle special case */
+ if (count[s] == 0) {
+ normalizedCounter[s] = 0;
+ continue;
+ }
+ if (count[s] <= lowThreshold) {
+ normalizedCounter[s] = -1;
+ stillToDistribute--;
+ } else {
+ short proba = (short)((count[s] * step) >> scale);
+ if (proba < 8) {
+ U64 restToBeat = vStep * rtbTable[proba];
+ proba += (count[s] * step) - ((U64)proba << scale) > restToBeat;
+ }
+ if (proba > largestP)
+ largestP = proba, largest = s;
+ normalizedCounter[s] = proba;
+ stillToDistribute -= proba;
+ }
+ }
+ if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
+ /* corner case, need another normalization method */
+ size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
+ if (FSE_isError(errorCode))
+ return errorCode;
+ } else
+ normalizedCounter[largest] += (short)stillToDistribute;
+ }
+
+ return tableLog;
+}
+
+/* fake FSE_CTable, for raw (uncompressed) input */
+size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits)
+{
+ const unsigned tableSize = 1 << nbBits;
+ const unsigned tableMask = tableSize - 1;
+ const unsigned maxSymbolValue = tableMask;
+ void *const ptr = ct;
+ U16 *const tableU16 = ((U16 *)ptr) + 2;
+ void *const FSCT = ((U32 *)ptr) + 1 /* header */ + (tableSize >> 1); /* assumption : tableLog >= 1 */
+ FSE_symbolCompressionTransform *const symbolTT = (FSE_symbolCompressionTransform *)(FSCT);
+ unsigned s;
+
+ /* Sanity checks */
+ if (nbBits < 1)
+ return ERROR(GENERIC); /* min size */
+
+ /* header */
+ tableU16[-2] = (U16)nbBits;
+ tableU16[-1] = (U16)maxSymbolValue;
+
+ /* Build table */
+ for (s = 0; s < tableSize; s++)
+ tableU16[s] = (U16)(tableSize + s);
+
+ /* Build Symbol Transformation Table */
+ {
+ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
+ for (s = 0; s <= maxSymbolValue; s++) {
+ symbolTT[s].deltaNbBits = deltaNbBits;
+ symbolTT[s].deltaFindState = s - 1;
+ }
+ }
+
+ return 0;
+}
+
+/* fake FSE_CTable, for rle input (always same symbol) */
+size_t FSE_buildCTable_rle(FSE_CTable *ct, BYTE symbolValue)
+{
+ void *ptr = ct;
+ U16 *tableU16 = ((U16 *)ptr) + 2;
+ void *FSCTptr = (U32 *)ptr + 2;
+ FSE_symbolCompressionTransform *symbolTT = (FSE_symbolCompressionTransform *)FSCTptr;
+
+ /* header */
+ tableU16[-2] = (U16)0;
+ tableU16[-1] = (U16)symbolValue;
+
+ /* Build table */
+ tableU16[0] = 0;
+ tableU16[1] = 0; /* just in case */
+
+ /* Build Symbol Transformation Table */
+ symbolTT[symbolValue].deltaNbBits = 0;
+ symbolTT[symbolValue].deltaFindState = 0;
+
+ return 0;
+}
+
+static size_t FSE_compress_usingCTable_generic(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct, const unsigned fast)
+{
+ const BYTE *const istart = (const BYTE *)src;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *ip = iend;
+
+ BIT_CStream_t bitC;
+ FSE_CState_t CState1, CState2;
+
+ /* init */
+ if (srcSize <= 2)
+ return 0;
+ {
+ size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
+ if (FSE_isError(initError))
+ return 0; /* not enough space available to write a bitstream */
+ }
+
+#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
+
+ if (srcSize & 1) {
+ FSE_initCState2(&CState1, ct, *--ip);
+ FSE_initCState2(&CState2, ct, *--ip);
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+ FSE_FLUSHBITS(&bitC);
+ } else {
+ FSE_initCState2(&CState2, ct, *--ip);
+ FSE_initCState2(&CState1, ct, *--ip);
+ }
+
+ /* join to mod 4 */
+ srcSize -= 2;
+ if ((sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) && (srcSize & 2)) { /* test bit 2 */
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+ FSE_FLUSHBITS(&bitC);
+ }
+
+ /* 2 or 4 encoding per loop */
+ while (ip > istart) {
+
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
+
+ if (sizeof(bitC.bitContainer) * 8 < FSE_MAX_TABLELOG * 2 + 7) /* this test must be static */
+ FSE_FLUSHBITS(&bitC);
+
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+
+ if (sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) { /* this test must be static */
+ FSE_encodeSymbol(&bitC, &CState2, *--ip);
+ FSE_encodeSymbol(&bitC, &CState1, *--ip);
+ }
+
+ FSE_FLUSHBITS(&bitC);
+ }
+
+ FSE_flushCState(&bitC, &CState2);
+ FSE_flushCState(&bitC, &CState1);
+ return BIT_closeCStream(&bitC);
+}
+
+size_t FSE_compress_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const FSE_CTable *ct)
+{
+ unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
+
+ if (fast)
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
+ else
+ return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
+}
+
+size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
diff --git a/lib/zstd/fse_decompress.c b/lib/zstd/fse_decompress.c
new file mode 100644
index 000000000000..a84300e5a013
--- /dev/null
+++ b/lib/zstd/fse_decompress.c
@@ -0,0 +1,332 @@
+/*
+ * FSE : Finite State Entropy decoder
+ * Copyright (C) 2013-2015, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ */
+
+/* **************************************************************
+* Compiler specifics
+****************************************************************/
+#define FORCE_INLINE static __always_inline
+
+/* **************************************************************
+* Includes
+****************************************************************/
+#include "bitstream.h"
+#include "fse.h"
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/string.h> /* memcpy, memset */
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define FSE_isError ERR_isError
+#define FSE_STATIC_ASSERT(c) \
+ { \
+ enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
+ } /* use only *after* variable declarations */
+
+/* check and forward error code */
+#define CHECK_F(f) \
+ { \
+ size_t const e = f; \
+ if (FSE_isError(e)) \
+ return e; \
+ }
+
+/* **************************************************************
+* Templates
+****************************************************************/
+/*
+ designed to be included
+ for type-specific functions (template emulation in C)
+ Objective is to write these functions only once, for improved maintenance
+*/
+
+/* safety checks */
+#ifndef FSE_FUNCTION_EXTENSION
+#error "FSE_FUNCTION_EXTENSION must be defined"
+#endif
+#ifndef FSE_FUNCTION_TYPE
+#error "FSE_FUNCTION_TYPE must be defined"
+#endif
+
+/* Function names */
+#define FSE_CAT(X, Y) X##Y
+#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
+#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
+
+/* Function templates */
+
+size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize)
+{
+ void *const tdPtr = dt + 1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
+ FSE_DECODE_TYPE *const tableDecode = (FSE_DECODE_TYPE *)(tdPtr);
+ U16 *symbolNext = (U16 *)workspace;
+
+ U32 const maxSV1 = maxSymbolValue + 1;
+ U32 const tableSize = 1 << tableLog;
+ U32 highThreshold = tableSize - 1;
+
+ /* Sanity Checks */
+ if (workspaceSize < sizeof(U16) * (FSE_MAX_SYMBOL_VALUE + 1))
+ return ERROR(tableLog_tooLarge);
+ if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE)
+ return ERROR(maxSymbolValue_tooLarge);
+ if (tableLog > FSE_MAX_TABLELOG)
+ return ERROR(tableLog_tooLarge);
+
+ /* Init, lay down lowprob symbols */
+ {
+ FSE_DTableHeader DTableH;
+ DTableH.tableLog = (U16)tableLog;
+ DTableH.fastMode = 1;
+ {
+ S16 const largeLimit = (S16)(1 << (tableLog - 1));
+ U32 s;
+ for (s = 0; s < maxSV1; s++) {
+ if (normalizedCounter[s] == -1) {
+ tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
+ symbolNext[s] = 1;
+ } else {
+ if (normalizedCounter[s] >= largeLimit)
+ DTableH.fastMode = 0;
+ symbolNext[s] = normalizedCounter[s];
+ }
+ }
+ }
+ memcpy(dt, &DTableH, sizeof(DTableH));
+ }
+
+ /* Spread symbols */
+ {
+ U32 const tableMask = tableSize - 1;
+ U32 const step = FSE_TABLESTEP(tableSize);
+ U32 s, position = 0;
+ for (s = 0; s < maxSV1; s++) {
+ int i;
+ for (i = 0; i < normalizedCounter[s]; i++) {
+ tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
+ position = (position + step) & tableMask;
+ while (position > highThreshold)
+ position = (position + step) & tableMask; /* lowprob area */
+ }
+ }
+ if (position != 0)
+ return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
+ }
+
+ /* Build Decoding table */
+ {
+ U32 u;
+ for (u = 0; u < tableSize; u++) {
+ FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
+ U16 nextState = symbolNext[symbol]++;
+ tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState));
+ tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize);
+ }
+ }
+
+ return 0;
+}
+
+/*-*******************************************************
+* Decompression (Byte symbols)
+*********************************************************/
+size_t FSE_buildDTable_rle(FSE_DTable *dt, BYTE symbolValue)
+{
+ void *ptr = dt;
+ FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
+ void *dPtr = dt + 1;
+ FSE_decode_t *const cell = (FSE_decode_t *)dPtr;
+
+ DTableH->tableLog = 0;
+ DTableH->fastMode = 0;
+
+ cell->newState = 0;
+ cell->symbol = symbolValue;
+ cell->nbBits = 0;
+
+ return 0;
+}
+
+size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits)
+{
+ void *ptr = dt;
+ FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
+ void *dPtr = dt + 1;
+ FSE_decode_t *const dinfo = (FSE_decode_t *)dPtr;
+ const unsigned tableSize = 1 << nbBits;
+ const unsigned tableMask = tableSize - 1;
+ const unsigned maxSV1 = tableMask + 1;
+ unsigned s;
+
+ /* Sanity checks */
+ if (nbBits < 1)
+ return ERROR(GENERIC); /* min size */
+
+ /* Build Decoding Table */
+ DTableH->tableLog = (U16)nbBits;
+ DTableH->fastMode = 1;
+ for (s = 0; s < maxSV1; s++) {
+ dinfo[s].newState = 0;
+ dinfo[s].symbol = (BYTE)s;
+ dinfo[s].nbBits = (BYTE)nbBits;
+ }
+
+ return 0;
+}
+
+FORCE_INLINE size_t FSE_decompress_usingDTable_generic(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt,
+ const unsigned fast)
+{
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *op = ostart;
+ BYTE *const omax = op + maxDstSize;
+ BYTE *const olimit = omax - 3;
+
+ BIT_DStream_t bitD;
+ FSE_DState_t state1;
+ FSE_DState_t state2;
+
+ /* Init */
+ CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
+
+ FSE_initDState(&state1, &bitD, dt);
+ FSE_initDState(&state2, &bitD, dt);
+
+#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
+
+ /* 4 symbols per loop */
+ for (; (BIT_reloadDStream(&bitD) == BIT_DStream_unfinished) & (op < olimit); op += 4) {
+ op[0] = FSE_GETSYMBOL(&state1);
+
+ if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
+ BIT_reloadDStream(&bitD);
+
+ op[1] = FSE_GETSYMBOL(&state2);
+
+ if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
+ {
+ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) {
+ op += 2;
+ break;
+ }
+ }
+
+ op[2] = FSE_GETSYMBOL(&state1);
+
+ if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
+ BIT_reloadDStream(&bitD);
+
+ op[3] = FSE_GETSYMBOL(&state2);
+ }
+
+ /* tail */
+ /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
+ while (1) {
+ if (op > (omax - 2))
+ return ERROR(dstSize_tooSmall);
+ *op++ = FSE_GETSYMBOL(&state1);
+ if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
+ *op++ = FSE_GETSYMBOL(&state2);
+ break;
+ }
+
+ if (op > (omax - 2))
+ return ERROR(dstSize_tooSmall);
+ *op++ = FSE_GETSYMBOL(&state2);
+ if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
+ *op++ = FSE_GETSYMBOL(&state1);
+ break;
+ }
+ }
+
+ return op - ostart;
+}
+
+size_t FSE_decompress_usingDTable(void *dst, size_t originalSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt)
+{
+ const void *ptr = dt;
+ const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr;
+ const U32 fastMode = DTableH->fastMode;
+
+ /* select fast mode (static) */
+ if (fastMode)
+ return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
+ return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
+}
+
+size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize)
+{
+ const BYTE *const istart = (const BYTE *)cSrc;
+ const BYTE *ip = istart;
+ unsigned tableLog;
+ unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
+ size_t NCountLength;
+
+ FSE_DTable *dt;
+ short *counting;
+ size_t spaceUsed32 = 0;
+
+ FSE_STATIC_ASSERT(sizeof(FSE_DTable) == sizeof(U32));
+
+ dt = (FSE_DTable *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += FSE_DTABLE_SIZE_U32(maxLog);
+ counting = (short *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += ALIGN(sizeof(short) * (FSE_MAX_SYMBOL_VALUE + 1), sizeof(U32)) >> 2;
+
+ if ((spaceUsed32 << 2) > workspaceSize)
+ return ERROR(tableLog_tooLarge);
+ workspace = (U32 *)workspace + spaceUsed32;
+ workspaceSize -= (spaceUsed32 << 2);
+
+ /* normal FSE decoding mode */
+ NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
+ if (FSE_isError(NCountLength))
+ return NCountLength;
+ // if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining
+ // case : NCountLength==cSrcSize */
+ if (tableLog > maxLog)
+ return ERROR(tableLog_tooLarge);
+ ip += NCountLength;
+ cSrcSize -= NCountLength;
+
+ CHECK_F(FSE_buildDTable_wksp(dt, counting, maxSymbolValue, tableLog, workspace, workspaceSize));
+
+ return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, dt); /* always return, even if it is an error code */
+}
diff --git a/lib/zstd/huf.h b/lib/zstd/huf.h
new file mode 100644
index 000000000000..2143da28d952
--- /dev/null
+++ b/lib/zstd/huf.h
@@ -0,0 +1,212 @@
+/*
+ * Huffman coder, part of New Generation Entropy library
+ * header file
+ * Copyright (C) 2013-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ */
+#ifndef HUF_H_298734234
+#define HUF_H_298734234
+
+/* *** Dependencies *** */
+#include <linux/types.h> /* size_t */
+
+/* *** Tool functions *** */
+#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
+size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
+
+/* Error Management */
+unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
+
+/* *** Advanced function *** */
+
+/** HUF_compress4X_wksp() :
+* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
+size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
+ size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
+
+/* *** Dependencies *** */
+#include "mem.h" /* U32 */
+
+/* *** Constants *** */
+#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
+#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
+#define HUF_SYMBOLVALUE_MAX 255
+
+#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
+#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
+#error "HUF_TABLELOG_MAX is too large !"
+#endif
+
+/* ****************************************
+* Static allocation
+******************************************/
+/* HUF buffer bounds */
+#define HUF_CTABLEBOUND 129
+#define HUF_BLOCKBOUND(size) (size + (size >> 8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
+#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
+
+/* static allocation of HUF's Compression Table */
+#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
+ U32 name##hb[maxSymbolValue + 1]; \
+ void *name##hv = &(name##hb); \
+ HUF_CElt *name = (HUF_CElt *)(name##hv) /* no final ; */
+
+/* static allocation of HUF's DTable */
+typedef U32 HUF_DTable;
+#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog)))
+#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = {((U32)((maxTableLog)-1) * 0x01000001)}
+#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = {((U32)(maxTableLog)*0x01000001)}
+
+/* The workspace must have alignment at least 4 and be at least this large */
+#define HUF_COMPRESS_WORKSPACE_SIZE (6 << 10)
+#define HUF_COMPRESS_WORKSPACE_SIZE_U32 (HUF_COMPRESS_WORKSPACE_SIZE / sizeof(U32))
+
+/* The workspace must have alignment at least 4 and be at least this large */
+#define HUF_DECOMPRESS_WORKSPACE_SIZE (3 << 10)
+#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
+
+/* ****************************************
+* Advanced decompression functions
+******************************************/
+size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize); /**< decodes RLE and uncompressed */
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
+ size_t workspaceSize); /**< considers RLE and uncompressed as errors */
+size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
+ size_t workspaceSize); /**< single-symbol decoder */
+size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
+ size_t workspaceSize); /**< double-symbols decoder */
+
+/* ****************************************
+* HUF detailed API
+******************************************/
+/*!
+HUF_compress() does the following:
+1. count symbol occurrence from source[] into table count[] using FSE_count()
+2. (optional) refine tableLog using HUF_optimalTableLog()
+3. build Huffman table from count using HUF_buildCTable()
+4. save Huffman table to memory buffer using HUF_writeCTable_wksp()
+5. encode the data stream using HUF_compress4X_usingCTable()
+
+The following API allows targeting specific sub-functions for advanced tasks.
+For example, it's possible to compress several blocks using the same 'CTable',
+or to save and regenerate 'CTable' using external methods.
+*/
+/* FSE_count() : find it within "fse.h" */
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
+typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
+size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, unsigned maxSymbolValue, unsigned huffLog, void *workspace, size_t workspaceSize);
+size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable);
+
+typedef enum {
+ HUF_repeat_none, /**< Cannot use the previous table */
+ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1,
+ 4}X_repeat */
+ HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */
+} HUF_repeat;
+/** HUF_compress4X_repeat() :
+* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+* If it uses hufTable it does not modify hufTable or repeat.
+* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+* If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
+ int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
+
+/** HUF_buildCTable_wksp() :
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
+ */
+size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize);
+
+/*! HUF_readStats() :
+ Read compact Huffman tree, saved by HUF_writeCTable().
+ `huffWeight` is destination buffer.
+ @return : size read from `src` , or an error Code .
+ Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
+size_t HUF_readStats_wksp(BYTE *huffWeight, size_t hwSize, U32 *rankStats, U32 *nbSymbolsPtr, U32 *tableLogPtr, const void *src, size_t srcSize,
+ void *workspace, size_t workspaceSize);
+
+/** HUF_readCTable() :
+* Loading a CTable saved with HUF_writeCTable() */
+size_t HUF_readCTable_wksp(HUF_CElt *CTable, unsigned maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize);
+
+/*
+HUF_decompress() does the following:
+1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
+2. build Huffman table from save, using HUF_readDTableXn()
+3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
+*/
+
+/** HUF_selectDecoder() :
+* Tells which decoder is likely to decode faster,
+* based on a set of pre-determined metrics.
+* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
+* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
+U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize);
+
+size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize);
+size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize);
+
+size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
+size_t HUF_decompress4X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
+size_t HUF_decompress4X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
+
+/* single stream variants */
+
+size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
+ size_t wkspSize); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
+size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable);
+/** HUF_compress1X_repeat() :
+* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+* If it uses hufTable it does not modify hufTable or repeat.
+* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+* If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void *workSpace,
+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat,
+ int preferRepeat); /**< `workSpace` must be a table of at least HUF_COMPRESS_WORKSPACE_SIZE_U32 unsigned */
+
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize);
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
+ size_t workspaceSize); /**< single-symbol decoder */
+size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace,
+ size_t workspaceSize); /**< double-symbols decoder */
+
+size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize,
+ const HUF_DTable *DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
+size_t HUF_decompress1X2_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
+size_t HUF_decompress1X4_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable);
+
+#endif /* HUF_H_298734234 */
diff --git a/lib/zstd/huf_compress.c b/lib/zstd/huf_compress.c
new file mode 100644
index 000000000000..40055a7016e6
--- /dev/null
+++ b/lib/zstd/huf_compress.c
@@ -0,0 +1,770 @@
+/*
+ * Huffman encoder, part of New Generation Entropy library
+ * Copyright (C) 2013-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ */
+
+/* **************************************************************
+* Includes
+****************************************************************/
+#include "bitstream.h"
+#include "fse.h" /* header compression */
+#include "huf.h"
+#include <linux/kernel.h>
+#include <linux/string.h> /* memcpy, memset */
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define HUF_STATIC_ASSERT(c) \
+ { \
+ enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
+ } /* use only *after* variable declarations */
+#define CHECK_V_F(e, f) \
+ size_t const e = f; \
+ if (ERR_isError(e)) \
+ return f
+#define CHECK_F(f) \
+ { \
+ CHECK_V_F(_var_err__, f); \
+ }
+
+/* **************************************************************
+* Utils
+****************************************************************/
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
+{
+ return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+}
+
+/* *******************************************************
+* HUF : Huffman block compression
+*********************************************************/
+/* HUF_compressWeights() :
+ * Same as FSE_compress(), but dedicated to huff0's weights compression.
+ * The use case needs much less stack memory.
+ * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
+ */
+#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
+size_t HUF_compressWeights_wksp(void *dst, size_t dstSize, const void *weightTable, size_t wtSize, void *workspace, size_t workspaceSize)
+{
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *op = ostart;
+ BYTE *const oend = ostart + dstSize;
+
+ U32 maxSymbolValue = HUF_TABLELOG_MAX;
+ U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
+
+ FSE_CTable *CTable;
+ U32 *count;
+ S16 *norm;
+ size_t spaceUsed32 = 0;
+
+ HUF_STATIC_ASSERT(sizeof(FSE_CTable) == sizeof(U32));
+
+ CTable = (FSE_CTable *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX);
+ count = (U32 *)workspace + spaceUsed32;
+ spaceUsed32 += HUF_TABLELOG_MAX + 1;
+ norm = (S16 *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += ALIGN(sizeof(S16) * (HUF_TABLELOG_MAX + 1), sizeof(U32)) >> 2;
+
+ if ((spaceUsed32 << 2) > workspaceSize)
+ return ERROR(tableLog_tooLarge);
+ workspace = (U32 *)workspace + spaceUsed32;
+ workspaceSize -= (spaceUsed32 << 2);
+
+ /* init conditions */
+ if (wtSize <= 1)
+ return 0; /* Not compressible */
+
+ /* Scan input and build symbol stats */
+ {
+ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize));
+ if (maxCount == wtSize)
+ return 1; /* only a single symbol in src : rle */
+ if (maxCount == 1)
+ return 0; /* each symbol present maximum once => not compressible */
+ }
+
+ tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
+ CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue));
+
+ /* Write table description header */
+ {
+ CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog));
+ op += hSize;
+ }
+
+ /* Compress */
+ CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, workspace, workspaceSize));
+ {
+ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable));
+ if (cSize == 0)
+ return 0; /* not enough space for compressed data */
+ op += cSize;
+ }
+
+ return op - ostart;
+}
+
+struct HUF_CElt_s {
+ U16 val;
+ BYTE nbBits;
+}; /* typedef'd to HUF_CElt within "huf.h" */
+
+/*! HUF_writeCTable_wksp() :
+ `CTable` : Huffman tree to save, using huf representation.
+ @return : size of saved CTable */
+size_t HUF_writeCTable_wksp(void *dst, size_t maxDstSize, const HUF_CElt *CTable, U32 maxSymbolValue, U32 huffLog, void *workspace, size_t workspaceSize)
+{
+ BYTE *op = (BYTE *)dst;
+ U32 n;
+
+ BYTE *bitsToWeight;
+ BYTE *huffWeight;
+ size_t spaceUsed32 = 0;
+
+ bitsToWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += ALIGN(HUF_TABLELOG_MAX + 1, sizeof(U32)) >> 2;
+ huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX, sizeof(U32)) >> 2;
+
+ if ((spaceUsed32 << 2) > workspaceSize)
+ return ERROR(tableLog_tooLarge);
+ workspace = (U32 *)workspace + spaceUsed32;
+ workspaceSize -= (spaceUsed32 << 2);
+
+ /* check conditions */
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
+ return ERROR(maxSymbolValue_tooLarge);
+
+ /* convert to weight */
+ bitsToWeight[0] = 0;
+ for (n = 1; n < huffLog + 1; n++)
+ bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
+ for (n = 0; n < maxSymbolValue; n++)
+ huffWeight[n] = bitsToWeight[CTable[n].nbBits];
+
+ /* attempt weights compression by FSE */
+ {
+ CHECK_V_F(hSize, HUF_compressWeights_wksp(op + 1, maxDstSize - 1, huffWeight, maxSymbolValue, workspace, workspaceSize));
+ if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */
+ op[0] = (BYTE)hSize;
+ return hSize + 1;
+ }
+ }
+
+ /* write raw values as 4-bits (max : 15) */
+ if (maxSymbolValue > (256 - 128))
+ return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
+ if (((maxSymbolValue + 1) / 2) + 1 > maxDstSize)
+ return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
+ op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1));
+ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
+ for (n = 0; n < maxSymbolValue; n += 2)
+ op[(n / 2) + 1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n + 1]);
+ return ((maxSymbolValue + 1) / 2) + 1;
+}
+
+size_t HUF_readCTable_wksp(HUF_CElt *CTable, U32 maxSymbolValue, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
+{
+ U32 *rankVal;
+ BYTE *huffWeight;
+ U32 tableLog = 0;
+ U32 nbSymbols = 0;
+ size_t readSize;
+ size_t spaceUsed32 = 0;
+
+ rankVal = (U32 *)workspace + spaceUsed32;
+ spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1;
+ huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
+
+ if ((spaceUsed32 << 2) > workspaceSize)
+ return ERROR(tableLog_tooLarge);
+ workspace = (U32 *)workspace + spaceUsed32;
+ workspaceSize -= (spaceUsed32 << 2);
+
+ /* get symbol weights */
+ readSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize);
+ if (ERR_isError(readSize))
+ return readSize;
+
+ /* check result */
+ if (tableLog > HUF_TABLELOG_MAX)
+ return ERROR(tableLog_tooLarge);
+ if (nbSymbols > maxSymbolValue + 1)
+ return ERROR(maxSymbolValue_tooSmall);
+
+ /* Prepare base value per rank */
+ {
+ U32 n, nextRankStart = 0;
+ for (n = 1; n <= tableLog; n++) {
+ U32 curr = nextRankStart;
+ nextRankStart += (rankVal[n] << (n - 1));
+ rankVal[n] = curr;
+ }
+ }
+
+ /* fill nbBits */
+ {
+ U32 n;
+ for (n = 0; n < nbSymbols; n++) {
+ const U32 w = huffWeight[n];
+ CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
+ }
+ }
+
+ /* fill val */
+ {
+ U16 nbPerRank[HUF_TABLELOG_MAX + 2] = {0}; /* support w=0=>n=tableLog+1 */
+ U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0};
+ {
+ U32 n;
+ for (n = 0; n < nbSymbols; n++)
+ nbPerRank[CTable[n].nbBits]++;
+ }
+ /* determine stating value per rank */
+ valPerRank[tableLog + 1] = 0; /* for w==0 */
+ {
+ U16 min = 0;
+ U32 n;
+ for (n = tableLog; n > 0; n--) { /* start at n=tablelog <-> w=1 */
+ valPerRank[n] = min; /* get starting value within each rank */
+ min += nbPerRank[n];
+ min >>= 1;
+ }
+ }
+ /* assign value within rank, symbol order */
+ {
+ U32 n;
+ for (n = 0; n <= maxSymbolValue; n++)
+ CTable[n].val = valPerRank[CTable[n].nbBits]++;
+ }
+ }
+
+ return readSize;
+}
+
+typedef struct nodeElt_s {
+ U32 count;
+ U16 parent;
+ BYTE byte;
+ BYTE nbBits;
+} nodeElt;
+
+static U32 HUF_setMaxHeight(nodeElt *huffNode, U32 lastNonNull, U32 maxNbBits)
+{
+ const U32 largestBits = huffNode[lastNonNull].nbBits;
+ if (largestBits <= maxNbBits)
+ return largestBits; /* early exit : no elt > maxNbBits */
+
+ /* there are several too large elements (at least >= 2) */
+ {
+ int totalCost = 0;
+ const U32 baseCost = 1 << (largestBits - maxNbBits);
+ U32 n = lastNonNull;
+
+ while (huffNode[n].nbBits > maxNbBits) {
+ totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
+ huffNode[n].nbBits = (BYTE)maxNbBits;
+ n--;
+ } /* n stops at huffNode[n].nbBits <= maxNbBits */
+ while (huffNode[n].nbBits == maxNbBits)
+ n--; /* n end at index of smallest symbol using < maxNbBits */
+
+ /* renorm totalCost */
+ totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
+
+ /* repay normalized cost */
+ {
+ U32 const noSymbol = 0xF0F0F0F0;
+ U32 rankLast[HUF_TABLELOG_MAX + 2];
+ int pos;
+
+ /* Get pos of last (smallest) symbol per rank */
+ memset(rankLast, 0xF0, sizeof(rankLast));
+ {
+ U32 currNbBits = maxNbBits;
+ for (pos = n; pos >= 0; pos--) {
+ if (huffNode[pos].nbBits >= currNbBits)
+ continue;
+ currNbBits = huffNode[pos].nbBits; /* < maxNbBits */
+ rankLast[maxNbBits - currNbBits] = pos;
+ }
+ }
+
+ while (totalCost > 0) {
+ U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
+ for (; nBitsToDecrease > 1; nBitsToDecrease--) {
+ U32 highPos = rankLast[nBitsToDecrease];
+ U32 lowPos = rankLast[nBitsToDecrease - 1];
+ if (highPos == noSymbol)
+ continue;
+ if (lowPos == noSymbol)
+ break;
+ {
+ U32 const highTotal = huffNode[highPos].count;
+ U32 const lowTotal = 2 * huffNode[lowPos].count;
+ if (highTotal <= lowTotal)
+ break;
+ }
+ }
+ /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
+ /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
+ while ((nBitsToDecrease <= HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol))
+ nBitsToDecrease++;
+ totalCost -= 1 << (nBitsToDecrease - 1);
+ if (rankLast[nBitsToDecrease - 1] == noSymbol)
+ rankLast[nBitsToDecrease - 1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
+ huffNode[rankLast[nBitsToDecrease]].nbBits++;
+ if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
+ rankLast[nBitsToDecrease] = noSymbol;
+ else {
+ rankLast[nBitsToDecrease]--;
+ if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease)
+ rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
+ }
+ } /* while (totalCost > 0) */
+
+ while (totalCost < 0) { /* Sometimes, cost correction overshoot */
+ if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0
+ (using maxNbBits) */
+ while (huffNode[n].nbBits == maxNbBits)
+ n--;
+ huffNode[n + 1].nbBits--;
+ rankLast[1] = n + 1;
+ totalCost++;
+ continue;
+ }
+ huffNode[rankLast[1] + 1].nbBits--;
+ rankLast[1]++;
+ totalCost++;
+ }
+ }
+ } /* there are several too large elements (at least >= 2) */
+
+ return maxNbBits;
+}
+
+typedef struct {
+ U32 base;
+ U32 curr;
+} rankPos;
+
+static void HUF_sort(nodeElt *huffNode, const U32 *count, U32 maxSymbolValue)
+{
+ rankPos rank[32];
+ U32 n;
+
+ memset(rank, 0, sizeof(rank));
+ for (n = 0; n <= maxSymbolValue; n++) {
+ U32 r = BIT_highbit32(count[n] + 1);
+ rank[r].base++;
+ }
+ for (n = 30; n > 0; n--)
+ rank[n - 1].base += rank[n].base;
+ for (n = 0; n < 32; n++)
+ rank[n].curr = rank[n].base;
+ for (n = 0; n <= maxSymbolValue; n++) {
+ U32 const c = count[n];
+ U32 const r = BIT_highbit32(c + 1) + 1;
+ U32 pos = rank[r].curr++;
+ while ((pos > rank[r].base) && (c > huffNode[pos - 1].count))
+ huffNode[pos] = huffNode[pos - 1], pos--;
+ huffNode[pos].count = c;
+ huffNode[pos].byte = (BYTE)n;
+ }
+}
+
+/** HUF_buildCTable_wksp() :
+ * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
+ * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
+ */
+#define STARTNODE (HUF_SYMBOLVALUE_MAX + 1)
+typedef nodeElt huffNodeTable[2 * HUF_SYMBOLVALUE_MAX + 1 + 1];
+size_t HUF_buildCTable_wksp(HUF_CElt *tree, const U32 *count, U32 maxSymbolValue, U32 maxNbBits, void *workSpace, size_t wkspSize)
+{
+ nodeElt *const huffNode0 = (nodeElt *)workSpace;
+ nodeElt *const huffNode = huffNode0 + 1;
+ U32 n, nonNullRank;
+ int lowS, lowN;
+ U16 nodeNb = STARTNODE;
+ U32 nodeRoot;
+
+ /* safety checks */
+ if (wkspSize < sizeof(huffNodeTable))
+ return ERROR(GENERIC); /* workSpace is not large enough */
+ if (maxNbBits == 0)
+ maxNbBits = HUF_TABLELOG_DEFAULT;
+ if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
+ return ERROR(GENERIC);
+ memset(huffNode0, 0, sizeof(huffNodeTable));
+
+ /* sort, decreasing order */
+ HUF_sort(huffNode, count, maxSymbolValue);
+
+ /* init for parents */
+ nonNullRank = maxSymbolValue;
+ while (huffNode[nonNullRank].count == 0)
+ nonNullRank--;
+ lowS = nonNullRank;
+ nodeRoot = nodeNb + lowS - 1;
+ lowN = nodeNb;
+ huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count;
+ huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb;
+ nodeNb++;
+ lowS -= 2;
+ for (n = nodeNb; n <= nodeRoot; n++)
+ huffNode[n].count = (U32)(1U << 30);
+ huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */
+
+ /* create parents */
+ while (nodeNb <= nodeRoot) {
+ U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
+ U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
+ huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
+ huffNode[n1].parent = huffNode[n2].parent = nodeNb;
+ nodeNb++;
+ }
+
+ /* distribute weights (unlimited tree height) */
+ huffNode[nodeRoot].nbBits = 0;
+ for (n = nodeRoot - 1; n >= STARTNODE; n--)
+ huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1;
+ for (n = 0; n <= nonNullRank; n++)
+ huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1;
+
+ /* enforce maxTableLog */
+ maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
+
+ /* fill result into tree (val, nbBits) */
+ {
+ U16 nbPerRank[HUF_TABLELOG_MAX + 1] = {0};
+ U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0};
+ if (maxNbBits > HUF_TABLELOG_MAX)
+ return ERROR(GENERIC); /* check fit into table */
+ for (n = 0; n <= nonNullRank; n++)
+ nbPerRank[huffNode[n].nbBits]++;
+ /* determine stating value per rank */
+ {
+ U16 min = 0;
+ for (n = maxNbBits; n > 0; n--) {
+ valPerRank[n] = min; /* get starting value within each rank */
+ min += nbPerRank[n];
+ min >>= 1;
+ }
+ }
+ for (n = 0; n <= maxSymbolValue; n++)
+ tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
+ for (n = 0; n <= maxSymbolValue; n++)
+ tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
+ }
+
+ return maxNbBits;
+}
+
+static size_t HUF_estimateCompressedSize(HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue)
+{
+ size_t nbBits = 0;
+ int s;
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
+ nbBits += CTable[s].nbBits * count[s];
+ }
+ return nbBits >> 3;
+}
+
+static int HUF_validateCTable(const HUF_CElt *CTable, const unsigned *count, unsigned maxSymbolValue)
+{
+ int bad = 0;
+ int s;
+ for (s = 0; s <= (int)maxSymbolValue; ++s) {
+ bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
+ }
+ return !bad;
+}
+
+static void HUF_encodeSymbol(BIT_CStream_t *bitCPtr, U32 symbol, const HUF_CElt *CTable)
+{
+ BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
+}
+
+size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
+
+#define HUF_FLUSHBITS(s) BIT_flushBits(s)
+
+#define HUF_FLUSHBITS_1(stream) \
+ if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \
+ HUF_FLUSHBITS(stream)
+
+#define HUF_FLUSHBITS_2(stream) \
+ if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 4 + 7) \
+ HUF_FLUSHBITS(stream)
+
+size_t HUF_compress1X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
+{
+ const BYTE *ip = (const BYTE *)src;
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *const oend = ostart + dstSize;
+ BYTE *op = ostart;
+ size_t n;
+ BIT_CStream_t bitC;
+
+ /* init */
+ if (dstSize < 8)
+ return 0; /* not enough space to compress */
+ {
+ size_t const initErr = BIT_initCStream(&bitC, op, oend - op);
+ if (HUF_isError(initErr))
+ return 0;
+ }
+
+ n = srcSize & ~3; /* join to mod 4 */
+ switch (srcSize & 3) {
+ case 3: HUF_encodeSymbol(&bitC, ip[n + 2], CTable); HUF_FLUSHBITS_2(&bitC);
+ case 2: HUF_encodeSymbol(&bitC, ip[n + 1], CTable); HUF_FLUSHBITS_1(&bitC);
+ case 1: HUF_encodeSymbol(&bitC, ip[n + 0], CTable); HUF_FLUSHBITS(&bitC);
+ case 0:
+ default:;
+ }
+
+ for (; n > 0; n -= 4) { /* note : n&3==0 at this stage */
+ HUF_encodeSymbol(&bitC, ip[n - 1], CTable);
+ HUF_FLUSHBITS_1(&bitC);
+ HUF_encodeSymbol(&bitC, ip[n - 2], CTable);
+ HUF_FLUSHBITS_2(&bitC);
+ HUF_encodeSymbol(&bitC, ip[n - 3], CTable);
+ HUF_FLUSHBITS_1(&bitC);
+ HUF_encodeSymbol(&bitC, ip[n - 4], CTable);
+ HUF_FLUSHBITS(&bitC);
+ }
+
+ return BIT_closeCStream(&bitC);
+}
+
+size_t HUF_compress4X_usingCTable(void *dst, size_t dstSize, const void *src, size_t srcSize, const HUF_CElt *CTable)
+{
+ size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */
+ const BYTE *ip = (const BYTE *)src;
+ const BYTE *const iend = ip + srcSize;
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *const oend = ostart + dstSize;
+ BYTE *op = ostart;
+
+ if (dstSize < 6 + 1 + 1 + 1 + 8)
+ return 0; /* minimum space to compress successfully */
+ if (srcSize < 12)
+ return 0; /* no saving possible : too small input */
+ op += 6; /* jumpTable */
+
+ {
+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
+ if (cSize == 0)
+ return 0;
+ ZSTD_writeLE16(ostart, (U16)cSize);
+ op += cSize;
+ }
+
+ ip += segmentSize;
+ {
+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
+ if (cSize == 0)
+ return 0;
+ ZSTD_writeLE16(ostart + 2, (U16)cSize);
+ op += cSize;
+ }
+
+ ip += segmentSize;
+ {
+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, segmentSize, CTable));
+ if (cSize == 0)
+ return 0;
+ ZSTD_writeLE16(ostart + 4, (U16)cSize);
+ op += cSize;
+ }
+
+ ip += segmentSize;
+ {
+ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend - op, ip, iend - ip, CTable));
+ if (cSize == 0)
+ return 0;
+ op += cSize;
+ }
+
+ return op - ostart;
+}
+
+static size_t HUF_compressCTable_internal(BYTE *const ostart, BYTE *op, BYTE *const oend, const void *src, size_t srcSize, unsigned singleStream,
+ const HUF_CElt *CTable)
+{
+ size_t const cSize =
+ singleStream ? HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
+ if (HUF_isError(cSize)) {
+ return cSize;
+ }
+ if (cSize == 0) {
+ return 0;
+ } /* uncompressible */
+ op += cSize;
+ /* check compressibility */
+ if ((size_t)(op - ostart) >= srcSize - 1) {
+ return 0;
+ }
+ return op - ostart;
+}
+
+/* `workSpace` must a table of at least 1024 unsigned */
+static size_t HUF_compress_internal(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog,
+ unsigned singleStream, void *workSpace, size_t wkspSize, HUF_CElt *oldHufTable, HUF_repeat *repeat, int preferRepeat)
+{
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *const oend = ostart + dstSize;
+ BYTE *op = ostart;
+
+ U32 *count;
+ size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
+ HUF_CElt *CTable;
+ size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
+
+ /* checks & inits */
+ if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize)
+ return ERROR(GENERIC);
+ if (!srcSize)
+ return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
+ if (!dstSize)
+ return 0; /* cannot fit within dst budget */
+ if (srcSize > HUF_BLOCKSIZE_MAX)
+ return ERROR(srcSize_wrong); /* curr block size limit */
+ if (huffLog > HUF_TABLELOG_MAX)
+ return ERROR(tableLog_tooLarge);
+ if (!maxSymbolValue)
+ maxSymbolValue = HUF_SYMBOLVALUE_MAX;
+ if (!huffLog)
+ huffLog = HUF_TABLELOG_DEFAULT;
+
+ count = (U32 *)workSpace;
+ workSpace = (BYTE *)workSpace + countSize;
+ wkspSize -= countSize;
+ CTable = (HUF_CElt *)workSpace;
+ workSpace = (BYTE *)workSpace + CTableSize;
+ wkspSize -= CTableSize;
+
+ /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
+ if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+ }
+
+ /* Scan input and build symbol stats */
+ {
+ CHECK_V_F(largest, FSE_count_wksp(count, &maxSymbolValue, (const BYTE *)src, srcSize, (U32 *)workSpace));
+ if (largest == srcSize) {
+ *ostart = ((const BYTE *)src)[0];
+ return 1;
+ } /* single symbol, rle */
+ if (largest <= (srcSize >> 7) + 1)
+ return 0; /* Fast heuristic : not compressible enough */
+ }
+
+ /* Check validity of previous table */
+ if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
+ *repeat = HUF_repeat_none;
+ }
+ /* Heuristic : use existing table for small inputs */
+ if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+ }
+
+ /* Build Huffman Tree */
+ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
+ {
+ CHECK_V_F(maxBits, HUF_buildCTable_wksp(CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize));
+ huffLog = (U32)maxBits;
+ /* Zero the unused symbols so we can check it for validity */
+ memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
+ }
+
+ /* Write table description header */
+ {
+ CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, CTable, maxSymbolValue, huffLog, workSpace, wkspSize));
+ /* Check if using the previous table will be beneficial */
+ if (repeat && *repeat != HUF_repeat_none) {
+ size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
+ size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
+ if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+ }
+ }
+ /* Use the new table */
+ if (hSize + 12ul >= srcSize) {
+ return 0;
+ }
+ op += hSize;
+ if (repeat) {
+ *repeat = HUF_repeat_none;
+ }
+ if (oldHufTable) {
+ memcpy(oldHufTable, CTable, CTableSize);
+ } /* Save the new table */
+ }
+ return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
+}
+
+size_t HUF_compress1X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
+ size_t wkspSize)
+{
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
+}
+
+size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
+{
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat,
+ preferRepeat);
+}
+
+size_t HUF_compress4X_wksp(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
+ size_t wkspSize)
+{
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
+}
+
+size_t HUF_compress4X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace,
+ size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat)
+{
+ return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat,
+ preferRepeat);
+}
diff --git a/lib/zstd/huf_decompress.c b/lib/zstd/huf_decompress.c
new file mode 100644
index 000000000000..6526482047dc
--- /dev/null
+++ b/lib/zstd/huf_decompress.c
@@ -0,0 +1,960 @@
+/*
+ * Huffman decoder, part of New Generation Entropy library
+ * Copyright (C) 2013-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at :
+ * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+ */
+
+/* **************************************************************
+* Compiler specifics
+****************************************************************/
+#define FORCE_INLINE static __always_inline
+
+/* **************************************************************
+* Dependencies
+****************************************************************/
+#include "bitstream.h" /* BIT_* */
+#include "fse.h" /* header compression */
+#include "huf.h"
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/string.h> /* memcpy, memset */
+
+/* **************************************************************
+* Error Management
+****************************************************************/
+#define HUF_STATIC_ASSERT(c) \
+ { \
+ enum { HUF_static_assert = 1 / (int)(!!(c)) }; \
+ } /* use only *after* variable declarations */
+
+/*-***************************/
+/* generic DTableDesc */
+/*-***************************/
+
+typedef struct {
+ BYTE maxTableLog;
+ BYTE tableType;
+ BYTE tableLog;
+ BYTE reserved;
+} DTableDesc;
+
+static DTableDesc HUF_getDTableDesc(const HUF_DTable *table)
+{
+ DTableDesc dtd;
+ memcpy(&dtd, table, sizeof(dtd));
+ return dtd;
+}
+
+/*-***************************/
+/* single-symbol decoding */
+/*-***************************/
+
+typedef struct {
+ BYTE byte;
+ BYTE nbBits;
+} HUF_DEltX2; /* single-symbol decoding */
+
+size_t HUF_readDTableX2_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
+{
+ U32 tableLog = 0;
+ U32 nbSymbols = 0;
+ size_t iSize;
+ void *const dtPtr = DTable + 1;
+ HUF_DEltX2 *const dt = (HUF_DEltX2 *)dtPtr;
+
+ U32 *rankVal;
+ BYTE *huffWeight;
+ size_t spaceUsed32 = 0;
+
+ rankVal = (U32 *)workspace + spaceUsed32;
+ spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1;
+ huffWeight = (BYTE *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
+
+ if ((spaceUsed32 << 2) > workspaceSize)
+ return ERROR(tableLog_tooLarge);
+ workspace = (U32 *)workspace + spaceUsed32;
+ workspaceSize -= (spaceUsed32 << 2);
+
+ HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
+ /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
+
+ iSize = HUF_readStats_wksp(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize);
+ if (HUF_isError(iSize))
+ return iSize;
+
+ /* Table header */
+ {
+ DTableDesc dtd = HUF_getDTableDesc(DTable);
+ if (tableLog > (U32)(dtd.maxTableLog + 1))
+ return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */
+ dtd.tableType = 0;
+ dtd.tableLog = (BYTE)tableLog;
+ memcpy(DTable, &dtd, sizeof(dtd));
+ }
+
+ /* Calculate starting value for each rank */
+ {
+ U32 n, nextRankStart = 0;
+ for (n = 1; n < tableLog + 1; n++) {
+ U32 const curr = nextRankStart;
+ nextRankStart += (rankVal[n] << (n - 1));
+ rankVal[n] = curr;
+ }
+ }
+
+ /* fill DTable */
+ {
+ U32 n;
+ for (n = 0; n < nbSymbols; n++) {
+ U32 const w = huffWeight[n];
+ U32 const length = (1 << w) >> 1;
+ U32 u;
+ HUF_DEltX2 D;
+ D.byte = (BYTE)n;
+ D.nbBits = (BYTE)(tableLog + 1 - w);
+ for (u = rankVal[w]; u < rankVal[w] + length; u++)
+ dt[u] = D;
+ rankVal[w] += length;
+ }
+ }
+
+ return iSize;
+}
+
+static BYTE HUF_decodeSymbolX2(BIT_DStream_t *Dstream, const HUF_DEltX2 *dt, const U32 dtLog)
+{
+ size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
+ BYTE const c = dt[val].byte;
+ BIT_skipBits(Dstream, dt[val].nbBits);
+ return c;
+}
+
+#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
+
+#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
+ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
+ HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
+
+#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
+ if (ZSTD_64bits()) \
+ HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
+
+FORCE_INLINE size_t HUF_decodeStreamX2(BYTE *p, BIT_DStream_t *const bitDPtr, BYTE *const pEnd, const HUF_DEltX2 *const dt, const U32 dtLog)
+{
+ BYTE *const pStart = p;
+
+ /* up to 4 symbols at a time */
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd - 4)) {
+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+ }
+
+ /* closer to the end */
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd))
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+
+ /* no more data to retrieve from bitstream, hence no need to reload */
+ while (p < pEnd)
+ HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
+
+ return pEnd - pStart;
+}
+
+static size_t HUF_decompress1X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ BYTE *op = (BYTE *)dst;
+ BYTE *const oend = op + dstSize;
+ const void *dtPtr = DTable + 1;
+ const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr;
+ BIT_DStream_t bitD;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ U32 const dtLog = dtd.tableLog;
+
+ {
+ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+
+ HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog);
+
+ /* check */
+ if (!BIT_endOfDStream(&bitD))
+ return ERROR(corruption_detected);
+
+ return dstSize;
+}
+
+size_t HUF_decompress1X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ DTableDesc dtd = HUF_getDTableDesc(DTable);
+ if (dtd.tableType != 0)
+ return ERROR(GENERIC);
+ return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
+{
+ const BYTE *ip = (const BYTE *)cSrc;
+
+ size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize);
+ if (HUF_isError(hSize))
+ return hSize;
+ if (hSize >= cSrcSize)
+ return ERROR(srcSize_wrong);
+ ip += hSize;
+ cSrcSize -= hSize;
+
+ return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
+}
+
+static size_t HUF_decompress4X2_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ /* Check */
+ if (cSrcSize < 10)
+ return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
+
+ {
+ const BYTE *const istart = (const BYTE *)cSrc;
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *const oend = ostart + dstSize;
+ const void *const dtPtr = DTable + 1;
+ const HUF_DEltX2 *const dt = (const HUF_DEltX2 *)dtPtr;
+
+ /* Init */
+ BIT_DStream_t bitD1;
+ BIT_DStream_t bitD2;
+ BIT_DStream_t bitD3;
+ BIT_DStream_t bitD4;
+ size_t const length1 = ZSTD_readLE16(istart);
+ size_t const length2 = ZSTD_readLE16(istart + 2);
+ size_t const length3 = ZSTD_readLE16(istart + 4);
+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
+ const BYTE *const istart1 = istart + 6; /* jumpTable */
+ const BYTE *const istart2 = istart1 + length1;
+ const BYTE *const istart3 = istart2 + length2;
+ const BYTE *const istart4 = istart3 + length3;
+ const size_t segmentSize = (dstSize + 3) / 4;
+ BYTE *const opStart2 = ostart + segmentSize;
+ BYTE *const opStart3 = opStart2 + segmentSize;
+ BYTE *const opStart4 = opStart3 + segmentSize;
+ BYTE *op1 = ostart;
+ BYTE *op2 = opStart2;
+ BYTE *op3 = opStart3;
+ BYTE *op4 = opStart4;
+ U32 endSignal;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ U32 const dtLog = dtd.tableLog;
+
+ if (length4 > cSrcSize)
+ return ERROR(corruption_detected); /* overflow */
+ {
+ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+ {
+ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+ {
+ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+ {
+ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+
+ /* 16-32 symbols per loop (4-8 symbols per stream) */
+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
+ for (; (endSignal == BIT_DStream_unfinished) && (op4 < (oend - 7));) {
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
+ HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
+ HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
+ HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
+ }
+
+ /* check corruption */
+ if (op1 > opStart2)
+ return ERROR(corruption_detected);
+ if (op2 > opStart3)
+ return ERROR(corruption_detected);
+ if (op3 > opStart4)
+ return ERROR(corruption_detected);
+ /* note : op4 supposed already verified within main loop */
+
+ /* finish bitStreams one by one */
+ HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
+ HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
+ HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
+ HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
+
+ /* check */
+ endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
+ if (!endSignal)
+ return ERROR(corruption_detected);
+
+ /* decoded size */
+ return dstSize;
+ }
+}
+
+size_t HUF_decompress4X2_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ DTableDesc dtd = HUF_getDTableDesc(DTable);
+ if (dtd.tableType != 0)
+ return ERROR(GENERIC);
+ return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
+{
+ const BYTE *ip = (const BYTE *)cSrc;
+
+ size_t const hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize);
+ if (HUF_isError(hSize))
+ return hSize;
+ if (hSize >= cSrcSize)
+ return ERROR(srcSize_wrong);
+ ip += hSize;
+ cSrcSize -= hSize;
+
+ return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
+}
+
+/* *************************/
+/* double-symbols decoding */
+/* *************************/
+typedef struct {
+ U16 sequence;
+ BYTE nbBits;
+ BYTE length;
+} HUF_DEltX4; /* double-symbols decoding */
+
+typedef struct {
+ BYTE symbol;
+ BYTE weight;
+} sortedSymbol_t;
+
+/* HUF_fillDTableX4Level2() :
+ * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
+static void HUF_fillDTableX4Level2(HUF_DEltX4 *DTable, U32 sizeLog, const U32 consumed, const U32 *rankValOrigin, const int minWeight,
+ const sortedSymbol_t *sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq)
+{
+ HUF_DEltX4 DElt;
+ U32 rankVal[HUF_TABLELOG_MAX + 1];
+
+ /* get pre-calculated rankVal */
+ memcpy(rankVal, rankValOrigin, sizeof(rankVal));
+
+ /* fill skipped values */
+ if (minWeight > 1) {
+ U32 i, skipSize = rankVal[minWeight];
+ ZSTD_writeLE16(&(DElt.sequence), baseSeq);
+ DElt.nbBits = (BYTE)(consumed);
+ DElt.length = 1;
+ for (i = 0; i < skipSize; i++)
+ DTable[i] = DElt;
+ }
+
+ /* fill DTable */
+ {
+ U32 s;
+ for (s = 0; s < sortedListSize; s++) { /* note : sortedSymbols already skipped */
+ const U32 symbol = sortedSymbols[s].symbol;
+ const U32 weight = sortedSymbols[s].weight;
+ const U32 nbBits = nbBitsBaseline - weight;
+ const U32 length = 1 << (sizeLog - nbBits);
+ const U32 start = rankVal[weight];
+ U32 i = start;
+ const U32 end = start + length;
+
+ ZSTD_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
+ DElt.nbBits = (BYTE)(nbBits + consumed);
+ DElt.length = 2;
+ do {
+ DTable[i++] = DElt;
+ } while (i < end); /* since length >= 1 */
+
+ rankVal[weight] += length;
+ }
+ }
+}
+
+typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
+typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
+
+static void HUF_fillDTableX4(HUF_DEltX4 *DTable, const U32 targetLog, const sortedSymbol_t *sortedList, const U32 sortedListSize, const U32 *rankStart,
+ rankVal_t rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline)
+{
+ U32 rankVal[HUF_TABLELOG_MAX + 1];
+ const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
+ const U32 minBits = nbBitsBaseline - maxWeight;
+ U32 s;
+
+ memcpy(rankVal, rankValOrigin, sizeof(rankVal));
+
+ /* fill DTable */
+ for (s = 0; s < sortedListSize; s++) {
+ const U16 symbol = sortedList[s].symbol;
+ const U32 weight = sortedList[s].weight;
+ const U32 nbBits = nbBitsBaseline - weight;
+ const U32 start = rankVal[weight];
+ const U32 length = 1 << (targetLog - nbBits);
+
+ if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */
+ U32 sortedRank;
+ int minWeight = nbBits + scaleLog;
+ if (minWeight < 1)
+ minWeight = 1;
+ sortedRank = rankStart[minWeight];
+ HUF_fillDTableX4Level2(DTable + start, targetLog - nbBits, nbBits, rankValOrigin[nbBits], minWeight, sortedList + sortedRank,
+ sortedListSize - sortedRank, nbBitsBaseline, symbol);
+ } else {
+ HUF_DEltX4 DElt;
+ ZSTD_writeLE16(&(DElt.sequence), symbol);
+ DElt.nbBits = (BYTE)(nbBits);
+ DElt.length = 1;
+ {
+ U32 const end = start + length;
+ U32 u;
+ for (u = start; u < end; u++)
+ DTable[u] = DElt;
+ }
+ }
+ rankVal[weight] += length;
+ }
+}
+
+size_t HUF_readDTableX4_wksp(HUF_DTable *DTable, const void *src, size_t srcSize, void *workspace, size_t workspaceSize)
+{
+ U32 tableLog, maxW, sizeOfSort, nbSymbols;
+ DTableDesc dtd = HUF_getDTableDesc(DTable);
+ U32 const maxTableLog = dtd.maxTableLog;
+ size_t iSize;
+ void *dtPtr = DTable + 1; /* force compiler to avoid strict-aliasing */
+ HUF_DEltX4 *const dt = (HUF_DEltX4 *)dtPtr;
+ U32 *rankStart;
+
+ rankValCol_t *rankVal;
+ U32 *rankStats;
+ U32 *rankStart0;
+ sortedSymbol_t *sortedSymbol;
+ BYTE *weightList;
+ size_t spaceUsed32 = 0;
+
+ HUF_STATIC_ASSERT((sizeof(rankValCol_t) & 3) == 0);
+
+ rankVal = (rankValCol_t *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2;
+ rankStats = (U32 *)workspace + spaceUsed32;
+ spaceUsed32 += HUF_TABLELOG_MAX + 1;
+ rankStart0 = (U32 *)workspace + spaceUsed32;
+ spaceUsed32 += HUF_TABLELOG_MAX + 2;
+ sortedSymbol = (sortedSymbol_t *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2;
+ weightList = (BYTE *)((U32 *)workspace + spaceUsed32);
+ spaceUsed32 += ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
+
+ if ((spaceUsed32 << 2) > workspaceSize)
+ return ERROR(tableLog_tooLarge);
+ workspace = (U32 *)workspace + spaceUsed32;
+ workspaceSize -= (spaceUsed32 << 2);
+
+ rankStart = rankStart0 + 1;
+ memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1));
+
+ HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
+ if (maxTableLog > HUF_TABLELOG_MAX)
+ return ERROR(tableLog_tooLarge);
+ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
+
+ iSize = HUF_readStats_wksp(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize, workspace, workspaceSize);
+ if (HUF_isError(iSize))
+ return iSize;
+
+ /* check result */
+ if (tableLog > maxTableLog)
+ return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
+
+ /* find maxWeight */
+ for (maxW = tableLog; rankStats[maxW] == 0; maxW--) {
+ } /* necessarily finds a solution before 0 */
+
+ /* Get start index of each weight */
+ {
+ U32 w, nextRankStart = 0;
+ for (w = 1; w < maxW + 1; w++) {
+ U32 curr = nextRankStart;
+ nextRankStart += rankStats[w];
+ rankStart[w] = curr;
+ }
+ rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
+ sizeOfSort = nextRankStart;
+ }
+
+ /* sort symbols by weight */
+ {
+ U32 s;
+ for (s = 0; s < nbSymbols; s++) {
+ U32 const w = weightList[s];
+ U32 const r = rankStart[w]++;
+ sortedSymbol[r].symbol = (BYTE)s;
+ sortedSymbol[r].weight = (BYTE)w;
+ }
+ rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
+ }
+
+ /* Build rankVal */
+ {
+ U32 *const rankVal0 = rankVal[0];
+ {
+ int const rescale = (maxTableLog - tableLog) - 1; /* tableLog <= maxTableLog */
+ U32 nextRankVal = 0;
+ U32 w;
+ for (w = 1; w < maxW + 1; w++) {
+ U32 curr = nextRankVal;
+ nextRankVal += rankStats[w] << (w + rescale);
+ rankVal0[w] = curr;
+ }
+ }
+ {
+ U32 const minBits = tableLog + 1 - maxW;
+ U32 consumed;
+ for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
+ U32 *const rankValPtr = rankVal[consumed];
+ U32 w;
+ for (w = 1; w < maxW + 1; w++) {
+ rankValPtr[w] = rankVal0[w] >> consumed;
+ }
+ }
+ }
+ }
+
+ HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog + 1);
+
+ dtd.tableLog = (BYTE)maxTableLog;
+ dtd.tableType = 1;
+ memcpy(DTable, &dtd, sizeof(dtd));
+ return iSize;
+}
+
+static U32 HUF_decodeSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog)
+{
+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
+ memcpy(op, dt + val, 2);
+ BIT_skipBits(DStream, dt[val].nbBits);
+ return dt[val].length;
+}
+
+static U32 HUF_decodeLastSymbolX4(void *op, BIT_DStream_t *DStream, const HUF_DEltX4 *dt, const U32 dtLog)
+{
+ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
+ memcpy(op, dt + val, 1);
+ if (dt[val].length == 1)
+ BIT_skipBits(DStream, dt[val].nbBits);
+ else {
+ if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) {
+ BIT_skipBits(DStream, dt[val].nbBits);
+ if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8))
+ /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
+ DStream->bitsConsumed = (sizeof(DStream->bitContainer) * 8);
+ }
+ }
+ return 1;
+}
+
+#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
+
+#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
+ if (ZSTD_64bits() || (HUF_TABLELOG_MAX <= 12)) \
+ ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
+
+#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
+ if (ZSTD_64bits()) \
+ ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
+
+FORCE_INLINE size_t HUF_decodeStreamX4(BYTE *p, BIT_DStream_t *bitDPtr, BYTE *const pEnd, const HUF_DEltX4 *const dt, const U32 dtLog)
+{
+ BYTE *const pStart = p;
+
+ /* up to 8 symbols at a time */
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd - (sizeof(bitDPtr->bitContainer) - 1))) {
+ HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX4_1(p, bitDPtr);
+ HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
+ HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
+ }
+
+ /* closer to end : up to 2 symbols at a time */
+ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd - 2))
+ HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
+
+ while (p <= pEnd - 2)
+ HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
+
+ if (p < pEnd)
+ p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
+
+ return p - pStart;
+}
+
+static size_t HUF_decompress1X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ BIT_DStream_t bitD;
+
+ /* Init */
+ {
+ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+
+ /* decode */
+ {
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *const oend = ostart + dstSize;
+ const void *const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */
+ const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog);
+ }
+
+ /* check */
+ if (!BIT_endOfDStream(&bitD))
+ return ERROR(corruption_detected);
+
+ /* decoded size */
+ return dstSize;
+}
+
+size_t HUF_decompress1X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ DTableDesc dtd = HUF_getDTableDesc(DTable);
+ if (dtd.tableType != 1)
+ return ERROR(GENERIC);
+ return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable *DCtx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
+{
+ const BYTE *ip = (const BYTE *)cSrc;
+
+ size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize, workspace, workspaceSize);
+ if (HUF_isError(hSize))
+ return hSize;
+ if (hSize >= cSrcSize)
+ return ERROR(srcSize_wrong);
+ ip += hSize;
+ cSrcSize -= hSize;
+
+ return HUF_decompress1X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx);
+}
+
+static size_t HUF_decompress4X4_usingDTable_internal(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ if (cSrcSize < 10)
+ return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
+
+ {
+ const BYTE *const istart = (const BYTE *)cSrc;
+ BYTE *const ostart = (BYTE *)dst;
+ BYTE *const oend = ostart + dstSize;
+ const void *const dtPtr = DTable + 1;
+ const HUF_DEltX4 *const dt = (const HUF_DEltX4 *)dtPtr;
+
+ /* Init */
+ BIT_DStream_t bitD1;
+ BIT_DStream_t bitD2;
+ BIT_DStream_t bitD3;
+ BIT_DStream_t bitD4;
+ size_t const length1 = ZSTD_readLE16(istart);
+ size_t const length2 = ZSTD_readLE16(istart + 2);
+ size_t const length3 = ZSTD_readLE16(istart + 4);
+ size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
+ const BYTE *const istart1 = istart + 6; /* jumpTable */
+ const BYTE *const istart2 = istart1 + length1;
+ const BYTE *const istart3 = istart2 + length2;
+ const BYTE *const istart4 = istart3 + length3;
+ size_t const segmentSize = (dstSize + 3) / 4;
+ BYTE *const opStart2 = ostart + segmentSize;
+ BYTE *const opStart3 = opStart2 + segmentSize;
+ BYTE *const opStart4 = opStart3 + segmentSize;
+ BYTE *op1 = ostart;
+ BYTE *op2 = opStart2;
+ BYTE *op3 = opStart3;
+ BYTE *op4 = opStart4;
+ U32 endSignal;
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ U32 const dtLog = dtd.tableLog;
+
+ if (length4 > cSrcSize)
+ return ERROR(corruption_detected); /* overflow */
+ {
+ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+ {
+ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+ {
+ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+ {
+ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
+ if (HUF_isError(errorCode))
+ return errorCode;
+ }
+
+ /* 16-32 symbols per loop (4-8 symbols per stream) */
+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
+ for (; (endSignal == BIT_DStream_unfinished) & (op4 < (oend - (sizeof(bitD4.bitContainer) - 1)));) {
+ HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX4_1(op1, &bitD1);
+ HUF_DECODE_SYMBOLX4_1(op2, &bitD2);
+ HUF_DECODE_SYMBOLX4_1(op3, &bitD3);
+ HUF_DECODE_SYMBOLX4_1(op4, &bitD4);
+ HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
+ HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
+ HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
+ HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
+ HUF_DECODE_SYMBOLX4_0(op1, &bitD1);
+ HUF_DECODE_SYMBOLX4_0(op2, &bitD2);
+ HUF_DECODE_SYMBOLX4_0(op3, &bitD3);
+ HUF_DECODE_SYMBOLX4_0(op4, &bitD4);
+
+ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
+ }
+
+ /* check corruption */
+ if (op1 > opStart2)
+ return ERROR(corruption_detected);
+ if (op2 > opStart3)
+ return ERROR(corruption_detected);
+ if (op3 > opStart4)
+ return ERROR(corruption_detected);
+ /* note : op4 already verified within main loop */
+
+ /* finish bitStreams one by one */
+ HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
+ HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
+ HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
+ HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
+
+ /* check */
+ {
+ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
+ if (!endCheck)
+ return ERROR(corruption_detected);
+ }
+
+ /* decoded size */
+ return dstSize;
+ }
+}
+
+size_t HUF_decompress4X4_usingDTable(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ DTableDesc dtd = HUF_getDTableDesc(DTable);
+ if (dtd.tableType != 1)
+ return ERROR(GENERIC);
+ return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
+}
+
+size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
+{
+ const BYTE *ip = (const BYTE *)cSrc;
+
+ size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize, workspace, workspaceSize);
+ if (HUF_isError(hSize))
+ return hSize;
+ if (hSize >= cSrcSize)
+ return ERROR(srcSize_wrong);
+ ip += hSize;
+ cSrcSize -= hSize;
+
+ return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
+}
+
+/* ********************************/
+/* Generic decompression selector */
+/* ********************************/
+
+size_t HUF_decompress1X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
+ : HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
+}
+
+size_t HUF_decompress4X_usingDTable(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable)
+{
+ DTableDesc const dtd = HUF_getDTableDesc(DTable);
+ return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable)
+ : HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
+}
+
+typedef struct {
+ U32 tableTime;
+ U32 decode256Time;
+} algo_time_t;
+static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = {
+ /* single, double, quad */
+ {{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */
+ {{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */
+ {{38, 130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */
+ {{448, 128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */
+ {{556, 128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */
+ {{714, 128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */
+ {{883, 128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */
+ {{897, 128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */
+ {{926, 128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */
+ {{947, 128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */
+ {{1107, 128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */
+ {{1177, 128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */
+ {{1242, 128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */
+ {{1349, 128}, {2644, 106}, {5260, 106}}, /* Q ==13 : 81-87% */
+ {{1455, 128}, {2422, 124}, {4174, 124}}, /* Q ==14 : 87-93% */
+ {{722, 128}, {1891, 145}, {1936, 146}}, /* Q ==15 : 93-99% */
+};
+
+/** HUF_selectDecoder() :
+* Tells which decoder is likely to decode faster,
+* based on a set of pre-determined metrics.
+* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
+* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
+U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize)
+{
+ /* decoder timing evaluation */
+ U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
+ U32 const D256 = (U32)(dstSize >> 8);
+ U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
+ U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
+ DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */
+
+ return DTime1 < DTime0;
+}
+
+typedef size_t (*decompressionAlgo)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize);
+
+size_t HUF_decompress4X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
+{
+ /* validation checks */
+ if (dstSize == 0)
+ return ERROR(dstSize_tooSmall);
+ if (cSrcSize > dstSize)
+ return ERROR(corruption_detected); /* invalid */
+ if (cSrcSize == dstSize) {
+ memcpy(dst, cSrc, dstSize);
+ return dstSize;
+ } /* not compressed */
+ if (cSrcSize == 1) {
+ memset(dst, *(const BYTE *)cSrc, dstSize);
+ return dstSize;
+ } /* RLE */
+
+ {
+ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
+ return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize)
+ : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize);
+ }
+}
+
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
+{
+ /* validation checks */
+ if (dstSize == 0)
+ return ERROR(dstSize_tooSmall);
+ if ((cSrcSize >= dstSize) || (cSrcSize <= 1))
+ return ERROR(corruption_detected); /* invalid */
+
+ {
+ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
+ return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize)
+ : HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize);
+ }
+}
+
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable *dctx, void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, void *workspace, size_t workspaceSize)
+{
+ /* validation checks */
+ if (dstSize == 0)
+ return ERROR(dstSize_tooSmall);
+ if (cSrcSize > dstSize)
+ return ERROR(corruption_detected); /* invalid */
+ if (cSrcSize == dstSize) {
+ memcpy(dst, cSrc, dstSize);
+ return dstSize;
+ } /* not compressed */
+ if (cSrcSize == 1) {
+ memset(dst, *(const BYTE *)cSrc, dstSize);
+ return dstSize;
+ } /* RLE */
+
+ {
+ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
+ return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize)
+ : HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workspace, workspaceSize);
+ }
+}
diff --git a/lib/zstd/mem.h b/lib/zstd/mem.h
new file mode 100644
index 000000000000..3a0f34c8706c
--- /dev/null
+++ b/lib/zstd/mem.h
@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
+ * An additional grant of patent rights can be found in the PATENTS file in the
+ * same directory.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ */
+
+#ifndef MEM_H_MODULE
+#define MEM_H_MODULE
+
+/*-****************************************
+* Dependencies
+******************************************/
+#include <asm/unaligned.h>
+#include <linux/string.h> /* memcpy */
+#include <linux/types.h> /* size_t, ptrdiff_t */
+
+/*-****************************************
+* Compiler specifics
+******************************************/
+#define ZSTD_STATIC static __inline __attribute__((unused))
+
+/*-**************************************************************
+* Basic Types
+*****************************************************************/
+typedef uint8_t BYTE;
+typedef uint16_t U16;
+typedef int16_t S16;
+typedef uint32_t U32;
+typedef int32_t S32;
+typedef uint64_t U64;
+typedef int64_t S64;
+typedef ptrdiff_t iPtrDiff;
+typedef uintptr_t uPtrDiff;
+
+/*-**************************************************************
+* Memory I/O
+*****************************************************************/
+ZSTD_STATIC unsigned ZSTD_32bits(void) { return sizeof(size_t) == 4; }
+ZSTD_STATIC unsigned ZSTD_64bits(void) { return sizeof(size_t) == 8; }
+
+#if defined(__LITTLE_ENDIAN)
+#define ZSTD_LITTLE_ENDIAN 1
+#else
+#define ZSTD_LITTLE_ENDIAN 0
+#endif
+
+ZSTD_STATIC unsigned ZSTD_isLittleEndian(void) { return ZSTD_LITTLE_ENDIAN; }
+
+ZSTD_STATIC U16 ZSTD_read16(const void *memPtr) { return get_unaligned((const U16 *)memPtr); }
+
+ZSTD_STATIC U32 ZSTD_read32(const void *memPtr) { return get_unaligned((const U32 *)memPtr); }
+
+ZSTD_STATIC U64 ZSTD_read64(const void *memPtr) { return get_unaligned((const U64 *)memPtr); }
+
+ZSTD_STATIC size_t ZSTD_readST(const void *memPtr) { return get_unaligned((const size_t *)memPtr); }
+
+ZSTD_STATIC void ZSTD_write16(void *memPtr, U16 value) { put_unaligned(value, (U16 *)memPtr); }
+
+ZSTD_STATIC void ZSTD_write32(void *memPtr, U32 value) { put_unaligned(value, (U32 *)memPtr); }
+
+ZSTD_STATIC void ZSTD_write64(void *memPtr, U64 value) { put_unaligned(value, (U64 *)memPtr); }
+
+/*=== Little endian r/w ===*/
+
+ZSTD_STATIC U16 ZSTD_readLE16(const void *memPtr) { return get_unaligned_le16(memPtr); }
+
+ZSTD_STATIC void ZSTD_writeLE16(void *memPtr, U16 val) { put_unaligned_le16(val, memPtr); }
+
+ZSTD_STATIC U32 ZSTD_readLE24(const void *memPtr) { return ZSTD_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16); }
+
+ZSTD_STATIC void ZSTD_writeLE24(void *memPtr, U32 val)
+{
+ ZSTD_writeLE16(memPtr, (U16)val);
+ ((BYTE *)memPtr)[2] = (BYTE)(val >> 16);
+}
+
+ZSTD_STATIC U32 ZSTD_readLE32(const void *memPtr) { return get_unaligned_le32(memPtr); }
+
+ZSTD_STATIC void ZSTD_writeLE32(void *memPtr, U32 val32) { put_unaligned_le32(val32, memPtr); }
+
+ZSTD_STATIC U64 ZSTD_readLE64(const void *memPtr) { return get_unaligned_le64(memPtr); }
+
+ZSTD_STATIC void ZSTD_writeLE64(void *memPtr, U64 val64) { put_unaligned_le64(val64, memPtr); }
+
+ZSTD_STATIC size_t ZSTD_readLEST(const void *memPtr)
+{
+ if (ZSTD_32bits())
+ return (size_t)ZSTD_readLE32(memPtr);
+ else
+ return (size_t)ZSTD_readLE64(memPtr);
+}
+
+ZSTD_STATIC void ZSTD_writeLEST(void *memPtr, size_t val)
+{
+ if (ZSTD_32bits())
+ ZSTD_writeLE32(memPtr, (U32)val);
+ else
+ ZSTD_writeLE64(memPtr, (U64)val);
+}
+
+/*=== Big endian r/w ===*/
+
+ZSTD_STATIC U32 ZSTD_readBE32(const void *memPtr) { return get_unaligned_be32(memPtr); }
+
+ZSTD_STATIC void ZSTD_writeBE32(void *memPtr, U32 val32) { put_unaligned_be32(val32, memPtr); }
+
+ZSTD_STATIC U64 ZSTD_readBE64(const void *memPtr) { return get_unaligned_be64(memPtr); }
+
+ZSTD_STATIC void ZSTD_writeBE64(void *memPtr, U64 val64) { put_unaligned_be64(val64, memPtr); }
+
+ZSTD_STATIC size_t ZSTD_readBEST(const void *memPtr)
+{
+ if (ZSTD_32bits())
+ return (size_t)ZSTD_readBE32(memPtr);
+ else
+ return (size_t)ZSTD_readBE64(memPtr);
+}
+
+ZSTD_STATIC void ZSTD_writeBEST(void *memPtr, size_t val)
+{
+ if (ZSTD_32bits())
+ ZSTD_writeBE32(memPtr, (U32)val);
+ else
+ ZSTD_writeBE64(memPtr, (U64)val);
+}
+
+/* function safe only for comparisons */
+ZSTD_STATIC U32 ZSTD_readMINMATCH(const void *memPtr, U32 length)
+{
+ switch (length) {
+ default:
+ case 4: return ZSTD_read32(memPtr);
+ case 3:
+ if (ZSTD_isLittleEndian())
+ return ZSTD_read32(memPtr) << 8;
+ else
+ return ZSTD_read32(memPtr) >> 8;
+ }
+}
+
+#endif /* MEM_H_MODULE */
diff --git a/lib/zstd/zstd_common.c b/lib/zstd/zstd_common.c
new file mode 100644
index 000000000000..a282624ee155
--- /dev/null
+++ b/lib/zstd/zstd_common.c
@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
+ * An additional grant of patent rights can be found in the PATENTS file in the
+ * same directory.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ */
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "error_private.h"
+#include "zstd_internal.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
+#include <linux/kernel.h>
+
+/*=**************************************************************
+* Custom allocator
+****************************************************************/
+
+#define stack_push(stack, size) \
+ ({ \
+ void *const ptr = ZSTD_PTR_ALIGN((stack)->ptr); \
+ (stack)->ptr = (char *)ptr + (size); \
+ (stack)->ptr <= (stack)->end ? ptr : NULL; \
+ })
+
+ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize)
+{
+ ZSTD_customMem stackMem = {ZSTD_stackAlloc, ZSTD_stackFree, workspace};
+ ZSTD_stack *stack = (ZSTD_stack *)workspace;
+ /* Verify preconditions */
+ if (!workspace || workspaceSize < sizeof(ZSTD_stack) || workspace != ZSTD_PTR_ALIGN(workspace)) {
+ ZSTD_customMem error = {NULL, NULL, NULL};
+ return error;
+ }
+ /* Initialize the stack */
+ stack->ptr = workspace;
+ stack->end = (char *)workspace + workspaceSize;
+ stack_push(stack, sizeof(ZSTD_stack));
+ return stackMem;
+}
+
+void *ZSTD_stackAllocAll(void *opaque, size_t *size)
+{
+ ZSTD_stack *stack = (ZSTD_stack *)opaque;
+ *size = (BYTE const *)stack->end - (BYTE *)ZSTD_PTR_ALIGN(stack->ptr);
+ return stack_push(stack, *size);
+}
+
+void *ZSTD_stackAlloc(void *opaque, size_t size)
+{
+ ZSTD_stack *stack = (ZSTD_stack *)opaque;
+ return stack_push(stack, size);
+}
+void ZSTD_stackFree(void *opaque, void *address)
+{
+ (void)opaque;
+ (void)address;
+}
+
+void *ZSTD_malloc(size_t size, ZSTD_customMem customMem) { return customMem.customAlloc(customMem.opaque, size); }
+
+void ZSTD_free(void *ptr, ZSTD_customMem customMem)
+{
+ if (ptr != NULL)
+ customMem.customFree(customMem.opaque, ptr);
+}
diff --git a/lib/zstd/zstd_internal.h b/lib/zstd/zstd_internal.h
new file mode 100644
index 000000000000..1a79fab9e13a
--- /dev/null
+++ b/lib/zstd/zstd_internal.h
@@ -0,0 +1,263 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
+ * An additional grant of patent rights can be found in the PATENTS file in the
+ * same directory.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ */
+
+#ifndef ZSTD_CCOMMON_H_MODULE
+#define ZSTD_CCOMMON_H_MODULE
+
+/*-*******************************************************
+* Compiler specifics
+*********************************************************/
+#define FORCE_INLINE static __always_inline
+#define FORCE_NOINLINE static noinline
+
+/*-*************************************
+* Dependencies
+***************************************/
+#include "error_private.h"
+#include "mem.h"
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/xxhash.h>
+#include <linux/zstd.h>
+
+/*-*************************************
+* shared macros
+***************************************/
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define CHECK_F(f) \
+ { \
+ size_t const errcod = f; \
+ if (ERR_isError(errcod)) \
+ return errcod; \
+ } /* check and Forward error code */
+#define CHECK_E(f, e) \
+ { \
+ size_t const errcod = f; \
+ if (ERR_isError(errcod)) \
+ return ERROR(e); \
+ } /* check and send Error code */
+#define ZSTD_STATIC_ASSERT(c) \
+ { \
+ enum { ZSTD_static_assert = 1 / (int)(!!(c)) }; \
+ }
+
+/*-*************************************
+* Common constants
+***************************************/
+#define ZSTD_OPT_NUM (1 << 12)
+#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */
+
+#define ZSTD_REP_NUM 3 /* number of repcodes */
+#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */
+#define ZSTD_REP_MOVE (ZSTD_REP_NUM - 1)
+#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM)
+static const U32 repStartValue[ZSTD_REP_NUM] = {1, 4, 8};
+
+#define KB *(1 << 10)
+#define MB *(1 << 20)
+#define GB *(1U << 30)
+
+#define BIT7 128
+#define BIT6 64
+#define BIT5 32
+#define BIT4 16
+#define BIT1 2
+#define BIT0 1
+
+#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
+static const size_t ZSTD_fcs_fieldSize[4] = {0, 2, 4, 8};
+static const size_t ZSTD_did_fieldSize[4] = {0, 1, 2, 4};
+
+#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
+static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
+typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
+
+#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
+#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
+
+#define HufLog 12
+typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
+
+#define LONGNBSEQ 0x7F00
+
+#define MINMATCH 3
+#define EQUAL_READ32 4
+
+#define Litbits 8
+#define MaxLit ((1 << Litbits) - 1)
+#define MaxML 52
+#define MaxLL 35
+#define MaxOff 28
+#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
+#define MLFSELog 9
+#define LLFSELog 9
+#define OffFSELog 8
+
+static const U32 LL_bits[MaxLL + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+static const S16 LL_defaultNorm[MaxLL + 1] = {4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1};
+#define LL_DEFAULTNORMLOG 6 /* for static allocation */
+static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
+
+static const U32 ML_bits[MaxML + 1] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+static const S16 ML_defaultNorm[MaxML + 1] = {1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1};
+#define ML_DEFAULTNORMLOG 6 /* for static allocation */
+static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
+
+static const S16 OF_defaultNorm[MaxOff + 1] = {1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1};
+#define OF_DEFAULTNORMLOG 5 /* for static allocation */
+static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
+
+/*-*******************************************
+* Shared functions to include for inlining
+*********************************************/
+ZSTD_STATIC void ZSTD_copy8(void *dst, const void *src) {
+ memcpy(dst, src, 8);
+}
+/*! ZSTD_wildcopy() :
+* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
+#define WILDCOPY_OVERLENGTH 8
+ZSTD_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
+{
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + length;
+ /* Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81388.
+ * Avoid the bad case where the loop only runs once by handling the
+ * special case separately. This doesn't trigger the bug because it
+ * doesn't involve pointer/integer overflow.
+ */
+ if (length <= 8)
+ return ZSTD_copy8(dst, src);
+ do {
+ ZSTD_copy8(op, ip);
+ op += 8;
+ ip += 8;
+ } while (op < oend);
+}
+
+/*-*******************************************
+* Private interfaces
+*********************************************/
+typedef struct ZSTD_stats_s ZSTD_stats_t;
+
+typedef struct {
+ U32 off;
+ U32 len;
+} ZSTD_match_t;
+
+typedef struct {
+ U32 price;
+ U32 off;
+ U32 mlen;
+ U32 litlen;
+ U32 rep[ZSTD_REP_NUM];
+} ZSTD_optimal_t;
+
+typedef struct seqDef_s {
+ U32 offset;
+ U16 litLength;
+ U16 matchLength;
+} seqDef;
+
+typedef struct {
+ seqDef *sequencesStart;
+ seqDef *sequences;
+ BYTE *litStart;
+ BYTE *lit;
+ BYTE *llCode;
+ BYTE *mlCode;
+ BYTE *ofCode;
+ U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
+ U32 longLengthPos;
+ /* opt */
+ ZSTD_optimal_t *priceTable;
+ ZSTD_match_t *matchTable;
+ U32 *matchLengthFreq;
+ U32 *litLengthFreq;
+ U32 *litFreq;
+ U32 *offCodeFreq;
+ U32 matchLengthSum;
+ U32 matchSum;
+ U32 litLengthSum;
+ U32 litSum;
+ U32 offCodeSum;
+ U32 log2matchLengthSum;
+ U32 log2matchSum;
+ U32 log2litLengthSum;
+ U32 log2litSum;
+ U32 log2offCodeSum;
+ U32 factor;
+ U32 staticPrices;
+ U32 cachedPrice;
+ U32 cachedLitLength;
+ const BYTE *cachedLiterals;
+} seqStore_t;
+
+const seqStore_t *ZSTD_getSeqStore(const ZSTD_CCtx *ctx);
+void ZSTD_seqToCodes(const seqStore_t *seqStorePtr);
+int ZSTD_isSkipFrame(ZSTD_DCtx *dctx);
+
+/*= Custom memory allocation functions */
+typedef void *(*ZSTD_allocFunction)(void *opaque, size_t size);
+typedef void (*ZSTD_freeFunction)(void *opaque, void *address);
+typedef struct {
+ ZSTD_allocFunction customAlloc;
+ ZSTD_freeFunction customFree;
+ void *opaque;
+} ZSTD_customMem;
+
+void *ZSTD_malloc(size_t size, ZSTD_customMem customMem);
+void ZSTD_free(void *ptr, ZSTD_customMem customMem);
+
+/*====== stack allocation ======*/
+
+typedef struct {
+ void *ptr;
+ const void *end;
+} ZSTD_stack;
+
+#define ZSTD_ALIGN(x) ALIGN(x, sizeof(size_t))
+#define ZSTD_PTR_ALIGN(p) PTR_ALIGN(p, sizeof(size_t))
+
+ZSTD_customMem ZSTD_initStack(void *workspace, size_t workspaceSize);
+
+void *ZSTD_stackAllocAll(void *opaque, size_t *size);
+void *ZSTD_stackAlloc(void *opaque, size_t size);
+void ZSTD_stackFree(void *opaque, void *address);
+
+/*====== common function ======*/
+
+ZSTD_STATIC U32 ZSTD_highbit32(U32 val) { return 31 - __builtin_clz(val); }
+
+/* hidden functions */
+
+/* ZSTD_invalidateRepCodes() :
+ * ensures next compression will not use repcodes from previous block.
+ * Note : only works with regular variant;
+ * do not use with extDict variant ! */
+void ZSTD_invalidateRepCodes(ZSTD_CCtx *cctx);
+
+size_t ZSTD_freeCCtx(ZSTD_CCtx *cctx);
+size_t ZSTD_freeDCtx(ZSTD_DCtx *dctx);
+size_t ZSTD_freeCDict(ZSTD_CDict *cdict);
+size_t ZSTD_freeDDict(ZSTD_DDict *cdict);
+size_t ZSTD_freeCStream(ZSTD_CStream *zcs);
+size_t ZSTD_freeDStream(ZSTD_DStream *zds);
+
+#endif /* ZSTD_CCOMMON_H_MODULE */
diff --git a/lib/zstd/zstd_opt.h b/lib/zstd/zstd_opt.h
new file mode 100644
index 000000000000..55e1b4cba808
--- /dev/null
+++ b/lib/zstd/zstd_opt.h
@@ -0,0 +1,1014 @@
+/**
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of https://github.com/facebook/zstd.
+ * An additional grant of patent rights can be found in the PATENTS file in the
+ * same directory.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ */
+
+/* Note : this file is intended to be included within zstd_compress.c */
+
+#ifndef ZSTD_OPT_H_91842398743
+#define ZSTD_OPT_H_91842398743
+
+#define ZSTD_LITFREQ_ADD 2
+#define ZSTD_FREQ_DIV 4
+#define ZSTD_MAX_PRICE (1 << 30)
+
+/*-*************************************
+* Price functions for optimal parser
+***************************************/
+FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t *ssPtr)
+{
+ ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum + 1);
+ ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum + 1);
+ ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum + 1);
+ ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum + 1);
+ ssPtr->factor = 1 + ((ssPtr->litSum >> 5) / ssPtr->litLengthSum) + ((ssPtr->litSum << 1) / (ssPtr->litSum + ssPtr->matchSum));
+}
+
+ZSTD_STATIC void ZSTD_rescaleFreqs(seqStore_t *ssPtr, const BYTE *src, size_t srcSize)
+{
+ unsigned u;
+
+ ssPtr->cachedLiterals = NULL;
+ ssPtr->cachedPrice = ssPtr->cachedLitLength = 0;
+ ssPtr->staticPrices = 0;
+
+ if (ssPtr->litLengthSum == 0) {
+ if (srcSize <= 1024)
+ ssPtr->staticPrices = 1;
+
+ for (u = 0; u <= MaxLit; u++)
+ ssPtr->litFreq[u] = 0;
+ for (u = 0; u < srcSize; u++)
+ ssPtr->litFreq[src[u]]++;
+
+ ssPtr->litSum = 0;
+ ssPtr->litLengthSum = MaxLL + 1;
+ ssPtr->matchLengthSum = MaxML + 1;
+ ssPtr->offCodeSum = (MaxOff + 1);
+ ssPtr->matchSum = (ZSTD_LITFREQ_ADD << Litbits);
+
+ for (u = 0; u <= MaxLit; u++) {
+ ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> ZSTD_FREQ_DIV);
+ ssPtr->litSum += ssPtr->litFreq[u];
+ }
+ for (u = 0; u <= MaxLL; u++)
+ ssPtr->litLengthFreq[u] = 1;
+ for (u = 0; u <= MaxML; u++)
+ ssPtr->matchLengthFreq[u] = 1;
+ for (u = 0; u <= MaxOff; u++)
+ ssPtr->offCodeFreq[u] = 1;
+ } else {
+ ssPtr->matchLengthSum = 0;
+ ssPtr->litLengthSum = 0;
+ ssPtr->offCodeSum = 0;
+ ssPtr->matchSum = 0;
+ ssPtr->litSum = 0;
+
+ for (u = 0; u <= MaxLit; u++) {
+ ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u] >> (ZSTD_FREQ_DIV + 1));
+ ssPtr->litSum += ssPtr->litFreq[u];
+ }
+ for (u = 0; u <= MaxLL; u++) {
+ ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u] >> (ZSTD_FREQ_DIV + 1));
+ ssPtr->litLengthSum += ssPtr->litLengthFreq[u];
+ }
+ for (u = 0; u <= MaxML; u++) {
+ ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV);
+ ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u];
+ ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3);
+ }
+ ssPtr->matchSum *= ZSTD_LITFREQ_ADD;
+ for (u = 0; u <= MaxOff; u++) {
+ ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV);
+ ssPtr->offCodeSum += ssPtr->offCodeFreq[u];
+ }
+ }
+
+ ZSTD_setLog2Prices(ssPtr);
+}
+
+FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t *ssPtr, U32 litLength, const BYTE *literals)
+{
+ U32 price, u;
+
+ if (ssPtr->staticPrices)
+ return ZSTD_highbit32((U32)litLength + 1) + (litLength * 6);
+
+ if (litLength == 0)
+ return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0] + 1);
+
+ /* literals */
+ if (ssPtr->cachedLiterals == literals) {
+ U32 const additional = litLength - ssPtr->cachedLitLength;
+ const BYTE *literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
+ price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
+ for (u = 0; u < additional; u++)
+ price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]] + 1);
+ ssPtr->cachedPrice = price;
+ ssPtr->cachedLitLength = litLength;
+ } else {
+ price = litLength * ssPtr->log2litSum;
+ for (u = 0; u < litLength; u++)
+ price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]] + 1);
+
+ if (litLength >= 12) {
+ ssPtr->cachedLiterals = literals;
+ ssPtr->cachedPrice = price;
+ ssPtr->cachedLitLength = litLength;
+ }
+ }
+
+ /* literal Length */
+ {
+ const BYTE LL_deltaCode = 19;
+ const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
+ price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode] + 1);
+ }
+
+ return price;
+}
+
+FORCE_INLINE U32 ZSTD_getPrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength, const int ultra)
+{
+ /* offset */
+ U32 price;
+ BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1);
+
+ if (seqStorePtr->staticPrices)
+ return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength + 1) + 16 + offCode;
+
+ price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode] + 1);
+ if (!ultra && offCode >= 20)
+ price += (offCode - 19) * 2;
+
+ /* match Length */
+ {
+ const BYTE ML_deltaCode = 36;
+ const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
+ price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode] + 1);
+ }
+
+ return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
+}
+
+ZSTD_STATIC void ZSTD_updatePrice(seqStore_t *seqStorePtr, U32 litLength, const BYTE *literals, U32 offset, U32 matchLength)
+{
+ U32 u;
+
+ /* literals */
+ seqStorePtr->litSum += litLength * ZSTD_LITFREQ_ADD;
+ for (u = 0; u < litLength; u++)
+ seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
+
+ /* literal Length */
+ {
+ const BYTE LL_deltaCode = 19;
+ const BYTE llCode = (litLength > 63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
+ seqStorePtr->litLengthFreq[llCode]++;
+ seqStorePtr->litLengthSum++;
+ }
+
+ /* match offset */
+ {
+ BYTE const offCode = (BYTE)ZSTD_highbit32(offset + 1);
+ seqStorePtr->offCodeSum++;
+ seqStorePtr->offCodeFreq[offCode]++;
+ }
+
+ /* match Length */
+ {
+ const BYTE ML_deltaCode = 36;
+ const BYTE mlCode = (matchLength > 127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
+ seqStorePtr->matchLengthFreq[mlCode]++;
+ seqStorePtr->matchLengthSum++;
+ }
+
+ ZSTD_setLog2Prices(seqStorePtr);
+}
+
+#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \
+ { \
+ while (last_pos < pos) { \
+ opt[last_pos + 1].price = ZSTD_MAX_PRICE; \
+ last_pos++; \
+ } \
+ opt[pos].mlen = mlen_; \
+ opt[pos].off = offset_; \
+ opt[pos].litlen = litlen_; \
+ opt[pos].price = price_; \
+ }
+
+/* Update hashTable3 up to ip (excluded)
+ Assumption : always within prefix (i.e. not within extDict) */
+FORCE_INLINE
+U32 ZSTD_insertAndFindFirstIndexHash3(ZSTD_CCtx *zc, const BYTE *ip)
+{
+ U32 *const hashTable3 = zc->hashTable3;
+ U32 const hashLog3 = zc->hashLog3;
+ const BYTE *const base = zc->base;
+ U32 idx = zc->nextToUpdate3;
+ const U32 target = zc->nextToUpdate3 = (U32)(ip - base);
+ const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3);
+
+ while (idx < target) {
+ hashTable3[ZSTD_hash3Ptr(base + idx, hashLog3)] = idx;
+ idx++;
+ }
+
+ return hashTable3[hash3];
+}
+
+/*-*************************************
+* Binary Tree search
+***************************************/
+static U32 ZSTD_insertBtAndGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, U32 nbCompares, const U32 mls, U32 extDict,
+ ZSTD_match_t *matches, const U32 minMatchLen)
+{
+ const BYTE *const base = zc->base;
+ const U32 curr = (U32)(ip - base);
+ const U32 hashLog = zc->params.cParams.hashLog;
+ const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
+ U32 *const hashTable = zc->hashTable;
+ U32 matchIndex = hashTable[h];
+ U32 *const bt = zc->chainTable;
+ const U32 btLog = zc->params.cParams.chainLog - 1;
+ const U32 btMask = (1U << btLog) - 1;
+ size_t commonLengthSmaller = 0, commonLengthLarger = 0;
+ const BYTE *const dictBase = zc->dictBase;
+ const U32 dictLimit = zc->dictLimit;
+ const BYTE *const dictEnd = dictBase + dictLimit;
+ const BYTE *const prefixStart = base + dictLimit;
+ const U32 btLow = btMask >= curr ? 0 : curr - btMask;
+ const U32 windowLow = zc->lowLimit;
+ U32 *smallerPtr = bt + 2 * (curr & btMask);
+ U32 *largerPtr = bt + 2 * (curr & btMask) + 1;
+ U32 matchEndIdx = curr + 8;
+ U32 dummy32; /* to be nullified at the end */
+ U32 mnum = 0;
+
+ const U32 minMatch = (mls == 3) ? 3 : 4;
+ size_t bestLength = minMatchLen - 1;
+
+ if (minMatch == 3) { /* HC3 match finder */
+ U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(zc, ip);
+ if (matchIndex3 > windowLow && (curr - matchIndex3 < (1 << 18))) {
+ const BYTE *match;
+ size_t currMl = 0;
+ if ((!extDict) || matchIndex3 >= dictLimit) {
+ match = base + matchIndex3;
+ if (match[bestLength] == ip[bestLength])
+ currMl = ZSTD_count(ip, match, iLimit);
+ } else {
+ match = dictBase + matchIndex3;
+ if (ZSTD_readMINMATCH(match, MINMATCH) ==
+ ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
+ currMl = ZSTD_count_2segments(ip + MINMATCH, match + MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH;
+ }
+
+ /* save best solution */
+ if (currMl > bestLength) {
+ bestLength = currMl;
+ matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex3;
+ matches[mnum].len = (U32)currMl;
+ mnum++;
+ if (currMl > ZSTD_OPT_NUM)
+ goto update;
+ if (ip + currMl == iLimit)
+ goto update; /* best possible, and avoid read overflow*/
+ }
+ }
+ }
+
+ hashTable[h] = curr; /* Update Hash Table */
+
+ while (nbCompares-- && (matchIndex > windowLow)) {
+ U32 *nextPtr = bt + 2 * (matchIndex & btMask);
+ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
+ const BYTE *match;
+
+ if ((!extDict) || (matchIndex + matchLength >= dictLimit)) {
+ match = base + matchIndex;
+ if (match[matchLength] == ip[matchLength]) {
+ matchLength += ZSTD_count(ip + matchLength + 1, match + matchLength + 1, iLimit) + 1;
+ }
+ } else {
+ match = dictBase + matchIndex;
+ matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iLimit, dictEnd, prefixStart);
+ if (matchIndex + matchLength >= dictLimit)
+ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
+ }
+
+ if (matchLength > bestLength) {
+ if (matchLength > matchEndIdx - matchIndex)
+ matchEndIdx = matchIndex + (U32)matchLength;
+ bestLength = matchLength;
+ matches[mnum].off = ZSTD_REP_MOVE_OPT + curr - matchIndex;
+ matches[mnum].len = (U32)matchLength;
+ mnum++;
+ if (matchLength > ZSTD_OPT_NUM)
+ break;
+ if (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */
+ break; /* drop, to guarantee consistency (miss a little bit of compression) */
+ }
+
+ if (match[matchLength] < ip[matchLength]) {
+ /* match is smaller than curr */
+ *smallerPtr = matchIndex; /* update smaller idx */
+ commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
+ if (matchIndex <= btLow) {
+ smallerPtr = &dummy32;
+ break;
+ } /* beyond tree size, stop the search */
+ smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */
+ matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to curr) */
+ } else {
+ /* match is larger than curr */
+ *largerPtr = matchIndex;
+ commonLengthLarger = matchLength;
+ if (matchIndex <= btLow) {
+ largerPtr = &dummy32;
+ break;
+ } /* beyond tree size, stop the search */
+ largerPtr = nextPtr;
+ matchIndex = nextPtr[0];
+ }
+ }
+
+ *smallerPtr = *largerPtr = 0;
+
+update:
+ zc->nextToUpdate = (matchEndIdx > curr + 8) ? matchEndIdx - 8 : curr + 1;
+ return mnum;
+}
+
+/** Tree updater, providing best match */
+static U32 ZSTD_BtGetAllMatches(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t *matches,
+ const U32 minMatchLen)
+{
+ if (ip < zc->base + zc->nextToUpdate)
+ return 0; /* skipped area */
+ ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
+ return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen);
+}
+
+static U32 ZSTD_BtGetAllMatches_selectMLS(ZSTD_CCtx *zc, /* Index table will be updated */
+ const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
+ ZSTD_match_t *matches, const U32 minMatchLen)
+{
+ switch (matchLengthSearch) {
+ case 3: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
+ default:
+ case 4: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
+ case 5: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
+ case 7:
+ case 6: return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
+ }
+}
+
+/** Tree updater, providing best match */
+static U32 ZSTD_BtGetAllMatches_extDict(ZSTD_CCtx *zc, const BYTE *const ip, const BYTE *const iLimit, const U32 maxNbAttempts, const U32 mls,
+ ZSTD_match_t *matches, const U32 minMatchLen)
+{
+ if (ip < zc->base + zc->nextToUpdate)
+ return 0; /* skipped area */
+ ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
+ return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen);
+}
+
+static U32 ZSTD_BtGetAllMatches_selectMLS_extDict(ZSTD_CCtx *zc, /* Index table will be updated */
+ const BYTE *ip, const BYTE *const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch,
+ ZSTD_match_t *matches, const U32 minMatchLen)
+{
+ switch (matchLengthSearch) {
+ case 3: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
+ default:
+ case 4: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
+ case 5: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
+ case 7:
+ case 6: return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
+ }
+}
+
+/*-*******************************
+* Optimal parser
+*********************************/
+FORCE_INLINE
+void ZSTD_compressBlock_opt_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
+{
+ seqStore_t *seqStorePtr = &(ctx->seqStore);
+ const BYTE *const istart = (const BYTE *)src;
+ const BYTE *ip = istart;
+ const BYTE *anchor = istart;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *const ilimit = iend - 8;
+ const BYTE *const base = ctx->base;
+ const BYTE *const prefixStart = base + ctx->dictLimit;
+
+ const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
+ const U32 sufficient_len = ctx->params.cParams.targetLength;
+ const U32 mls = ctx->params.cParams.searchLength;
+ const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
+
+ ZSTD_optimal_t *opt = seqStorePtr->priceTable;
+ ZSTD_match_t *matches = seqStorePtr->matchTable;
+ const BYTE *inr;
+ U32 offset, rep[ZSTD_REP_NUM];
+
+ /* init */
+ ctx->nextToUpdate3 = ctx->nextToUpdate;
+ ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
+ ip += (ip == prefixStart);
+ {
+ U32 i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ rep[i] = ctx->rep[i];
+ }
+
+ /* Match Loop */
+ while (ip < ilimit) {
+ U32 cur, match_num, last_pos, litlen, price;
+ U32 u, mlen, best_mlen, best_off, litLength;
+ memset(opt, 0, sizeof(ZSTD_optimal_t));
+ last_pos = 0;
+ litlen = (U32)(ip - anchor);
+
+ /* check repCode */
+ {
+ U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
+ for (i = (ip == anchor); i < last_i; i++) {
+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
+ if ((repCur > 0) && (repCur < (S32)(ip - prefixStart)) &&
+ (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) {
+ mlen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repCur, iend) + minMatch;
+ if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
+ best_mlen = mlen;
+ best_off = i;
+ cur = 0;
+ last_pos = 1;
+ goto _storeSequence;
+ }
+ best_off = i - (ip == anchor);
+ do {
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+ if (mlen > last_pos || price < opt[mlen].price)
+ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
+ mlen--;
+ } while (mlen >= minMatch);
+ }
+ }
+ }
+
+ match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch);
+
+ if (!last_pos && !match_num) {
+ ip++;
+ continue;
+ }
+
+ if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
+ best_mlen = matches[match_num - 1].len;
+ best_off = matches[match_num - 1].off;
+ cur = 0;
+ last_pos = 1;
+ goto _storeSequence;
+ }
+
+ /* set prices using matches at position = 0 */
+ best_mlen = (last_pos) ? last_pos : minMatch;
+ for (u = 0; u < match_num; u++) {
+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
+ best_mlen = matches[u].len;
+ while (mlen <= best_mlen) {
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
+ if (mlen > last_pos || price < opt[mlen].price)
+ SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
+ mlen++;
+ }
+ }
+
+ if (last_pos < minMatch) {
+ ip++;
+ continue;
+ }
+
+ /* initialize opt[0] */
+ {
+ U32 i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ opt[0].rep[i] = rep[i];
+ }
+ opt[0].mlen = 1;
+ opt[0].litlen = litlen;
+
+ /* check further positions */
+ for (cur = 1; cur <= last_pos; cur++) {
+ inr = ip + cur;
+
+ if (opt[cur - 1].mlen == 1) {
+ litlen = opt[cur - 1].litlen + 1;
+ if (cur > litlen) {
+ price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen);
+ } else
+ price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
+ } else {
+ litlen = 1;
+ price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1);
+ }
+
+ if (cur > last_pos || price <= opt[cur].price)
+ SET_PRICE(cur, 1, 0, litlen, price);
+
+ if (cur == last_pos)
+ break;
+
+ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
+ continue;
+
+ mlen = opt[cur].mlen;
+ if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
+ opt[cur].rep[2] = opt[cur - mlen].rep[1];
+ opt[cur].rep[1] = opt[cur - mlen].rep[0];
+ opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
+ } else {
+ opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
+ opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
+ opt[cur].rep[0] =
+ ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
+ }
+
+ best_mlen = minMatch;
+ {
+ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
+ for (i = (opt[cur].mlen != 1); i < last_i; i++) { /* check rep */
+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
+ if ((repCur > 0) && (repCur < (S32)(inr - prefixStart)) &&
+ (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) {
+ mlen = (U32)ZSTD_count(inr + minMatch, inr + minMatch - repCur, iend) + minMatch;
+
+ if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
+ best_mlen = mlen;
+ best_off = i;
+ last_pos = cur + 1;
+ goto _storeSequence;
+ }
+
+ best_off = i - (opt[cur].mlen != 1);
+ if (mlen > best_mlen)
+ best_mlen = mlen;
+
+ do {
+ if (opt[cur].mlen == 1) {
+ litlen = opt[cur].litlen;
+ if (cur > litlen) {
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
+ best_off, mlen - MINMATCH, ultra);
+ } else
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+ } else {
+ litlen = 0;
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
+ }
+
+ if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
+ SET_PRICE(cur + mlen, mlen, i, litlen, price);
+ mlen--;
+ } while (mlen >= minMatch);
+ }
+ }
+ }
+
+ match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen);
+
+ if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
+ best_mlen = matches[match_num - 1].len;
+ best_off = matches[match_num - 1].off;
+ last_pos = cur + 1;
+ goto _storeSequence;
+ }
+
+ /* set prices using matches at position = cur */
+ for (u = 0; u < match_num; u++) {
+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
+ best_mlen = matches[u].len;
+
+ while (mlen <= best_mlen) {
+ if (opt[cur].mlen == 1) {
+ litlen = opt[cur].litlen;
+ if (cur > litlen)
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
+ matches[u].off - 1, mlen - MINMATCH, ultra);
+ else
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
+ } else {
+ litlen = 0;
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra);
+ }
+
+ if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
+ SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
+
+ mlen++;
+ }
+ }
+ }
+
+ best_mlen = opt[last_pos].mlen;
+ best_off = opt[last_pos].off;
+ cur = last_pos - best_mlen;
+
+ /* store sequence */
+_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
+ opt[0].mlen = 1;
+
+ while (1) {
+ mlen = opt[cur].mlen;
+ offset = opt[cur].off;
+ opt[cur].mlen = best_mlen;
+ opt[cur].off = best_off;
+ best_mlen = mlen;
+ best_off = offset;
+ if (mlen > cur)
+ break;
+ cur -= mlen;
+ }
+
+ for (u = 0; u <= last_pos;) {
+ u += opt[u].mlen;
+ }
+
+ for (cur = 0; cur < last_pos;) {
+ mlen = opt[cur].mlen;
+ if (mlen == 1) {
+ ip++;
+ cur++;
+ continue;
+ }
+ offset = opt[cur].off;
+ cur += mlen;
+ litLength = (U32)(ip - anchor);
+
+ if (offset > ZSTD_REP_MOVE_OPT) {
+ rep[2] = rep[1];
+ rep[1] = rep[0];
+ rep[0] = offset - ZSTD_REP_MOVE_OPT;
+ offset--;
+ } else {
+ if (offset != 0) {
+ best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
+ if (offset != 1)
+ rep[2] = rep[1];
+ rep[1] = rep[0];
+ rep[0] = best_off;
+ }
+ if (litLength == 0)
+ offset--;
+ }
+
+ ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
+ anchor = ip = ip + mlen;
+ }
+ } /* for (cur=0; cur < last_pos; ) */
+
+ /* Save reps for next block */
+ {
+ int i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ ctx->repToConfirm[i] = rep[i];
+ }
+
+ /* Last Literals */
+ {
+ size_t const lastLLSize = iend - anchor;
+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+ }
+}
+
+FORCE_INLINE
+void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx *ctx, const void *src, size_t srcSize, const int ultra)
+{
+ seqStore_t *seqStorePtr = &(ctx->seqStore);
+ const BYTE *const istart = (const BYTE *)src;
+ const BYTE *ip = istart;
+ const BYTE *anchor = istart;
+ const BYTE *const iend = istart + srcSize;
+ const BYTE *const ilimit = iend - 8;
+ const BYTE *const base = ctx->base;
+ const U32 lowestIndex = ctx->lowLimit;
+ const U32 dictLimit = ctx->dictLimit;
+ const BYTE *const prefixStart = base + dictLimit;
+ const BYTE *const dictBase = ctx->dictBase;
+ const BYTE *const dictEnd = dictBase + dictLimit;
+
+ const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
+ const U32 sufficient_len = ctx->params.cParams.targetLength;
+ const U32 mls = ctx->params.cParams.searchLength;
+ const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
+
+ ZSTD_optimal_t *opt = seqStorePtr->priceTable;
+ ZSTD_match_t *matches = seqStorePtr->matchTable;
+ const BYTE *inr;
+
+ /* init */
+ U32 offset, rep[ZSTD_REP_NUM];
+ {
+ U32 i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ rep[i] = ctx->rep[i];
+ }
+
+ ctx->nextToUpdate3 = ctx->nextToUpdate;
+ ZSTD_rescaleFreqs(seqStorePtr, (const BYTE *)src, srcSize);
+ ip += (ip == prefixStart);
+
+ /* Match Loop */
+ while (ip < ilimit) {
+ U32 cur, match_num, last_pos, litlen, price;
+ U32 u, mlen, best_mlen, best_off, litLength;
+ U32 curr = (U32)(ip - base);
+ memset(opt, 0, sizeof(ZSTD_optimal_t));
+ last_pos = 0;
+ opt[0].litlen = (U32)(ip - anchor);
+
+ /* check repCode */
+ {
+ U32 i, last_i = ZSTD_REP_CHECK + (ip == anchor);
+ for (i = (ip == anchor); i < last_i; i++) {
+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i];
+ const U32 repIndex = (U32)(curr - repCur);
+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE *const repMatch = repBase + repIndex;
+ if ((repCur > 0 && repCur <= (S32)curr) &&
+ (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
+ && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
+ /* repcode detected we should take it */
+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ mlen = (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
+
+ if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
+ best_mlen = mlen;
+ best_off = i;
+ cur = 0;
+ last_pos = 1;
+ goto _storeSequence;
+ }
+
+ best_off = i - (ip == anchor);
+ litlen = opt[0].litlen;
+ do {
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+ if (mlen > last_pos || price < opt[mlen].price)
+ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
+ mlen--;
+ } while (mlen >= minMatch);
+ }
+ }
+ }
+
+ match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */
+
+ if (!last_pos && !match_num) {
+ ip++;
+ continue;
+ }
+
+ {
+ U32 i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ opt[0].rep[i] = rep[i];
+ }
+ opt[0].mlen = 1;
+
+ if (match_num && (matches[match_num - 1].len > sufficient_len || matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
+ best_mlen = matches[match_num - 1].len;
+ best_off = matches[match_num - 1].off;
+ cur = 0;
+ last_pos = 1;
+ goto _storeSequence;
+ }
+
+ best_mlen = (last_pos) ? last_pos : minMatch;
+
+ /* set prices using matches at position = 0 */
+ for (u = 0; u < match_num; u++) {
+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
+ best_mlen = matches[u].len;
+ litlen = opt[0].litlen;
+ while (mlen <= best_mlen) {
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
+ if (mlen > last_pos || price < opt[mlen].price)
+ SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
+ mlen++;
+ }
+ }
+
+ if (last_pos < minMatch) {
+ ip++;
+ continue;
+ }
+
+ /* check further positions */
+ for (cur = 1; cur <= last_pos; cur++) {
+ inr = ip + cur;
+
+ if (opt[cur - 1].mlen == 1) {
+ litlen = opt[cur - 1].litlen + 1;
+ if (cur > litlen) {
+ price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - litlen);
+ } else
+ price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
+ } else {
+ litlen = 1;
+ price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr - 1);
+ }
+
+ if (cur > last_pos || price <= opt[cur].price)
+ SET_PRICE(cur, 1, 0, litlen, price);
+
+ if (cur == last_pos)
+ break;
+
+ if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */
+ continue;
+
+ mlen = opt[cur].mlen;
+ if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
+ opt[cur].rep[2] = opt[cur - mlen].rep[1];
+ opt[cur].rep[1] = opt[cur - mlen].rep[0];
+ opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
+ } else {
+ opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur - mlen].rep[1] : opt[cur - mlen].rep[2];
+ opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur - mlen].rep[0] : opt[cur - mlen].rep[1];
+ opt[cur].rep[0] =
+ ((opt[cur].off == ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur - mlen].rep[0] - 1) : (opt[cur - mlen].rep[opt[cur].off]);
+ }
+
+ best_mlen = minMatch;
+ {
+ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
+ for (i = (mlen != 1); i < last_i; i++) {
+ const S32 repCur = (i == ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
+ const U32 repIndex = (U32)(curr + cur - repCur);
+ const BYTE *const repBase = repIndex < dictLimit ? dictBase : base;
+ const BYTE *const repMatch = repBase + repIndex;
+ if ((repCur > 0 && repCur <= (S32)(curr + cur)) &&
+ (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
+ && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) {
+ /* repcode detected */
+ const BYTE *const repEnd = repIndex < dictLimit ? dictEnd : iend;
+ mlen = (U32)ZSTD_count_2segments(inr + minMatch, repMatch + minMatch, iend, repEnd, prefixStart) + minMatch;
+
+ if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
+ best_mlen = mlen;
+ best_off = i;
+ last_pos = cur + 1;
+ goto _storeSequence;
+ }
+
+ best_off = i - (opt[cur].mlen != 1);
+ if (mlen > best_mlen)
+ best_mlen = mlen;
+
+ do {
+ if (opt[cur].mlen == 1) {
+ litlen = opt[cur].litlen;
+ if (cur > litlen) {
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr - litlen,
+ best_off, mlen - MINMATCH, ultra);
+ } else
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
+ } else {
+ litlen = 0;
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
+ }
+
+ if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
+ SET_PRICE(cur + mlen, mlen, i, litlen, price);
+ mlen--;
+ } while (mlen >= minMatch);
+ }
+ }
+ }
+
+ match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
+
+ if (match_num > 0 && (matches[match_num - 1].len > sufficient_len || cur + matches[match_num - 1].len >= ZSTD_OPT_NUM)) {
+ best_mlen = matches[match_num - 1].len;
+ best_off = matches[match_num - 1].off;
+ last_pos = cur + 1;
+ goto _storeSequence;
+ }
+
+ /* set prices using matches at position = cur */
+ for (u = 0; u < match_num; u++) {
+ mlen = (u > 0) ? matches[u - 1].len + 1 : best_mlen;
+ best_mlen = matches[u].len;
+
+ while (mlen <= best_mlen) {
+ if (opt[cur].mlen == 1) {
+ litlen = opt[cur].litlen;
+ if (cur > litlen)
+ price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip + cur - litlen,
+ matches[u].off - 1, mlen - MINMATCH, ultra);
+ else
+ price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off - 1, mlen - MINMATCH, ultra);
+ } else {
+ litlen = 0;
+ price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off - 1, mlen - MINMATCH, ultra);
+ }
+
+ if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
+ SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
+
+ mlen++;
+ }
+ }
+ } /* for (cur = 1; cur <= last_pos; cur++) */
+
+ best_mlen = opt[last_pos].mlen;
+ best_off = opt[last_pos].off;
+ cur = last_pos - best_mlen;
+
+ /* store sequence */
+_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
+ opt[0].mlen = 1;
+
+ while (1) {
+ mlen = opt[cur].mlen;
+ offset = opt[cur].off;
+ opt[cur].mlen = best_mlen;
+ opt[cur].off = best_off;
+ best_mlen = mlen;
+ best_off = offset;
+ if (mlen > cur)
+ break;
+ cur -= mlen;
+ }
+
+ for (u = 0; u <= last_pos;) {
+ u += opt[u].mlen;
+ }
+
+ for (cur = 0; cur < last_pos;) {
+ mlen = opt[cur].mlen;
+ if (mlen == 1) {
+ ip++;
+ cur++;
+ continue;
+ }
+ offset = opt[cur].off;
+ cur += mlen;
+ litLength = (U32)(ip - anchor);
+
+ if (offset > ZSTD_REP_MOVE_OPT) {
+ rep[2] = rep[1];
+ rep[1] = rep[0];
+ rep[0] = offset - ZSTD_REP_MOVE_OPT;
+ offset--;
+ } else {
+ if (offset != 0) {
+ best_off = (offset == ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]);
+ if (offset != 1)
+ rep[2] = rep[1];
+ rep[1] = rep[0];
+ rep[0] = best_off;
+ }
+
+ if (litLength == 0)
+ offset--;
+ }
+
+ ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
+ ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen - MINMATCH);
+ anchor = ip = ip + mlen;
+ }
+ } /* for (cur=0; cur < last_pos; ) */
+
+ /* Save reps for next block */
+ {
+ int i;
+ for (i = 0; i < ZSTD_REP_NUM; i++)
+ ctx->repToConfirm[i] = rep[i];
+ }
+
+ /* Last Literals */
+ {
+ size_t lastLLSize = iend - anchor;
+ memcpy(seqStorePtr->lit, anchor, lastLLSize);
+ seqStorePtr->lit += lastLLSize;
+ }
+}
+
+#endif /* ZSTD_OPT_H_91842398743 */