diff --git a/js/compressionLib/index.js b/js/compressionLib/index.js index 450f22cf..6d64e00d 100644 --- a/js/compressionLib/index.js +++ b/js/compressionLib/index.js @@ -14,7 +14,7 @@ const libraryFolderPath = isDevEnv ? path.join(__dirname, '..', '..', 'library') const winArchPath = process.arch === 'ia32' ? 'lz4-win-x86.exe' : 'lz4-win-x64.exe'; const productionPath = path.join(execPath, libraryFolderPath, winArchPath); const devPath = path.join(__dirname, '..', '..', 'library', winArchPath); - +const macLibraryPath = path.join(execPath, '..', 'library'); const lz4Path = isDevEnv ? devPath : productionPath; /** @@ -27,7 +27,7 @@ const lz4Path = isDevEnv ? devPath : productionPath; */ function compression(pathToFolder, outputPath, callback) { if (isMac) { - child.exec(`cd "${DATA_FOLDER_PATH}" && tar cf - "${pathToFolder}" | lz4 > "${outputPath}.tar.lz4"`, (error, stdout, stderr) => { + child.exec(`cd "${DATA_FOLDER_PATH}" && tar cf - "${pathToFolder}" | "${macLibraryPath}/lz4.exec" > "${outputPath}.tar.lz4"`, (error, stdout, stderr) => { if (error) { return callback(new Error(error), null); } @@ -58,7 +58,7 @@ function compression(pathToFolder, outputPath, callback) { */ function deCompression(pathName, callback) { if (isMac) { - child.exec(`cd "${DATA_FOLDER_PATH}" && lz4 -d "${pathName}" | tar -xf - `, (error, stdout, stderr) => { + child.exec(`cd "${DATA_FOLDER_PATH}" && "${macLibraryPath}/lz4.exec" -d "${pathName}" | tar -xf - `, (error, stdout, stderr) => { if (error) { return callback(new Error(error), null); } diff --git a/library/lz4-mac/.gitattributes b/library/lz4-mac/.gitattributes deleted file mode 100755 index 6212bd40..00000000 --- a/library/lz4-mac/.gitattributes +++ /dev/null @@ -1,21 +0,0 @@ -# Set the default behavior -* text eol=lf - -# Explicitly declare source files -*.c text eol=lf -*.h text eol=lf - -# Denote files that should not be modified. -*.odt binary -*.png binary - -# Visual Studio -*.sln text eol=crlf -*.vcxproj* text eol=crlf -*.vcproj* text eol=crlf -*.suo binary -*.rc text eol=crlf - -# Windows -*.bat text eol=crlf -*.cmd text eol=crlf diff --git a/library/lz4-mac/.gitignore b/library/lz4-mac/.gitignore deleted file mode 100755 index 117b02d1..00000000 --- a/library/lz4-mac/.gitignore +++ /dev/null @@ -1,31 +0,0 @@ -# Object files -*.o -*.ko - -# Libraries -*.lib -*.a - -# Shared objects (inc. Windows DLLs) -*.dll -*.so -*.so.* -*.dylib -*.dSYM # apple - -# Executables -*.exe -*.out -*.app -lz4 - -# IDE / editors files -.clang_complete -_codelite/ -_codelite_lz4/ -bin/ -*.zip - -# Mac -.DS_Store -*.dSYM diff --git a/library/lz4-mac/INSTALL b/library/lz4-mac/INSTALL deleted file mode 100755 index 6aab067b..00000000 --- a/library/lz4-mac/INSTALL +++ /dev/null @@ -1,16 +0,0 @@ -Installation -============= - -``` -make -make install # this command may require root access -``` - -LZ4's `Makefile` supports standard [Makefile conventions], -including [staged installs], [redirection], or [command redefinition]. -It is compatible with parallel builds (`-j#`). - -[Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html -[staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html -[redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html -[command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html diff --git a/library/lz4-mac/LICENSE b/library/lz4-mac/LICENSE deleted file mode 100755 index c221aebd..00000000 --- a/library/lz4-mac/LICENSE +++ /dev/null @@ -1,11 +0,0 @@ -This repository uses 2 different licenses : -- all files in the `lib` directory use a BSD 2-Clause license -- all other files use a GPLv2 license, unless explicitly stated otherwise - -Relevant license is reminded at the top of each source file, -and with presence of COPYING or LICENSE file in associated directories. - -This model is selected to emphasize that -files in the `lib` directory are designed to be included into 3rd party applications, -while all other files, in `programs`, `tests` or `examples`, -receive more limited attention and support for such scenario. diff --git a/library/lz4-mac/Makefile b/library/lz4-mac/Makefile deleted file mode 100755 index da485a18..00000000 --- a/library/lz4-mac/Makefile +++ /dev/null @@ -1,181 +0,0 @@ -# ################################################################ -# LZ4 - Makefile -# Copyright (C) Yann Collet 2011-2016 -# All rights reserved. -# -# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets -# -# BSD license -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, this -# list of conditions and the following disclaimer in the documentation and/or -# other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# You can contact the author at : -# - LZ4 source repository : https://github.com/lz4/lz4 -# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c -# ################################################################ - -LZ4DIR = lib -PRGDIR = programs -TESTDIR = tests -EXDIR = examples - - -# Define nul output -ifneq (,$(filter Windows%,$(OS))) -EXT = .exe -VOID = nul -else -EXT = -VOID = /dev/null -endif - - -.PHONY: default -default: lib-release lz4-release - -.PHONY: all -all: allmost manuals - -.PHONY: allmost -allmost: lib lz4 examples - -.PHONY: lib lib-release -lib lib-release: - @$(MAKE) -C $(LZ4DIR) $@ - -.PHONY: lz4 lz4-release -lz4 : lib -lz4-release : lib-release -lz4 lz4-release : - @$(MAKE) -C $(PRGDIR) $@ - @cp $(PRGDIR)/lz4$(EXT) . - -.PHONY: examples -examples: lib lz4 - $(MAKE) -C $(EXDIR) test - -.PHONY: manuals -manuals: - @$(MAKE) -C contrib/gen_manual $@ - -.PHONY: clean -clean: - @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) - @$(MAKE) -C $(PRGDIR) $@ > $(VOID) - @$(MAKE) -C $(TESTDIR) $@ > $(VOID) - @$(MAKE) -C $(EXDIR) $@ > $(VOID) - @$(MAKE) -C contrib/gen_manual $@ - @$(RM) lz4$(EXT) - @echo Cleaning completed - - -#----------------------------------------------------------------------------- -# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets -#----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) -HOST_OS = POSIX - -.PHONY: install uninstall -install uninstall: - @$(MAKE) -C $(LZ4DIR) $@ - @$(MAKE) -C $(PRGDIR) $@ - -travis-install: - $(MAKE) -j1 install DESTDIR=~/install_test_dir - -cmake: - @cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE) - -endif - - -ifneq (,$(filter MSYS%,$(shell uname))) -HOST_OS = MSYS -CMAKE_PARAMS = -G"MSYS Makefiles" -endif - - -#------------------------------------------------------------------------ -#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets -#------------------------------------------------------------------------ -ifneq (,$(filter $(HOST_OS),MSYS POSIX)) - -.PHONY: list -list: - @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs - -.PHONY: test -test: - $(MAKE) -C $(TESTDIR) $@ - -clangtest: clean - clang -v - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(LZ4DIR) all CC=clang - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(PRGDIR) all CC=clang - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) all CC=clang - -clangtest-native: clean - clang -v - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(LZ4DIR) all CC=clang - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(PRGDIR) native CC=clang - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) native CC=clang - -usan: clean - CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1 - -usan32: clean - CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T1mn" NB_LOOPS=-i1 - -staticAnalyze: clean - CFLAGS=-g scan-build --status-bugs -v $(MAKE) all - -platformTest: clean - @echo "\n ---- test lz4 with $(CC) compiler ----" - @$(CC) -v - CFLAGS="-O3 -Werror" $(MAKE) -C $(LZ4DIR) all - CFLAGS="-O3 -Werror -static" $(MAKE) -C $(PRGDIR) all - CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) all - $(MAKE) -C $(TESTDIR) test-platform - -.PHONY: versionsTest -versionsTest: clean - $(MAKE) -C $(TESTDIR) $@ - -gpptest: clean - g++ -v - CC=g++ $(MAKE) -C $(LZ4DIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - CC=g++ $(MAKE) -C $(TESTDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - -gpptest32: clean - g++ -v - CC=g++ $(MAKE) -C $(LZ4DIR) all CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - CC=g++ $(MAKE) -C $(PRGDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - -c_standards: clean - # note : lz4 is not C90 compatible, because it requires long long support - CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost - CFLAGS="-std=c99 -Werror" $(MAKE) clean allmost - CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost - CFLAGS="-std=c11 -Werror" $(MAKE) clean allmost - -endif diff --git a/library/lz4-mac/NEWS b/library/lz4-mac/NEWS deleted file mode 100755 index ff20e7e7..00000000 --- a/library/lz4-mac/NEWS +++ /dev/null @@ -1,236 +0,0 @@ -v1.8.1 -fix : decompression failed when using a combination of extDict + low memory address (#397), reported and fixed by Julian Scheid (@jscheid) -install: fix : correct man page directory (#387), reported by Stuart Cardall (@itoffshore) -build : `make` and `make test` compatible with `-jX`, reported by @mwgamera - -v1.8.0 -cli : fix : do not modify /dev/null permissions, reported by @Maokaman1 -cli : added GNU separator -- specifying that all following arguments are files -API : added LZ4_compress_HC_destSize(), by Oleg (@remittor) -API : added LZ4F_resetDecompressionContext() -API : lz4frame : negative compression levels trigger fast acceleration, request by Lawrence Chan -API : lz4frame : can control block checksum and dictionary ID -API : fix : expose obsolete decoding functions, reported by Chen Yufei -API : experimental : lz4frame_static : new dictionary compression API -build : fix : static lib installation, by Ido Rosen -build : dragonFlyBSD, OpenBSD, NetBSD supported -build : LZ4_MEMORY_USAGE can be modified at compile time, through external define -doc : Updated LZ4 Frame format to v1.6.0, restoring Dictionary-ID field -doc : lz4 api manual, by Przemyslaw Skibinski - -v1.7.5 -lz4hc : new high compression mode : levels 10-12 compress more and slower, by Przemyslaw Skibinski -lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) -cli : fix minor notification when using -r recursive mode -API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) -doc : markdown version of man page, by Takayuki Matsuoka (#279) -build : Makefile : fix make -jX lib+exe concurrency (#277) -build : cmake : improvements by Michał Górny (#296) - -v1.7.4.2 -fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) - -v1.7.4 -Improved : much better speed in -mx32 mode -cli : fix : Large file support in 32-bits mode on Mac OS-X -fix : compilation on gcc 4.4 (#272), reported by Antoine Martin - -v1.7.3 -Changed : moved to versioning; package, cli and library have same version number -Improved: Small decompression speed boost -Improved: Small compression speed improvement on 64-bits systems -Improved: Small compression ratio and speed improvement on small files -Improved: Significant speed boost on ARMv6 and ARMv7 -Fix : better ratio on 64-bits big-endian targets -Improved cmake build script, by Evan Nemerson -New liblz4-dll project, by Przemyslaw Skibinki -Makefile: Generates object files (*.o) for faster (re)compilation on low power systems -cli : new : --rm and --help commands -cli : new : preserved file attributes, by Przemyslaw Skibinki -cli : fix : crash on some invalid inputs -cli : fix : -t correctly validates lz4-compressed files, by Nick Terrell -cli : fix : detects and reports fread() errors, thanks to Hiroshi Fujishima report #243 -cli : bench : new : -r recursive mode -lz4cat : can cat multiple files in a single command line (#184) -Added : doc/lz4_manual.html, by Przemyslaw Skibinski -Added : dictionary compression and frame decompression examples, by Nick Terrell -Added : Debianization, by Evgeniy Polyakov - -r131 -New : Dos/DJGPP target, thanks to Louis Santillan (#114) -Added : Example using lz4frame library, by Zbigniew Jędrzejewski-Szmek (#118) -Changed: xxhash symbols are modified (namespace emulation) within liblz4 - -r130: -Fixed : incompatibility sparse mode vs console, reported by Yongwoon Cho (#105) -Fixed : LZ4IO exits too early when frame crc not present, reported by Yongwoon Cho (#106) -Fixed : incompatibility sparse mode vs append mode, reported by Takayuki Matsuoka (#110) -Performance fix : big compression speed boost for clang (+30%) -New : cross-version test, by Takayuki Matsuoka - -r129: -Added : LZ4_compress_fast(), LZ4_compress_fast_continue() -Added : LZ4_compress_destSize() -Changed: New lz4 and lz4hc compression API. Previous function prototypes still supported. -Changed: Sparse file support enabled by default -New : LZ4 CLI improved performance compressing/decompressing multiple files (#86, kind contribution from Kyle J. Harper & Takayuki Matsuoka) -Fixed : GCC 4.9+ optimization bug - Reported by Markus Trippelsdorf, Greg Slazinski & Evan Nemerson -Changed: Enums converted to LZ4F_ namespace convention - by Takayuki Matsuoka -Added : AppVeyor CI environment, for Visual tests - Suggested by Takayuki Matsuoka -Modified:Obsolete functions generate warnings - Suggested by Evan Nemerson, contributed by Takayuki Matsuoka -Fixed : Bug #75 (unfinished stream), reported by Yongwoon Cho -Updated: Documentation converted to MarkDown format - -r128: -New : lz4cli sparse file support (Requested by Neil Wilson, and contributed by Takayuki Matsuoka) -New : command -m, to compress multiple files in a single command (suggested by Kyle J. Harper) -Fixed : Restored lz4hc compression ratio (slightly lower since r124) -New : lz4 cli supports long commands (suggested by Takayuki Matsuoka) -New : lz4frame & lz4cli frame content size support -New : lz4frame supports skippable frames, as requested by Sergey Cherepanov -Changed: Default "make install" directory is /usr/local, as notified by Ron Johnson -New : lz4 cli supports "pass-through" mode, requested by Neil Wilson -New : datagen can generate sparse files -New : scan-build tests, thanks to kind help by Takayuki Matsuoka -New : g++ compatibility tests -New : arm cross-compilation test, thanks to kind help by Takayuki Matsuoka -Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48, reported by Thomas Klausner) -Added : Visual project directory -Updated: Man page & Specification - -r127: -N/A : added a file on SVN - -r126: -New : lz4frame API is now integrated into liblz4 -Fixed : GCC 4.9 bug on highest performance settings, reported by Greg Slazinski -Fixed : bug within LZ4 HC streaming mode, reported by James Boyle -Fixed : older compiler don't like nameless unions, reported by Cheyi Lin -Changed : lz4 is C90 compatible -Changed : added -pedantic option, fixed a few mminor warnings - -r125: -Changed : endian and alignment code -Changed : directory structure : new "lib" directory -Updated : lz4io, now uses lz4frame -Improved: slightly improved decoding speed -Fixed : LZ4_compress_limitedOutput(); Special thanks to Christopher Speller ! -Fixed : some alignment warnings under clang -Fixed : deprecated function LZ4_slideInputBufferHC() - -r124: -New : LZ4 HC streaming mode -Fixed : LZ4F_compressBound() using null preferencesPtr -Updated : xxHash to r38 -Updated library number, to 1.4.0 - -r123: -Added : experimental lz4frame API, thanks to Takayuki Matsuoka and Christopher Jackson for testings -Fix : s390x support, thanks to Nobuhiro Iwamatsu -Fix : test mode (-t) no longer requires confirmation, thanks to Thary Nguyen - -r122: -Fix : AIX & AIX64 support (SamG) -Fix : mips 64-bits support (lew van) -Added : Examples directory, using code examples from Takayuki Matsuoka -Updated : Framing specification, to v1.4.1 -Updated : xxHash, to r36 - -r121: -Added : Makefile : install for kFreeBSD and Hurd (Nobuhiro Iwamatsu) -Fix : Makefile : install for OS-X and BSD, thanks to Takayuki Matsuoka - -r120: -Modified : Streaming API, using strong types -Added : LZ4_versionNumber(), thanks to Takayuki Matsuoka -Fix : OS-X : library install name, thanks to Clemens Lang -Updated : Makefile : synchronize library version number with lz4.h, thanks to Takayuki Matsuoka -Updated : Makefile : stricter compilation flags -Added : pkg-config, thanks to Zbigniew Jędrzejewski-Szmek (issue 135) -Makefile : lz4-test only test native binaries, as suggested by Michał Górny (issue 136) -Updated : xxHash to r35 - -r119: -Fix : Issue 134 : extended malicious address space overflow in 32-bits mode for some specific configurations - -r118: -New : LZ4 Streaming API (Fast version), special thanks to Takayuki Matsuoka -New : datagen : parametrable synthetic data generator for tests -Improved : fuzzer, support more test cases, more parameters, ability to jump to specific test -fix : support ppc64le platform (issue 131) -fix : Issue 52 (malicious address space overflow in 32-bits mode when using large custom format) -fix : Makefile : minor issue 130 : header files permissions - -r117: -Added : man pages for lz4c and lz4cat -Added : automated tests on Travis, thanks to Takayuki Matsuoka ! -fix : block-dependency command line (issue 127) -fix : lz4fullbench (issue 128) - -r116: -hotfix (issue 124 & 125) - -r115: -Added : lz4cat utility, installed on POSX systems (issue 118) -OS-X compatible compilation of dynamic library (issue 115) - -r114: -Makefile : library correctly compiled with -O3 switch (issue 114) -Makefile : library compilation compatible with clang -Makefile : library is versioned and linked (issue 119) -lz4.h : no more static inline prototypes (issue 116) -man : improved header/footer (issue 111) -Makefile : Use system default $(CC) & $(MAKE) variables (issue 112) -xxhash : updated to r34 - -r113: -Large decompression speed improvement for GCC 32-bits. Thanks to Valery Croizier ! -LZ4HC : Compression Level is now a programmable parameter (CLI from 4 to 9) -Separated IO routines from command line (lz4io.c) -Version number into lz4.h (suggested by Francesc Alted) - -r112: -quickfix - -r111 : -Makefile : added capability to install libraries -Modified Directory tree, to better separate libraries from programs. - -r110 : -lz4 & lz4hc : added capability to allocate state & stream state with custom allocator (issue 99) -fuzzer & fullbench : updated to test new functions -man : documented -l command (Legacy format, for Linux kernel compression) (issue 102) -cmake : improved version by Mika Attila, building programs and libraries (issue 100) -xxHash : updated to r33 -Makefile : clean also delete local package .tar.gz - -r109 : -lz4.c : corrected issue 98 (LZ4_compress_limitedOutput()) -Makefile : can specify version number from makefile - -r108 : -lz4.c : corrected compression efficiency issue 97 in 64-bits chained mode (-BD) for streams > 4 GB (thanks Roman Strashkin for reporting) - -r107 : -Makefile : support DESTDIR for staged installs. Thanks Jorge Aparicio. -Makefile : make install installs both lz4 and lz4c (Jorge Aparicio) -Makefile : removed -Wno-implicit-declaration compilation switch -lz4cli.c : include for isatty() (Luca Barbato) -lz4.h : introduced LZ4_MAX_INPUT_SIZE constant (Shay Green) -lz4.h : LZ4_compressBound() : unified macro and inline definitions (Shay Green) -lz4.h : LZ4_decompressSafe_partial() : clarify comments (Shay Green) -lz4.c : LZ4_compress() verify input size condition (Shay Green) -bench.c : corrected a bug in free memory size evaluation -cmake : install into bin/ directory (Richard Yao) -cmake : check for just C compiler (Elan Ruusamae) - -r106 : -Makefile : make dist modify text files in the package to respect Unix EoL convention -lz4cli.c : corrected small display bug in HC mode - -r105 : -Makefile : New install script and man page, contributed by Prasad Pandit -lz4cli.c : Minor modifications, for easier extensibility -COPYING : added license file -LZ4_Streaming_Format.odt : modified file name to remove white space characters -Makefile : .exe suffix now properly added only for Windows target diff --git a/library/lz4-mac/README.md b/library/lz4-mac/README.md deleted file mode 100755 index 596fdac7..00000000 --- a/library/lz4-mac/README.md +++ /dev/null @@ -1,115 +0,0 @@ -LZ4 - Extremely fast compression -================================ - -LZ4 is lossless compression algorithm, -providing compression speed at 400 MB/s per core, -scalable with multi-cores CPU. -It features an extremely fast decoder, -with speed in multiple GB/s per core, -typically reaching RAM speed limits on multi-core systems. - -Speed can be tuned dynamically, selecting an "acceleration" factor -which trades compression ratio for more speed up. -On the other end, a high compression derivative, LZ4_HC, is also provided, -trading CPU time for improved compression ratio. -All versions feature the same decompression speed. - -LZ4 library is provided as open-source software using BSD 2-Clause license. - - -|Branch |Status | -|------------|---------| -|master | [![Build Status][travisMasterBadge]][travisLink] [![Build status][AppveyorMasterBadge]][AppveyorLink] [![coverity][coverBadge]][coverlink] | -|dev | [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] | - -[travisMasterBadge]: https://travis-ci.org/lz4/lz4.svg?branch=master "Continuous Integration test suite" -[travisDevBadge]: https://travis-ci.org/lz4/lz4.svg?branch=dev "Continuous Integration test suite" -[travisLink]: https://travis-ci.org/lz4/lz4 -[AppveyorMasterBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=master&svg=true "Windows test suite" -[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=dev&svg=true "Windows test suite" -[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/lz4-1lndh -[coverBadge]: https://scan.coverity.com/projects/4735/badge.svg "Static code analysis of Master branch" -[coverlink]: https://scan.coverity.com/projects/4735 - -> **Branch Policy:** -> - The "master" branch is considered stable, at all times. -> - The "dev" branch is the one where all contributions must be merged - before being promoted to master. -> + If you plan to propose a patch, please commit into the "dev" branch, - or its own feature branch. - Direct commit to "master" are not permitted. - -Benchmarks -------------------------- - -The benchmark uses [lzbench], from @inikep -compiled with GCC v6.2.0 on Linux 64-bits. -The reference system uses a Core i7-3930K CPU @ 4.5GHz. -Benchmark evaluates the compression of reference [Silesia Corpus] -in single-thread mode. - -[lzbench]: https://github.com/inikep/lzbench -[Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia - -| Compressor | Ratio | Compression | Decompression | -| ---------- | ----- | ----------- | ------------- | -| memcpy | 1.000 | 7300 MB/s | 7300 MB/s | -|**LZ4 fast 8 (v1.7.3)**| 1.799 |**911 MB/s** | **3360 MB/s** | -|**LZ4 default (v1.7.3)**|**2.101**|**625 MB/s** | **3220 MB/s** | -| LZO 2.09 | 2.108 | 620 MB/s | 845 MB/s | -| QuickLZ 1.5.0 | 2.238 | 510 MB/s | 600 MB/s | -| Snappy 1.1.3 | 2.091 | 450 MB/s | 1550 MB/s | -| LZF v3.6 | 2.073 | 365 MB/s | 820 MB/s | -| [Zstandard] 1.1.1 -1 | 2.876 | 330 MB/s | 930 MB/s | -| [Zstandard] 1.1.1 -3 | 3.164 | 200 MB/s | 810 MB/s | -| [zlib] deflate 1.2.8 -1| 2.730 | 100 MB/s | 370 MB/s | -|**LZ4 HC -9 (v1.7.3)** |**2.720**| 34 MB/s | **3240 MB/s** | -| [zlib] deflate 1.2.8 -6| 3.099 | 33 MB/s | 390 MB/s | - -[zlib]: http://www.zlib.net/ -[Zstandard]: http://www.zstd.net/ - -LZ4 is also compatible and well optimized for x32 mode, for which it provides an additional +10% speed performance. - - -Installation -------------------------- - -``` -make -make install # this command may require root access -``` - -LZ4's `Makefile` supports standard [Makefile conventions], -including [staged installs], [redirection], or [command redefinition]. -It is compatible with parallel builds (`-j#`). - -[Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html -[staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html -[redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html -[command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html - - -Documentation -------------------------- - -The raw LZ4 block compression format is detailed within [lz4_Block_format]. - -To compress an arbitrarily long file or data stream, multiple blocks are required. -Organizing these blocks and providing a common header format to handle their content -is the purpose of the Frame format, defined into [lz4_Frame_format]. -Interoperable versions of LZ4 must respect this frame format. - -[lz4_Block_format]: doc/lz4_Block_format.md -[lz4_Frame_format]: doc/lz4_Frame_format.md - - -Other source versions -------------------------- - -Beyond the C reference source, -many contributors have created versions of lz4 in multiple languages -(Java, C#, Python, Perl, Ruby, etc.). -A list of known source ports is maintained on the [LZ4 Homepage]. - -[LZ4 Homepage]: http://www.lz4.org diff --git a/library/lz4-mac/lib/.gitignore b/library/lz4-mac/lib/.gitignore deleted file mode 100755 index 5d6f134c..00000000 --- a/library/lz4-mac/lib/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# make install artefact -liblz4.pc diff --git a/library/lz4-mac/lib/LICENSE b/library/lz4-mac/lib/LICENSE deleted file mode 100755 index 74c2cdd7..00000000 --- a/library/lz4-mac/lib/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -LZ4 Library -Copyright (c) 2011-2016, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/library/lz4-mac/lib/Makefile b/library/lz4-mac/lib/Makefile deleted file mode 100755 index dd33f503..00000000 --- a/library/lz4-mac/lib/Makefile +++ /dev/null @@ -1,182 +0,0 @@ -# ################################################################ -# LZ4 library - Makefile -# Copyright (C) Yann Collet 2011-2016 -# All rights reserved. -# -# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets -# -# BSD license -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, this -# list of conditions and the following disclaimer in the documentation and/or -# other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# You can contact the author at : -# - LZ4 source repository : https://github.com/Cyan4973/lz4 -# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c -# ################################################################ - -# Version numbers -LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h` -LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h` -LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h` -LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) -LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) -LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) -LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) -LIBVER := $(shell echo $(LIBVER_SCRIPT)) - -BUILD_STATIC:=yes - -CPPFLAGS+= -DXXH_NAMESPACE=LZ4_ -CFLAGS ?= -O3 -DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ - -Wundef -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) - -SRCFILES := $(sort $(wildcard *.c)) - - -# OS X linker doesn't support -soname, and use different extension -# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html -ifeq ($(shell uname), Darwin) - SHARED_EXT = dylib - SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) - SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) - SONAME_FLAGS = -install_name $(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) -else - SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR) - SHARED_EXT = so - SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) - SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) -endif - -LIBLZ4 = liblz4.$(SHARED_EXT_VER) - -.PHONY: default -default: lib-release - -lib-release: DEBUGFLAGS := -lib-release: lib - -lib: liblz4.a liblz4 - -all: lib - -all32: CFLAGS+=-m32 -all32: all - -liblz4.a: $(SRCFILES) -ifeq ($(BUILD_STATIC),yes) # can be disabled on command line - @echo compiling static library - @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ - @$(AR) rcs $@ *.o -endif - -$(LIBLZ4): $(SRCFILES) - @echo compiling dynamic library $(LIBVER) -ifneq (,$(filter Windows%,$(OS))) - @$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll - dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib -else - @$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@ - @echo creating versioned links - @ln -sf $@ liblz4.$(SHARED_EXT_MAJOR) - @ln -sf $@ liblz4.$(SHARED_EXT) -endif - -liblz4: $(LIBLZ4) - -clean: - @$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib - @$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER) - @echo Cleaning library completed - - -#----------------------------------------------------------------------------- -# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets -#----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) - -DESTDIR ?= -# directory variables : GNU conventions prefer lowercase -# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html -# support both lower and uppercase (BSD), use uppercase in script -prefix ?= /usr/local -PREFIX ?= $(prefix) -exec_prefix ?= $(PREFIX) -libdir ?= $(exec_prefix)/lib -LIBDIR ?= $(libdir) -includedir ?= $(PREFIX)/include -INCLUDEDIR ?= $(includedir) - -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly)) -PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig -else -PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig -endif - -ifneq (,$(filter $(shell uname),SunOS)) -INSTALL ?= ginstall -else -INSTALL ?= install -endif - -INSTALL_PROGRAM ?= $(INSTALL) -INSTALL_DATA ?= $(INSTALL) -m 644 - -liblz4.pc: liblz4.pc.in Makefile - @echo creating pkgconfig - @sed -e 's|@PREFIX@|$(PREFIX)|' \ - -e 's|@LIBDIR@|$(LIBDIR)|' \ - -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ - -e 's|@VERSION@|$(LIBVER)|' \ - $< >$@ - -install: lib liblz4.pc - @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/ - @$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/ - @echo Installing libraries -ifeq ($(BUILD_STATIC),yes) - @$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a - @$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h -endif - @$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) - @echo Installing headers in $(INCLUDEDIR) - @$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h - @echo lz4 libraries installed - -uninstall: - @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc - @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) - @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) - @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a - @$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h - @$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4frame.h - @echo lz4 libraries successfully uninstalled - -endif diff --git a/library/lz4-mac/lib/README.md b/library/lz4-mac/lib/README.md deleted file mode 100755 index 7082fe3e..00000000 --- a/library/lz4-mac/lib/README.md +++ /dev/null @@ -1,73 +0,0 @@ -LZ4 - Library Files -================================ - -The `/lib` directory contains many files, but depending on project's objectives, -not all of them are necessary. - -#### Minimal LZ4 build - -The minimum required is **`lz4.c`** and **`lz4.h`**, -which provides the fast compression and decompression algorithm. -They generate and decode data using [LZ4 block format]. - - -#### High Compression variant - -For more compression ratio at the cost of compression speed, -the High Compression variant called **lz4hc** is available. -Add files **`lz4hc.c`** and **`lz4hc.h`**. -The variant still depends on regular `lib/lz4.*` source files. - - -#### Frame variant, for interoperability - -In order to produce compressed data compatible with `lz4` command line utility, -it's necessary to encode lz4-compressed blocks using the [official interoperable frame format]. -This format is generated and decoded automatically by the **lz4frame** library. -Its public API is described in `lib/lz4frame.h`. -In order to work properly, lz4frame needs all other modules present in `/lib`, -including, lz4 and lz4hc, and also **xxhash**. -So it's necessary to include all `*.c` and `*.h` files present in `/lib`. - - -#### Advanced / Experimental API - -A complex API defined in `lz4frame_static.h` contains definitions -which are not guaranteed to remain stable in future versions. -As a consequence, it must be used with static linking ***only***. - - -#### Windows : using MinGW+MSYS to create DLL - -DLL can be created using MinGW+MSYS with the `make liblz4` command. -This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`. -The import library is only required with Visual C++. -The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library -`dll\liblz4.dll` are required to compile a project using gcc/MinGW. -The dynamic library has to be added to linking options. -It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be linked with `dll\liblz4.dll`. For example: -``` - gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll -``` -The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. - - -#### Miscellaneous - -Other files present in the directory are not source code. There are : - - - `LICENSE` : contains the BSD license text - - `Makefile` : `make` script to compile and install lz4 library (static and dynamic) - - `liblz4.pc.in` : for `pkg-config` (used in `make install`) - - `README.md` : this file - -[official interoperable frame format]: ../doc/lz4_Frame_format.md -[LZ4 block format]: ../doc/lz4_Block_format.md - - -#### License - -All source material within __lib__ directory are BSD 2-Clause licensed. -See [LICENSE](LICENSE) for details. -The license is also reminded at the top of each source file. diff --git a/library/lz4-mac/lib/dll/example/Makefile b/library/lz4-mac/lib/dll/example/Makefile deleted file mode 100755 index e9879568..00000000 --- a/library/lz4-mac/lib/dll/example/Makefile +++ /dev/null @@ -1,63 +0,0 @@ -# ########################################################################## -# LZ4 programs - Makefile -# Copyright (C) Yann Collet 2016 -# -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - LZ4 homepage : http://www.lz4.org -# - LZ4 source repository : https://github.com/lz4/lz4 -# ########################################################################## - -VOID := /dev/null -LZ4DIR := ../include -LIBDIR := ../static -DLLDIR := ../dll - -CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make -CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ - -Wdeclaration-after-statement -Wstrict-prototypes \ - -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(MOREFLAGS) -CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ -FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) - - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -else -EXT = -endif - -.PHONY: default fullbench-dll fullbench-lib - - -default: all - -all: fullbench-dll fullbench-lib - - -fullbench-lib: fullbench.c xxhash.c - $(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/liblz4_static.lib - -fullbench-dll: fullbench.c xxhash.c - $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(DLLDIR)/liblz4.dll - -clean: - @$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \ - @echo Cleaning completed diff --git a/library/lz4-mac/lib/dll/example/README.md b/library/lz4-mac/lib/dll/example/README.md deleted file mode 100755 index 223e4738..00000000 --- a/library/lz4-mac/lib/dll/example/README.md +++ /dev/null @@ -1,69 +0,0 @@ -LZ4 Windows binary package -==================================== - -#### The package contents - -- `lz4.exe` : Command Line Utility, supporting gzip-like arguments -- `dll\liblz4.dll` : The DLL of LZ4 library -- `dll\liblz4.lib` : The import library of LZ4 library for Visual C++ -- `example\` : The example of usage of LZ4 library -- `include\` : Header files required with LZ4 library -- `static\liblz4_static.lib` : The static LZ4 library - - -#### Usage of Command Line Interface - -Command Line Interface (CLI) supports gzip-like arguments. -By default CLI takes an input file and compresses it to an output file: -``` - Usage: lz4 [arg] [input] [output] -``` -The full list of commands for CLI can be obtained with `-h` or `-H`. The ratio can -be improved with commands from `-3` to `-16` but higher levels also have slower -compression. CLI includes in-memory compression benchmark module with compression -levels starting from `-b` and ending with `-e` with iteration time of `-i` seconds. -CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined -into `-b1e18i1`. - - -#### The example of usage of static and dynamic LZ4 libraries with gcc/MinGW - -Use `cd example` and `make` to build `fullbench-dll` and `fullbench-lib`. -`fullbench-dll` uses a dynamic LZ4 library from the `dll` directory. -`fullbench-lib` uses a static LZ4 library from the `lib` directory. - - -#### Using LZ4 DLL with gcc/MinGW - -The header files from `include\` and the dynamic library `dll\liblz4.dll` -are required to compile a project using gcc/MinGW. -The dynamic library has to be added to linking options. -It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be linked with `dll\liblz4.dll`. For example: -``` - gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\liblz4.dll -``` -The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. - - -#### The example of usage of static and dynamic LZ4 libraries with Visual C++ - -Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a -dynamic LZ4 library from the `dll` directory. The solution works with Visual C++ -2010 or newer. When one will open the solution with Visual C++ newer than 2010 -then the solution will upgraded to the current version. - - -#### Using LZ4 DLL with Visual C++ - -The header files from `include\` and the import library `dll\liblz4.lib` -are required to compile a project using Visual C++. - -1. The header files should be added to `Additional Include Directories` that can - be found in project properties `C/C++` then `General`. -2. The import library has to be added to `Additional Dependencies` that can - be found in project properties `Linker` then `Input`. - If one will provide only the name `liblz4.lib` without a full path to the library - the directory has to be added to `Linker\General\Additional Library Directories`. - -The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. diff --git a/library/lz4-mac/lib/dll/example/fullbench-dll.sln b/library/lz4-mac/lib/dll/example/fullbench-dll.sln deleted file mode 100755 index 72e302e7..00000000 --- a/library/lz4-mac/lib/dll/example/fullbench-dll.sln +++ /dev/null @@ -1,25 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Express 2012 for Windows Desktop -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32 - {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32 - {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64 - {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64 - {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32 - {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32 - {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64 - {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/library/lz4-mac/lib/dll/example/fullbench-dll.vcxproj b/library/lz4-mac/lib/dll/example/fullbench-dll.vcxproj deleted file mode 100755 index cdb55342..00000000 --- a/library/lz4-mac/lib/dll/example/fullbench-dll.vcxproj +++ /dev/null @@ -1,182 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {13992FD2-077E-4954-B065-A428198201A9} - Win32Proj - fullbench-dll - $(SolutionDir)bin\$(Platform)_$(Configuration)\ - $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ - - - - Application - true - Unicode - - - Application - true - Unicode - - - Application - false - true - Unicode - - - Application - false - true - Unicode - - - - - - - - - - - - - - - - - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) - true - false - ..\include - - - Console - true - $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) - liblz4.lib;%(AdditionalDependencies) - false - - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) - true - true - /analyze:stacksize295252 %(AdditionalOptions) - ..\include - - - Console - true - $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) - liblz4.lib;%(AdditionalDependencies) - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) - false - false - ..\include - - - Console - true - true - true - $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) - liblz4.lib;%(AdditionalDependencies) - false - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) - false - true - /analyze:stacksize295252 %(AdditionalOptions) - ..\include - - - Console - true - true - true - $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) - liblz4.lib;%(AdditionalDependencies) - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/library/lz4-mac/lib/dll/liblz4.def b/library/lz4-mac/lib/dll/liblz4.def deleted file mode 100755 index 0ace2235..00000000 --- a/library/lz4-mac/lib/dll/liblz4.def +++ /dev/null @@ -1,62 +0,0 @@ -LIBRARY liblz4.dll -EXPORTS - LZ4F_compressBegin - LZ4F_compressBound - LZ4F_compressEnd - LZ4F_compressFrame - LZ4F_compressFrameBound - LZ4F_compressUpdate - LZ4F_createCompressionContext - LZ4F_createDecompressionContext - LZ4F_decompress - LZ4F_flush - LZ4F_freeCompressionContext - LZ4F_freeDecompressionContext - LZ4F_getErrorName - LZ4F_getFrameInfo - LZ4F_getVersion - LZ4F_isError - LZ4_compress - LZ4_compressBound - LZ4_compressHC - LZ4_compressHC_continue - LZ4_compressHC_limitedOutput - LZ4_compressHC_limitedOutput_continue - LZ4_compressHC_limitedOutput_withStateHC - LZ4_compressHC_withStateHC - LZ4_compress_HC - LZ4_compress_HC_continue - LZ4_compress_HC_extStateHC - LZ4_compress_continue - LZ4_compress_default - LZ4_compress_destSize - LZ4_compress_fast - LZ4_compress_fast_continue - LZ4_compress_fast_extState - LZ4_compress_limitedOutput - LZ4_compress_limitedOutput_continue - LZ4_compress_limitedOutput_withState - LZ4_compress_withState - LZ4_createStream - LZ4_createStreamDecode - LZ4_createStreamHC - LZ4_decompress_fast - LZ4_decompress_fast_continue - LZ4_decompress_fast_usingDict - LZ4_decompress_safe - LZ4_decompress_safe_continue - LZ4_decompress_safe_partial - LZ4_decompress_safe_usingDict - LZ4_freeStream - LZ4_freeStreamDecode - LZ4_freeStreamHC - LZ4_loadDict - LZ4_loadDictHC - LZ4_resetStream - LZ4_resetStreamHC - LZ4_saveDict - LZ4_saveDictHC - LZ4_setStreamDecode - LZ4_sizeofState - LZ4_sizeofStateHC - LZ4_versionNumber diff --git a/library/lz4-mac/lib/liblz4.pc.in b/library/lz4-mac/lib/liblz4.pc.in deleted file mode 100755 index cb31cd78..00000000 --- a/library/lz4-mac/lib/liblz4.pc.in +++ /dev/null @@ -1,14 +0,0 @@ -# LZ4 - Fast LZ compression algorithm -# Copyright (C) 2011-2014, Yann Collet. -# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -prefix=@PREFIX@ -libdir=@LIBDIR@ -includedir=@INCLUDEDIR@ - -Name: lz4 -Description: extremely fast lossless compression algorithm library -URL: http://www.lz4.org/ -Version: @VERSION@ -Libs: -L@LIBDIR@ -llz4 -Cflags: -I@INCLUDEDIR@ diff --git a/library/lz4-mac/lib/lz4.c b/library/lz4-mac/lib/lz4.c deleted file mode 100755 index 179408da..00000000 --- a/library/lz4-mac/lib/lz4.c +++ /dev/null @@ -1,1516 +0,0 @@ -/* - LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2017, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ - - -/*-************************************ -* Tuning parameters -**************************************/ -/* - * LZ4_HEAPMODE : - * Select how default compression functions will allocate memory for their hash table, - * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). - */ -#ifndef LZ4_HEAPMODE -# define LZ4_HEAPMODE 0 -#endif - -/* - * ACCELERATION_DEFAULT : - * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 - */ -#define ACCELERATION_DEFAULT 1 - - -/*-************************************ -* CPU Feature Detection -**************************************/ -/* LZ4_FORCE_MEMORY_ACCESS - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets which assembly generation depends on alignment. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define LZ4_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) -# define LZ4_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/* - * LZ4_FORCE_SW_BITCOUNT - * Define this parameter if your target system or compiler does not support hardware bit count - */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ -# define LZ4_FORCE_SW_BITCOUNT -#endif - - -/*-************************************ -* Dependency -**************************************/ -#include "lz4.h" -/* see also "memory routines" below */ - - -/*-************************************ -* Compiler Options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ -#endif /* _MSC_VER */ - -#ifndef LZ4_FORCE_INLINE -# ifdef _MSC_VER /* Visual Studio */ -# define LZ4_FORCE_INLINE static __forceinline -# else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define LZ4_FORCE_INLINE static inline -# endif -# else -# define LZ4_FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -# endif /* _MSC_VER */ -#endif /* LZ4_FORCE_INLINE */ - -/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE - * Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy, - * together with a simple 8-byte copy loop as a fall-back path. - * However, this optimization hurts the decompression speed by >30%, - * because the execution does not go to the optimized loop - * for typical compressible data, and all of the preamble checks - * before going to the fall-back path become useless overhead. - * This optimization happens only with the -O3 flag, and -O2 generates - * a simple 8-byte copy loop. - * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy - * functions are annotated with __attribute__((optimize("O2"))), - * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute - * of LZ4_wildCopy does not affect the compression speed. - */ -#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) -# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) -# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE -#else -# define LZ4_FORCE_O2_GCC_PPC64LE -# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static -#endif - -#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) -# define expect(expr,value) (__builtin_expect ((expr),(value)) ) -#else -# define expect(expr,value) (expr) -#endif - -#define likely(expr) expect((expr) != 0, 1) -#define unlikely(expr) expect((expr) != 0, 0) - - -/*-************************************ -* Memory routines -**************************************/ -#include /* malloc, calloc, free */ -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free -#include /* memset, memcpy */ -#define MEM_INIT memset - - -/*-************************************ -* Basic Types -**************************************/ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef uintptr_t uptrval; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef size_t uptrval; /* generally true, except OpenVMS-64 */ -#endif - -#if defined(__x86_64__) - typedef U64 reg_t; /* 64-bits in x32 mode */ -#else - typedef size_t reg_t; /* 32-bits in x32 mode */ -#endif - -/*-************************************ -* Reading and writing into memory -**************************************/ -static unsigned LZ4_isLittleEndian(void) -{ - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} - - -#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) -/* lie to the compiler about data alignment; use with caution */ - -static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } -static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } -static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } - -static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } - -#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; - -static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } - -static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } - -#else /* safe and portable access through memcpy() */ - -static U16 LZ4_read16(const void* memPtr) -{ - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static U32 LZ4_read32(const void* memPtr) -{ - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static reg_t LZ4_read_ARCH(const void* memPtr) -{ - reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static void LZ4_write16(void* memPtr, U16 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -static void LZ4_write32(void* memPtr, U32 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -#endif /* LZ4_FORCE_MEMORY_ACCESS */ - - -static U16 LZ4_readLE16(const void* memPtr) -{ - if (LZ4_isLittleEndian()) { - return LZ4_read16(memPtr); - } else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)((U16)p[0] + (p[1]<<8)); - } -} - -static void LZ4_writeLE16(void* memPtr, U16 value) -{ - if (LZ4_isLittleEndian()) { - LZ4_write16(memPtr, value); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); - } -} - -static void LZ4_copy8(void* dst, const void* src) -{ - memcpy(dst,src,8); -} - -/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -LZ4_FORCE_O2_INLINE_GCC_PPC64LE -void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) -{ - BYTE* d = (BYTE*)dstPtr; - const BYTE* s = (const BYTE*)srcPtr; - BYTE* const e = (BYTE*)dstEnd; - - do { LZ4_copy8(d,s); d+=8; s+=8; } while (d=2) -# include -# define DEBUGLOG(l, ...) { \ - if (l<=LZ4_DEBUG) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ - } } -#else -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif - - -/*-************************************ -* Common functions -**************************************/ -static unsigned LZ4_NbCommonBytes (register reg_t val) -{ - if (LZ4_isLittleEndian()) { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else /* Big Endian CPU */ { - if (sizeof(val)==8) { /* 64-bits */ -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); -# else - static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. - Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. - Note that this code path is never triggered in 32-bits mode. */ - unsigned r; - if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } - } -} - -#define STEPSIZE sizeof(reg_t) -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) -{ - const BYTE* const pStart = pIn; - - while (likely(pIn compression run slower on incompressible data */ - - -/*-************************************ -* Local Structures and types -**************************************/ -typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; -typedef enum { byPtr, byU32, byU16 } tableType_t; - -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; -typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; - -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - - -/*-************************************ -* Local Utils -**************************************/ -int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } -const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } -int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } - - -/*-****************************** -* Compression functions -********************************/ -static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) -{ - if (tableType == byU16) - return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); - else - return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); -} - -static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) -{ - static const U64 prime5bytes = 889523592379ULL; - static const U64 prime8bytes = 11400714785074694791ULL; - const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) - return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - else - return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); -} - -LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) -{ - if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); - return LZ4_hash4(LZ4_read32(p), tableType); -} - -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) -{ - switch (tableType) - { - case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } - case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } - } -} - -LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - U32 const h = LZ4_hashPosition(p, tableType); - LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); -} - -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } - if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ -} - -LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - U32 const h = LZ4_hashPosition(p, tableType); - return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); -} - - -/** LZ4_compress_generic() : - inlined, to ensure branches are decided at compilation time */ -LZ4_FORCE_INLINE int LZ4_compress_generic( - LZ4_stream_t_internal* const cctx, - const char* const source, - char* const dest, - const int inputSize, - const int maxOutputSize, - const limitedOutput_directive outputLimited, - const tableType_t tableType, - const dict_directive dict, - const dictIssue_directive dictIssue, - const U32 acceleration) -{ - const BYTE* ip = (const BYTE*) source; - const BYTE* base; - const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - cctx->dictSize; - const BYTE* const dictionary = cctx->dictionary; - const BYTE* const dictEnd = dictionary + cctx->dictSize; - const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; - const BYTE* anchor = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dest; - BYTE* const olimit = op + maxOutputSize; - - U32 forwardH; - - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - switch(dict) - { - case noDict: - default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; - case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; - case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; - } - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (inputSizehashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - ptrdiff_t refDelta = 0; - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; - do { - U32 const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } - - /* Encode Literals */ - { unsigned const litLength = (unsigned)(ip - anchor); - token = op++; - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; - if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; - *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; - if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); - matchCode += more; - ip += more; - } - } else { - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; - } - - if ( outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) - return 0; - if (matchCode >= ML_MASK) { - *token += ML_MASK; - matchCode -= ML_MASK; - LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4*255) { - op+=4; - LZ4_write32(op, 0xFFFFFFFF); - matchCode -= 4*255; - } - op += matchCode / 255; - *op++ = (BYTE)(matchCode % 255); - } else - *token += (BYTE)(matchCode); - } - - anchor = ip; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { size_t const lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - return 0; - if (lastRun >= RUN_MASK) { - size_t accumulator = lastRun - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRun<internal_donotuse; - LZ4_resetStream((LZ4_stream_t*)state); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); - } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); - } -} - - -int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ -#if (LZ4_HEAPMODE) - void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ -#else - LZ4_stream_t ctx; - void* const ctxPtr = &ctx; -#endif - - int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); - -#if (LZ4_HEAPMODE) - FREEMEM(ctxPtr); -#endif - return result; -} - - -int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); -} - - -/* hidden debug function */ -/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t ctx; - LZ4_resetStream(&ctx); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); -} - - -/*-****************************** -* *_destSize() variant -********************************/ - -static int LZ4_compress_destSize_generic( - LZ4_stream_t_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) -{ - const BYTE* ip = (const BYTE*) src; - const BYTE* base = (const BYTE*) src; - const BYTE* lowLimit = (const BYTE*) src; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + targetDstSize; - BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; - BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); - BYTE* const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - - /* Init conditions */ - if (targetDstSize < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (*srcSizePtrhashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = 1 << LZ4_skipTrigger; - - do { - U32 h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - - } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - - /* Encode Literal length */ - { unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if (op + ((litLength+240)/255) + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength>=RUN_MASK) { - unsigned len = litLength - RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15-1) + (oMaxMatch-op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength>=ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; - } - else *token += (BYTE)(matchLength); - } - - anchor = ip; - - /* Test end of block */ - if (ip > mflimit) break; - if (op > oMaxSeq) break; - - /* Fill table */ - LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); - if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend-op) - 1; - lastRunSize -= (lastRunSize+240)/255; - } - ip = anchor + lastRunSize; - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ - return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); - } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } -} - - -int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) -{ -#if (LZ4_HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ -#else - LZ4_stream_t ctxBody; - LZ4_stream_t* ctx = &ctxBody; -#endif - - int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); - -#if (LZ4_HEAPMODE) - FREEMEM(ctx); -#endif - return result; -} - - - -/*-****************************** -* Streaming functions -********************************/ - -LZ4_stream_t* LZ4_createStream(void) -{ - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); - return lz4s; -} - -void LZ4_resetStream (LZ4_stream_t* LZ4_stream) -{ - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); -} - -int LZ4_freeStream (LZ4_stream_t* LZ4_stream) -{ - if (!LZ4_stream) return 0; /* support free on NULL */ - FREEMEM(LZ4_stream); - return (0); -} - - -#define HASH_UNIT sizeof(reg_t) -int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) -{ - LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; - const BYTE* p = (const BYTE*)dictionary; - const BYTE* const dictEnd = p + dictSize; - const BYTE* base; - - if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); - - if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; - return 0; - } - - if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; - base = p - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; - - while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); - p+=3; - } - - return dict->dictSize; -} - - -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) -{ - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ - /* rescale hash table */ - U32 const delta = LZ4_dict->currentOffset - 64 KB; - const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; - int i; - for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; - else LZ4_dict->hashTable[i] -= delta; - } - LZ4_dict->currentOffset = 64 KB; - if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; - LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; - } -} - - -int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; - if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; - } - } - - /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) { - int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } - - /* external dictionary mode */ - { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } -} - - -/* Hidden debug function, to force external dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) -{ - LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; - int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT(streamPtr, smallest); - - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); - - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - - return result; -} - - -/*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -{ - LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; - - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); - - dict->dictionary = (const BYTE*)safeBuffer; - dict->dictSize = (U32)dictSize; - - return dictSize; -} - - - -/*-***************************** -* Decompression functions -*******************************/ -/*! LZ4_decompress_generic() : - * This generic decompression function covers all use cases. - * It shall be instantiated several times, using different sets of directives. - * Note that it is important for performance that this function really get inlined, - * in order to remove useless branches during compilation optimization. - */ -LZ4_FORCE_O2_GCC_PPC64LE -LZ4_FORCE_INLINE int LZ4_decompress_generic( - const char* const src, - char* const dst, - int srcSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ - - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dst when no prefix */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) -{ - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + outputSize; - BYTE* cpy; - BYTE* oexit = op + targetOutputSize; - - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; - const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; - - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - - - /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); - - /* Main Loop : decode sequences */ - while (1) { - size_t length; - const BYTE* match; - size_t offset; - - /* get literal length */ - unsigned const token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) { - unsigned s; - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ - } - memcpy(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; - - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; - - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ - - if (length <= (size_t)(lowPrefix-match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } - - /* copy match within block */ - cpy = op + length; - if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - memcpy(op+4, match, 4); - match -= dec64; - } else { LZ4_copy8(op, match); match+=8; } - op += 8; - - if (unlikely(cpy>oend-12)) { - BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op16) LZ4_wildCopy(op+8, match+8, cpy); - } - op=cpy; /* correction */ - } - - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-src); /* Nb of input bytes read */ - - /* Overflow error detected */ -_output_error: - return (int) (-(((const char*)ip)-src))-1; -} - - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); -} - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); -} - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_fast(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); -} - - -/*===== streaming decompression functions =====*/ - -LZ4_streamDecode_t* LZ4_createStreamDecode(void) -{ - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); - return lz4s; -} - -int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) -{ - if (!LZ4_stream) return 0; /* support free on NULL */ - FREEMEM(LZ4_stream); - return 0; -} - -/*! - * LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error - */ -int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; - lz4sd->externalDict = NULL; - lz4sd->extDictSize = 0; - return 1; -} - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LZ4_setStreamDecode() -*/ -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += result; - lz4sd->prefixEnd += result; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE*)dest + result; - } - - return result; -} - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += originalSize; - lz4sd->prefixEnd += originalSize; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = originalSize; - lz4sd->prefixEnd = (BYTE*)dest + originalSize; - } - - return result; -} - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as "_continue" ones, - the dictionary must be explicitly provided within parameters -*/ - -LZ4_FORCE_O2_GCC_PPC64LE -LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) -{ - if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); - if (dictStart+dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); - } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); -} - -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); -} - -/* debug function */ -LZ4_FORCE_O2_GCC_PPC64LE -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - - -/*=************************************************* -* Obsolete Functions -***************************************************/ -/* obsolete compression functions */ -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } -int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } -int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } - -/* -These function names are deprecated and should no longer be used. -They are only provided here for compatibility with older user programs. -- LZ4_uncompress is totally equivalent to LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe -*/ -int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - - -/* Obsolete Streaming functions */ - -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } - -static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) -{ - MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); - lz4ds->internal_donotuse.bufferStart = base; -} - -int LZ4_resetStreamState(void* state, char* inputBuffer) -{ - if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); - return 0; -} - -void* LZ4_create (char* inputBuffer) -{ - LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); - LZ4_init (lz4ds, (BYTE*)inputBuffer); - return lz4ds; -} - -char* LZ4_slideInputBuffer (void* LZ4_Data) -{ - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; - int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); - return (char*)(ctx->bufferStart + dictSize); -} - -/* Obsolete streaming decompression functions */ - -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/library/lz4-mac/lib/lz4.h b/library/lz4-mac/lib/lz4.h deleted file mode 100755 index d284d630..00000000 --- a/library/lz4-mac/lib/lz4.h +++ /dev/null @@ -1,475 +0,0 @@ -/* - * LZ4 - Fast LZ compression algorithm - * Header File - * Copyright (C) 2011-2017, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ -#if defined (__cplusplus) -extern "C" { -#endif - -#ifndef LZ4_H_2983827168210 -#define LZ4_H_2983827168210 - -/* --- Dependency --- */ -#include /* size_t */ - - -/** - Introduction - - LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, - scalable with multi-cores CPU. It features an extremely fast decoder, with speed in - multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. - - The LZ4 compression library provides in-memory compression and decompression functions. - Compression can be done in: - - a single step (described as Simple Functions) - - a single step, reusing a context (described in Advanced Functions) - - unbounded multiple steps (described as Streaming compression) - - lz4.h provides block compression functions. It gives full buffer control to user. - Decompressing an lz4-compressed block also requires metadata (such as compressed size). - Each application is free to encode such metadata in whichever way it wants. - - An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), - take care of encoding standard metadata alongside LZ4-compressed blocks. - If your application requires interoperability, it's recommended to use it. - A library is provided to take care of it, see lz4frame.h. -*/ - -/*^*************************************************************** -* Export parameters -*****************************************************************/ -/* -* LZ4_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -* LZ4LIB_API : -* Control library symbols visibility. -*/ -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4LIB_API __declspec(dllexport) -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#elif defined(__GNUC__) && (__GNUC__ >= 4) -# define LZ4LIB_API __attribute__ ((__visibility__ ("default"))) -#else -# define LZ4LIB_API -#endif - - -/*------ Version ------*/ -#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ - -#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) - -#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE -#define LZ4_QUOTE(str) #str -#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) -#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) - -LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; to be used when checking dll version */ -LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; to be used when checking dll version */ - - -/*-************************************ -* Tuning parameter -**************************************/ -/*! - * LZ4_MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect - * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache - */ -#ifndef LZ4_MEMORY_USAGE -# define LZ4_MEMORY_USAGE 14 -#endif - -/*-************************************ -* Simple Functions -**************************************/ -/*! LZ4_compress_default() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, - compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); - -/*! LZ4_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, including malicious data packets. - It never writes outside output buffer, nor reads outside input buffer. -*/ -LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); - - -/*-************************************ -* Advanced Functions -**************************************/ -#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) - -/*! -LZ4_compressBound() : - Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (destination buffer size). - Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) - inputSize : max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) -*/ -LZ4LIB_API int LZ4_compressBound(int inputSize); - -/*! -LZ4_compress_fast() : - Same as LZ4_compress_default(), but allows to select an "acceleration" factor. - The larger the acceleration value, the faster the algorithm, but also the lesser the compression. - It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. - An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. -*/ -LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); - - -/*! -LZ4_compress_fast_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. -*/ -LZ4LIB_API int LZ4_sizeofState(void); -LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); - - -/*! -LZ4_compress_destSize() : - Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails -*/ -LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); - - -/*! -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). -*/ -LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); - -/*! -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets -*/ -LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); - - -/*-********************************************* -* Streaming Compression Functions -***********************************************/ -typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ - -/*! LZ4_createStream() and LZ4_freeStream() : - * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. - * LZ4_freeStream() releases its memory. - */ -LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); -LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); - -/*! LZ4_resetStream() : - * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to start compressing a new stream. - */ -LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); - -/*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream_t. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed, and is the same as reset. - * @return : dictionary size, in bytes (necessarily <= 64 KB) - */ -LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); - -/*! LZ4_compress_fast_continue() : - * Compress content into 'src' using data from previously compressed blocks, improving compression ratio. - * 'dst' buffer must be already allocated. - * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * - * Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! - * Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. - * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. - * - * @return : size of compressed block - * or 0 if there is an error (typically, compressed data cannot fit into 'dst') - * After an error, the stream status is invalid, it can only be reset or freed. - */ -LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); - -/*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its current memory location, - * save it into a safer place (char* safeBuffer). - * Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable. - * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); - - -/*-********************************************** -* Streaming Decompression Functions -* Bufferless synchronous API -************************************************/ -typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ - -/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : - * creation / destruction of streaming decompression tracking structure. - * A tracking structure can be re-used multiple times sequentially. */ -LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); -LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); - -/*! LZ4_setStreamDecode() : - * An LZ4_streamDecode_t structure can be allocated once and re-used multiple times. - * Use this function to start decompression of a new stream of blocks. - * A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order. - * @return : 1 if OK, 0 if error - */ -LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); - -/*! LZ4_decompress_*_continue() : - * These decoding functions allow decompression of consecutive blocks in "streaming" mode. - * A block is an unsplittable entity, it must be presented entirely to a decompression function. - * Decompression functions only accept one block at a time. - * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB). - * - * Special : if application sets a ring buffer for decompression, it must respect one of the following conditions : - * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - * maxBlockSize is implementation dependent. It's the maximum size of any single block. - * In which case, encoding and decoding buffers do not need to be synchronized, - * and encoding ring buffer can have any size, including small ones ( < 64 KB). - * - _At least_ 64 KB + 8 bytes + maxBlockSize. - * In which case, encoding and decoding buffers do not need to be synchronized, - * and encoding ring buffer can have any size, including larger than decoding buffer. - * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - * and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block. -*/ -LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); - - -/*! LZ4_decompress_*_usingDict() : - * These decoding functions work the same as - * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() - * They are stand-alone, and don't need an LZ4_streamDecode_t structure. - */ -LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); - - -/*^********************************************** - * !!!!!! STATIC LINKING ONLY !!!!!! - ***********************************************/ -/*-************************************ - * Private definitions - ************************************** - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Using these definitions will expose code to API and/or ABI break in future versions of the library. - **************************************/ -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) -#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ - -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint32_t initCheck; - const uint8_t* dictionary; - uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ - uint32_t dictSize; -} LZ4_stream_t_internal; - -typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - -#else - -typedef struct { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int initCheck; - const unsigned char* dictionary; - unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ - unsigned int dictSize; -} LZ4_stream_t_internal; - -typedef struct { - const unsigned char* externalDict; - size_t extDictSize; - const unsigned char* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - -#endif - -/*! - * LZ4_stream_t : - * information structure to track an LZ4 stream. - * init this structure before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * it may change in a future version ! - */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) -union LZ4_stream_u { - unsigned long long table[LZ4_STREAMSIZE_U64]; - LZ4_stream_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_stream_t */ - - -/*! - * LZ4_streamDecode_t : - * information structure to track an LZ4 stream during decompression. - * init this structure using LZ4_setStreamDecode (or memset()) before first use - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! - */ -#define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) -union LZ4_streamDecode_u { - unsigned long long table[LZ4_STREAMDECODESIZE_U64]; - LZ4_streamDecode_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_streamDecode_t */ - - -/*-************************************ -* Obsolete Functions -**************************************/ - -/*! Deprecation warnings - Should deprecation warnings be a problem, - it is generally possible to disable them, - typically with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual. - Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ -#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS -# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ -#else -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */ -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ -# define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) -# elif defined(_MSC_VER) -# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) -# else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) -# endif -#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ - -/* Obsolete compression functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); - -/* Obsolete decompression functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); - -/* Obsolete streaming functions; use new streaming interface whenever possible */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); - -/* Obsolete streaming decoding functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); - -#endif /* LZ4_H_2983827168210 */ - - -#if defined (__cplusplus) -} -#endif diff --git a/library/lz4-mac/lib/lz4frame.c b/library/lz4-mac/lib/lz4frame.c deleted file mode 100755 index 3adbdd9f..00000000 --- a/library/lz4-mac/lib/lz4frame.c +++ /dev/null @@ -1,1673 +0,0 @@ -/* -LZ4 auto-framing library -Copyright (C) 2011-2016, Yann Collet. - -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- LZ4 homepage : http://www.lz4.org -- LZ4 source repository : https://github.com/lz4/lz4 -*/ - -/* LZ4F is a stand-alone API to create LZ4-compressed Frames -* in full conformance with specification v1.5.0 -* All related operations, including memory management, are handled by the library. -* */ - - -/*-************************************ -* Compiler Options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - -/*-************************************ -* Memory routines -**************************************/ -#include /* malloc, calloc, free */ -#define ALLOCATOR(s) calloc(1,s) -#define FREEMEM free -#include /* memset, memcpy, memmove */ -#define MEM_INIT memset - - -/*-************************************ -* Includes -**************************************/ -#include "lz4frame_static.h" -#include "lz4.h" -#define LZ4_HC_STATIC_LINKING_ONLY -#include "lz4hc.h" -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" - - -/*-************************************ -* Debug -**************************************/ -#define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/*-************************************ -* Basic Types -**************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/* unoptimized version; solves endianess & alignment issues */ -static U32 LZ4F_readLE32 (const void* src) -{ - const BYTE* const srcPtr = (const BYTE*)src; - U32 value32 = srcPtr[0]; - value32 += (srcPtr[1]<<8); - value32 += (srcPtr[2]<<16); - value32 += ((U32)srcPtr[3])<<24; - return value32; -} - -static void LZ4F_writeLE32 (void* dst, U32 value32) -{ - BYTE* const dstPtr = (BYTE*)dst; - dstPtr[0] = (BYTE)value32; - dstPtr[1] = (BYTE)(value32 >> 8); - dstPtr[2] = (BYTE)(value32 >> 16); - dstPtr[3] = (BYTE)(value32 >> 24); -} - -static U64 LZ4F_readLE64 (const void* src) -{ - const BYTE* const srcPtr = (const BYTE*)src; - U64 value64 = srcPtr[0]; - value64 += ((U64)srcPtr[1]<<8); - value64 += ((U64)srcPtr[2]<<16); - value64 += ((U64)srcPtr[3]<<24); - value64 += ((U64)srcPtr[4]<<32); - value64 += ((U64)srcPtr[5]<<40); - value64 += ((U64)srcPtr[6]<<48); - value64 += ((U64)srcPtr[7]<<56); - return value64; -} - -static void LZ4F_writeLE64 (void* dst, U64 value64) -{ - BYTE* const dstPtr = (BYTE*)dst; - dstPtr[0] = (BYTE)value64; - dstPtr[1] = (BYTE)(value64 >> 8); - dstPtr[2] = (BYTE)(value64 >> 16); - dstPtr[3] = (BYTE)(value64 >> 24); - dstPtr[4] = (BYTE)(value64 >> 32); - dstPtr[5] = (BYTE)(value64 >> 40); - dstPtr[6] = (BYTE)(value64 >> 48); - dstPtr[7] = (BYTE)(value64 >> 56); -} - - -/*-************************************ -* Constants -**************************************/ -#define KB *(1<<10) -#define MB *(1<<20) -#define GB *(1<<30) - -#define _1BIT 0x01 -#define _2BITS 0x03 -#define _3BITS 0x07 -#define _4BITS 0x0F -#define _8BITS 0xFF - -#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U -#define LZ4F_MAGICNUMBER 0x184D2204U -#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U -#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB - -static const size_t minFHSize = 7; -static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ -static const size_t BHSize = 4; - - -/*-************************************ -* Structures and local types -**************************************/ -typedef struct LZ4F_cctx_s -{ - LZ4F_preferences_t prefs; - U32 version; - U32 cStage; - const LZ4F_CDict* cdict; - size_t maxBlockSize; - size_t maxBufferSize; - BYTE* tmpBuff; - BYTE* tmpIn; - size_t tmpInSize; - U64 totalInSize; - XXH32_state_t xxh; - void* lz4CtxPtr; - U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ -} LZ4F_cctx_t; - - -/*-************************************ -* Error management -**************************************/ -#define LZ4F_GENERATE_STRING(STRING) #STRING, -static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; - - -unsigned LZ4F_isError(LZ4F_errorCode_t code) -{ - return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode)); -} - -const char* LZ4F_getErrorName(LZ4F_errorCode_t code) -{ - static const char* codeError = "Unspecified error code"; - if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)]; - return codeError; -} - -LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult) -{ - if (!LZ4F_isError(functionResult)) return LZ4F_OK_NoError; - return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult); -} - -static LZ4F_errorCode_t err0r(LZ4F_errorCodes code) -{ - /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ - LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); - return (LZ4F_errorCode_t)-(ptrdiff_t)code; -} - -unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } - -int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } - - -/*-************************************ -* Private functions -**************************************/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -static size_t LZ4F_getBlockSize(unsigned blockSizeID) -{ - static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; - - if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; - blockSizeID -= 4; - if (blockSizeID > 3) return err0r(LZ4F_ERROR_maxBlockSize_invalid); - return blockSizes[blockSizeID]; -} - -static BYTE LZ4F_headerChecksum (const void* header, size_t length) -{ - U32 const xxh = XXH32(header, length, 0); - return (BYTE)(xxh >> 8); -} - - -/*-************************************ -* Simple-pass compression functions -**************************************/ -static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, - const size_t srcSize) -{ - LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; - size_t maxBlockSize = 64 KB; - while (requestedBSID > proposedBSID) { - if (srcSize <= maxBlockSize) - return proposedBSID; - proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1); - maxBlockSize <<= 2; - } - return requestedBSID; -} - -/*! LZ4F_compressBound_internal() : - * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. - * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario. - * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers. - * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. - */ -static size_t LZ4F_compressBound_internal(size_t srcSize, - const LZ4F_preferences_t* preferencesPtr, - size_t alreadyBuffered) -{ - LZ4F_preferences_t prefsNull; - memset(&prefsNull, 0, sizeof(prefsNull)); - prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ - { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - U32 const flush = prefsPtr->autoFlush | (srcSize==0); - LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID; - size_t const blockSize = LZ4F_getBlockSize(blockID); - size_t const maxBuffered = blockSize - 1; - size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); - size_t const maxSrcSize = srcSize + bufferedSize; - unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); - size_t const partialBlockSize = (srcSize - (srcSize==0)) & (blockSize-1); /* 0 => -1 == MAX => blockSize-1 */ - size_t const lastBlockSize = flush ? partialBlockSize : 0; - unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); - - size_t const blockHeaderSize = 4; - size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag; - size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - - return ((blockHeaderSize + blockCRCSize) * nbBlocks) + - (blockSize * nbFullBlocks) + lastBlockSize + frameEnd; - } -} - -size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) -{ - LZ4F_preferences_t prefs; - size_t const headerSize = maxFHSize; /* max header size, including optional fields */ - - if (preferencesPtr!=NULL) prefs = *preferencesPtr; - else memset(&prefs, 0, sizeof(prefs)); - prefs.autoFlush = 1; - - return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);; -} - - -/*! LZ4F_compressFrame_usingCDict() : - * Compress srcBuffer using a dictionary, in a single step. - * cdict can be NULL, in which case, no dictionary is used. - * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). - * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, - * however, it's the only way to provide a dictID, so it's not recommended. - * @return : number of bytes written into dstBuffer, - * or an error code if it fails (can be tested using LZ4F_isError()) - */ -size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity, - const void* srcBuffer, size_t srcSize, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* preferencesPtr) -{ - LZ4F_cctx_t cctxI; - LZ4_stream_t lz4ctx; - LZ4F_preferences_t prefs; - LZ4F_compressOptions_t options; - BYTE* const dstStart = (BYTE*) dstBuffer; - BYTE* dstPtr = dstStart; - BYTE* const dstEnd = dstStart + dstCapacity; - - memset(&cctxI, 0, sizeof(cctxI)); - cctxI.version = LZ4F_VERSION; - cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ - - if (preferencesPtr!=NULL) - prefs = *preferencesPtr; - else - memset(&prefs, 0, sizeof(prefs)); - if (prefs.frameInfo.contentSize != 0) - prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - - prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); - prefs.autoFlush = 1; - if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) - prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */ - - if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - cctxI.lz4CtxPtr = &lz4ctx; - cctxI.lz4CtxLevel = 1; - } /* fast compression context pre-created on stack */ - - memset(&options, 0, sizeof(options)); - options.stableSrc = 1; - - if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */ - return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - - { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs); /* write header */ - if (LZ4F_isError(headerSize)) return headerSize; - dstPtr += headerSize; /* header size */ } - - { size_t const cSize = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); - if (LZ4F_isError(cSize)) return cSize; - dstPtr += cSize; } - - { size_t const tailSize = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ - if (LZ4F_isError(tailSize)) return tailSize; - dstPtr += tailSize; } - - if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */ - FREEMEM(cctxI.lz4CtxPtr); - - return (dstPtr - dstStart); -} - - -/*! LZ4F_compressFrame() : - * Compress an entire srcBuffer into a valid LZ4 frame, in a single step. - * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. - * @return : number of bytes written into dstBuffer. - * or an error code if it fails (can be tested using LZ4F_isError()) - */ -size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, - const void* srcBuffer, size_t srcSize, - const LZ4F_preferences_t* preferencesPtr) -{ - return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity, - srcBuffer, srcSize, - NULL, preferencesPtr); -} - - -/*-*************************************************** -* Dictionary compression -*****************************************************/ - -struct LZ4F_CDict_s { - void* dictContent; - LZ4_stream_t* fastCtx; - LZ4_streamHC_t* HCCtx; -}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */ - -/*! LZ4F_createCDict() : - * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. - * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. - * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict - * @return : digested dictionary for compression, or NULL if failed */ -LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) -{ - const char* dictStart = (const char*)dictBuffer; - LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict)); - if (!cdict) return NULL; - if (dictSize > 64 KB) { - dictStart += dictSize - 64 KB; - dictSize = 64 KB; - } - cdict->dictContent = ALLOCATOR(dictSize); - cdict->fastCtx = LZ4_createStream(); - cdict->HCCtx = LZ4_createStreamHC(); - if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { - LZ4F_freeCDict(cdict); - return NULL; - } - memcpy(cdict->dictContent, dictStart, dictSize); - LZ4_resetStream(cdict->fastCtx); - LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize); - LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT); - LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize); - return cdict; -} - -void LZ4F_freeCDict(LZ4F_CDict* cdict) -{ - if (cdict==NULL) return; /* support free on NULL */ - FREEMEM(cdict->dictContent); - LZ4_freeStream(cdict->fastCtx); - LZ4_freeStreamHC(cdict->HCCtx); - FREEMEM(cdict); -} - - -/*-********************************* -* Advanced compression functions -***********************************/ - -/*! LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries. - * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. - * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. - * Object can release its memory using LZ4F_freeCompressionContext(); - */ -LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) -{ - LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t)); - if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed); - - cctxPtr->version = version; - cctxPtr->cStage = 0; /* Next stage : init stream */ - - *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; - - return LZ4F_OK_NoError; -} - - -LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) -{ - LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; - - if (cctxPtr != NULL) { /* support free on NULL */ - FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */ - FREEMEM(cctxPtr->tmpBuff); - FREEMEM(LZ4F_compressionContext); - } - - return LZ4F_OK_NoError; -} - - -/*! LZ4F_compressBegin_usingCDict() : - * init streaming compression and writes frame header into dstBuffer. - * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. - * @return : number of bytes written into dstBuffer for the header - * or an error code (can be tested using LZ4F_isError()) - */ -size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, - void* dstBuffer, size_t dstCapacity, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* preferencesPtr) -{ - LZ4F_preferences_t prefNull; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - BYTE* headerStart; - - if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - memset(&prefNull, 0, sizeof(prefNull)); - if (preferencesPtr == NULL) preferencesPtr = &prefNull; - cctxPtr->prefs = *preferencesPtr; - - /* Ctx Management */ - { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ - if (cctxPtr->lz4CtxLevel < tableID) { - FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) - cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); - else - cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); - if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed); - cctxPtr->lz4CtxLevel = tableID; - } } - - /* Buffer Management */ - if (cctxPtr->prefs.frameInfo.blockSizeID == 0) - cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; - cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); - - { size_t const requiredBuffSize = preferencesPtr->autoFlush ? - (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */ - cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); - - if (cctxPtr->maxBufferSize < requiredBuffSize) { - cctxPtr->maxBufferSize = 0; - FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); - if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed); - cctxPtr->maxBufferSize = requiredBuffSize; - } } - cctxPtr->tmpIn = cctxPtr->tmpBuff; - cctxPtr->tmpInSize = 0; - XXH32_reset(&(cctxPtr->xxh), 0); - - /* context init */ - cctxPtr->cdict = cdict; - if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) { - /* frame init only for blockLinked : blockIndependent will be init at each block */ - if (cdict) { - if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx)); - } else { - memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx)); - LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); - } - } else { - if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) - LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); - else - LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); - } - } - - /* Magic Number */ - LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); - dstPtr += 4; - headerStart = dstPtr; - - /* FLG Byte */ - *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ - + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) - + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) - + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3) - + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) - + (cctxPtr->prefs.frameInfo.dictID > 0) ); - /* BD Byte */ - *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); - /* Optional Frame content size field */ - if (cctxPtr->prefs.frameInfo.contentSize) { - LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); - dstPtr += 8; - cctxPtr->totalInSize = 0; - } - /* Optional dictionary ID field */ - if (cctxPtr->prefs.frameInfo.dictID) { - LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID); - dstPtr += 4; - } - /* Header CRC Byte */ - *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); - dstPtr++; - - cctxPtr->cStage = 1; /* header written, now request input data block */ - return (dstPtr - dstStart); -} - - -/*! LZ4F_compressBegin() : - * init streaming compression and writes frame header into dstBuffer. - * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. - * preferencesPtr can be NULL, in which case default parameters are selected. - * @return : number of bytes written into dstBuffer for the header - * or an error code (can be tested using LZ4F_isError()) - */ -size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, - void* dstBuffer, size_t dstCapacity, - const LZ4F_preferences_t* preferencesPtr) -{ - return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity, - NULL, preferencesPtr); -} - - -/* LZ4F_compressBound() : - * @ return size of Dst buffer given a srcSize to handle worst case situations. - * The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations. - * This function cannot fail. - */ -size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) -{ - return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1); -} - - -typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict); - - -/*! LZ4F_makeBlock(): - * compress a single block, add header and checksum - * assumption : dst buffer capacity is >= srcSize */ -static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, - compressFunc_t compress, void* lz4ctx, int level, - const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag) -{ - BYTE* const cSizePtr = (BYTE*)dst; - U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), - (int)(srcSize), (int)(srcSize-1), - level, cdict); - LZ4F_writeLE32(cSizePtr, cSize); - if (cSize == 0) { /* compression failed */ - cSize = (U32)srcSize; - LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); - memcpy(cSizePtr+4, src, srcSize); - } - if (crcFlag) { - U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */ - LZ4F_writeLE32(cSizePtr+4+cSize, crc32); - } - return 4 + cSize + ((U32)crcFlag)*4; -} - - -static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) -{ - int const acceleration = (level < -1) ? -level : 1; - if (cdict) { - memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx)); - return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); - } - return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration); -} - -static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) -{ - int const acceleration = (level < -1) ? -level : 1; - (void)cdict; /* init once at beginning of frame */ - return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); -} - -static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) -{ - if (cdict) { - memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx)); - LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level); - return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); - } - return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level); -} - -static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) -{ - (void)level; (void)cdict; /* init once at beginning of frame */ - return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); -} - -static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) -{ - if (level < LZ4HC_CLEVEL_MIN) { - if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock; - return LZ4F_compressBlock_continue; - } - if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC; - return LZ4F_compressBlockHC_continue; -} - -static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) -{ - if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) - return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); - return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); -} - -typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; - -/*! LZ4F_compressUpdate() : - * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). - * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. - * or an error code if it fails (which can be tested using LZ4F_isError()) - */ -size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, - void* dstBuffer, size_t dstCapacity, - const void* srcBuffer, size_t srcSize, - const LZ4F_compressOptions_t* compressOptionsPtr) -{ - LZ4F_compressOptions_t cOptionsNull; - size_t const blockSize = cctxPtr->maxBlockSize; - const BYTE* srcPtr = (const BYTE*)srcBuffer; - const BYTE* const srcEnd = srcPtr + srcSize; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - LZ4F_lastBlockStatus lastBlockCompressed = notDone; - compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - - - if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - memset(&cOptionsNull, 0, sizeof(cOptionsNull)); - if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; - - /* complete tmp buffer */ - if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ - size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; - if (sizeToCopy > srcSize) { - /* add src to tmpIn buffer */ - memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); - srcPtr = srcEnd; - cctxPtr->tmpInSize += srcSize; - /* still needs some CRC */ - } else { - /* complete tmpIn block and then compress it */ - lastBlockCompressed = fromTmpBuffer; - memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); - srcPtr += sizeToCopy; - - dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, - compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); - - if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; - cctxPtr->tmpInSize = 0; - } - } - - while ((size_t)(srcEnd - srcPtr) >= blockSize) { - /* compress full blocks */ - lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, - compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); - srcPtr += blockSize; - } - - if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { - /* compress remaining input < blockSize */ - lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, - compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); - srcPtr = srcEnd; - } - - /* preserve dictionary if necessary */ - if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { - if (compressOptionsPtr->stableSrc) { - cctxPtr->tmpIn = cctxPtr->tmpBuff; - } else { - int const realDictSize = LZ4F_localSaveDict(cctxPtr); - if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC); - cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; - } - } - - /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */ - && !(cctxPtr->prefs.autoFlush)) - { - int const realDictSize = LZ4F_localSaveDict(cctxPtr); - cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; - } - - /* some input data left, necessarily < blockSize */ - if (srcPtr < srcEnd) { - /* fill tmp buffer */ - size_t const sizeToCopy = srcEnd - srcPtr; - memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); - cctxPtr->tmpInSize = sizeToCopy; - } - - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) - XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); - - cctxPtr->totalInSize += srcSize; - return dstPtr - dstStart; -} - - -/*! LZ4F_flush() : - * Should you need to create compressed data immediately, without waiting for a block to be filled, - * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. - * The result of the function is the number of bytes written into dstBuffer - * (it can be zero, this means there was no data left within compressionContext) - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - */ -size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr) -{ - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - compressFunc_t compress; - - if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ - if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +4 : block header(4) */ - (void)compressOptionsPtr; /* not yet useful */ - - /* select compression function */ - compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - - /* compress tmp buffer */ - dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, - compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag); - if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; - cctxPtr->tmpInSize = 0; - - /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */ - int realDictSize = LZ4F_localSaveDict(cctxPtr); - cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; - } - - return dstPtr - dstStart; -} - - -/*! LZ4F_compressEnd() : - * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). - * It will flush whatever data remained within compressionContext (like LZ4_flush()) - * but also properly finalize the frame, with an endMark and a checksum. - * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. - */ -size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) -{ - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - - size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr); - if (LZ4F_isError(flushSize)) return flushSize; - dstPtr += flushSize; - - LZ4F_writeLE32(dstPtr, 0); - dstPtr+=4; /* endMark */ - - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) { - U32 const xxh = XXH32_digest(&(cctxPtr->xxh)); - LZ4F_writeLE32(dstPtr, xxh); - dstPtr+=4; /* content Checksum */ - } - - cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ - cctxPtr->maxBufferSize = 0; /* reuse HC context */ - - if (cctxPtr->prefs.frameInfo.contentSize) { - if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) - return err0r(LZ4F_ERROR_frameSize_wrong); - } - - return dstPtr - dstStart; -} - - -/*-*************************************************** -* Frame Decompression -*****************************************************/ - -typedef enum { - dstage_getFrameHeader=0, dstage_storeFrameHeader, - dstage_init, - dstage_getBlockHeader, dstage_storeBlockHeader, - dstage_copyDirect, dstage_getBlockChecksum, - dstage_getCBlock, dstage_storeCBlock, - dstage_decodeCBlock, dstage_decodeCBlock_intoDst, - dstage_decodeCBlock_intoTmp, dstage_flushOut, - dstage_getSuffix, dstage_storeSuffix, - dstage_getSFrameSize, dstage_storeSFrameSize, - dstage_skipSkippable -} dStage_t; - -struct LZ4F_dctx_s { - LZ4F_frameInfo_t frameInfo; - U32 version; - dStage_t dStage; - U64 frameRemainingSize; - size_t maxBlockSize; - size_t maxBufferSize; - BYTE* tmpIn; - size_t tmpInSize; - size_t tmpInTarget; - BYTE* tmpOutBuffer; - const BYTE* dict; - size_t dictSize; - BYTE* tmpOut; - size_t tmpOutSize; - size_t tmpOutStart; - XXH32_state_t xxh; - XXH32_state_t blockChecksum; - BYTE header[LZ4F_HEADER_SIZE_MAX]; -}; /* typedef'd to LZ4F_dctx in lz4frame.h */ - - -/*! LZ4F_createDecompressionContext() : - * Create a decompressionContext object, which will track all decompression operations. - * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. - * Object can later be released using LZ4F_freeDecompressionContext(). - * @return : if != 0, there was an error during context creation. - */ -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) -{ - LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx)); - if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC); - - dctx->version = versionNumber; - *LZ4F_decompressionContextPtr = dctx; - return LZ4F_OK_NoError; -} - -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx) -{ - LZ4F_errorCode_t result = LZ4F_OK_NoError; - if (dctx != NULL) { /* can accept NULL input, like free() */ - result = (LZ4F_errorCode_t)dctx->dStage; - FREEMEM(dctx->tmpIn); - FREEMEM(dctx->tmpOutBuffer); - FREEMEM(dctx); - } - return result; -} - - -/*==--- Streaming Decompression operations ---==*/ - -void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) -{ - dctx->dStage = dstage_getFrameHeader; - dctx->dict = NULL; - dctx->dictSize = 0; -} - - -/*! LZ4F_headerSize() : - * @return : size of frame header - * or an error code, which can be tested using LZ4F_isError() - */ -static size_t LZ4F_headerSize(const void* src, size_t srcSize) -{ - /* minimal srcSize to determine header size */ - if (srcSize < 5) return err0r(LZ4F_ERROR_frameHeader_incomplete); - - /* special case : skippable frames */ - if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8; - - /* control magic number */ - if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) - return err0r(LZ4F_ERROR_frameType_unknown); - - /* Frame Header Size */ - { BYTE const FLG = ((const BYTE*)src)[4]; - U32 const contentSizeFlag = (FLG>>3) & _1BIT; - U32 const dictIDFlag = FLG & _1BIT; - return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); - } -} - - -/*! LZ4F_decodeHeader() : - * input : `src` points at the **beginning of the frame** - * output : set internal values of dctx, such as - * dctx->frameInfo and dctx->dStage. - * Also allocates internal buffers. - * @return : nb Bytes read from src (necessarily <= srcSize) - * or an error code (testable with LZ4F_isError()) - */ -static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize) -{ - unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID; - size_t frameHeaderSize; - const BYTE* srcPtr = (const BYTE*)src; - - /* need to decode header to get frameInfo */ - if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */ - memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo)); - - /* special case : skippable frames */ - if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) { - dctx->frameInfo.frameType = LZ4F_skippableFrame; - if (src == (void*)(dctx->header)) { - dctx->tmpInSize = srcSize; - dctx->tmpInTarget = 8; - dctx->dStage = dstage_storeSFrameSize; - return srcSize; - } else { - dctx->dStage = dstage_getSFrameSize; - return 4; - } - } - - /* control magic number */ - if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) - return err0r(LZ4F_ERROR_frameType_unknown); - dctx->frameInfo.frameType = LZ4F_frame; - - /* Flags */ - { U32 const FLG = srcPtr[4]; - U32 const version = (FLG>>6) & _2BITS; - blockChecksumFlag = (FLG>>4) & _1BIT; - blockMode = (FLG>>5) & _1BIT; - contentSizeFlag = (FLG>>3) & _1BIT; - contentChecksumFlag = (FLG>>2) & _1BIT; - dictIDFlag = FLG & _1BIT; - /* validate */ - if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */ - if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */ - } - - /* Frame Header Size */ - frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4); - - if (srcSize < frameHeaderSize) { - /* not enough input to fully decode frame header */ - if (srcPtr != dctx->header) - memcpy(dctx->header, srcPtr, srcSize); - dctx->tmpInSize = srcSize; - dctx->tmpInTarget = frameHeaderSize; - dctx->dStage = dstage_storeFrameHeader; - return srcSize; - } - - { U32 const BD = srcPtr[5]; - blockSizeID = (BD>>4) & _3BITS; - /* validate */ - if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */ - if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */ - } - - /* check header */ - { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); - if (HC != srcPtr[frameHeaderSize-1]) - return err0r(LZ4F_ERROR_headerChecksum_invalid); - } - - /* save */ - dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; - dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag; - dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; - dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; - dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID); - if (contentSizeFlag) - dctx->frameRemainingSize = - dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); - if (dictIDFlag) - dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5); - - dctx->dStage = dstage_init; - - return frameHeaderSize; -} - - -/*! LZ4F_getFrameInfo() : - * This function extracts frame parameters (max blockSize, frame checksum, etc.). - * Usage is optional. Objective is to provide relevant information for allocation purposes. - * This function works in 2 situations : - * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process. - * Amount of input data provided must be large enough to successfully decode the frame header. - * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum. - * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx. - * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * Decompression must resume from (srcBuffer + *srcSizePtr). - * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError() - * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. - * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. - */ -LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr, - const void* srcBuffer, size_t* srcSizePtr) -{ - if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */ - /* frameInfo already decoded */ - size_t o=0, i=0; - *srcSizePtr = 0; - *frameInfoPtr = dctx->frameInfo; - /* returns : recommended nb of bytes for LZ4F_decompress() */ - return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL); - } else { - if (dctx->dStage == dstage_storeFrameHeader) { - /* frame decoding already started, in the middle of header => automatic fail */ - *srcSizePtr = 0; - return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted); - } else { - size_t decodeResult; - size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); - if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; } - if (*srcSizePtr < hSize) { - *srcSizePtr=0; - return err0r(LZ4F_ERROR_frameHeader_incomplete); - } - - decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); - if (LZ4F_isError(decodeResult)) { - *srcSizePtr = 0; - } else { - *srcSizePtr = decodeResult; - decodeResult = BHSize; /* block header size */ - } - *frameInfoPtr = dctx->frameInfo; - return decodeResult; - } } -} - - -/* LZ4F_updateDict() : - * only used for LZ4F_blockLinked mode */ -static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) -{ - if (dctx->dictSize==0) - dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ - - if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity */ - dctx->dictSize += dstSize; - return; - } - - if (dstPtr - dstPtr0 + dstSize >= 64 KB) { /* dstBuffer large enough to become dictionary */ - dctx->dict = (const BYTE*)dstPtr0; - dctx->dictSize = dstPtr - dstPtr0 + dstSize; - return; - } - - if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { - /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */ - dctx->dictSize += dstSize; - return; - } - - if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ - size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; - size_t copySize = 64 KB - dctx->tmpOutSize; - const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; - if (dctx->tmpOutSize > 64 KB) copySize = 0; - if (copySize > preserveSize) copySize = preserveSize; - - memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - - dctx->dict = dctx->tmpOutBuffer; - dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize; - return; - } - - if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */ - if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */ - size_t const preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ - memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); - dctx->dictSize = preserveSize; - } - memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize); - dctx->dictSize += dstSize; - return; - } - - /* join dict & dest into tmp */ - { size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ - if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize; - memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); - memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize); - dctx->dict = dctx->tmpOutBuffer; - dctx->dictSize = preserveSize + dstSize; - } -} - - - -/*! LZ4F_decompress() : - * Call this function repetitively to regenerate compressed data in srcBuffer. - * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer - * into dstBuffer of capacity *dstSizePtr. - * - * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). - * - * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * If number of bytes read is < number of bytes provided, then decompression operation is not complete. - * Remaining data will have to be presented again in a subsequent invocation. - * - * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. - * Schematically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling. - * Note that this is just a hint, and it's always possible to any srcSize value. - * When a frame is fully decoded, @return will be 0. - * If decompression failed, @return is an error code which can be tested using LZ4F_isError(). - */ -size_t LZ4F_decompress(LZ4F_dctx* dctx, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const LZ4F_decompressOptions_t* decompressOptionsPtr) -{ - LZ4F_decompressOptions_t optionsNull; - const BYTE* const srcStart = (const BYTE*)srcBuffer; - const BYTE* const srcEnd = srcStart + *srcSizePtr; - const BYTE* srcPtr = srcStart; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* const dstEnd = dstStart + *dstSizePtr; - BYTE* dstPtr = dstStart; - const BYTE* selectedIn = NULL; - unsigned doAnotherStage = 1; - size_t nextSrcSizeHint = 1; - - - memset(&optionsNull, 0, sizeof(optionsNull)); - if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; - *srcSizePtr = 0; - *dstSizePtr = 0; - - /* behaves as a state machine */ - - while (doAnotherStage) { - - switch(dctx->dStage) - { - - case dstage_getFrameHeader: - if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ - size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */ - if (LZ4F_isError(hSize)) return hSize; - srcPtr += hSize; - break; - } - dctx->tmpInSize = 0; - if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */ - dctx->tmpInTarget = minFHSize; /* minimum to attempt decode */ - dctx->dStage = dstage_storeFrameHeader; - /* fall-through */ - - case dstage_storeFrameHeader: - { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr)); - memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); - dctx->tmpInSize += sizeToCopy; - srcPtr += sizeToCopy; - } - if (dctx->tmpInSize < dctx->tmpInTarget) { - nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ - doAnotherStage = 0; /* not enough src data, ask for some more */ - break; - } - { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */ - if (LZ4F_isError(hSize)) return hSize; - } - break; - - case dstage_init: - if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0); - /* internal buffers allocation */ - { size_t const bufferNeeded = dctx->maxBlockSize - + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); - if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ - dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ - FREEMEM(dctx->tmpIn); - dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize + 4 /* block checksum */); - if (dctx->tmpIn == NULL) - return err0r(LZ4F_ERROR_allocation_failed); - FREEMEM(dctx->tmpOutBuffer); - dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded); - if (dctx->tmpOutBuffer== NULL) - return err0r(LZ4F_ERROR_allocation_failed); - dctx->maxBufferSize = bufferNeeded; - } } - dctx->tmpInSize = 0; - dctx->tmpInTarget = 0; - dctx->tmpOut = dctx->tmpOutBuffer; - dctx->tmpOutStart = 0; - dctx->tmpOutSize = 0; - - dctx->dStage = dstage_getBlockHeader; - /* fall-through */ - - case dstage_getBlockHeader: - if ((size_t)(srcEnd - srcPtr) >= BHSize) { - selectedIn = srcPtr; - srcPtr += BHSize; - } else { - /* not enough input to read cBlockSize field */ - dctx->tmpInSize = 0; - dctx->dStage = dstage_storeBlockHeader; - } - - if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */ - case dstage_storeBlockHeader: - { size_t const remainingInput = (size_t)(srcEnd - srcPtr); - size_t const wantedData = BHSize - dctx->tmpInSize; - size_t const sizeToCopy = MIN(wantedData, remainingInput); - memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctx->tmpInSize += sizeToCopy; - - if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */ - nextSrcSizeHint = BHSize - dctx->tmpInSize; - doAnotherStage = 0; - break; - } - selectedIn = dctx->tmpIn; - } /* if (dctx->dStage == dstage_storeBlockHeader) */ - - /* decode block header */ - { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; - size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4; - if (nextCBlockSize==0) { /* frameEnd signal, no more block */ - dctx->dStage = dstage_getSuffix; - break; - } - if (nextCBlockSize > dctx->maxBlockSize) - return err0r(LZ4F_ERROR_maxBlockSize_invalid); - if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) { - /* next block is uncompressed */ - dctx->tmpInTarget = nextCBlockSize; - if (dctx->frameInfo.blockChecksumFlag) { - XXH32_reset(&dctx->blockChecksum, 0); - } - dctx->dStage = dstage_copyDirect; - break; - } - /* next block is a compressed block */ - dctx->tmpInTarget = nextCBlockSize + crcSize; - dctx->dStage = dstage_getCBlock; - if (dstPtr==dstEnd) { - nextSrcSizeHint = nextCBlockSize + crcSize + BHSize; - doAnotherStage = 0; - } - break; - } - - case dstage_copyDirect: /* uncompressed block */ - { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr)); - size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); - memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctx->frameInfo.blockChecksumFlag) { - XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); - } - if (dctx->frameInfo.contentChecksumFlag) - XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); - if (dctx->frameInfo.contentSize) - dctx->frameRemainingSize -= sizeToCopy; - - /* history management (linked blocks only)*/ - if (dctx->frameInfo.blockMode == LZ4F_blockLinked) - LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0); - - srcPtr += sizeToCopy; - dstPtr += sizeToCopy; - if (sizeToCopy == dctx->tmpInTarget) { /* all done */ - if (dctx->frameInfo.blockChecksumFlag) { - dctx->tmpInSize = 0; - dctx->dStage = dstage_getBlockChecksum; - } else - dctx->dStage = dstage_getBlockHeader; /* new block */ - break; - } - dctx->tmpInTarget -= sizeToCopy; /* need to copy more */ - nextSrcSizeHint = dctx->tmpInTarget + - + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */ - + BHSize /* next header size */; - doAnotherStage = 0; - break; - } - - /* check block checksum for recently transferred uncompressed block */ - case dstage_getBlockChecksum: - { const void* crcSrc; - if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) { - crcSrc = srcPtr; - srcPtr += 4; - } else { - size_t const stillToCopy = 4 - dctx->tmpInSize; - size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr)); - memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); - dctx->tmpInSize += sizeToCopy; - srcPtr += sizeToCopy; - if (dctx->tmpInSize < 4) { /* all input consumed */ - doAnotherStage = 0; - break; - } - crcSrc = dctx->header; - } - { U32 const readCRC = LZ4F_readLE32(crcSrc); - U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); - if (readCRC != calcCRC) - return err0r(LZ4F_ERROR_blockChecksum_invalid); - } - } - dctx->dStage = dstage_getBlockHeader; /* new block */ - break; - - case dstage_getCBlock: - if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) { - dctx->tmpInSize = 0; - dctx->dStage = dstage_storeCBlock; - break; - } - /* input large enough to read full block directly */ - selectedIn = srcPtr; - srcPtr += dctx->tmpInTarget; - dctx->dStage = dstage_decodeCBlock; - break; - - case dstage_storeCBlock: - { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize; - size_t const inputLeft = (size_t)(srcEnd-srcPtr); - size_t const sizeToCopy = MIN(wantedData, inputLeft); - memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); - dctx->tmpInSize += sizeToCopy; - srcPtr += sizeToCopy; - if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */ - nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; - doAnotherStage=0; - break; - } - selectedIn = dctx->tmpIn; - dctx->dStage = dstage_decodeCBlock; - } - /* fall-through */ - - /* At this stage, input is large enough to decode a block */ - case dstage_decodeCBlock: - if (dctx->frameInfo.blockChecksumFlag) { - dctx->tmpInTarget -= 4; - { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); - U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); - if (readBlockCrc != calcBlockCrc) - return err0r(LZ4F_ERROR_blockChecksum_invalid); - } } - if ((size_t)(dstEnd-dstPtr) < dctx->maxBlockSize) /* not enough place into dst : decode into tmpOut */ - dctx->dStage = dstage_decodeCBlock_intoTmp; - else - dctx->dStage = dstage_decodeCBlock_intoDst; - break; - - case dstage_decodeCBlock_intoDst: - { int const decodedSize = LZ4_decompress_safe_usingDict( - (const char*)selectedIn, (char*)dstPtr, - (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, - (const char*)dctx->dict, (int)dctx->dictSize); - if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */ - if (dctx->frameInfo.contentChecksumFlag) - XXH32_update(&(dctx->xxh), dstPtr, decodedSize); - if (dctx->frameInfo.contentSize) - dctx->frameRemainingSize -= decodedSize; - - /* dictionary management */ - if (dctx->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0); - - dstPtr += decodedSize; - dctx->dStage = dstage_getBlockHeader; - break; - } - - case dstage_decodeCBlock_intoTmp: - /* not enough place into dst : decode into tmpOut */ - /* ensure enough place for tmpOut */ - if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { - if (dctx->dict == dctx->tmpOutBuffer) { - if (dctx->dictSize > 128 KB) { - memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB); - dctx->dictSize = 64 KB; - } - dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize; - } else { /* dict not within tmp */ - size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB); - dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace; - } - } - - /* Decode block */ - { int const decodedSize = LZ4_decompress_safe_usingDict( - (const char*)selectedIn, (char*)dctx->tmpOut, - (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, - (const char*)dctx->dict, (int)dctx->dictSize); - if (decodedSize < 0) /* decompression failed */ - return err0r(LZ4F_ERROR_decompressionFailed); - if (dctx->frameInfo.contentChecksumFlag) - XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize); - if (dctx->frameInfo.contentSize) - dctx->frameRemainingSize -= decodedSize; - dctx->tmpOutSize = decodedSize; - dctx->tmpOutStart = 0; - dctx->dStage = dstage_flushOut; - } - /* fall-through */ - - case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ - { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr)); - memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy); - - /* dictionary management */ - if (dctx->frameInfo.blockMode==LZ4F_blockLinked) - LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1); - - dctx->tmpOutStart += sizeToCopy; - dstPtr += sizeToCopy; - - if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */ - dctx->dStage = dstage_getBlockHeader; /* get next block */ - break; - } - nextSrcSizeHint = BHSize; - doAnotherStage = 0; /* still some data to flush */ - break; - } - - case dstage_getSuffix: - if (dctx->frameRemainingSize) - return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */ - if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */ - nextSrcSizeHint = 0; - LZ4F_resetDecompressionContext(dctx); - doAnotherStage = 0; - break; - } - if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ - dctx->tmpInSize = 0; - dctx->dStage = dstage_storeSuffix; - } else { - selectedIn = srcPtr; - srcPtr += 4; - } - - if (dctx->dStage == dstage_storeSuffix) /* can be skipped */ - case dstage_storeSuffix: - { size_t const remainingInput = (size_t)(srcEnd - srcPtr); - size_t const wantedData = 4 - dctx->tmpInSize; - size_t const sizeToCopy = MIN(wantedData, remainingInput); - memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctx->tmpInSize += sizeToCopy; - if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */ - nextSrcSizeHint = 4 - dctx->tmpInSize; - doAnotherStage=0; - break; - } - selectedIn = dctx->tmpIn; - } /* if (dctx->dStage == dstage_storeSuffix) */ - - /* case dstage_checkSuffix: */ /* no direct call, avoid scan-build warning */ - { U32 const readCRC = LZ4F_readLE32(selectedIn); - U32 const resultCRC = XXH32_digest(&(dctx->xxh)); - if (readCRC != resultCRC) - return err0r(LZ4F_ERROR_contentChecksum_invalid); - nextSrcSizeHint = 0; - LZ4F_resetDecompressionContext(dctx); - doAnotherStage = 0; - break; - } - - case dstage_getSFrameSize: - if ((srcEnd - srcPtr) >= 4) { - selectedIn = srcPtr; - srcPtr += 4; - } else { - /* not enough input to read cBlockSize field */ - dctx->tmpInSize = 4; - dctx->tmpInTarget = 8; - dctx->dStage = dstage_storeSFrameSize; - } - - if (dctx->dStage == dstage_storeSFrameSize) - case dstage_storeSFrameSize: - { - size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, - (size_t)(srcEnd - srcPtr) ); - memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctx->tmpInSize += sizeToCopy; - if (dctx->tmpInSize < dctx->tmpInTarget) { - /* not enough input to get full sBlockSize; wait for more */ - nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize; - doAnotherStage = 0; - break; - } - selectedIn = dctx->header + 4; - } /* if (dctx->dStage == dstage_storeSFrameSize) */ - - /* case dstage_decodeSFrameSize: */ /* no direct access */ - { size_t const SFrameSize = LZ4F_readLE32(selectedIn); - dctx->frameInfo.contentSize = SFrameSize; - dctx->tmpInTarget = SFrameSize; - dctx->dStage = dstage_skipSkippable; - break; - } - - case dstage_skipSkippable: - { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr)); - srcPtr += skipSize; - dctx->tmpInTarget -= skipSize; - doAnotherStage = 0; - nextSrcSizeHint = dctx->tmpInTarget; - if (nextSrcSizeHint) break; /* still more to skip */ - /* frame fully skipped : prepare context for a new frame */ - LZ4F_resetDecompressionContext(dctx); - break; - } - } - } /* while (doAnotherStage) */ - - /* preserve history within tmp whenever necessary */ - LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2); - if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */ - && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */ - && (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */ - && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */ - { - if (dctx->dStage == dstage_flushOut) { - size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer; - size_t copySize = 64 KB - dctx->tmpOutSize; - const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; - if (dctx->tmpOutSize > 64 KB) copySize = 0; - if (copySize > preserveSize) copySize = preserveSize; - - memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - - dctx->dict = dctx->tmpOutBuffer; - dctx->dictSize = preserveSize + dctx->tmpOutStart; - } else { - const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize; - size_t const newDictSize = MIN(dctx->dictSize, 64 KB); - - memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); - - dctx->dict = dctx->tmpOutBuffer; - dctx->dictSize = newDictSize; - dctx->tmpOut = dctx->tmpOutBuffer + newDictSize; - } - } - - *srcSizePtr = (srcPtr - srcStart); - *dstSizePtr = (dstPtr - dstStart); - return nextSrcSizeHint; -} - -/*! LZ4F_decompress_usingDict() : - * Same as LZ4F_decompress(), using a predefined dictionary. - * Dictionary is used "in place", without any preprocessing. - * It must remain accessible throughout the entire frame decoding. - */ -size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const void* dict, size_t dictSize, - const LZ4F_decompressOptions_t* decompressOptionsPtr) -{ - if (dctx->dStage <= dstage_init) { - dctx->dict = (const BYTE*)dict; - dctx->dictSize = dictSize; - } - return LZ4F_decompress(dctx, dstBuffer, dstSizePtr, - srcBuffer, srcSizePtr, - decompressOptionsPtr); -} diff --git a/library/lz4-mac/lib/lz4frame.h b/library/lz4-mac/lib/lz4frame.h deleted file mode 100755 index 88a6513c..00000000 --- a/library/lz4-mac/lib/lz4frame.h +++ /dev/null @@ -1,391 +0,0 @@ -/* - LZ4 auto-framing library - Header File - Copyright (C) 2011-2017, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/* LZ4F is a stand-alone API to create LZ4-compressed frames - * conformant with specification v1.5.1. - * It also offers streaming capabilities. - * lz4.h is not required when using lz4frame.h. - * */ - -#ifndef LZ4F_H_09782039843 -#define LZ4F_H_09782039843 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* --- Dependency --- */ -#include /* size_t */ - - -/** - Introduction - - lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md). - lz4frame.h provides frame compression functions that take care - of encoding standard metadata alongside LZ4-compressed blocks. -*/ - -/*-*************************************************************** - * Compiler specifics - *****************************************************************/ -/* LZ4_DLL_EXPORT : - * Enable exporting of functions when building a Windows DLL - * LZ4FLIB_API : - * Control library symbols visibility. - */ -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4FLIB_API __declspec(dllexport) -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4FLIB_API __declspec(dllimport) -#elif defined(__GNUC__) && (__GNUC__ >= 4) -# define LZ4FLIB_API __attribute__ ((__visibility__ ("default"))) -#else -# define LZ4FLIB_API -#endif - -#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS -# define LZ4F_DEPRECATE(x) x -#else -# if defined(_MSC_VER) -# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */ -# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6)) -# define LZ4F_DEPRECATE(x) x __attribute__((deprecated)) -# else -# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */ -# endif -#endif - - -/*-************************************ - * Error management - **************************************/ -typedef size_t LZ4F_errorCode_t; - -LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells if a `LZ4F_errorCode_t` function result is an error code */ -LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; useful for debugging */ - - -/*-************************************ - * Frame compression types - **************************************/ -/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */ -#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS -# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x -#else -# define LZ4F_OBSOLETE_ENUM(x) -#endif - -/* The larger the block size, the (slightly) better the compression ratio, - * though there are diminishing returns. - * Larger blocks also increase memory usage on both compression and decompression sides. */ -typedef enum { - LZ4F_default=0, - LZ4F_max64KB=4, - LZ4F_max256KB=5, - LZ4F_max1MB=6, - LZ4F_max4MB=7 - LZ4F_OBSOLETE_ENUM(max64KB) - LZ4F_OBSOLETE_ENUM(max256KB) - LZ4F_OBSOLETE_ENUM(max1MB) - LZ4F_OBSOLETE_ENUM(max4MB) -} LZ4F_blockSizeID_t; - -/* Linked blocks sharply reduce inefficiencies when using small blocks, - * they compress better. - * However, some LZ4 decoders are only compatible with independent blocks */ -typedef enum { - LZ4F_blockLinked=0, - LZ4F_blockIndependent - LZ4F_OBSOLETE_ENUM(blockLinked) - LZ4F_OBSOLETE_ENUM(blockIndependent) -} LZ4F_blockMode_t; - -typedef enum { - LZ4F_noContentChecksum=0, - LZ4F_contentChecksumEnabled - LZ4F_OBSOLETE_ENUM(noContentChecksum) - LZ4F_OBSOLETE_ENUM(contentChecksumEnabled) -} LZ4F_contentChecksum_t; - -typedef enum { - LZ4F_noBlockChecksum=0, - LZ4F_blockChecksumEnabled -} LZ4F_blockChecksum_t; - -typedef enum { - LZ4F_frame=0, - LZ4F_skippableFrame - LZ4F_OBSOLETE_ENUM(skippableFrame) -} LZ4F_frameType_t; - -#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS -typedef LZ4F_blockSizeID_t blockSizeID_t; -typedef LZ4F_blockMode_t blockMode_t; -typedef LZ4F_frameType_t frameType_t; -typedef LZ4F_contentChecksum_t contentChecksum_t; -#endif - -/*! LZ4F_frameInfo_t : - * makes it possible to set or read frame parameters. - * It's not required to set all fields, as long as the structure was initially memset() to zero. - * For all fields, 0 sets it to default value */ -typedef struct { - LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */ - LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */ - LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */ - unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */ - unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */ - LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */ -} LZ4F_frameInfo_t; - -/*! LZ4F_preferences_t : - * makes it possible to supply detailed compression parameters to the stream interface. - * It's not required to set all fields, as long as the structure was initially memset() to zero. - * All reserved fields must be set to zero. */ -typedef struct { - LZ4F_frameInfo_t frameInfo; - int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */ - unsigned autoFlush; /* 1 == always flush, to reduce usage of internal buffers */ - unsigned reserved[4]; /* must be zero for forward compatibility */ -} LZ4F_preferences_t; - -LZ4FLIB_API int LZ4F_compressionLevel_max(void); - - -/*-********************************* -* Simple compression function -***********************************/ -/*! LZ4F_compressFrameBound() : - * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. - * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. - */ -LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); - -/*! LZ4F_compressFrame() : - * Compress an entire srcBuffer into a valid LZ4 frame. - * dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. - * @return : number of bytes written into dstBuffer. - * or an error code if it fails (can be tested using LZ4F_isError()) - */ -LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, - const void* srcBuffer, size_t srcSize, - const LZ4F_preferences_t* preferencesPtr); - - -/*-*********************************** -* Advanced compression functions -*************************************/ -typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */ -typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with previous API version */ - -typedef struct { - unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */ - unsigned reserved[3]; -} LZ4F_compressOptions_t; - -/*--- Resource Management ---*/ - -#define LZ4F_VERSION 100 -LZ4FLIB_API unsigned LZ4F_getVersion(void); -/*! LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. - * The function will provide a pointer to a fully allocated LZ4F_cctx object. - * If @return != zero, there was an error during context creation. - * Object can release its memory using LZ4F_freeCompressionContext(); - */ -LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); -LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); - - -/*---- Compression ----*/ - -#define LZ4F_HEADER_SIZE_MAX 19 -/*! LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. - * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. - * @return : number of bytes written into dstBuffer for the header - * or an error code (which can be tested using LZ4F_isError()) - */ -LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_preferences_t* prefsPtr); - -/*! LZ4F_compressBound() : - * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. - * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. - * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. - * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. - */ -LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); - -/*! LZ4F_compressUpdate() : - * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. - * This value is provided by LZ4F_compressBound(). - * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). - * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. - * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. - * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). - * or an error code if it fails (which can be tested using LZ4F_isError()) - */ -LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); - -/*! LZ4F_flush() : - * When data must be generated and sent immediately, without waiting for a block to be completely filled, - * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. - * `dstCapacity` must be large enough to ensure the operation will be successful. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. - * @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx) - * or an error code if it fails (which can be tested using LZ4F_isError()) - */ -LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); - -/*! LZ4F_compressEnd() : - * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). - * It will flush whatever data remained within `cctx` (like LZ4_flush()) - * and properly finalize the frame, with an endMark and a checksum. - * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. - * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) - * or an error code if it fails (which can be tested using LZ4F_isError()) - * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. - */ -LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); - - -/*-********************************* -* Decompression functions -***********************************/ -typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */ -typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */ - -typedef struct { - unsigned stableDst; /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */ - unsigned reserved[3]; /* must be set to zero for forward compatibility */ -} LZ4F_decompressOptions_t; - - -/* Resource management */ - -/*!LZ4F_createDecompressionContext() : - * Create an LZ4F_dctx object, to track all decompression operations. - * The version provided MUST be LZ4F_VERSION. - * The function provides a pointer to an allocated and initialized LZ4F_dctx object. - * The result is an errorCode, which can be tested using LZ4F_isError(). - * dctx memory can be released using LZ4F_freeDecompressionContext(); - * The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. - * That is, it should be == 0 if decompression has been completed fully and correctly. - */ -LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); -LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); - - -/*-*********************************** -* Streaming decompression functions -*************************************/ - -/*! LZ4F_getFrameInfo() : - * This function extracts frame parameters (max blockSize, dictID, etc.). - * Its usage is optional. - * Extracted information is typically useful for allocation and dictionary. - * This function works in 2 situations : - * - At the beginning of a new frame, in which case - * it will decode information from `srcBuffer`, starting the decoding process. - * Input size must be large enough to successfully decode the entire frame header. - * Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. - * It's allowed to provide more input data than this minimum. - * - After decoding has been started. - * In which case, no input is read, frame parameters are extracted from dctx. - * - If decoding has barely started, but not yet extracted information from header, - * LZ4F_getFrameInfo() will fail. - * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * Decompression must resume from (srcBuffer + *srcSizePtr). - * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError(). - * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. - * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. - */ -LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, - LZ4F_frameInfo_t* frameInfoPtr, - const void* srcBuffer, size_t* srcSizePtr); - -/*! LZ4F_decompress() : - * Call this function repetitively to regenerate compressed data from `srcBuffer`. - * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. - * - * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value). - * - * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value). - * Number of bytes consumed can be < number of bytes provided. - * It typically happens when dstBuffer is not large enough to contain all decoded data. - * Unconsumed source data must be presented again in subsequent invocations. - * - * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten. - * `dstBuffer` itself can be changed at will between each consecutive function invocation. - * - * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. - * Schematically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. - * This is just a hint though, it's always possible to provide any srcSize. - * When a frame is fully decoded, @return will be 0 (no more data expected). - * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). - * - * After a frame is fully decoded, dctx can be used again to decompress another frame. - * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state. - */ -LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const LZ4F_decompressOptions_t* dOptPtr); - - -/*! LZ4F_resetDecompressionContext() : v1.8.0 - * In case of an error, the context is left in "undefined" state. - * In which case, it's necessary to reset it, before re-using it. - * This method can also be used to abruptly stop an unfinished decompression, - * and start a new one using the same context. */ -LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */ - - - -#if defined (__cplusplus) -} -#endif - -#endif /* LZ4F_H_09782039843 */ diff --git a/library/lz4-mac/lib/lz4frame_static.h b/library/lz4-mac/lib/lz4frame_static.h deleted file mode 100755 index 1899f8e4..00000000 --- a/library/lz4-mac/lib/lz4frame_static.h +++ /dev/null @@ -1,143 +0,0 @@ -/* - LZ4 auto-framing library - Header File for static linking only - Copyright (C) 2011-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -#ifndef LZ4FRAME_STATIC_H_0398209384 -#define LZ4FRAME_STATIC_H_0398209384 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* lz4frame_static.h should be used solely in the context of static linking. - * It contains definitions which are not stable and may change in the future. - * Never use it in the context of DLL linking. - */ - - -/* --- Dependency --- */ -#include "lz4frame.h" - - -/* --- Error List --- */ -#define LZ4F_LIST_ERRORS(ITEM) \ - ITEM(OK_NoError) \ - ITEM(ERROR_GENERIC) \ - ITEM(ERROR_maxBlockSize_invalid) \ - ITEM(ERROR_blockMode_invalid) \ - ITEM(ERROR_contentChecksumFlag_invalid) \ - ITEM(ERROR_compressionLevel_invalid) \ - ITEM(ERROR_headerVersion_wrong) \ - ITEM(ERROR_blockChecksum_invalid) \ - ITEM(ERROR_reservedFlag_set) \ - ITEM(ERROR_allocation_failed) \ - ITEM(ERROR_srcSize_tooLarge) \ - ITEM(ERROR_dstMaxSize_tooSmall) \ - ITEM(ERROR_frameHeader_incomplete) \ - ITEM(ERROR_frameType_unknown) \ - ITEM(ERROR_frameSize_wrong) \ - ITEM(ERROR_srcPtr_wrong) \ - ITEM(ERROR_decompressionFailed) \ - ITEM(ERROR_headerChecksum_invalid) \ - ITEM(ERROR_contentChecksum_invalid) \ - ITEM(ERROR_frameDecoding_alreadyStarted) \ - ITEM(ERROR_maxCode) - -#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, - -/* enum list is exposed, to handle specific errors */ -typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; - -LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); - - - -/********************************** - * Bulk processing dictionary API - *********************************/ -typedef struct LZ4F_CDict_s LZ4F_CDict; - -/*! LZ4_createCDict() : - * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. - * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. - * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ -LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); -void LZ4F_freeCDict(LZ4F_CDict* CDict); - - -/*! LZ4_compressFrame_usingCDict() : - * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary. - * If cdict==NULL, compress without a dictionary. - * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). - * If this condition is not respected, function will fail (@return an errorCode). - * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, - * but it's not recommended, as it's the only way to provide dictID in the frame header. - * @return : number of bytes written into dstBuffer. - * or an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* preferencesPtr); - - -/*! LZ4F_compressBegin_usingCDict() : - * Inits streaming dictionary compression, and writes the frame header into dstBuffer. - * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. - * `prefsPtr` is optional : you may provide NULL as argument, - * however, it's the only way to provide dictID in the frame header. - * @return : number of bytes written into dstBuffer for the header, - * or an error code (which can be tested using LZ4F_isError()) */ -size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* prefsPtr); - - -/*! LZ4F_decompress_usingDict() : - * Same as LZ4F_decompress(), using a predefined dictionary. - * Dictionary is used "in place", without any preprocessing. - * It must remain accessible throughout the entire frame decoding. */ -size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const void* dict, size_t dictSize, - const LZ4F_decompressOptions_t* decompressOptionsPtr); - - -#if defined (__cplusplus) -} -#endif - -#endif /* LZ4FRAME_STATIC_H_0398209384 */ diff --git a/library/lz4-mac/lib/lz4hc.c b/library/lz4-mac/lib/lz4hc.c deleted file mode 100755 index d7f8d23b..00000000 --- a/library/lz4-mac/lib/lz4hc.c +++ /dev/null @@ -1,807 +0,0 @@ -/* - LZ4 HC - High Compression Mode of LZ4 - Copyright (C) 2011-2017, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ - - -/* ************************************* -* Tuning Parameter -***************************************/ - -/*! HEAPMODE : - * Select how default compression function will allocate workplace memory, - * in stack (0:fastest), or in heap (1:requires malloc()). - * Since workplace is rather large, heap mode is recommended. - */ -#ifndef LZ4HC_HEAPMODE -# define LZ4HC_HEAPMODE 1 -#endif - - -/*=== Dependency ===*/ -#include "lz4hc.h" - - -/*=== Common LZ4 definitions ===*/ -#if defined(__GNUC__) -# pragma GCC diagnostic ignored "-Wunused-function" -#endif -#if defined (__clang__) -# pragma clang diagnostic ignored "-Wunused-function" -#endif - -#define LZ4_COMMONDEFS_ONLY -#include "lz4.c" /* LZ4_count, constants, mem */ - - -/*=== Constants ===*/ -#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) - - -/*=== Macros ===*/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) -#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) -#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) -#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ -#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ - -static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } - - - -/************************************** -* HC Compression -**************************************/ -static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) -{ - MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); - MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = 64 KB; - hc4->base = start - 64 KB; - hc4->end = start; - hc4->dictBase = start - 64 KB; - hc4->dictLimit = 64 KB; - hc4->lowLimit = 64 KB; -} - - -/* Update chains up to ip (excluded) */ -LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) -{ - U16* const chainTable = hc4->chainTable; - U32* const hashTable = hc4->hashTable; - const BYTE* const base = hc4->base; - U32 const target = (U32)(ip - base); - U32 idx = hc4->nextToUpdate; - - while (idx < target) { - U32 const h = LZ4HC_hashPtr(base+idx); - size_t delta = idx - hashTable[h]; - if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; - DELTANEXTU16(chainTable, idx) = (U16)delta; - hashTable[h] = idx; - idx++; - } - - hc4->nextToUpdate = target; -} - - -LZ4_FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ - const BYTE* const ip, const BYTE* const iLimit, - const BYTE** matchpos, - const int maxNbAttempts) -{ - U16* const chainTable = hc4->chainTable; - U32* const HashTable = hc4->hashTable; - const BYTE* const base = hc4->base; - const BYTE* const dictBase = hc4->dictBase; - const U32 dictLimit = hc4->dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); - U32 matchIndex; - int nbAttempts = maxNbAttempts; - size_t ml = 0; - - /* HC4 match finder */ - LZ4HC_Insert(hc4, ip); - matchIndex = HashTable[LZ4HC_hashPtr(ip)]; - - while ((matchIndex>=lowLimit) && (nbAttempts)) { - nbAttempts--; - if (matchIndex >= dictLimit) { - const BYTE* const match = base + matchIndex; - if ( (*(match+ml) == *(ip+ml)) /* can be longer */ - && (LZ4_read32(match) == LZ4_read32(ip)) ) - { - size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = match; } - } - } else { - const BYTE* const match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) { - size_t mlt; - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iLimit) vLimit = iLimit; - mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); - if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ - } - } - matchIndex -= DELTANEXTU16(chainTable, matchIndex); - } - - return (int)ml; -} - - -LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( - LZ4HC_CCtx_internal* hc4, - const BYTE* const ip, - const BYTE* const iLowLimit, - const BYTE* const iHighLimit, - int longest, - const BYTE** matchpos, - const BYTE** startpos, - const int maxNbAttempts) -{ - U16* const chainTable = hc4->chainTable; - U32* const HashTable = hc4->hashTable; - const BYTE* const base = hc4->base; - const U32 dictLimit = hc4->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); - const BYTE* const dictBase = hc4->dictBase; - int const delta = (int)(ip-iLowLimit); - int nbAttempts = maxNbAttempts; - U32 matchIndex; - - - /* First Match */ - LZ4HC_Insert(hc4, ip); - matchIndex = HashTable[LZ4HC_hashPtr(ip)]; - - while ((matchIndex>=lowLimit) && (nbAttempts)) { - nbAttempts--; - if (matchIndex >= dictLimit) { - const BYTE* const matchPtr = base + matchIndex; - if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) { - if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { - int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); - int back = 0; - - while ( (ip+back > iLowLimit) - && (matchPtr+back > lowPrefixPtr) - && (ip[back-1] == matchPtr[back-1])) { - back--; - } - - mlt -= back; - - if (mlt > longest) { - longest = mlt; - *matchpos = matchPtr+back; - *startpos = ip+back; - } } } - } else { - const BYTE* const matchPtr = dictBase + matchIndex; - if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { - int mlt; - int back=0; - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--; - mlt -= back; - if (mlt > longest) { longest = mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } - } - } - matchIndex -= DELTANEXTU16(chainTable, matchIndex); - } - - return longest; -} - - -typedef enum { - noLimit = 0, - limitedOutput = 1, - limitedDestSize = 2, -} limitedOutput_directive; - -#ifndef LZ4HC_DEBUG -# define LZ4HC_DEBUG 0 -#endif - -/* LZ4HC_encodeSequence() : - * @return : 0 if ok, - * 1 if buffer issue detected */ -LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( - const BYTE** ip, - BYTE** op, - const BYTE** anchor, - int matchLength, - const BYTE* const match, - limitedOutput_directive limit, - BYTE* oend) -{ - size_t length; - BYTE* const token = (*op)++; - -#if LZ4HC_DEBUG - printf("literal : %u -- match : %u -- offset : %u\n", - (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); -#endif - - /* Encode Literal length */ - length = (size_t)(*ip - *anchor); - if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ - if (length >= RUN_MASK) { - size_t len = length - RUN_MASK; - *token = (RUN_MASK << ML_BITS); - for(; len >= 255 ; len -= 255) *(*op)++ = 255; - *(*op)++ = (BYTE)len; - } else { - *token = (BYTE)(length << ML_BITS); - } - - /* Copy Literals */ - LZ4_wildCopy(*op, *anchor, (*op) + length); - *op += length; - - /* Encode Offset */ - LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2; - - /* Encode MatchLength */ - length = (size_t)(matchLength - MINMATCH); - if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ - if (length >= ML_MASK) { - *token += ML_MASK; - length -= ML_MASK; - for(; length >= 510 ; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; } - if (length >= 255) { length -= 255; *(*op)++ = 255; } - *(*op)++ = (BYTE)length; - } else { - *token += (BYTE)(length); - } - - /* Prepare next loop */ - *ip += matchLength; - *anchor = *ip; - - return 0; -} - -/* btopt */ -#include "lz4opt.h" - - -static int LZ4HC_compress_hashChain ( - LZ4HC_CCtx_internal* const ctx, - const char* const source, - char* const dest, - int* srcSizePtr, - int const maxOutputSize, - unsigned maxNbAttempts, - limitedOutput_directive limit - ) -{ - const int inputSize = *srcSizePtr; - - const BYTE* ip = (const BYTE*) source; - const BYTE* anchor = ip; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - - BYTE* optr = (BYTE*) dest; - BYTE* op = (BYTE*) dest; - BYTE* oend = op + maxOutputSize; - - int ml, ml2, ml3, ml0; - const BYTE* ref = NULL; - const BYTE* start2 = NULL; - const BYTE* ref2 = NULL; - const BYTE* start3 = NULL; - const BYTE* ref3 = NULL; - const BYTE* start0; - const BYTE* ref0; - - /* init */ - *srcSizePtr = 0; - if (limit == limitedDestSize && maxOutputSize < 1) return 0; /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - - ctx->end += inputSize; - if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support limitations LZ4 decompressor */ - if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ - - ip++; - - /* Main Loop */ - while (ip < mflimit) { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); - if (!ml) { ip++; continue; } - - /* saved, in case we would skip too much */ - start0 = ip; - ref0 = ref; - ml0 = ml; - -_Search2: - if (ip+ml < mflimit) - ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts); - else - ml2 = ml; - - if (ml2 == ml) { /* No better match */ - optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; - continue; - } - - if (start0 < ip) { - if (start2 < ip + ml0) { /* empirical */ - ip = start0; - ref = ref0; - ml = ml0; - } - } - - /* Here, start0==ip */ - if ((start2 - ip) < 3) { /* First Match too small : removed */ - ml = ml2; - ip = start2; - ref =ref2; - goto _Search2; - } - -_Search3: - /* At this stage, we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) */ - if ((start2 - ip) < OPTIMAL_ML) { - int correction; - int new_ml = ml; - if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; - if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; - correction = new_ml - (int)(start2 - ip); - if (correction > 0) { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ - - if (start2 + ml2 < mflimit) - ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); - else - ml3 = ml2; - - if (ml3 == ml2) { /* No better match : 2 sequences to encode */ - /* ip & ref are known; Now for ml */ - if (start2 < ip+ml) ml = (int)(start2 - ip); - /* Now, encode 2 sequences */ - optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; - ip = start2; - optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) goto _dest_overflow; - continue; - } - - if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */ - if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ - if (start2 < ip+ml) { - int correction = (int)(ip+ml - start2); - start2 += correction; - ref2 += correction; - ml2 -= correction; - if (ml2 < MINMATCH) { - start2 = start3; - ref2 = ref3; - ml2 = ml3; - } - } - - optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; - ip = start3; - ref = ref3; - ml = ml3; - - start0 = start2; - ref0 = ref2; - ml0 = ml2; - goto _Search2; - } - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - goto _Search3; - } - - /* - * OK, now we have 3 ascending matches; let's write at least the first one - * ip & ref are known; Now for ml - */ - if (start2 < ip+ml) { - if ((start2 - ip) < (int)ML_MASK) { - int correction; - if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; - if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; - correction = ml - (int)(start2 - ip); - if (correction > 0) { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } else { - ml = (int)(start2 - ip); - } - } - optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; - - ip = start2; - ref = ref2; - ml = ml2; - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - - goto _Search3; - } - -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ - size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255; - size_t const totalSize = 1 + litLength + lastRunSize; - if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */ - if (limit && (op + totalSize > oend)) { - if (limit == limitedOutput) return 0; /* Check output limit */ - /* adapt lastRunSize to fill 'dest' */ - lastRunSize = (size_t)(oend - op) - 1; - litLength = (lastRunSize + 255 - RUN_MASK) / 255; - lastRunSize -= litLength; - } - ip = anchor + lastRunSize; - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = (RUN_MASK << ML_BITS); - for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize << ML_BITS); - } - memcpy(op, anchor, lastRunSize); - op += lastRunSize; - } - - /* End */ - *srcSizePtr = (int) (((const char*)ip) - source); - return (int) (((char*)op)-dest); - -_dest_overflow: - if (limit == limitedDestSize) { - op = optr; /* restore correct out pointer */ - goto _last_literals; - } - return 0; -} - -static int LZ4HC_getSearchNum(int compressionLevel) -{ - switch (compressionLevel) { - default: return 0; /* unused */ - case 11: return 128; - case 12: return 1<<10; - } -} - -static int LZ4HC_compress_generic ( - LZ4HC_CCtx_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - int const dstCapacity, - int cLevel, - limitedOutput_directive limit - ) -{ - if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe to reconsider */ - if (cLevel > 9) { - if (limit == limitedDestSize) cLevel = 10; - switch (cLevel) { - case 10: - return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << 12, limit); - case 11: - ctx->searchNum = LZ4HC_getSearchNum(cLevel); - return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 128, 0); - default: - cLevel = 12; - /* fall-through */ - case 12: - ctx->searchNum = LZ4HC_getSearchNum(cLevel); - return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, LZ4_OPT_NUM, 1); - } - } - return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (cLevel-1), limit); /* levels 1-9 */ -} - - -int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } - -int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) -{ - LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4HC_init (ctx, (const BYTE*)src); - if (dstCapacity < LZ4_compressBound(srcSize)) - return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); - else - return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit); -} - -int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) -{ -#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 - LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); -#else - LZ4_streamHC_t state; - LZ4_streamHC_t* const statePtr = &state; -#endif - int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); -#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 - free(statePtr); -#endif - return cSize; -} - -/* LZ4_compress_HC_destSize() : - * currently, only compatible with Hash Chain implementation, - * hence limit compression level to LZ4HC_CLEVEL_OPT_MIN-1*/ -int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) -{ - LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; - LZ4HC_init(ctx, (const BYTE*) source); - return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize); -} - - - -/************************************** -* Streaming Functions -**************************************/ -/* allocation */ -LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { - if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ - free(LZ4_streamHCPtr); - return 0; -} - - -/* initialization */ -void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) -{ - LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ - LZ4_streamHCPtr->internal_donotuse.base = NULL; - if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; /* cap compression level */ - LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; - LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel); -} - -void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) -{ - int const currentCLevel = LZ4_streamHCPtr->internal_donotuse.compressionLevel; - int const minCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? 1 : LZ4HC_CLEVEL_OPT_MIN; - int const maxCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? LZ4HC_CLEVEL_OPT_MIN-1 : LZ4HC_CLEVEL_MAX; - compressionLevel = MIN(compressionLevel, minCLevel); - compressionLevel = MAX(compressionLevel, maxCLevel); - LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel; -} - -int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) -{ - LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - if (dictSize > 64 KB) { - dictionary += dictSize - 64 KB; - dictSize = 64 KB; - } - LZ4HC_init (ctxPtr, (const BYTE*)dictionary); - ctxPtr->end = (const BYTE*)dictionary + dictSize; - if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) - LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); - else - if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); - return dictSize; -} - - -/* compression */ - -static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) -{ - if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) - LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); - else - if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ - - /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ - ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ -} - -static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, - const char* src, char* dst, - int* srcSizePtr, int dstCapacity, - limitedOutput_directive limit) -{ - LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - /* auto-init if forgotten */ - if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src); - - /* Check overflow */ - if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { - size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; - if (dictSize > 64 KB) dictSize = 64 KB; - LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); - } - - /* Check if blocks follow each other */ - if ((const BYTE*)src != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src); - - /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr; - const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; - const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; - if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) { - if (sourceEnd > dictEnd) sourceEnd = dictEnd; - ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); - if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; - } - } - - return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit); -} - -int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity) -{ - if (dstCapacity < LZ4_compressBound(srcSize)) - return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput); - else - return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, noLimit); -} - -int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize) -{ - LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_init(ctxPtr, (const BYTE*)src); /* not compatible with btopt implementation */ - return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize); -} - - - -/* dictionary saving */ - -int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) -{ - LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; - int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); - if (dictSize > 64 KB) dictSize = 64 KB; - if (dictSize < 4) dictSize = 0; - if (dictSize > prefixSize) dictSize = prefixSize; - memmove(safeBuffer, streamPtr->end - dictSize, dictSize); - { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base); - streamPtr->end = (const BYTE*)safeBuffer + dictSize; - streamPtr->base = streamPtr->end - endIndex; - streamPtr->dictLimit = endIndex - dictSize; - streamPtr->lowLimit = endIndex - dictSize; - if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; - } - return dictSize; -} - - -/*********************************** -* Deprecated Functions -***********************************/ -/* These functions currently generate deprecation warnings */ -/* Deprecated compression functions */ -int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } -int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } -int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } -int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); } -int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); } -int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } -int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } -int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); } -int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); } -int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } - - -/* Deprecated streaming functions */ -int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } - -int LZ4_resetStreamStateHC(void* state, char* inputBuffer) -{ - LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; - if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ - LZ4HC_init(ctx, (const BYTE*)inputBuffer); - ctx->inputBuffer = (BYTE*)inputBuffer; - return 0; -} - -void* LZ4_createHC (char* inputBuffer) -{ - LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t)); - if (hc4 == NULL) return NULL; /* not enough memory */ - LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); - hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer; - return hc4; -} - -int LZ4_freeHC (void* LZ4HC_Data) { - if (!LZ4HC_Data) return 0; /* support free on NULL */ - FREEMEM(LZ4HC_Data); - return 0; -} - -int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) -{ - return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, noLimit); -} - -int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel) -{ - return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput); -} - -char* LZ4_slideInputBufferHC(void* LZ4HC_Data) -{ - LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; - int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); - return (char*)(hc4->inputBuffer + dictSize); -} diff --git a/library/lz4-mac/lib/lz4hc.h b/library/lz4-mac/lib/lz4hc.h deleted file mode 100755 index 66d56364..00000000 --- a/library/lz4-mac/lib/lz4hc.h +++ /dev/null @@ -1,278 +0,0 @@ -/* - LZ4 HC - High Compression Mode of LZ4 - Header File - Copyright (C) 2011-2017, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -#ifndef LZ4_HC_H_19834876238432 -#define LZ4_HC_H_19834876238432 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* --- Dependency --- */ -/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ -#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ - - -/* --- Useful constants --- */ -#define LZ4HC_CLEVEL_MIN 3 -#define LZ4HC_CLEVEL_DEFAULT 9 -#define LZ4HC_CLEVEL_OPT_MIN 11 -#define LZ4HC_CLEVEL_MAX 12 - - -/*-************************************ - * Block Compression - **************************************/ -/*! LZ4_compress_HC() : - * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. - * `dst` must be already allocated. - * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") - * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") - * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_CLEVEL_MAX will work. - * Values >LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. - * @return : the number of bytes written into 'dst' - * or 0 if compression fails. - */ -LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); - - -/* Note : - * Decompression functions are provided within "lz4.h" (BSD license) - */ - - -/*! LZ4_compress_HC_extStateHC() : - * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. - * `state` size is provided by LZ4_sizeofStateHC(). - * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). - */ -LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); -LZ4LIB_API int LZ4_sizeofStateHC(void); - - -/*-************************************ - * Streaming Compression - * Bufferless synchronous API - **************************************/ - typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ - -/*! LZ4_createStreamHC() and LZ4_freeStreamHC() : - * These functions create and release memory for LZ4 HC streaming state. - * Newly created states are automatically initialized. - * Existing states can be re-used several times, using LZ4_resetStreamHC(). - * These methods are API and ABI stable, they can be used in combination with a DLL. - */ -LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); -LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); - -LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); -LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); - -LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); - -LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); - -/* - These functions compress data in successive blocks of any size, using previous blocks as dictionary. - One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. - There is an exception for ring buffers, which can be smaller than 64 KB. - Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue(). - - Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). - A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). - - Then, use LZ4_compress_HC_continue() to compress each successive block. - Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. - 'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success. - Because in case of failure, the API does not guarantee context recovery, and context will have to be reset. - If `dst` buffer budget cannot be >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead. - - If, for any reason, previous data block can't be preserved unmodified in memory for next compression block, - you can save it to a more stable memory space, using LZ4_saveDictHC(). - Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. -*/ - - - /*-************************************* - * PRIVATE DEFINITIONS : - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_streamHC_t`. - * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 - **************************************/ -#define LZ4HC_DICTIONARY_LOGSIZE 17 /* because of btopt, hc would only need 16 */ -#define LZ4HC_MAXD (1<= 199901L) /* C99 */) -#include - -typedef struct -{ - uint32_t hashTable[LZ4HC_HASHTABLESIZE]; - uint16_t chainTable[LZ4HC_MAXD]; - const uint8_t* end; /* next block here to continue on current prefix */ - const uint8_t* base; /* All index relative to this position */ - const uint8_t* dictBase; /* alternate base for extDict */ - uint8_t* inputBuffer; /* deprecated */ - uint32_t dictLimit; /* below that point, need extDict */ - uint32_t lowLimit; /* below that point, no more dict */ - uint32_t nextToUpdate; /* index from which to continue dictionary update */ - uint32_t searchNum; /* only for optimal parser */ - uint32_t compressionLevel; -} LZ4HC_CCtx_internal; - -#else - -typedef struct -{ - unsigned int hashTable[LZ4HC_HASHTABLESIZE]; - unsigned short chainTable[LZ4HC_MAXD]; - const unsigned char* end; /* next block here to continue on current prefix */ - const unsigned char* base; /* All index relative to this position */ - const unsigned char* dictBase; /* alternate base for extDict */ - unsigned char* inputBuffer; /* deprecated */ - unsigned int dictLimit; /* below that point, need extDict */ - unsigned int lowLimit; /* below that point, no more dict */ - unsigned int nextToUpdate; /* index from which to continue dictionary update */ - unsigned int searchNum; /* only for optimal parser */ - int compressionLevel; -} LZ4HC_CCtx_internal; - -#endif - -#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 393268 */ -#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) -union LZ4_streamHC_u { - size_t table[LZ4_STREAMHCSIZE_SIZET]; - LZ4HC_CCtx_internal internal_donotuse; -}; /* previously typedef'd to LZ4_streamHC_t */ -/* - LZ4_streamHC_t : - This structure allows static allocation of LZ4 HC streaming state. - State must be initialized using LZ4_resetStreamHC() before first use. - - Static allocation shall only be used in combination with static linking. - When invoking LZ4 from a DLL, use create/free functions instead, which are API and ABI stable. -*/ - - -/*-************************************ -* Deprecated Functions -**************************************/ -/* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ - -/* deprecated compression functions */ -/* these functions will trigger warning messages in future releases */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); - -/* Deprecated Streaming functions using older model; should no longer be used */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); - - -#if defined (__cplusplus) -} -#endif - -#endif /* LZ4_HC_H_19834876238432 */ - -/*-************************************************ - * !!!!! STATIC LINKING ONLY !!!!! - * Following definitions are considered experimental. - * They should not be linked from DLL, - * as there is no guarantee of API stability yet. - * Prototypes will be promoted to "stable" status - * after successfull usage in real-life scenarios. - *************************************************/ -#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */ -#ifndef LZ4_HC_SLO_098092834 -#define LZ4_HC_SLO_098092834 - -/*! LZ4_compress_HC_destSize() : v1.8.0 (experimental) - * Will try to compress as much data from `src` as possible - * that can fit into `targetDstSize` budget. - * Result is provided in 2 parts : - * @return : the number of bytes written into 'dst' - * or 0 if compression fails. - * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` - */ -int LZ4_compress_HC_destSize(void* LZ4HC_Data, - const char* src, char* dst, - int* srcSizePtr, int targetDstSize, - int compressionLevel); - -/*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental) - * Similar as LZ4_compress_HC_continue(), - * but will read a variable nb of bytes from `src` - * to fit into `targetDstSize` budget. - * Result is provided in 2 parts : - * @return : the number of bytes written into 'dst' - * or 0 if compression fails. - * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`. - * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN - * beyond that level, compression performance will be much reduced due to internal incompatibilities - */ -int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, - const char* src, char* dst, - int* srcSizePtr, int targetDstSize); - -/*! LZ4_setCompressionLevel() : v1.8.0 (experimental) - * It's possible to change compression level after LZ4_resetStreamHC(), between 2 invocations of LZ4_compress_HC_continue*(), - * but that requires to stay in the same mode (aka 1-10 or 11-12). - * This function ensures this condition. - */ -void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); - - - -#endif /* LZ4_HC_SLO_098092834 */ -#endif /* LZ4_HC_STATIC_LINKING_ONLY */ diff --git a/library/lz4-mac/lib/lz4opt.h b/library/lz4-mac/lib/lz4opt.h deleted file mode 100755 index 584dc97f..00000000 --- a/library/lz4-mac/lib/lz4opt.h +++ /dev/null @@ -1,366 +0,0 @@ -/* - lz4opt.h - Optimal Mode of LZ4 - Copyright (C) 2015-2017, Przemyslaw Skibinski - Note : this file is intended to be included within lz4hc.c - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -#define LZ4_OPT_NUM (1<<12) - - -typedef struct { - int off; - int len; -} LZ4HC_match_t; - -typedef struct { - int price; - int off; - int mlen; - int litlen; -} LZ4HC_optimal_t; - - -/* price in bytes */ -LZ4_FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen) -{ - size_t price = litlen; - if (litlen >= (size_t)RUN_MASK) - price += 1 + (litlen-RUN_MASK)/255; - return price; -} - - -/* requires mlen >= MINMATCH */ -LZ4_FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen) -{ - size_t price = 2 + 1; /* 16-bit offset + token */ - - price += LZ4HC_literalsPrice(litlen); - - if (mlen >= (size_t)(ML_MASK+MINMATCH)) - price+= 1 + (mlen-(ML_MASK+MINMATCH))/255; - - return price; -} - - -/*-************************************* -* Binary Tree search -***************************************/ -LZ4_FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( - LZ4HC_CCtx_internal* ctx, - const BYTE* const ip, - const BYTE* const iHighLimit, - size_t best_mlen, - LZ4HC_match_t* matches, - int* matchNum) -{ - U16* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const U32 current = (U32)(ip - base); - const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); - const BYTE* const dictBase = ctx->dictBase; - const BYTE* match; - int nbAttempts = ctx->searchNum; - int mnum = 0; - U16 *ptr0, *ptr1, delta0, delta1; - U32 matchIndex; - size_t matchLength = 0; - U32* HashPos; - - if (ip + MINMATCH > iHighLimit) return 1; - - /* HC4 match finder */ - HashPos = &HashTable[LZ4HC_hashPtr(ip)]; - matchIndex = *HashPos; - *HashPos = current; - - ptr0 = &DELTANEXTMAXD(current*2+1); - ptr1 = &DELTANEXTMAXD(current*2); - delta0 = delta1 = (U16)(current - matchIndex); - - while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { - nbAttempts--; - if (matchIndex >= dictLimit) { - match = base + matchIndex; - matchLength = LZ4_count(ip, match, iHighLimit); - } else { - const BYTE* vLimit = ip + (dictLimit - matchIndex); - match = dictBase + matchIndex; - if (vLimit > iHighLimit) vLimit = iHighLimit; - matchLength = LZ4_count(ip, match, vLimit); - if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) - matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit); - if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > best_mlen) { - best_mlen = matchLength; - if (matches) { - if (matchIndex >= dictLimit) - matches[mnum].off = (int)(ip - match); - else - matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */ - matches[mnum].len = (int)matchLength; - mnum++; - } - if (best_mlen > LZ4_OPT_NUM) break; - } - - if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ - - DEBUGLOG(6, "ip :%016llX", (U64)ip); - DEBUGLOG(6, "match:%016llX", (U64)match); - if (*(ip+matchLength) < *(match+matchLength)) { - *ptr0 = delta0; - ptr0 = &DELTANEXTMAXD(matchIndex*2); - if (*ptr0 == (U16)-1) break; - delta0 = *ptr0; - delta1 += delta0; - matchIndex -= delta0; - } else { - *ptr1 = delta1; - ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - if (*ptr1 == (U16)-1) break; - delta1 = *ptr1; - delta0 += delta1; - matchIndex -= delta1; - } - } - - *ptr0 = (U16)-1; - *ptr1 = (U16)-1; - if (matchNum) *matchNum = mnum; - /* if (best_mlen > 8) return best_mlen-8; */ - if (!matchNum) return 1; - return 1; -} - - -LZ4_FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit) -{ - const BYTE* const base = ctx->base; - const U32 target = (U32)(ip - base); - U32 idx = ctx->nextToUpdate; - while(idx < target) - idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); -} - - -/** Tree updater, providing best match */ -LZ4_FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( - LZ4HC_CCtx_internal* ctx, - const BYTE* const ip, const BYTE* const iHighLimit, - size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) -{ - int mnum = 0; - if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ - if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); - best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); - ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen); - return mnum; -} - - -#define SET_PRICE(pos, ml, offset, ll, cost) \ -{ \ - while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ - opt[pos].mlen = (int)ml; \ - opt[pos].off = (int)offset; \ - opt[pos].litlen = (int)ll; \ - opt[pos].price = (int)cost; \ -} - - -static int LZ4HC_compress_optimal ( - LZ4HC_CCtx_internal* ctx, - const char* const source, - char* dest, - int inputSize, - int maxOutputSize, - limitedOutput_directive limit, - size_t sufficient_len, - const int fullUpdate - ) -{ - LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; /* this uses a bit too much stack memory to my taste ... */ - LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; - - const BYTE* ip = (const BYTE*) source; - const BYTE* anchor = ip; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + maxOutputSize; - - /* init */ - DEBUGLOG(5, "LZ4HC_compress_optimal"); - if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; - ctx->end += inputSize; - ip++; - - /* Main Loop */ - while (ip < mflimit) { - size_t const llen = ip - anchor; - size_t last_pos = 0; - size_t match_num, cur, best_mlen, best_off; - memset(opt, 0, sizeof(LZ4HC_optimal_t)); /* memset only the first one */ - - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); - if (!match_num) { ip++; continue; } - - if ((size_t)matches[match_num-1].len > sufficient_len) { - /* good enough solution : immediate encoding */ - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - cur = 0; - last_pos = 1; - goto encode; - } - - /* set prices using matches at position = 0 */ - { size_t matchNb; - for (matchNb = 0; matchNb < match_num; matchNb++) { - size_t mlen = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH; - best_mlen = matches[matchNb].len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ - for ( ; mlen <= best_mlen ; mlen++) { - size_t const cost = LZ4HC_sequencePrice(llen, mlen) - LZ4HC_literalsPrice(llen); - SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost); /* updates last_pos and opt[pos] */ - } } } - - if (last_pos < MINMATCH) { ip++; continue; } /* note : on clang at least, this test improves performance */ - - /* check further positions */ - opt[0].mlen = opt[1].mlen = 1; - for (cur = 1; cur <= last_pos; cur++) { - const BYTE* const curPtr = ip + cur; - - /* establish baseline price if cur is literal */ - { size_t price, litlen; - if (opt[cur-1].mlen == 1) { - /* no match at previous position */ - litlen = opt[cur-1].litlen + 1; - if (cur > litlen) { - price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); - } else { - price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); - } - } else { - litlen = 1; - price = opt[cur - 1].price + LZ4HC_literalsPrice(1); - } - - if (price < (size_t)opt[cur].price) - SET_PRICE(cur, 1 /*mlen*/, 0 /*off*/, litlen, price); /* note : increases last_pos */ - } - - if (cur == last_pos || curPtr >= mflimit) break; - - match_num = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate); - if ((match_num > 0) && (size_t)matches[match_num-1].len > sufficient_len) { - /* immediate encoding */ - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - last_pos = cur + 1; - goto encode; - } - - /* set prices using matches at position = cur */ - { size_t matchNb; - for (matchNb = 0; matchNb < match_num; matchNb++) { - size_t ml = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH; - best_mlen = (cur + matches[matchNb].len < LZ4_OPT_NUM) ? - (size_t)matches[matchNb].len : LZ4_OPT_NUM - cur; - - for ( ; ml <= best_mlen ; ml++) { - size_t ll, price; - if (opt[cur].mlen == 1) { - ll = opt[cur].litlen; - if (cur > ll) - price = opt[cur - ll].price + LZ4HC_sequencePrice(ll, ml); - else - price = LZ4HC_sequencePrice(llen + ll, ml) - LZ4HC_literalsPrice(llen); - } else { - ll = 0; - price = opt[cur].price + LZ4HC_sequencePrice(0, ml); - } - - if (cur + ml > last_pos || price < (size_t)opt[cur + ml].price) { - SET_PRICE(cur + ml, ml, matches[matchNb].off, ll, price); - } } } } - } /* for (cur = 1; cur <= last_pos; cur++) */ - - best_mlen = opt[last_pos].mlen; - best_off = opt[last_pos].off; - cur = last_pos - best_mlen; - -encode: /* cur, last_pos, best_mlen, best_off must be set */ - opt[0].mlen = 1; - while (1) { /* from end to beginning */ - size_t const ml = opt[cur].mlen; - int const offset = opt[cur].off; - opt[cur].mlen = (int)best_mlen; - opt[cur].off = (int)best_off; - best_mlen = ml; - best_off = offset; - if (ml > cur) break; /* can this happen ? */ - cur -= ml; - } - - /* encode all recorded sequences */ - cur = 0; - while (cur < last_pos) { - int const ml = opt[cur].mlen; - int const offset = opt[cur].off; - if (ml == 1) { ip++; cur++; continue; } - cur += ml; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) return 0; - } - } /* while (ip < mflimit) */ - - /* Encode Last Literals */ - { int lastRun = (int)(iend - anchor); - if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun< 1 > 2) - */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define XXH_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : - */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ - -/*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. - * Results are therefore identical for little-endian and big-endian CPU. - * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independence be of no importance for your application, you may set the #define below to 1, - * to improve speed for Big-endian CPU. - * This option has no impact on Little_Endian CPU. - */ -#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ -# define XXH_FORCE_NATIVE_FORMAT 0 -#endif - -/*!XXH_FORCE_ALIGN_CHECK : - * This is a minor performance trick, only useful with lots of very small keys. - * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. - */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif -#endif - - -/* ************************************* -* Includes & Memory related functions -***************************************/ -/*! Modify the local functions below should you wish to use some other memory routines -* for malloc(), free() */ -#include -static void* XXH_malloc(size_t s) { return malloc(s); } -static void XXH_free (void* p) { free(p); } -/*! and for memcpy() */ -#include -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } - -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" - - -/* ************************************* -* Compiler Specific Options -***************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - -#ifndef XXH_FORCE_INLINE -# ifdef _MSC_VER /* Visual Studio */ -# define XXH_FORCE_INLINE static __forceinline -# else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define XXH_FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define XXH_FORCE_INLINE static inline -# endif -# else -# define XXH_FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -# endif /* _MSC_VER */ -#endif /* XXH_FORCE_INLINE */ - - -/* ************************************* -* Basic Types -***************************************/ -#ifndef MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; -# else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; -# endif -#endif - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; } __attribute__((packed)) unalign; -static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ -static U32 XXH_read32(const void* memPtr) -{ - U32 val; - memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - - -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -#else -static U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -#endif - - -/* ************************************* -* Architecture Macros -***************************************/ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; - -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ -#ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) -#endif - - -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - else - return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); -} - -XXH_FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -static U32 XXH_readBE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); -} - - -/* ************************************* -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } - - -/* ******************************************************************* -* 32-bits hash functions -*********************************************************************/ -static const U32 PRIME32_1 = 2654435761U; -static const U32 PRIME32_2 = 2246822519U; -static const U32 PRIME32_3 = 3266489917U; -static const U32 PRIME32_4 = 668265263U; -static const U32 PRIME32_5 = 374761393U; - -static U32 XXH32_round(U32 seed, U32 input) -{ - seed += input * PRIME32_2; - seed = XXH_rotl32(seed, 13); - seed *= PRIME32_1; - return seed; -} - -XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do { - v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; - v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; - v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; - v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, input, len); - return XXH32_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - - -/*====== Hash streaming ======*/ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) -{ - memcpy(dstState, srcState, sizeof(*dstState)); -} - -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) -{ - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -XXH_FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } - - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -XXH_FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; - U32 h32; - - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } else { - h32 = state->v3 /* == seed */ + PRIME32_5; - } - - h32 += state->total_len_32; - - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - - -/*====== Canonical representation ======*/ - -/*! Default XXH result types are basic unsigned 32 and 64 bits. -* The canonical representation follows human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. -*/ - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} - - -#ifndef XXH_NO_LONG_LONG - -/* ******************************************************************* -* 64-bits hash functions -*********************************************************************/ - -/*====== Memory access ======*/ - -#ifndef MEM_MODULE -# define MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint64_t U64; -# else - typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ -# endif -#endif - - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64; -static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ - -static U64 XXH_read64(const void* memPtr) -{ - U64 val; - memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap64 _byteswap_uint64 -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap64 __builtin_bswap64 -#else -static U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - -XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - else - return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); -} - -XXH_FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - -static U64 XXH_readBE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); -} - - -/*====== xxh64 ======*/ - -static const U64 PRIME64_1 = 11400714785074694791ULL; -static const U64 PRIME64_2 = 14029467366897019727ULL; -static const U64 PRIME64_3 = 1609587929392839161ULL; -static const U64 PRIME64_4 = 9650029242287828579ULL; -static const U64 PRIME64_5 = 2870177450012600261ULL; - -static U64 XXH64_round(U64 acc, U64 input) -{ - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; -} - -static U64 XXH64_mergeRound(U64 acc, U64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; -} - -XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif - - if (len>=32) { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do { - v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; - v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; - v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; - v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; - } while (p<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - - } else { - h64 = seed + PRIME64_5; - } - - h64 += (U64) len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_state_t state; - XXH64_reset(&state, seed); - XXH64_update(&state, input, len); - return XXH64_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - -/*====== Hash Streaming ======*/ - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) -{ - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) -{ - memcpy(dstState, srcState, sizeof(*dstState)); -} - -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) -{ - XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - -XXH_FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - -XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; - U64 h64; - - if (state->total_len >= 32) { - U64 const v1 = state->v1; - U64 const v2 = state->v2; - U64 const v3 = state->v3; - U64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->v3 + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - -XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - - -/*====== Canonical representation ======*/ - -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); -} - -#endif /* XXH_NO_LONG_LONG */ diff --git a/library/lz4-mac/lib/xxhash.h b/library/lz4-mac/lib/xxhash.h deleted file mode 100755 index 870a6d91..00000000 --- a/library/lz4-mac/lib/xxhash.h +++ /dev/null @@ -1,293 +0,0 @@ -/* - xxHash - Extremely Fast Hash algorithm - Header File - Copyright (C) 2012-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : https://github.com/Cyan4973/xxHash -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ - -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************** -* Definitions -******************************/ -#include /* size_t */ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - -/* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module. -*/ -#ifdef XXH_PRIVATE_API -# ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -# endif -# if defined(__GNUC__) -# define XXH_PUBLIC_API static __inline __attribute__((unused)) -# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define XXH_PUBLIC_API static inline -# elif defined(_MSC_VER) -# define XXH_PUBLIC_API static __inline -# else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ -# endif -#else -# define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ - -/*!XXH_NAMESPACE, aka Namespace Emulation : - -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with other libraries which may also include xxHash, - -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). - -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ -#ifdef XXH_NAMESPACE -# define XXH_CAT(A,B) A##B -# define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) -# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) -# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) -# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) -# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) -# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) -#endif - - -/* ************************************* -* Version -***************************************/ -#define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 2 -#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); - - -/*-********************************************************************** -* 32-bits hash -************************************************************************/ -typedef unsigned int XXH32_hash_t; - -/*! XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); - -/*====== Streaming ======*/ -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); - -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); - -/* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . - -XXH state must first be allocated, using XXH*_createState() . - -Start a new hash by initializing state with a seed, using XXH*_reset(). - -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. - -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). - -When done, free XXH state space if it was allocated dynamically. -*/ - -/*====== Canonical representation ======*/ - -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); - -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ - - -#ifndef XXH_NO_LONG_LONG -/*-********************************************************************** -* 64-bits hash -************************************************************************/ -typedef unsigned long long XXH64_hash_t; - -/*! XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". - "seed" can be used to alter the result predictably. - This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). -*/ -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); - -/*====== Streaming ======*/ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); - -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); - -/*====== Canonical representation ======*/ -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); -#endif /* XXH_NO_LONG_LONG */ - - -#ifdef XXH_STATIC_LINKING_ONLY - -/* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. - They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! -=================================================================================================== */ - -/* These definitions are only meant to allow allocation of XXH state - statically, on stack, or in a struct for example. - Do not use members directly. */ - - struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH32_state_t */ - -#ifndef XXH_NO_LONG_LONG - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH64_state_t */ -#endif - -# ifdef XXH_PRIVATE_API -# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ -# endif - -#endif /* XXH_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* XXHASH_H_5627135585666179 */ diff --git a/library/lz4-mac/programs/.gitignore b/library/lz4-mac/programs/.gitignore deleted file mode 100755 index daa7f148..00000000 --- a/library/lz4-mac/programs/.gitignore +++ /dev/null @@ -1,20 +0,0 @@ -# local binary (Makefile) -lz4 -unlz4 -lz4cat -lz4c -lz4c32 -datagen -frametest -frametest32 -fullbench -fullbench32 -fuzzer -fuzzer32 -*.exe - -# tests files -tmp* - -# artefacts -*.dSYM diff --git a/library/lz4-mac/programs/COPYING b/library/lz4-mac/programs/COPYING deleted file mode 100755 index d159169d..00000000 --- a/library/lz4-mac/programs/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/library/lz4-mac/programs/Makefile b/library/lz4-mac/programs/Makefile deleted file mode 100755 index ed1a7796..00000000 --- a/library/lz4-mac/programs/Makefile +++ /dev/null @@ -1,174 +0,0 @@ -# ########################################################################## -# LZ4 programs - Makefile -# Copyright (C) Yann Collet 2011-2017 -# -# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets -# -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - LZ4 homepage : http://www.lz4.org -# - LZ4 source repository : https://github.com/Cyan4973/lz4 -# ########################################################################## -# lz4 : Command Line Utility, supporting gzip-like arguments -# lz4c : CLU, supporting also legacy lz4demo arguments -# lz4c32: Same as lz4c, but forced to compile in 32-bits mode -# ########################################################################## - -# Version numbers -LZ4DIR := ../lib -LIBVER_SRC := $(LZ4DIR)/lz4.h -LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` -LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` -LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` -LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) -LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) -LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) -LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) -LIBVER := $(shell echo $(LIBVER_SCRIPT)) - -SRCFILES := $(sort $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c)) -OBJFILES := $(SRCFILES:.c=.o) - -CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ -CFLAGS ?= -O3 -DEBUGFLAGS:=-Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ - -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) - -LZ4_VERSION=$(LIBVER) -MD2ROFF = ronn -MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)" - - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT :=.exe -VOID := nul -else -EXT := -VOID := /dev/null -endif - - - -default: lz4-release - -all: lz4 lz4c - -all32: CFLAGS+=-m32 -all32: all - -lz4: $(OBJFILES) - $(CC) $(FLAGS) $^ -o $@$(EXT) - -lz4-release: DEBUGFLAGS= -lz4-release: lz4 - -lz4c: lz4 - ln -s lz4 lz4c - -lz4c32: CFLAGS += -m32 -lz4c32 : $(SRCFILES) - $(CC) $(FLAGS) $^ -o $@$(EXT) - -lz4.1: lz4.1.md - cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ - -man: lz4.1 - -clean-man: - rm lz4.1 - -preview-man: clean-man man - man ./lz4.1 - -clean: - @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) - @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat - @echo Cleaning completed - - -#----------------------------------------------------------------------------- -# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets -#----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) - -unlz4: lz4 - ln -s lz4 unlz4 - -lz4cat: lz4 - ln -s lz4 lz4cat - -DESTDIR ?= -# directory variables : GNU conventions prefer lowercase -# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html -# support both lower and uppercase (BSD), use uppercase in script -prefix ?= /usr/local -PREFIX ?= $(prefix) -exec_prefix ?= $(PREFIX) -bindir ?= $(exec_prefix)/bin -BINDIR ?= $(bindir) -datarootdir ?= $(PREFIX)/share -mandir ?= $(datarootdir)/man -man1dir ?= $(mandir)/man1 - -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -MANDIR ?= $(PREFIX)/man/man1 -else -MANDIR ?= $(man1dir) -endif - -ifneq (,$(filter $(shell uname),SunOS)) -INSTALL ?= ginstall -else -INSTALL ?= install -endif - -INSTALL_PROGRAM ?= $(INSTALL) -m 755 -INSTALL_DATA ?= $(INSTALL) -m 644 - - -install: lz4 - @echo Installing binaries - @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4 - @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4c - @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat - @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 - @echo Installing man pages - @$(INSTALL_DATA) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 - @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 - @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1 - @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1 - @echo lz4 installation completed - -uninstall: - @$(RM) $(DESTDIR)$(BINDIR)/lz4cat - @$(RM) $(DESTDIR)$(BINDIR)/unlz4 - @$(RM) $(DESTDIR)$(BINDIR)/lz4 - @$(RM) $(DESTDIR)$(BINDIR)/lz4c - @$(RM) $(DESTDIR)$(MANDIR)/lz4.1 - @$(RM) $(DESTDIR)$(MANDIR)/lz4c.1 - @$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1 - @$(RM) $(DESTDIR)$(MANDIR)/unlz4.1 - @echo lz4 programs successfully uninstalled - -endif diff --git a/library/lz4-mac/programs/README.md b/library/lz4-mac/programs/README.md deleted file mode 100755 index 2ad04496..00000000 --- a/library/lz4-mac/programs/README.md +++ /dev/null @@ -1,71 +0,0 @@ -Command Line Interface for LZ4 library -============================================ - -Command Line Interface (CLI) can be created using the `make` command without any additional parameters. -There are also multiple targets that create different variations of CLI: -- `lz4` : default CLI, with a command line syntax close to gzip -- `lz4c` : Same as `lz4` with additional support legacy lz4 commands (incompatible with gzip) -- `lz4c32` : Same as `lz4c`, but forced to compile in 32-bits mode - - -#### Aggregation of parameters -CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. - - - -#### Benchmark in Command Line Interface -CLI includes in-memory compression benchmark module for lz4. -The benchmark is conducted using a given filename. -The file is read into memory. -It makes benchmark more precise as it eliminates I/O overhead. - -The benchmark measures ratio, compressed size, compression and decompression speed. -One can select compression levels starting from `-b` and ending with `-e`. -The `-i` parameter selects a number of seconds used for each of tested levels. - - - -#### Usage of Command Line Interface -The full list of commands can be obtained with `-h` or `-H` parameter: -``` -Usage : - lz4 [arg] [input] [output] - -input : a filename - with no FILE, or when FILE is - or stdin, read standard input -Arguments : - -1 : Fast compression (default) - -9 : High compression - -d : decompression (default for .lz4 extension) - -z : force compression - -f : overwrite output without prompting ---rm : remove source file(s) after successful de/compression - -h/-H : display help/long help and exit - -Advanced arguments : - -V : display Version number and exit - -v : verbose mode - -q : suppress warnings; specify twice to suppress errors too - -c : force write to standard output, even if it is the console - -t : test compressed file integrity - -m : multiple input files (implies automatic output filenames) - -r : operate recursively on directories (sets also -m) - -l : compress using Legacy format (Linux kernel compression) - -B# : Block size [4-7] (default : 7) - -BD : Block dependency (improve compression ratio) ---no-frame-crc : disable stream checksum (default:enabled) ---content-size : compressed frame includes original size (default:not present) ---[no-]sparse : sparse mode (default:enabled on file, disabled on stdout) -Benchmark arguments : - -b# : benchmark file(s), using # compression level (default : 1) - -e# : test all compression levels from -bX to # (default : 1) - -i# : minimum evaluation time in seconds (default : 3s) - -B# : cut file into independent blocks of size # bytes [32+] - or predefined block size [4-7] (default: 7) -``` - -#### License - -All files in this directory are licensed under GPL-v2. -See [COPYING](COPYING) for details. -The text of the license is also included at the top of each source file. diff --git a/library/lz4-mac/programs/bench.c b/library/lz4-mac/programs/bench.c deleted file mode 100755 index 5c83d59a..00000000 --- a/library/lz4-mac/programs/bench.c +++ /dev/null @@ -1,524 +0,0 @@ -/* - bench.c - Demo program to benchmark open-source compression algorithms - Copyright (C) Yann Collet 2012-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ - - -/*-************************************ -* Compiler options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - -/* ************************************* -* Includes -***************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ -#include /* malloc, free */ -#include /* memset */ -#include /* fprintf, fopen, ftello */ -#include /* clock_t, clock, CLOCKS_PER_SEC */ - -#include "datagen.h" /* RDG_genBuffer */ -#include "xxhash.h" - - -#include "lz4.h" -#define COMPRESSOR0 LZ4_compress_local -static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) { (void)clevel; return LZ4_compress_default(src, dst, srcSize, dstSize); } -#include "lz4hc.h" -#define COMPRESSOR1 LZ4_compress_HC -#define DEFAULTCOMPRESSOR COMPRESSOR0 -#define LZ4_isError(errcode) (errcode==0) - - -/* ************************************* -* Constants -***************************************/ -#ifndef LZ4_GIT_COMMIT_STRING -# define LZ4_GIT_COMMIT_STRING "" -#else -# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT) -#endif - -#define NBSECONDS 3 -#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ -#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ -#define COOLPERIOD_SEC 10 -#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */ - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); - -static U32 g_compressibilityDefault = 50; - - -/* ************************************* -* console display -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ - { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } -static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; -static clock_t g_time = 0; - - -/* ************************************* -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -/* ************************************* -* Benchmark Parameters -***************************************/ -static U32 g_nbSeconds = NBSECONDS; -static size_t g_blockSize = 0; -int g_additionalParam = 0; - -void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } - -void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } - -void BMK_SetNbSeconds(unsigned nbSeconds) -{ - g_nbSeconds = nbSeconds; - DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds); -} - -void BMK_SetBlockSize(size_t blockSize) -{ - g_blockSize = blockSize; -} - - -/* ******************************************************** -* Bench functions -**********************************************************/ -typedef struct { - const char* srcPtr; - size_t srcSize; - char* cPtr; - size_t cRoom; - size_t cSize; - char* resPtr; - size_t resSize; -} blockParam_t; - -struct compressionParameters -{ - int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); -}; - -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - -static int BMK_benchMem(const void* srcBuffer, size_t srcSize, - const char* displayName, int cLevel, - const size_t* fileSizes, U32 nbFiles) -{ - size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; - blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); - size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ - void* const compressedBuffer = malloc(maxCompressedSize); - void* const resultBuffer = malloc(srcSize); - U32 nbBlocks; - UTIL_time_t ticksPerSecond; - struct compressionParameters compP; - int cfunctionId; - - /* checks */ - if (!compressedBuffer || !resultBuffer || !blockTable) - EXM_THROW(31, "allocation error : not enough memory"); - - /* init */ - if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ - UTIL_initTimer(&ticksPerSecond); - - /* Init */ - if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1; - switch (cfunctionId) - { -#ifdef COMPRESSOR0 - case 0 : compP.compressionFunction = COMPRESSOR0; break; -#endif -#ifdef COMPRESSOR1 - case 1 : compP.compressionFunction = COMPRESSOR1; break; -#endif - default : compP.compressionFunction = DEFAULTCOMPRESSOR; - } - - /* Init blockTable data */ - { const char* srcPtr = (const char*)srcBuffer; - char* cPtr = (char*)compressedBuffer; - char* resPtr = (char*)resultBuffer; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb ACTIVEPERIOD_MICROSEC) { - DISPLAYLEVEL(2, "\rcooling down ... \r"); - UTIL_sleep(COOLPERIOD_SEC); - UTIL_getTime(&coolTime); - } - - /* Compression */ - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); - if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ - - UTIL_sleepMilli(1); /* give processor time to other processes */ - UTIL_waitForNextTick(ticksPerSecond); - UTIL_getTime(&clockStart); - - if (!cCompleted) { /* still some time to do compression tests */ - U32 nbLoops = 0; - do { - U32 blockNb; - for (blockNb=0; blockNbmaxTime; - } } - - cSize = 0; - { U32 blockNb; for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / fastestC ); - - (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ -#if 1 - /* Decompression */ - if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ - - UTIL_sleepMilli(1); /* give processor time to other processes */ - UTIL_waitForNextTick(ticksPerSecond); - UTIL_getTime(&clockStart); - - if (!dCompleted) { - U32 nbLoops = 0; - do { - U32 blockNb; - for (blockNb=0; blockNb(DECOMP_MULT*maxTime); - } } - - markNb = (markNb+1) % NB_MARKS; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / fastestC, - (double)srcSize / fastestD ); - - /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); - if (crcOrig!=crcCheck) { - size_t u; - DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); - for (u=0; u u) break; - bacc += blockTable[segNb].srcSize; - } - pos = (U32)(u - bacc); - bNb = pos / (128 KB); - DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos); - break; - } - if (u==srcSize-1) { /* should never happen */ - DISPLAY("no difference detected\n"); - } } - break; - } } /* CRC Checking */ -#endif - } /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */ - - if (g_displayLevel == 1) { - double cSpeed = (double)srcSize / fastestC; - double dSpeed = (double)srcSize / fastestD; - if (g_additionalParam) - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); - else - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); - } - DISPLAYLEVEL(2, "%2i#\n", cLevel); - } /* Bench */ - - /* clean up */ - free(blockTable); - free(compressedBuffer); - free(resultBuffer); - return 0; -} - - -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t step = 64 MB; - BYTE* testmem=NULL; - - requiredMem = (((requiredMem >> 26) + 1) << 26); - requiredMem += 2*step; - if (requiredMem > maxMemory) requiredMem = maxMemory; - - while (!testmem) { - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - testmem = (BYTE*) malloc ((size_t)requiredMem); - } - free (testmem); - - /* keep some space available */ - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - - return (size_t)requiredMem; -} - - -static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, - const char* displayName, int cLevel, int cLevelLast, - const size_t* fileSizes, unsigned nbFiles) -{ - int l; - - const char* pch = strrchr(displayName, '\\'); /* Windows */ - if (!pch) pch = strrchr(displayName, '/'); /* Linux */ - if (pch) displayName = pch+1; - - SET_REALTIME_PRIORITY; - - if (g_displayLevel == 1 && !g_additionalParam) - DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LZ4_VERSION_STRING, LZ4_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); - - if (cLevelLast < cLevel) cLevelLast = cLevel; - - for (l=cLevel; l <= cLevelLast; l++) { - BMK_benchMem(srcBuffer, benchedSize, - displayName, l, - fileSizes, nbFiles); - } -} - - -/*! BMK_loadFiles() : - Loads `buffer` with content of files listed within `fileNamesTable`. - At most, fills `buffer` entirely */ -static void BMK_loadFiles(void* buffer, size_t bufferSize, - size_t* fileSizes, - const char** fileNamesTable, unsigned nbFiles) -{ - size_t pos = 0, totalSize = 0; - unsigned n; - for (n=0; n bufferSize-pos) { /* buffer too small - stop after this file */ - fileSize = bufferSize-pos; - nbFiles=n; - } - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); - pos += readSize; } - fileSizes[n] = (size_t)fileSize; - totalSize += (size_t)fileSize; - fclose(f); - } - - if (totalSize == 0) EXM_THROW(12, "no data to bench"); -} - -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - int cLevel, int cLevelLast) -{ - void* srcBuffer; - size_t benchedSize; - size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); - char mfName[20] = {0}; - - if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); - - /* Memory allocation & restrictions */ - benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; - if (benchedSize==0) EXM_THROW(12, "not enough memory"); - if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; - if (benchedSize > LZ4_MAX_INPUT_SIZE) { - benchedSize = LZ4_MAX_INPUT_SIZE; - DISPLAY("File(s) bigger than LZ4's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20)); - } else { - if (benchedSize < totalSizeToLoad) - DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); - } - srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */ - if (!srcBuffer) EXM_THROW(12, "not enough memory"); - - /* Load input buffer */ - BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); - - /* Bench */ - snprintf (mfName, sizeof(mfName), " %u files", nbFiles); - { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; - BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, cLevelLast, - fileSizes, nbFiles); - } - - /* clean up */ - free(srcBuffer); - free(fileSizes); -} - - -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) -{ - char name[20] = {0}; - size_t benchedSize = 10000000; - void* const srcBuffer = malloc(benchedSize); - - /* Memory allocation */ - if (!srcBuffer) EXM_THROW(21, "not enough memory"); - - /* Fill input buffer */ - RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); - - /* Bench */ - snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1); - - /* clean up */ - free(srcBuffer); -} - - -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - int cLevel, int cLevelLast) -{ - double const compressibility = (double)g_compressibilityDefault / 100; - - if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; - if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; - if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); - - if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility); - else - BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast); - return 0; -} diff --git a/library/lz4-mac/programs/bench.h b/library/lz4-mac/programs/bench.h deleted file mode 100755 index 15def93f..00000000 --- a/library/lz4-mac/programs/bench.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - bench.h - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2016 - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -#ifndef BENCH_H_125623623633 -#define BENCH_H_125623623633 - -#include - -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - int cLevel, int cLevelLast); - -/* Set Parameters */ -void BMK_SetNbSeconds(unsigned nbLoops); -void BMK_SetBlockSize(size_t blockSize); -void BMK_setAdditionalParam(int additionalParam); -void BMK_setNotificationLevel(unsigned level); - -#endif /* BENCH_H_125623623633 */ diff --git a/library/lz4-mac/programs/datagen.c b/library/lz4-mac/programs/datagen.c deleted file mode 100755 index 7285d691..00000000 --- a/library/lz4-mac/programs/datagen.c +++ /dev/null @@ -1,192 +0,0 @@ -/* - datagen.c - compressible data generator test tool - Copyright (C) Yann Collet 2012-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/************************************** -* Includes -**************************************/ -#include "platform.h" /* Compiler options, SET_BINARY_MODE */ -#include "util.h" /* U32 */ -#include /* malloc */ -#include /* FILE, fwrite */ -#include /* memcpy */ - - -/************************************** -* Constants -**************************************/ -#define KB *(1 <<10) - -#define PRIME1 2654435761U -#define PRIME2 2246822519U - - -/************************************** -* Local types -**************************************/ -#define LTLOG 13 -#define LTSIZE (1<> (32 - r))) -static unsigned int RDG_rand(U32* src) -{ - U32 rand32 = *src; - rand32 *= PRIME1; - rand32 ^= PRIME2; - rand32 = RDG_rotl32(rand32, 13); - *src = rand32; - return rand32; -} - - -static void RDG_fillLiteralDistrib(litDistribTable lt, double ld) -{ - BYTE const firstChar = ld <= 0.0 ? 0 : '('; - BYTE const lastChar = ld <= 0.0 ? 255 : '}'; - BYTE character = ld <= 0.0 ? 0 : '0'; - U32 u = 0; - - while (u lastChar) character = firstChar; - } -} - - -static BYTE RDG_genChar(U32* seed, const litDistribTable lt) -{ - U32 id = RDG_rand(seed) & LTMASK; - return (lt[id]); -} - - -#define RDG_DICTSIZE (32 KB) -#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767) -#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) -void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr) -{ - BYTE* buffPtr = (BYTE*)buffer; - const U32 matchProba32 = (U32)(32768 * matchProba); - size_t pos = prefixSize; - U32* seed = seedPtr; - - /* special case */ - while (matchProba >= 1.0) - { - size_t size0 = RDG_rand(seed) & 3; - size0 = (size_t)1 << (16 + size0 * 2); - size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ - if (buffSize < pos + size0) - { - memset(buffPtr+pos, 0, buffSize-pos); - return; - } - memset(buffPtr+pos, 0, size0); - pos += size0; - buffPtr[pos-1] = RDG_genChar(seed, lt); - } - - /* init */ - if (pos==0) { - buffPtr[0] = RDG_genChar(seed, lt); - pos=1; - } - - /* Generate compressible data */ - while (pos < buffSize) - { - /* Select : Literal (char) or Match (within 32K) */ - if (RDG_RAND15BITS < matchProba32) - { - /* Copy (within 32K) */ - size_t match; - size_t d; - int length = RDG_RANDLENGTH + 4; - U32 offset = RDG_RAND15BITS + 1; - if (offset > pos) offset = (U32)pos; - match = pos - offset; - d = pos + length; - if (d > buffSize) d = buffSize; - while (pos < d) buffPtr[pos++] = buffPtr[match++]; - } - else - { - /* Literal (noise) */ - size_t d; - size_t length = RDG_RANDLENGTH; - d = pos + length; - if (d > buffSize) d = buffSize; - while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt); - } - } -} - - -void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) -{ - litDistribTable lt; - if (litProba==0.0) litProba = matchProba / 4.5; - RDG_fillLiteralDistrib(lt, litProba); - RDG_genBlock(buffer, size, 0, matchProba, lt, &seed); -} - - -#define RDG_BLOCKSIZE (128 KB) -void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed) -{ - BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE]; - U64 total = 0; - size_t genBlockSize = RDG_BLOCKSIZE; - litDistribTable lt; - - /* init */ - if (litProba==0.0) litProba = matchProba / 4.5; - RDG_fillLiteralDistrib(lt, litProba); - SET_BINARY_MODE(stdout); - - /* Generate dict */ - RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed); - - /* Generate compressible data */ - while (total < size) - { - RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed); - if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total); - total += genBlockSize; - fwrite(buff, 1, genBlockSize, stdout); - /* update dict */ - memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); - } -} diff --git a/library/lz4-mac/programs/datagen.h b/library/lz4-mac/programs/datagen.h deleted file mode 100755 index 91c5b02e..00000000 --- a/library/lz4-mac/programs/datagen.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - datagen.h - compressible data generator header - Copyright (C) Yann Collet 2012-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - - -#include /* size_t */ - -void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed); -void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); -/* RDG_genOut - Generate 'size' bytes of compressible data into stdout. - Compressibility can be controlled using 'matchProba'. - 'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used. - Generated data can be selected using 'seed'. - If (matchProba, litProba and seed) are equal, the function always generate the same content. - - RDG_genBuffer - Same as RDG_genOut, but generate data into provided buffer -*/ diff --git a/library/lz4-mac/programs/lz4.1 b/library/lz4-mac/programs/lz4.1 deleted file mode 100755 index 39d78cdb..00000000 --- a/library/lz4-mac/programs/lz4.1 +++ /dev/null @@ -1,220 +0,0 @@ -. -.TH "LZ4" "1" "July 2017" "lz4 1.8.0" "User Commands" -. -.SH "NAME" -\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files -. -.SH "SYNOPSIS" -\fBlz4\fR [\fIOPTIONS\fR] [\-|INPUT\-FILE] \fIOUTPUT\-FILE\fR -. -.P -\fBunlz4\fR is equivalent to \fBlz4 \-d\fR -. -.P -\fBlz4cat\fR is equivalent to \fBlz4 \-dcfm\fR -. -.P -When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\. -. -.SH "DESCRIPTION" -\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\. -. -.SS "Difference between lz4 and gzip" -\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are : -. -.IP "\(bu" 4 -\fBlz4\fR preserves original files -. -.IP "\(bu" 4 -\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files) -. -.IP "\(bu" 4 -\fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 -. -.IP "\(bu" 4 -\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression) -. -.IP "\(bu" 4 -\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them) -. -.IP "\(bu" 4 -If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\. -. -.IP "\(bu" 4 -If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\. -. -.IP "\(bu" 4 -As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\. -. -.IP "" 0 -. -.P -Default behaviors can be modified by opt\-in commands, detailed below\. -. -.IP "\(bu" 4 -\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\. -. -.IP "\(bu" 4 -Similarly, \fBlz4 \-m \-d\fR can decompress multiple \fB*\.lz4\fR files\. -. -.IP "\(bu" 4 -It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\. -. -.IP "\(bu" 4 -Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. -. -.IP "" 0 -. -.SS "Concatenation of \.lz4 files" -It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: -. -.IP "" 4 -. -.nf - -lz4 file1 > foo\.lz4 -lz4 file2 >> foo\.lz4 -. -.fi -. -.IP "" 0 -. -.P -Then \fBlz4cat foo\.lz4\fR is equivalent to \fBcat file1 file2\fR\. -. -.SH "OPTIONS" -. -.SS "Short commands concatenation" -In some cases, some options can be expressed using short command \fB\-x\fR or long command \fB\-\-long\-word\fR\. Short commands can be concatenated together\. For example, \fB\-d \-c\fR is equivalent to \fB\-dc\fR\. Long commands cannot be concatenated\. They must be clearly separated by a space\. -. -.SS "Multiple commands" -When multiple contradictory commands are issued on a same command line, only the latest one will be applied\. -. -.SS "Operation mode" -. -.TP -\fB\-z\fR \fB\-\-compress\fR -Compress\. This is the default operation mode when no operation mode option is specified, no other operation mode is implied from the command name (for example, \fBunlz4\fR implies \fB\-\-decompress\fR), nor from the input file name (for example, a file extension \fB\.lz4\fR implies \fB\-\-decompress\fR by default)\. \fB\-z\fR can also be used to force compression of an already compressed \fB\.lz4\fR file\. -. -.TP -\fB\-d\fR \fB\-\-decompress\fR \fB\-\-uncompress\fR -Decompress\. \fB\-\-decompress\fR is also the default operation when the input filename has an \fB\.lz4\fR extension\. -. -.TP -\fB\-t\fR \fB\-\-test\fR -Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is discarded\. No files are created nor removed\. -. -.TP -\fB\-b#\fR -Benchmark mode, using \fB#\fR compression level\. -. -.SS "Operation modifiers" -. -.TP -\fB\-#\fR -Compression level, with # being any value from 1 to 16\. Higher values trade compression speed for compression ratio\. Values above 16 are considered the same as 16\. Recommended values are 1 for fast compression (default), and 9 for high compression\. Speed/compression trade\-off will vary depending on data to compress\. Decompression speed remains fast at all settings\. -. -.TP -\fB\-f\fR \fB\-\-[no\-]force\fR -This option has several effects: -. -.IP -If the target file already exists, overwrite it without prompting\. -. -.IP -When used with \fB\-\-decompress\fR and \fBlz4\fR cannot recognize the type of the source file, copy the source file as is to standard output\. This allows \fBlz4cat \-\-force\fR to be used like \fBcat (1)\fR for files that have not been compressed with \fBlz4\fR\. -. -.TP -\fB\-c\fR \fB\-\-stdout\fR \fB\-\-to\-stdout\fR -Force write to standard output, even if it is the console\. -. -.TP -\fB\-m\fR \fB\-\-multiple\fR -Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\. -. -.TP -\fB\-r\fR -operate recursively on directories\. This mode also sets \fB\-m\fR (multiple input files)\. -. -.TP -\fB\-B#\fR -Block size [4\-7](default : 7) -. -.br -\fB\-B4\fR= 64KB ; \fB\-B5\fR= 256KB ; \fB\-B6\fR= 1MB ; \fB\-B7\fR= 4MB -. -.TP -\fB\-BD\fR -Block Dependency (improves compression ratio on small blocks) -. -.TP -\fB\-\-[no\-]frame\-crc\fR -Select frame checksum (default:enabled) -. -.TP -\fB\-\-[no\-]content\-size\fR -Header includes original size (default:not present) -. -.br -Note : this option can only be activated when the original size can be determined, hence for a file\. It won\'t work with unknown source size, such as stdin or pipe\. -. -.TP -\fB\-\-[no\-]sparse\fR -Sparse mode support (default:enabled on file, disabled on stdout) -. -.TP -\fB\-l\fR -Use Legacy format (typically for Linux Kernel compression) -. -.br -Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) nor \fB\-r\fR -. -.SS "Other options" -. -.TP -\fB\-v\fR \fB\-\-verbose\fR -Verbose mode -. -.TP -\fB\-q\fR \fB\-\-quiet\fR -Suppress warnings and real\-time statistics; specify twice to suppress errors too -. -.TP -\fB\-h\fR \fB\-H\fR \fB\-\-help\fR -Display help/long help and exit -. -.TP -\fB\-V\fR \fB\-\-version\fR -Display Version number and exit -. -.TP -\fB\-k\fR \fB\-\-keep\fR -Preserve source files (default behavior) -. -.TP -\fB\-\-rm\fR -Delete source files on successful compression or decompression -. -.TP -\fB\-\-\fR -Treat all subsequent arguments as files -. -.SS "Benchmark mode" -. -.TP -\fB\-b#\fR -Benchmark file(s), using # compression level -. -.TP -\fB\-e#\fR -Benchmark multiple compression levels, from b# to e# (included) -. -.TP -\fB\-i#\fR -Minimum evaluation in seconds [1\-9] (default : 3) -. -.SH "BUGS" -Report bugs at: https://github\.com/lz4/lz4/issues -. -.SH "AUTHOR" -Yann Collet diff --git a/library/lz4-mac/programs/lz4.1.md b/library/lz4-mac/programs/lz4.1.md deleted file mode 100755 index c6b99bcd..00000000 --- a/library/lz4-mac/programs/lz4.1.md +++ /dev/null @@ -1,218 +0,0 @@ -lz4(1) -- lz4, unlz4, lz4cat - Compress or decompress .lz4 files -================================================================ - -SYNOPSIS --------- - -`lz4` [*OPTIONS*] [-|INPUT-FILE] - -`unlz4` is equivalent to `lz4 -d` - -`lz4cat` is equivalent to `lz4 -dcfm` - -When writing scripts that need to decompress files, -it is recommended to always use the name `lz4` with appropriate arguments -(`lz4 -d` or `lz4 -dc`) instead of the names `unlz4` and `lz4cat`. - - -DESCRIPTION ------------ - -`lz4` is an extremely fast lossless compression algorithm, -based on **byte-aligned LZ77** family of compression scheme. -`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with -multi-core CPUs. -It features an extremely fast decoder, with speed in multiple GB/s per core, -typically reaching RAM speed limit on multi-core systems. -The native file format is the `.lz4` format. - -### Difference between lz4 and gzip - -`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`. -Differences are : - - * `lz4` preserves original files - * `lz4` compresses a single file by default (see `-m` for multiple files) - * `lz4 file1 file2` means : compress file1 _into_ file2 - * `lz4 file.lz4` will default to decompression (use `-z` to force compression) - * `lz4` shows real-time notification statistics - during compression or decompression of a single file - (use `-q` to silent them) - * If no destination name is provided, result is sent to `stdout` - _except if stdout is the console_. - * If no destination name is provided, __and__ if `stdout` is the console, - `file` is compressed into `file.lz4`. - * As a consequence of previous rules, note the following example : - `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, - hence it does _not_ create `file.lz4`. - -Default behaviors can be modified by opt-in commands, detailed below. - - * `lz4 -m` makes it possible to provide multiple input filenames, - which will be compressed into files using suffix `.lz4`. - Progress notifications are also disabled by default (use `-v` to enable them). - This mode has a behavior which more closely mimics `gzip` command line, - with the main remaining difference being that source files are preserved by default. - * Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files. - * It's possible to opt-in to erase source files - on successful compression or decompression, using `--rm` command. - * Consequently, `lz4 -m --rm` behaves the same as `gzip`. - -### Concatenation of .lz4 files - -It is possible to concatenate `.lz4` files as is. -`lz4` will decompress such files as if they were a single `.lz4` file. -For example: - - lz4 file1 > foo.lz4 - lz4 file2 >> foo.lz4 - -Then `lz4cat foo.lz4` is equivalent to `cat file1 file2`. - -OPTIONS -------- - -### Short commands concatenation - -In some cases, some options can be expressed using short command `-x` -or long command `--long-word`. -Short commands can be concatenated together. -For example, `-d -c` is equivalent to `-dc`. -Long commands cannot be concatenated. -They must be clearly separated by a space. - -### Multiple commands - -When multiple contradictory commands are issued on a same command line, -only the latest one will be applied. - -### Operation mode - -* `-z` `--compress`: - Compress. - This is the default operation mode when no operation mode option is - specified, no other operation mode is implied from the command name - (for example, `unlz4` implies `--decompress`), - nor from the input file name - (for example, a file extension `.lz4` implies `--decompress` by default). - `-z` can also be used to force compression of an already compressed - `.lz4` file. - -* `-d` `--decompress` `--uncompress`: - Decompress. - `--decompress` is also the default operation when the input filename has an - `.lz4` extension. - -* `-t` `--test`: - Test the integrity of compressed `.lz4` files. - The decompressed data is discarded. - No files are created nor removed. - -* `-b#`: - Benchmark mode, using `#` compression level. - -### Operation modifiers - -* `-#`: - Compression level, with # being any value from 1 to 16. - Higher values trade compression speed for compression ratio. - Values above 16 are considered the same as 16. - Recommended values are 1 for fast compression (default), - and 9 for high compression. - Speed/compression trade-off will vary depending on data to compress. - Decompression speed remains fast at all settings. - -* `-f` `--[no-]force`: - This option has several effects: - - If the target file already exists, overwrite it without prompting. - - When used with `--decompress` and `lz4` cannot recognize the type of - the source file, copy the source file as is to standard output. - This allows `lz4cat --force` to be used like `cat (1)` for files - that have not been compressed with `lz4`. - -* `-c` `--stdout` `--to-stdout`: - Force write to standard output, even if it is the console. - -* `-m` `--multiple`: - Multiple input files. - Compressed file names will be appended a `.lz4` suffix. - This mode also reduces notification level. - `lz4 -m` has a behavior equivalent to `gzip -k` - (it preserves source files by default). - -* `-r` : - operate recursively on directories. - This mode also sets `-m` (multiple input files). - -* `-B#`: - Block size \[4-7\](default : 7)
- `-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB - -* `-BD`: - Block Dependency (improves compression ratio on small blocks) - -* `--[no-]frame-crc`: - Select frame checksum (default:enabled) - -* `--[no-]content-size`: - Header includes original size (default:not present)
- Note : this option can only be activated when the original size can be - determined, hence for a file. It won't work with unknown source size, - such as stdin or pipe. - -* `--[no-]sparse`: - Sparse mode support (default:enabled on file, disabled on stdout) - -* `-l`: - Use Legacy format (typically for Linux Kernel compression)
- Note : `-l` is not compatible with `-m` (`--multiple`) nor `-r` - -### Other options - -* `-v` `--verbose`: - Verbose mode - -* `-q` `--quiet`: - Suppress warnings and real-time statistics; - specify twice to suppress errors too - -* `-h` `-H` `--help`: - Display help/long help and exit - -* `-V` `--version`: - Display Version number and exit - -* `-k` `--keep`: - Preserve source files (default behavior) - -* `--rm` : - Delete source files on successful compression or decompression - -* `--` : - Treat all subsequent arguments as files - - -### Benchmark mode - -* `-b#`: - Benchmark file(s), using # compression level - -* `-e#`: - Benchmark multiple compression levels, from b# to e# (included) - -* `-i#`: - Minimum evaluation in seconds \[1-9\] (default : 3) - - -BUGS ----- - -Report bugs at: https://github.com/lz4/lz4/issues - - -AUTHOR ------- - -Yann Collet diff --git a/library/lz4-mac/programs/lz4cli.c b/library/lz4-mac/programs/lz4cli.c deleted file mode 100755 index 362ba49f..00000000 --- a/library/lz4-mac/programs/lz4cli.c +++ /dev/null @@ -1,670 +0,0 @@ -/* - LZ4cli - LZ4 Command Line Interface - Copyright (C) Yann Collet 2011-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -/* - Note : this is stand-alone program. - It is not part of LZ4 compression library, it is a user program of the LZ4 library. - The license of LZ4 library is BSD. - The license of xxHash library is BSD. - The license of this compression CLI program is GPLv2. -*/ - - -/**************************** -* Includes -*****************************/ -#include "platform.h" /* Compiler options, IS_CONSOLE */ -#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ -#include /* fprintf, getchar */ -#include /* exit, calloc, free */ -#include /* strcmp, strlen */ -#include "bench.h" /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */ -#include "lz4io.h" /* LZ4IO_compressFilename, LZ4IO_decompressFilename, LZ4IO_compressMultipleFilenames */ -#include "lz4hc.h" /* LZ4HC_CLEVEL_MAX */ -#include "lz4.h" /* LZ4_VERSION_STRING */ - - -/***************************** -* Constants -******************************/ -#define COMPRESSOR_NAME "LZ4 command line interface" -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_versionString(), AUTHOR -#define LZ4_EXTENSION ".lz4" -#define LZ4CAT "lz4cat" -#define UNLZ4 "unlz4" -#define LZ4_LEGACY "lz4c" -static int g_lz4c_legacy_commands = 0; - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -#define LZ4_BLOCKSIZEID_DEFAULT 7 - - -/*-************************************ -* Macros -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */ - - -/*-************************************ -* Exceptions -***************************************/ -#define DEBUG 0 -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -/*-************************************ -* Version modifiers -***************************************/ -#define DEFAULT_COMPRESSOR LZ4IO_compressFilename -#define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename -int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */ - - -/*-*************************** -* Functions -*****************************/ -static int usage(const char* exeName) -{ - DISPLAY( "Usage : \n"); - DISPLAY( " %s [arg] [input] [output] \n", exeName); - DISPLAY( "\n"); - DISPLAY( "input : a filename \n"); - DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); - DISPLAY( "Arguments : \n"); - DISPLAY( " -1 : Fast compression (default) \n"); - DISPLAY( " -9 : High compression \n"); - DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); - DISPLAY( " -z : force compression \n"); - DISPLAY( " -D FILE: use dictionary in FILE \n"); - DISPLAY( " -f : overwrite output without prompting \n"); - DISPLAY( " -k : preserve source files(s) (default) \n"); - DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); - DISPLAY( " -h/-H : display help/long help and exit \n"); - return 0; -} - -static int usage_advanced(const char* exeName) -{ - DISPLAY(WELCOME_MESSAGE); - usage(exeName); - DISPLAY( "\n"); - DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit \n"); - DISPLAY( " -v : verbose mode \n"); - DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); - DISPLAY( " -c : force write to standard output, even if it is the console\n"); - DISPLAY( " -t : test compressed file integrity\n"); - DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); -#ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories (sets also -m) \n"); -#endif - DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); - DISPLAY( " -B# : Block size [4-7] (default : 7) \n"); - DISPLAY( " -BD : Block dependency (improve compression ratio) \n"); - DISPLAY( " -BX : enable block checksum (default:disabled) \n"); - DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); - DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); - DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); - DISPLAY( "Benchmark arguments : \n"); - DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); - DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); - DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n"); - DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n"); - DISPLAY( " or predefined block size [4-7] (default: 7) \n"); - if (g_lz4c_legacy_commands) { - DISPLAY( "Legacy arguments : \n"); - DISPLAY( " -c0 : fast compression \n"); - DISPLAY( " -c1 : high compression \n"); - DISPLAY( " -c2,-hc: very high compression \n"); - DISPLAY( " -y : overwrite output without prompting \n"); - } - return 0; -} - -static int usage_longhelp(const char* exeName) -{ - usage_advanced(exeName); - DISPLAY( "\n"); - DISPLAY( "****************************\n"); - DISPLAY( "***** Advanced comment *****\n"); - DISPLAY( "****************************\n"); - DISPLAY( "\n"); - DISPLAY( "Which values can [output] have ? \n"); - DISPLAY( "---------------------------------\n"); - DISPLAY( "[output] : a filename \n"); - DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark); - DISPLAY( " '%s' to discard output (test mode) \n", NULL_OUTPUT); - DISPLAY( "[output] can be left empty. In this case, it receives the following value :\n"); - DISPLAY( " - if stdout is not the console, then [output] = stdout \n"); - DISPLAY( " - if stdout is console : \n"); - DISPLAY( " + for compression, output to filename%s \n", LZ4_EXTENSION); - DISPLAY( " + for decompression, output to filename without '%s'\n", LZ4_EXTENSION); - DISPLAY( " > if input filename has no '%s' extension : error \n", LZ4_EXTENSION); - DISPLAY( "\n"); - DISPLAY( "Compression levels : \n"); - DISPLAY( "---------------------\n"); - DISPLAY( "-0 ... -2 => Fast compression, all identicals\n"); - DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX); - DISPLAY( "\n"); - DISPLAY( "stdin, stdout and the console : \n"); - DISPLAY( "--------------------------------\n"); - DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n"); - DISPLAY( "%s will refuse to read from console, or write to console \n", exeName); - DISPLAY( "except if '-c' command is specified, to force output to console \n"); - DISPLAY( "\n"); - DISPLAY( "Simple example :\n"); - DISPLAY( "----------------\n"); - DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n"); - DISPLAY( " %s filename\n", exeName); - DISPLAY( "\n"); - DISPLAY( "Short arguments can be aggregated. For example :\n"); - DISPLAY( "----------------------------------\n"); - DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n"); - DISPLAY( " %s -9 -f filename \n", exeName); - DISPLAY( " is equivalent to :\n"); - DISPLAY( " %s -9f filename \n", exeName); - DISPLAY( "\n"); - DISPLAY( "%s can be used in 'pure pipe mode'. For example :\n", exeName); - DISPLAY( "-------------------------------------\n"); - DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); - DISPLAY( " generator | %s | consumer \n", exeName); - if (g_lz4c_legacy_commands) { - DISPLAY( "\n"); - DISPLAY( "***** Warning ***** \n"); - DISPLAY( "Legacy arguments take precedence. Therefore : \n"); - DISPLAY( "--------------------------------- \n"); - DISPLAY( " %s -hc filename \n", exeName); - DISPLAY( "means 'compress filename in high compression mode' \n"); - DISPLAY( "It is not equivalent to : \n"); - DISPLAY( " %s -h -c filename \n", exeName); - DISPLAY( "which displays help text and exits \n"); - } - return 0; -} - -static int badusage(const char* exeName) -{ - DISPLAYLEVEL(1, "Incorrect parameters\n"); - if (displayLevel >= 1) usage(exeName); - exit(1); -} - - -static void waitEnter(void) -{ - DISPLAY("Press enter to continue...\n"); - (void)getchar(); -} - -static const char* lastNameFromPath(const char* path) -{ - const char* name = path; - if (strrchr(name, '/')) name = strrchr(name, '/') + 1; - if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */ - return name; -} - -/*! exeNameMatch() : - @return : a non-zero value if exeName matches test, excluding the extension - */ -static int exeNameMatch(const char* exeName, const char* test) -{ - return !strncmp(exeName, test, strlen(test)) && - (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.'); -} - -/*! readU32FromChar() : - @return : unsigned integer value read from input in `char` format - allows and interprets K, KB, KiB, M, MB and MiB suffix. - Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : function result can overflow if digit string > MAX_UINT */ -static unsigned readU32FromChar(const char** stringPtr) -{ - unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) { - result *= 10; - result += **stringPtr - '0'; - (*stringPtr)++ ; - } - if ((**stringPtr=='K') || (**stringPtr=='M')) { - result <<= 10; - if (**stringPtr=='M') result <<= 10; - (*stringPtr)++ ; - if (**stringPtr=='i') (*stringPtr)++; - if (**stringPtr=='B') (*stringPtr)++; - } - return result; -} - -typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e; - -int main(int argc, const char** argv) -{ - int i, - cLevel=1, - cLevelLast=1, - legacy_format=0, - forceStdout=0, - main_pause=0, - multiple_inputs=0, - all_arguments_are_files=0, - operationResult=0; - operationMode_e mode = om_auto; - const char* input_filename = NULL; - const char* output_filename= NULL; - const char* dictionary_filename = NULL; - char* dynNameSpace = NULL; - const char** inFileNames = (const char**) calloc(argc, sizeof(char*)); - unsigned ifnIdx=0; - const char nullOutput[] = NULL_OUTPUT; - const char extension[] = LZ4_EXTENSION; - size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); - const char* const exeName = lastNameFromPath(argv[0]); -#ifdef UTIL_HAS_CREATEFILELIST - const char** extendedFileList = NULL; - char* fileNamesBuf = NULL; - unsigned fileNamesNb, recursive=0; -#endif - - /* Init */ - if (inFileNames==NULL) { - DISPLAY("Allocation error : not enough memory \n"); - return 1; - } - inFileNames[0] = stdinmark; - LZ4IO_setOverwrite(0); - - /* predefined behaviors, based on binary/link name */ - if (exeNameMatch(exeName, LZ4CAT)) { - mode = om_decompress; - LZ4IO_setOverwrite(1); - LZ4IO_setRemoveSrcFile(0); - forceStdout=1; - output_filename=stdoutmark; - displayLevel=1; - multiple_inputs=1; - } - if (exeNameMatch(exeName, UNLZ4)) { mode = om_decompress; } - if (exeNameMatch(exeName, LZ4_LEGACY)) { g_lz4c_legacy_commands=1; } - - /* command switches */ - for(i=1; i='0') && (*argument<='9')) { - cLevel = readU32FromChar(&argument); - argument--; - continue; - } - - - switch(argument[0]) - { - /* Display help */ - case 'V': DISPLAY(WELCOME_MESSAGE); goto _cleanup; /* Version */ - case 'h': usage_advanced(exeName); goto _cleanup; - case 'H': usage_longhelp(exeName); goto _cleanup; - - case 'e': - argument++; - cLevelLast = readU32FromChar(&argument); - argument--; - break; - - /* Compression (default) */ - case 'z': mode = om_compress; break; - - case 'D': - if (argument[1] == '\0') { - /* path is next arg */ - if (i + 1 == argc) { - /* there is no next arg */ - badusage(exeName); - } - dictionary_filename = argv[++i]; - } else { - /* path follows immediately */ - dictionary_filename = argument + 1; - } - /* skip to end of argument so that we jump to parsing next argument */ - argument += strlen(argument) - 1; - break; - - /* Use Legacy format (ex : Linux kernel compression) */ - case 'l': legacy_format = 1; blockSize = 8 MB; break; - - /* Decoding */ - case 'd': mode = om_decompress; break; - - /* Force stdout, even if stdout==console */ - case 'c': forceStdout=1; output_filename=stdoutmark; break; - - /* Test integrity */ - case 't': mode = om_test; break; - - /* Overwrite */ - case 'f': LZ4IO_setOverwrite(1); break; - - /* Verbose mode */ - case 'v': displayLevel++; break; - - /* Quiet mode */ - case 'q': if (displayLevel) displayLevel--; break; - - /* keep source file (default anyway, so useless) (for xz/lzma compatibility) */ - case 'k': LZ4IO_setRemoveSrcFile(0); break; - - /* Modify Block Properties */ - case 'B': - while (argument[1]!=0) { - int exitBlockProperties=0; - switch(argument[1]) - { - case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break; - case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* disabled by default */ - default : - if (argument[1] < '0' || argument[1] > '9') { - exitBlockProperties=1; - break; - } else { - unsigned B; - argument++; - B = readU32FromChar(&argument); - argument--; - if (B < 4) badusage(exeName); - if (B <= 7) { - blockSize = LZ4IO_setBlockSizeID(B); - BMK_SetBlockSize(blockSize); - DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); - } else { - if (B < 32) badusage(exeName); - BMK_SetBlockSize(B); - if (B >= 1024) { - DISPLAYLEVEL(2, "bench: using blocks of size %u KB \n", (U32)(B>>10)); - } else { - DISPLAYLEVEL(2, "bench: using blocks of size %u bytes \n", (U32)(B)); - } - } - break; - } - } - if (exitBlockProperties) break; - } - break; - - /* Benchmark */ - case 'b': mode = om_bench; multiple_inputs=1; - break; - -#ifdef UTIL_HAS_CREATEFILELIST - /* recursive */ - case 'r': recursive=1; -#endif - /* fall-through */ - /* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */ - case 'm': multiple_inputs=1; - break; - - /* Modify Nb Seconds (benchmark only) */ - case 'i': - { unsigned iters; - argument++; - iters = readU32FromChar(&argument); - argument--; - BMK_setNotificationLevel(displayLevel); - BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */ - } - break; - - /* Pause at the end (hidden option) */ - case 'p': main_pause=1; break; - - /* Unrecognised command */ - default : badusage(exeName); - } - } - continue; - } - - /* Store in *inFileNames[] if -m is used. */ - if (multiple_inputs) { inFileNames[ifnIdx++]=argument; continue; } - - /* Store first non-option arg in input_filename to preserve original cli logic. */ - if (!input_filename) { input_filename=argument; continue; } - - /* Second non-option arg in output_filename to preserve original cli logic. */ - if (!output_filename) { - output_filename=argument; - if (!strcmp (output_filename, nullOutput)) output_filename = nulmark; - continue; - } - - /* 3rd non-option arg should not exist */ - DISPLAYLEVEL(1, "Warning : %s won't be used ! Do you want multiple input files (-m) ? \n", argument); - } - - DISPLAYLEVEL(3, WELCOME_MESSAGE); -#ifdef _POSIX_C_SOURCE - DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE); -#endif -#ifdef _POSIX_VERSION - DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION); -#endif -#ifdef PLATFORM_POSIX_VERSION - DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); -#endif -#ifdef _FILE_OFFSET_BITS - DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS); -#endif - if ((mode == om_compress) || (mode == om_bench)) - DISPLAYLEVEL(4, "Blocks size : %u KB\n", (U32)(blockSize>>10)); - - if (multiple_inputs) { - input_filename = inFileNames[0]; -#ifdef UTIL_HAS_CREATEFILELIST - if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ - extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb); - if (extendedFileList) { - unsigned u; - for (u=0; u try to select one automatically (when possible) */ - while ((!output_filename) && (multiple_inputs==0)) { - if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ - if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ - size_t const inSize = strlen(input_filename); - size_t const extSize = strlen(LZ4_EXTENSION); - size_t const extStart= (inSize > extSize) ? inSize-extSize : 0; - if (!strcmp(input_filename+extStart, LZ4_EXTENSION)) mode = om_decompress; - else mode = om_compress; - } - if (mode == om_compress) { /* compression to file */ - size_t const l = strlen(input_filename); - dynNameSpace = (char*)calloc(1,l+5); - if (dynNameSpace==NULL) { perror(exeName); exit(1); } - strcpy(dynNameSpace, input_filename); - strcat(dynNameSpace, LZ4_EXTENSION); - output_filename = dynNameSpace; - DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); - break; - } - if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */ - size_t outl; - size_t const inl = strlen(input_filename); - dynNameSpace = (char*)calloc(1,inl+1); - if (dynNameSpace==NULL) { perror(exeName); exit(1); } - strcpy(dynNameSpace, input_filename); - outl = inl; - if (inl>4) - while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0; - if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); } - output_filename = dynNameSpace; - DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); - } - break; - } - - /* Check if output is defined as console; trigger an error in this case */ - if (!output_filename) output_filename = "*\\dummy^!//"; - if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { - DISPLAYLEVEL(1, "refusing to write to console without -c\n"); - exit(1); - } - /* Downgrade notification level in stdout and multiple file mode */ - if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; - if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; - - /* IO Stream/File */ - LZ4IO_setNotificationLevel(displayLevel); - if (ifnIdx == 0) multiple_inputs = 0; - if (mode == om_decompress) { - if (multiple_inputs) - operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); - else - operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename); - } else { /* compression is default action */ - if (legacy_format) { - DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n"); - LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); - } else { - if (multiple_inputs) - operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel); - else - operationResult = DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); - } - } - -_cleanup: - if (main_pause) waitEnter(); - free(dynNameSpace); -#ifdef UTIL_HAS_CREATEFILELIST - if (extendedFileList) { - UTIL_freeFileList(extendedFileList, fileNamesBuf); - inFileNames = NULL; - } -#endif - free((void*)inFileNames); - return operationResult; -} diff --git a/library/lz4-mac/programs/lz4io.c b/library/lz4-mac/programs/lz4io.c deleted file mode 100755 index 57434f79..00000000 --- a/library/lz4-mac/programs/lz4io.c +++ /dev/null @@ -1,1148 +0,0 @@ -/* - LZ4io.c - LZ4 File/Stream Interface - Copyright (C) Yann Collet 2011-2017 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -/* - Note : this is stand-alone program. - It is not part of LZ4 compression library, it is a user code of the LZ4 library. - - The license of LZ4 library is BSD. - - The license of xxHash library is BSD. - - The license of this source file is GPLv2. -*/ - - -/*-************************************ -* Compiler options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ -#endif - - -/***************************** -* Includes -*****************************/ -#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */ -#include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ -#include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ -#include /* malloc, free */ -#include /* strerror, strcmp, strlen */ -#include /* clock */ -#include /* stat64 */ -#include /* stat64 */ -#include "lz4io.h" -#include "lz4.h" /* still required for legacy format */ -#include "lz4hc.h" /* still required for legacy format */ -#include "lz4frame.h" -#include "lz4frame_static.h" - - -/***************************** -* Constants -*****************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define _1BIT 0x01 -#define _2BITS 0x03 -#define _3BITS 0x07 -#define _4BITS 0x0F -#define _8BITS 0xFF - -#define MAGICNUMBER_SIZE 4 -#define LZ4IO_MAGICNUMBER 0x184D2204 -#define LZ4IO_SKIPPABLE0 0x184D2A50 -#define LZ4IO_SKIPPABLEMASK 0xFFFFFFF0 -#define LEGACY_MAGICNUMBER 0x184C2102 - -#define CACHELINE 64 -#define LEGACY_BLOCKSIZE (8 MB) -#define MIN_STREAM_BUFSIZE (192 KB) -#define LZ4IO_BLOCKSIZEID_DEFAULT 7 -#define LZ4_MAX_DICT_SIZE (64 KB) - - -/************************************** -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if (((clock_t)(g_time - clock()) > refreshRate) || (g_displayLevel>=4)) \ - { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } } -static const clock_t refreshRate = CLOCKS_PER_SEC / 6; -static clock_t g_time = 0; - - -/************************************** -* Local Parameters -**************************************/ -static int g_overwrite = 1; -static int g_testMode = 0; -static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; -static int g_blockChecksum = 0; -static int g_streamChecksum = 1; -static int g_blockIndependence = 1; -static int g_sparseFileSupport = 1; -static int g_contentSizeFlag = 0; -static int g_useDictionary = 0; -static const char* g_dictionaryFilename = NULL; - - -/************************************** -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, " \n"); \ - exit(error); \ -} - - -/************************************** -* Version modifiers -**************************************/ -#define EXTENDED_ARGUMENTS -#define EXTENDED_HELP -#define EXTENDED_FORMAT -#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F - - -/* ************************************************** */ -/* ****************** Parameters ******************** */ -/* ************************************************** */ - -int LZ4IO_setDictionaryFilename(const char* dictionaryFilename) { - g_dictionaryFilename = dictionaryFilename; - g_useDictionary = dictionaryFilename != NULL; - return g_useDictionary; -} - -/* Default setting : overwrite = 1; return : overwrite mode (0/1) */ -int LZ4IO_setOverwrite(int yes) -{ - g_overwrite = (yes!=0); - return g_overwrite; -} - -/* Default setting : testMode = 0; return : testMode (0/1) */ -int LZ4IO_setTestMode(int yes) -{ - g_testMode = (yes!=0); - return g_testMode; -} - -/* blockSizeID : valid values : 4-5-6-7 */ -size_t LZ4IO_setBlockSizeID(unsigned bsid) -{ - static const size_t blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; - static const unsigned minBlockSizeID = 4; - static const unsigned maxBlockSizeID = 7; - if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0; - g_blockSizeId = bsid; - return blockSizeTable[g_blockSizeId-minBlockSizeID]; -} - -int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) -{ - g_blockIndependence = (blockMode == LZ4IO_blockIndependent); - return g_blockIndependence; -} - -/* Default setting : no block checksum */ -int LZ4IO_setBlockChecksumMode(int enable) -{ - g_blockChecksum = (enable != 0); - return g_blockChecksum; -} - -/* Default setting : checksum enabled */ -int LZ4IO_setStreamChecksumMode(int enable) -{ - g_streamChecksum = (enable != 0); - return g_streamChecksum; -} - -/* Default setting : 0 (no notification) */ -int LZ4IO_setNotificationLevel(int level) -{ - g_displayLevel = level; - return g_displayLevel; -} - -/* Default setting : 0 (disabled) */ -int LZ4IO_setSparseFile(int enable) -{ - g_sparseFileSupport = (enable!=0); - return g_sparseFileSupport; -} - -/* Default setting : 0 (disabled) */ -int LZ4IO_setContentSize(int enable) -{ - g_contentSizeFlag = (enable!=0); - return g_contentSizeFlag; -} - -static U32 g_removeSrcFile = 0; -void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } - - - -/* ************************************************************************ ** -** ********************** LZ4 File / Pipe compression ********************* ** -** ************************************************************************ */ - -static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } -static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { - return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; -} - - -/** LZ4IO_openSrcFile() : - * condition : `srcFileName` must be non-NULL. - * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* LZ4IO_openSrcFile(const char* srcFileName) -{ - FILE* f; - - if (!strcmp (srcFileName, stdinmark)) { - DISPLAYLEVEL(4,"Using stdin for input\n"); - f = stdin; - SET_BINARY_MODE(stdin); - } else { - f = fopen(srcFileName, "rb"); - if ( f==NULL ) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno)); - } - - return f; -} - -/** FIO_openDstFile() : - * condition : `dstFileName` must be non-NULL. - * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* LZ4IO_openDstFile(const char* dstFileName) -{ - FILE* f; - - if (!strcmp (dstFileName, stdoutmark)) { - DISPLAYLEVEL(4,"Using stdout for output\n"); - f = stdout; - SET_BINARY_MODE(stdout); - if (g_sparseFileSupport==1) { - g_sparseFileSupport = 0; - DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); - } - } else { - if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ - f = fopen( dstFileName, "rb" ); - if (f != NULL) { /* dest exists, prompt for overwrite authorization */ - fclose(f); - if (g_displayLevel <= 1) { /* No interaction possible */ - DISPLAY("%s already exists; not overwritten \n", dstFileName); - return NULL; - } - DISPLAY("%s already exists; do you wish to overwrite (y/N) ? ", dstFileName); - { int ch = getchar(); - if ((ch!='Y') && (ch!='y')) { - DISPLAY(" not overwritten \n"); - return NULL; - } - while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ - } } } - f = fopen( dstFileName, "wb" ); - if (f==NULL) DISPLAYLEVEL(1, "%s: %s\n", dstFileName, strerror(errno)); - } - - /* sparse file */ - if (f && g_sparseFileSupport) { SET_SPARSE_FILE_MODE(f); } - - return f; -} - - - -/*************************************** -* Legacy Compression -***************************************/ - -/* unoptimized version; solves endianess & alignment issues */ -static void LZ4IO_writeLE32 (void* p, unsigned value32) -{ - unsigned char* const dstPtr = (unsigned char*)p; - dstPtr[0] = (unsigned char)value32; - dstPtr[1] = (unsigned char)(value32 >> 8); - dstPtr[2] = (unsigned char)(value32 >> 16); - dstPtr[3] = (unsigned char)(value32 >> 24); -} - -static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSize, int cLevel) -{ - (void)cLevel; - return LZ4_compress_fast(src, dst, srcSize, dstSize, 1); -} - -/* LZ4IO_compressFilename_Legacy : - * This function is intentionally "hidden" (not published in .h) - * It generates compressed streams using the old 'legacy' format */ -int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) -{ - int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); - unsigned long long filesize = 0; - unsigned long long compressedfilesize = MAGICNUMBER_SIZE; - char* in_buff; - char* out_buff; - const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE); - FILE* finput; - FILE* foutput; - clock_t clockEnd; - - /* Init */ - clock_t const clockStart = clock(); - compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC; - - finput = LZ4IO_openSrcFile(input_filename); - if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename); - foutput = LZ4IO_openDstFile(output_filename); - if (foutput == NULL) { fclose(finput); EXM_THROW(20, "%s : open file error ", input_filename); } - - /* Allocate Memory */ - in_buff = (char*)malloc(LEGACY_BLOCKSIZE); - out_buff = (char*)malloc(outBuffSize); - if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); - - /* Write Archive Header */ - LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); - { size_t const sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); } - - /* Main Loop */ - while (1) { - unsigned int outSize; - /* Read Block */ - size_t const inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); - if (inSize == 0) break; - if (inSize > LEGACY_BLOCKSIZE) EXM_THROW(23, "Read error : wrong fread() size report "); /* should be impossible */ - filesize += inSize; - - /* Compress Block */ - outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel); - compressedfilesize += outSize+4; - DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", - (int)(filesize>>20), (double)compressedfilesize/filesize*100); - - /* Write Block */ - LZ4IO_writeLE32(out_buff, outSize); - { size_t const sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); - if (sizeCheck!=(size_t)(outSize+4)) - EXM_THROW(24, "Write error : cannot write compressed block"); - } } - if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename); - - /* Status */ - clockEnd = clock(); - if (clockEnd==clockStart) clockEnd+=1; /* avoid division by zero (speed) */ - filesize += !filesize; /* avoid division by zero (ratio) */ - DISPLAYLEVEL(2, "\r%79s\r", ""); /* blank line */ - DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - filesize, compressedfilesize, (double)compressedfilesize / filesize * 100); - { double const seconds = (double)(clockEnd - clockStart) / CLOCKS_PER_SEC; - DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, - (double)filesize / seconds / 1024 / 1024); - } - - /* Close & Free */ - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - - return 0; -} - - -/********************************************* -* Compression using Frame format -*********************************************/ - -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; - LZ4F_compressionContext_t ctx; - LZ4F_CDict* cdict; -} cRess_t; - -static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { - size_t readSize; - size_t dictEnd = 0; - size_t dictLen = 0; - size_t dictStart; - size_t circularBufSize = LZ4_MAX_DICT_SIZE; - char* circularBuf; - char* dictBuf; - FILE* dictFile; - - if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided"); - - circularBuf = (char *) malloc(circularBufSize); - if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory"); - - dictFile = LZ4IO_openSrcFile(dictFilename); - if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file"); - - /* opportunistically seek to the part of the file we care about. If this */ - /* fails it's not a problem since we'll just read everything anyways. */ - if (strcmp(dictFilename, stdinmark)) { - UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END); - } - - do { - readSize = fread(circularBuf + dictEnd, 1, circularBufSize - dictEnd, dictFile); - dictEnd = (dictEnd + readSize) % circularBufSize; - dictLen += readSize; - } while (readSize>0); - - if (dictLen > LZ4_MAX_DICT_SIZE) { - dictLen = LZ4_MAX_DICT_SIZE; - } - - *dictSize = dictLen; - - dictStart = (circularBufSize + dictEnd - dictLen) % circularBufSize; - - if (dictStart == 0) { - /* We're in the simple case where the dict starts at the beginning of our circular buffer. */ - dictBuf = circularBuf; - circularBuf = NULL; - } else { - /* Otherwise, we will alloc a new buffer and copy our dict into that. */ - dictBuf = (char *) malloc(dictLen ? dictLen : 1); - if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory"); - - memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart); - memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart)); - } - - free(circularBuf); - - return dictBuf; -} - -static LZ4F_CDict* LZ4IO_createCDict(void) { - size_t dictionarySize; - void* dictionaryBuffer; - LZ4F_CDict* cdict; - if (!g_useDictionary) { - return NULL; - } - dictionaryBuffer = LZ4IO_createDict(g_dictionaryFilename, &dictionarySize); - if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); - cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize); - free(dictionaryBuffer); - return cdict; -} - -static cRess_t LZ4IO_createCResources(void) -{ - const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); - cRess_t ress; - - LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); - - /* Allocate Memory */ - ress.srcBuffer = malloc(blockSize); - ress.srcBufferSize = blockSize; - ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */ - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); - - ress.cdict = LZ4IO_createCDict(); - - return ress; -} - -static void LZ4IO_freeCResources(cRess_t ress) -{ - free(ress.srcBuffer); - free(ress.dstBuffer); - - LZ4F_freeCDict(ress.cdict); - ress.cdict = NULL; - - { LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx); - if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); } -} - -/* - * LZ4IO_compressFilename_extRess() - * result : 0 : compression completed correctly - * 1 : missing or pb opening srcFileName - */ -static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) -{ - unsigned long long filesize = 0; - unsigned long long compressedfilesize = 0; - FILE* srcFile; - FILE* dstFile; - void* const srcBuffer = ress.srcBuffer; - void* const dstBuffer = ress.dstBuffer; - const size_t dstBufferSize = ress.dstBufferSize; - const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); - size_t readSize; - LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */ - LZ4F_preferences_t prefs; - - /* Init */ - srcFile = LZ4IO_openSrcFile(srcFileName); - if (srcFile == NULL) return 1; - dstFile = LZ4IO_openDstFile(dstFileName); - if (dstFile == NULL) { fclose(srcFile); return 1; } - memset(&prefs, 0, sizeof(prefs)); - - - /* Set compression parameters */ - prefs.autoFlush = 1; - prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence; - prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; - prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum; - prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; - if (g_contentSizeFlag) { - U64 const fileSize = UTIL_getFileSize(srcFileName); - prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ - if (fileSize==0) - DISPLAYLEVEL(3, "Warning : cannot determine input content size \n"); - } - - /* read first block */ - readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); - if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName); - filesize += readSize; - - /* single-block file */ - if (readSize < blockSize) { - /* Compress in single pass */ - size_t cSize = LZ4F_compressFrame_usingCDict(dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs); - if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); - compressedfilesize = cSize; - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", - (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ - - /* Write Block */ - { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block"); - } } - - else - - /* multiple-blocks file */ - { - /* Write Archive Header */ - size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs); - if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); - { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } - compressedfilesize += headerSize; - - /* Main Loop */ - while (readSize>0) { - size_t outSize; - - /* Compress Block */ - outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); - if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize)); - compressedfilesize += outSize; - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); - - /* Write Block */ - { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); - if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } - - /* Read next block */ - readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); - filesize += readSize; - } - if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); - - /* End of Stream mark */ - headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); - if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); - - { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } - compressedfilesize += headerSize; - } - - /* Release files */ - fclose (srcFile); - fclose (dstFile); - - /* Copy owner, file permissions and modification time */ - { stat_t statbuf; - if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && UTIL_getFileStat(srcFileName, &statbuf)) - UTIL_setFileStat(dstFileName, &statbuf); - } - - if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ - - /* Final Status */ - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */ - - return 0; -} - - -int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) -{ - clock_t const start = clock(); - cRess_t const ress = LZ4IO_createCResources(); - - int const issueWithSrcFile = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); - - /* Free resources */ - LZ4IO_freeCResources(ress); - - /* Final Status */ - { clock_t const end = clock(); - double const seconds = (double)(end - start) / CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); - } - - return issueWithSrcFile; -} - - -#define FNSPACE 30 -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) -{ - int i; - int missed_files = 0; - char* dstFileName = (char*)malloc(FNSPACE); - size_t ofnSize = FNSPACE; - const size_t suffixSize = strlen(suffix); - cRess_t ress; - - if (dstFileName == NULL) return ifntSize; /* not enough memory */ - ress = LZ4IO_createCResources(); - - /* loop on each file */ - for (i=0; i 1 GB) { - int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR); - if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); - storedSkips -= 1 GB; - } - - while (ptrT < bufferTEnd) { - size_t seg0SizeT = segmentSizeT; - size_t nb0T; - - /* count leading zeros */ - if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; - bufferSizeT -= seg0SizeT; - for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; - storedSkips += (unsigned)(nb0T * sizeT); - - if (nb0T != seg0SizeT) { /* not all 0s */ - errno = 0; - { int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); - } - storedSkips = 0; - seg0SizeT -= nb0T; - ptrT += nb0T; - { size_t const sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file); - if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block"); - } } - ptrT += seg0SizeT; - } - - if (bufferSize & maskT) { /* size not multiple of sizeT : implies end of block */ - const char* const restStart = (const char*)bufferTEnd; - const char* restPtr = restStart; - size_t const restSize = bufferSize & maskT; - const char* const restEnd = restStart + restSize; - for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; - storedSkips += (unsigned) (restPtr - restStart); - if (restPtr != restEnd) { - int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); - storedSkips = 0; - { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); - if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); - } } - } - - return storedSkips; -} - -static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) -{ - if (storedSkips>0) { /* implies g_sparseFileSupport>0 */ - int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR); - if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); - { const char lastZeroByte[1] = { 0 }; - size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); - if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); - } } -} - - -static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */ -static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) -{ - unsigned long long streamSize = 0; - unsigned storedSkips = 0; - - /* Allocate Memory */ - char* const in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); - char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE); - if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); - - /* Main Loop */ - while (1) { - unsigned int blockSize; - - /* Block Size */ - { size_t const sizeCheck = fread(in_buff, 1, 4, finput); - if (sizeCheck == 0) break; /* Nothing to read : file read is completed */ - if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); } - blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */ - if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) { - /* Cannot read next block : maybe new stream ? */ - g_magicRead = blockSize; - break; - } - - /* Read Block */ - { size_t const sizeCheck = fread(in_buff, 1, blockSize, finput); - if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); } - - /* Decode Block */ - { int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); - if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); - streamSize += decodeSize; - /* Write Block */ - storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); /* success or die */ - } } - if (ferror(finput)) EXM_THROW(54, "Read error : ferror"); - - LZ4IO_fwriteSparseEnd(foutput, storedSkips); - - /* Free */ - free(in_buff); - free(out_buff); - - return streamSize; -} - - - -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; - FILE* dstFile; - LZ4F_decompressionContext_t dCtx; - void* dictBuffer; - size_t dictBufferSize; -} dRess_t; - -static void LZ4IO_loadDDict(dRess_t* ress) { - if (!g_useDictionary) { - ress->dictBuffer = NULL; - ress->dictBufferSize = 0; - return; - } - - ress->dictBuffer = LZ4IO_createDict(g_dictionaryFilename, &ress->dictBufferSize); - if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); -} - -static const size_t LZ4IO_dBufferSize = 64 KB; -static dRess_t LZ4IO_createDResources(void) -{ - dRess_t ress; - - /* init */ - LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); - - /* Allocate Memory */ - ress.srcBufferSize = LZ4IO_dBufferSize; - ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = LZ4IO_dBufferSize; - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - - LZ4IO_loadDDict(&ress); - - ress.dstFile = NULL; - return ress; -} - -static void LZ4IO_freeDResources(dRess_t ress) -{ - LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx); - if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); - free(ress.srcBuffer); - free(ress.dstBuffer); - free(ress.dictBuffer); -} - - -static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile) -{ - unsigned long long filesize = 0; - LZ4F_errorCode_t nextToLoad; - unsigned storedSkips = 0; - - /* Init feed with magic number (already consumed from FILE* sFile) */ - { size_t inSize = MAGICNUMBER_SIZE; - size_t outSize= 0; - LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER); - nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL); - if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); - } - - /* Main Loop */ - for (;nextToLoad;) { - size_t readSize; - size_t pos = 0; - size_t decodedBytes = ress.dstBufferSize; - - /* Read input */ - if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize; - readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile); - if (!readSize) break; /* reached end of file or stream */ - - while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) { /* still to read, or still to flush */ - /* Decode Input (at least partially) */ - size_t remaining = readSize - pos; - decodedBytes = ress.dstBufferSize; - nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL); - if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); - pos += remaining; - - /* Write Block */ - if (decodedBytes) { - if (!g_testMode) - storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); - filesize += decodedBytes; - DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); - } - - if (!nextToLoad) break; - } - } - /* can be out because readSize == 0, which could be an fread() error */ - if (ferror(srcFile)) EXM_THROW(67, "Read error"); - - if (!g_testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips); - if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream"); - - return filesize; -} - - -#define PTSIZE (64 KB) -#define PTSIZET (PTSIZE / sizeof(size_t)) -static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) -{ - size_t buffer[PTSIZET]; - size_t readBytes = 1; - unsigned long long total = MAGICNUMBER_SIZE; - unsigned storedSkips = 0; - - size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); - - while (readBytes) { - readBytes = fread(buffer, 1, PTSIZE, finput); - total += readBytes; - storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, storedSkips); - } - if (ferror(finput)) EXM_THROW(51, "Read Error") - - LZ4IO_fwriteSparseEnd(foutput, storedSkips); - return total; -} - - -/** Safely handle cases when (unsigned)offset > LONG_MAX */ -static int fseek_u32(FILE *fp, unsigned offset, int where) -{ - const unsigned stepMax = 1U << 30; - int errorNb = 0; - - if (where != SEEK_CUR) return -1; /* Only allows SEEK_CUR */ - while (offset > 0) { - unsigned s = offset; - if (s > stepMax) s = stepMax; - errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR); - if (errorNb != 0) break; - offset -= s; - } - return errorNb; -} - -#define ENDOFSTREAM ((unsigned long long)-1) -static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) -{ - unsigned char MNstore[MAGICNUMBER_SIZE]; - unsigned magicNumber; - static unsigned nbFrames = 0; - - /* init */ - nbFrames++; - - /* Check Archive Header */ - if (g_magicRead) { /* magic number already read from finput (see legacy frame)*/ - magicNumber = g_magicRead; - g_magicRead = 0; - } else { - size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes==0) { nbFrames = 0; return ENDOFSTREAM; } /* EOF */ - if (nbReadBytes != MAGICNUMBER_SIZE) - EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); - magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ - } - if (LZ4IO_isSkippableMagicNumber(magicNumber)) - magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ - - switch(magicNumber) - { - case LZ4IO_MAGICNUMBER: - return LZ4IO_decompressLZ4F(ress, finput, foutput); - case LEGACY_MAGICNUMBER: - DISPLAYLEVEL(4, "Detected : Legacy format \n"); - return LZ4IO_decodeLegacyStream(finput, foutput); - case LZ4IO_SKIPPABLE0: - DISPLAYLEVEL(4, "Skipping detected skippable area \n"); - { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); - if (nbReadBytes != 4) - EXM_THROW(42, "Stream error : skippable size unreadable"); - } - { unsigned const size = LZ4IO_readLE32(MNstore); - int const errorNb = fseek_u32(finput, size, SEEK_CUR); - if (errorNb != 0) - EXM_THROW(43, "Stream error : cannot skip skippable area"); - } - return 0; - EXTENDED_FORMAT; /* macro extension for custom formats */ - default: - if (nbFrames == 1) { /* just started */ - /* Wrong magic number at the beginning of 1st stream */ - if (!g_testMode && g_overwrite) { - nbFrames = 0; - return LZ4IO_passThrough(finput, foutput, MNstore); - } - EXM_THROW(44,"Unrecognized header : file cannot be decoded"); - } - { long int const position = ftell(finput); /* only works for files < 2 GB */ - DISPLAYLEVEL(2, "Stream followed by undecodable data "); - if (position != -1L) - DISPLAYLEVEL(2, "at position %i ", (int)position); - DISPLAYLEVEL(2, "\n"); - } - return ENDOFSTREAM; - } -} - - -static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename) -{ - FILE* const foutput = ress.dstFile; - unsigned long long filesize = 0; - - /* Init */ - FILE* const finput = LZ4IO_openSrcFile(input_filename); - if (finput==NULL) return 1; - - /* Loop over multiple streams */ - for ( ; ; ) { /* endless loop, see break condition */ - unsigned long long const decodedSize = - selectDecoder(ress, finput, foutput); - if (decodedSize == ENDOFSTREAM) break; - filesize += decodedSize; - } - - /* Close input */ - fclose(finput); - if (g_removeSrcFile) { /* --rm */ - if (remove(input_filename)) - EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); - } - - /* Final Status */ - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%-20.20s : decoded %llu bytes \n", input_filename, filesize); - (void)output_filename; - - return 0; -} - - -static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename) -{ - FILE* const foutput = LZ4IO_openDstFile(output_filename); - if (foutput==NULL) return 1; /* failure */ - - ress.dstFile = foutput; - LZ4IO_decompressSrcFile(ress, input_filename, output_filename); - - fclose(foutput); - - /* Copy owner, file permissions and modification time */ - { stat_t statbuf; - if ( strcmp (input_filename, stdinmark) - && strcmp (output_filename, stdoutmark) - && strcmp (output_filename, nulmark) - && UTIL_getFileStat(input_filename, &statbuf) ) { - UTIL_setFileStat(output_filename, &statbuf); - /* should return value be read ? or is silent fail good enough ? */ - } } - - return 0; -} - - -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) -{ - dRess_t const ress = LZ4IO_createDResources(); - clock_t const start = clock(); - - int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename); - - clock_t const end = clock(); - double const seconds = (double)(end - start) / CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); - - LZ4IO_freeDResources(ress); - return missingFiles; -} - - -int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) -{ - int i; - int skippedFiles = 0; - int missingFiles = 0; - char* outFileName = (char*)malloc(FNSPACE); - size_t ofnSize = FNSPACE; - size_t const suffixSize = strlen(suffix); - dRess_t ress = LZ4IO_createDResources(); - - if (outFileName==NULL) return ifntSize; /* not enough memory */ - ress.dstFile = LZ4IO_openDstFile(stdoutmark); - - for (i=0; i /* size_t */ - - -/* ************************************************** */ -/* Special input/output values */ -/* ************************************************** */ -#define NULL_OUTPUT "null" -static const char stdinmark[] = "stdin"; -static const char stdoutmark[] = "stdout"; -#ifdef _WIN32 -static const char nulmark[] = "nul"; -#else -static const char nulmark[] = "/dev/null"; -#endif - - -/* ************************************************** */ -/* ****************** Functions ********************* */ -/* ************************************************** */ - -int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel); -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename); - -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); -int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix); - - -/* ************************************************** */ -/* ****************** Parameters ******************** */ -/* ************************************************** */ - -int LZ4IO_setDictionaryFilename(const char* dictionaryFilename); - -/* Default setting : overwrite = 1; - return : overwrite mode (0/1) */ -int LZ4IO_setOverwrite(int yes); - -/* Default setting : testMode = 0; - return : testMode (0/1) */ -int LZ4IO_setTestMode(int yes); - -/* blockSizeID : valid values : 4-5-6-7 - return : 0 if error, blockSize if OK */ -size_t LZ4IO_setBlockSizeID(unsigned blockSizeID); - -/* Default setting : independent blocks */ -typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t; -int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode); - -/* Default setting : no block checksum */ -int LZ4IO_setBlockChecksumMode(int xxhash); - -/* Default setting : stream checksum enabled */ -int LZ4IO_setStreamChecksumMode(int xxhash); - -/* Default setting : 0 (no notification) */ -int LZ4IO_setNotificationLevel(int level); - -/* Default setting : 0 (disabled) */ -int LZ4IO_setSparseFile(int enable); - -/* Default setting : 0 (disabled) */ -int LZ4IO_setContentSize(int enable); - -void LZ4IO_setRemoveSrcFile(unsigned flag); - - -#endif /* LZ4IO_H_237902873 */ diff --git a/library/lz4-mac/programs/platform.h b/library/lz4-mac/programs/platform.h deleted file mode 100755 index 66491b68..00000000 --- a/library/lz4-mac/programs/platform.h +++ /dev/null @@ -1,154 +0,0 @@ -/* - platform.h - compiler and OS detection - Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ - -#ifndef PLATFORM_H_MODULE -#define PLATFORM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/* ************************************** -* Compiler Options -****************************************/ -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ -# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ -# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -# endif -#endif - - -/* ************************************** -* Detect 64-bit OS -* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros -****************************************/ -#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ - || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ - || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ - || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ - || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ - || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ - || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ - || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ -# if !defined(__64BIT__) -# define __64BIT__ 1 -# endif -#endif - - -/* ********************************************************* -* Turn on Large Files support (>4GB) for 32-bit Linux/Unix -***********************************************************/ -#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ -# if !defined(_FILE_OFFSET_BITS) -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# endif -# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ -# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ -# endif -# if defined(_AIX) || defined(__hpux) -# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ -# endif -#endif - - -/* ************************************************************ -* Detect POSIX version -* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows -* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX -* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION -***************************************************************/ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ - || defined(__midipix__) || defined(__VMS)) -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ - || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ -# define PLATFORM_POSIX_VERSION 200112L -# else -# if defined(__linux__) || defined(__linux) -# ifndef _POSIX_C_SOURCE -# define _POSIX_C_SOURCE 200112L /* use feature test macro */ -# endif -# endif -# include /* declares _POSIX_VERSION */ -# if defined(_POSIX_VERSION) /* POSIX compliant */ -# define PLATFORM_POSIX_VERSION _POSIX_VERSION -# else -# define PLATFORM_POSIX_VERSION 0 -# endif -# endif -#endif -#if !defined(PLATFORM_POSIX_VERSION) -# define PLATFORM_POSIX_VERSION -1 -#endif - - -/*-********************************************* -* Detect if isatty() and fileno() are available -************************************************/ -#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__) -# include /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#elif defined(WIN32) || defined(_WIN32) -# include /* _isatty */ -# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include /* FILE */ -static __inline int IS_CONSOLE(FILE* stdStream) -{ - DWORD dummy; - return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); -} -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - -/****************************** -* OS-specific Includes -******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include /* _O_BINARY */ -# include /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include /* FSCTL_SET_SPARSE */ -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) -#endif - - - -#if defined (__cplusplus) -} -#endif - -#endif /* PLATFORM_H_MODULE */ diff --git a/library/lz4-mac/programs/util.h b/library/lz4-mac/programs/util.h deleted file mode 100755 index 5a69c55c..00000000 --- a/library/lz4-mac/programs/util.h +++ /dev/null @@ -1,494 +0,0 @@ -/* - util.h - utility functions - Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ - -#ifndef UTIL_H_MODULE -#define UTIL_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/*-**************************************** -* Dependencies -******************************************/ -#include "platform.h" /* PLATFORM_POSIX_VERSION */ -#include /* malloc */ -#include /* size_t, ptrdiff_t */ -#include /* fprintf */ -#include /* stat, utime */ -#include /* stat */ -#if defined(_MSC_VER) -# include /* utime */ -# include /* _chmod */ -#else -# include /* chown, stat */ -# include /* utime */ -#endif -#include /* time */ -#include - - - -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif - - -/* ************************************************************ -* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW -***************************************************************/ -#if defined(_MSC_VER) && (_MSC_VER >= 1400) -# define UTIL_fseek _fseeki64 -#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define UTIL_fseek fseeko -#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) -# define UTIL_fseek fseeko64 -#else -# define UTIL_fseek fseek -#endif - - -/*-**************************************** -* Sleep functions: Windows - Posix - others -******************************************/ -#if defined(_WIN32) -# include -# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) -# define UTIL_sleep(s) Sleep(1000*s) -# define UTIL_sleepMilli(milli) Sleep(milli) -#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ -# include -# include /* setpriority */ -# include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ -# if defined(PRIO_PROCESS) -# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20) -# else -# define SET_REALTIME_PRIORITY /* disabled */ -# endif -# define UTIL_sleep(s) sleep(s) -# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ -# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } -# else -# define UTIL_sleepMilli(milli) /* disabled */ -# endif -#else -# define SET_REALTIME_PRIORITY /* disabled */ -# define UTIL_sleep(s) /* disabled */ -# define UTIL_sleepMilli(milli) /* disabled */ -#endif - - -/* ************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) - - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ -#endif -#if defined(__GNUC__) -# define UTIL_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define UTIL_STATIC static inline -#elif defined(_MSC_VER) -# define UTIL_STATIC static __inline -#else -# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - -/*-**************************************** -* Time functions -******************************************/ -#if (PLATFORM_POSIX_VERSION >= 1) -#include -#include /* times */ - typedef U64 UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; } -#elif defined(_WIN32) /* Windows */ - typedef LARGE_INTEGER UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } -#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ - typedef clock_t UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } -#endif - - -/* returns time span in microseconds */ -UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond ) -{ - UTIL_time_t clockEnd; - UTIL_getTime(&clockEnd); - return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); -} - - -UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) -{ - UTIL_time_t clockStart, clockEnd; - UTIL_getTime(&clockStart); - do { - UTIL_getTime(&clockEnd); - } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); -} - - - -/*-**************************************** -* File functions -******************************************/ -#if defined(_MSC_VER) - #define chmod _chmod - typedef struct __stat64 stat_t; -#else - typedef struct stat stat_t; -#endif - - -UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) -{ - int res = 0; - struct utimbuf timebuf; - - timebuf.actime = time(NULL); - timebuf.modtime = statbuf->st_mtime; - res += utime(filename, &timebuf); /* set access and modification times */ - -#if !defined(_WIN32) - res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ -#endif - - res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */ - - errno = 0; - return -res; /* number of errors is returned */ -} - - -UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) -{ - int r; -#if defined(_MSC_VER) - r = _stat64(infilename, statbuf); - if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ -#else - r = stat(infilename, statbuf); - if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ -#endif - return 1; -} - - -UTIL_STATIC int UTIL_isRegFile(const char* infilename) -{ - stat_t statbuf; - return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ -} - - -UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) -{ - int r; - stat_t statbuf; -#if defined(_MSC_VER) - r = _stat64(infilename, &statbuf); - if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; -#else - r = stat(infilename, &statbuf); - if (!r && S_ISDIR(statbuf.st_mode)) return 1; -#endif - return 0; -} - - -UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct __stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#elif defined(__MINGW32__) && defined (__MSVCRT__) - struct _stati64 statbuf; - r = _stati64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ -#endif - return (U64)statbuf.st_size; -} - - -UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) -{ - U64 total = 0; - unsigned n; - for (n=0; n= *bufEnd) { - ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; - *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); - *bufEnd = *bufStart + newListSize; - if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } - } - if (*bufStart + *pos + pathLength < *bufEnd) { - strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos)); - *pos += pathLength + 1; - nbFiles++; - } - } - free(path); - } while (FindNextFileA(hFile, &cFile)); - - FindClose(hFile); - return nbFiles; -} - -#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ -# define UTIL_HAS_CREATEFILELIST -# include /* opendir, readdir */ -# include /* strerror, memcpy */ - -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) -{ - DIR *dir; - struct dirent *entry; - char* path; - int dirLength, fnameLength, pathLength, nbFiles = 0; - - if (!(dir = opendir(dirName))) { - fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); - return 0; - } - - dirLength = (int)strlen(dirName); - errno = 0; - while ((entry = readdir(dir)) != NULL) { - if (strcmp (entry->d_name, "..") == 0 || - strcmp (entry->d_name, ".") == 0) continue; - fnameLength = (int)strlen(entry->d_name); - path = (char*) malloc(dirLength + fnameLength + 2); - if (!path) { closedir(dir); return 0; } - memcpy(path, dirName, dirLength); - path[dirLength] = '/'; - memcpy(path+dirLength+1, entry->d_name, fnameLength); - pathLength = dirLength+1+fnameLength; - path[pathLength] = 0; - - if (UTIL_isDirectory(path)) { - nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ - if (*bufStart == NULL) { free(path); closedir(dir); return 0; } - } else { - if (*bufStart + *pos + pathLength >= *bufEnd) { - ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; - *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); - *bufEnd = *bufStart + newListSize; - if (*bufStart == NULL) { free(path); closedir(dir); return 0; } - } - if (*bufStart + *pos + pathLength < *bufEnd) { - strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos)); - *pos += pathLength + 1; - nbFiles++; - } - } - free(path); - errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ - } - - if (errno != 0) { - fprintf(stderr, "readdir(%s) error: %s\n", dirName, strerror(errno)); - free(*bufStart); - *bufStart = NULL; - } - closedir(dir); - return nbFiles; -} - -#else - -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) -{ - (void)bufStart; (void)bufEnd; (void)pos; - fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); - return 0; -} - -#endif /* #ifdef _WIN32 */ - -/* - * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, - * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb). - * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) - * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. - */ -UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb) -{ - size_t pos; - unsigned i, nbFiles; - char* buf = (char*)malloc(LIST_SIZE_INCREASE); - char* bufend = buf + LIST_SIZE_INCREASE; - const char** fileTable; - - if (!buf) return NULL; - - for (i=0, pos=0, nbFiles=0; i= bufend) { - ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; - buf = (char*)UTIL_realloc(buf, newListSize); - bufend = buf + newListSize; - if (!buf) return NULL; - } - if (buf + pos + len < bufend) { - strncpy(buf + pos, inputNames[i], bufend - (buf + pos)); - pos += len + 1; - nbFiles++; - } - } else { - nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend); - if (buf == NULL) return NULL; - } } - - if (nbFiles == 0) { free(buf); return NULL; } - - fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); - if (!fileTable) { free(buf); return NULL; } - - for (i=0, pos=0; i bufend) { free(buf); free((void*)fileTable); return NULL; } - - *allocatedBuffer = buf; - *allocatedNamesNb = nbFiles; - - return fileTable; -} - - -UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) -{ - if (allocatedBuffer) free(allocatedBuffer); - if (filenameTable) free((void*)filenameTable); -} - - -#if defined (__cplusplus) -} -#endif - -#endif /* UTIL_H_MODULE */ diff --git a/library/lz4-mac/tests/.gitignore b/library/lz4-mac/tests/.gitignore deleted file mode 100755 index 4c0f3114..00000000 --- a/library/lz4-mac/tests/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ - -# test build artefacts -datagen -frametest -frametest32 -fullbench -fullbench32 -fuzzer -fuzzer32 -fasttest - -# test artefacts -tmp* -versionsTest diff --git a/library/lz4-mac/tests/COPYING b/library/lz4-mac/tests/COPYING deleted file mode 100755 index d159169d..00000000 --- a/library/lz4-mac/tests/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/library/lz4-mac/tests/Makefile b/library/lz4-mac/tests/Makefile deleted file mode 100755 index 1a907b72..00000000 --- a/library/lz4-mac/tests/Makefile +++ /dev/null @@ -1,398 +0,0 @@ -# ########################################################################## -# LZ4 programs - Makefile -# Copyright (C) Yann Collet 2011-2017 -# -# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets -# -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - LZ4 homepage : http://www.lz4.org -# - LZ4 source repository : https://github.com/lz4/lz4 -# ########################################################################## -# fuzzer : Test tool, to check lz4 integrity on target platform -# frametest : Test tool, to check lz4frame integrity on target platform -# fullbench : Precisely measure speed for each LZ4 function variant -# datagen : generates synthetic data samples for tests & benchmarks -# ########################################################################## - -LZ4DIR := ../lib -PRGDIR := ../programs -TESTDIR := versionsTest -PYTHON ?= python3 - -CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make -CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ - -Wpointer-arith -Wstrict-aliasing=1 -CFLAGS += $(MOREFLAGS) -CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_ -FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) - - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -VOID = nul -else -EXT = -VOID = /dev/null -endif -LZ4 := $(PRGDIR)/lz4$(EXT) - - -# Default test parameters -TEST_FILES := COPYING -FUZZER_TIME := -T3mn -NB_LOOPS ?= -i1 - - -default: all - -all: fullbench fuzzer frametest datagen - -all32: CFLAGS+=-m32 -all32: all - -lz4: - $(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" - -lz4c unlz4 lz4cat: lz4 - ln -sf $(LZ4) $(PRGDIR)/$@ - -lz4c32: # create a 32-bits version for 32/64 interop tests - $(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)" - -fullbench : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -fullbench-lib: fullbench.c $(LZ4DIR)/xxhash.c - $(MAKE) -C $(LZ4DIR) liblz4.a - $(CC) $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a - -fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c - $(MAKE) -C $(LZ4DIR) liblz4 - $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll - -fuzzer : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -datagen : $(PRGDIR)/datagen.c datagencli.c - $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) - -clean: - @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) - @$(MAKE) -C $(PRGDIR) $@ > $(VOID) - @$(RM) core *.o *.test tmp* \ - fullbench-dll$(EXT) fullbench-lib$(EXT) \ - fullbench$(EXT) fullbench32$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) \ - frametest$(EXT) frametest32$(EXT) \ - fasttest$(EXT) datagen$(EXT) - @rm -fR $(TESTDIR) - @echo Cleaning completed - -.PHONY: versionsTest -versionsTest: - $(PYTHON) test-lz4-versions.py - - -#----------------------------------------------------------------------------- -# validated only for Linux, OSX, BSD, Hurd and Solaris targets -#----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) - -MD5:=md5sum -ifneq (,$(filter $(shell uname), Darwin )) -MD5:=md5 -r -endif - -DIFF:=diff -ifneq (,$(filter $(shell uname),SunOS)) -DIFF:=gdiff -endif - -DD:=dd - - -test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer - -test32: CFLAGS+=-m32 -test32: test - -test-lz4-sparse: lz4 datagen - @echo "\n ---- test sparse file support ----" - ./datagen -g5M -P100 > tmplsdg5M - $(LZ4) -B4D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB4 - $(DIFF) -s tmplsdg5M tmplscB4 - $(LZ4) -B5D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB5 - $(DIFF) -s tmplsdg5M tmplscB5 - $(LZ4) -B6D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB6 - $(DIFF) -s tmplsdg5M tmplscB6 - $(LZ4) -B7D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB7 - $(DIFF) -s tmplsdg5M tmplscB7 - $(LZ4) tmplsdg5M | $(LZ4) -dv --no-sparse > tmplsnosparse - $(DIFF) -s tmplsdg5M tmplsnosparse - ls -ls tmpls* - ./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block) - ./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd - ls -ls tmplsodd - @$(RM) tmpls* - @echo "\n Compatibility with Console :" - echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c - echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat - echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c - @echo "\n Compatibility with Append :" - ./datagen -P100 -g1M > tmplsdg1M - cat tmplsdg1M tmplsdg1M > tmpls2M - $(LZ4) -B5 -v tmplsdg1M tmplsc - $(LZ4) -d -v tmplsc tmplsr - $(LZ4) -d -v tmplsc >> tmplsr - ls -ls tmp* - $(DIFF) tmpls2M tmplsr - @$(RM) tmpls* - -test-lz4-contentSize: lz4 datagen - @echo "\n ---- test original size support ----" - ./datagen -g15M > tmplc1 - $(LZ4) -v tmplc1 | $(LZ4) -t - $(LZ4) -v --content-size tmplc1 | $(LZ4) -d > tmplc2 - $(DIFF) -s tmplc1 tmplc2 - # test large size [2-4] GB - @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmplc1 - @ls -ls tmplc1 - @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmplc2 - @ls -ls tmplc2 - $(DIFF) -s tmplc1 tmplc2 - @$(RM) tmplc* - -test-lz4-frame-concatenation: lz4 datagen - @echo "\n ---- test frame concatenation ----" - @echo -n > tmp-lfc-empty - @echo hi > tmp-lfc-nonempty - cat tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src - @$(LZ4) -zq tmp-lfc-empty > tmp-lfc-empty.lz4 - @$(LZ4) -zq tmp-lfc-nonempty > tmp-lfc-nonempty.lz4 - cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4 - $(LZ4) -d tmp-lfc-concat.lz4 > tmp-lfc-result - sdiff tmp-lfc-src tmp-lfc-result - @$(RM) tmp-lfc-* - @echo frame concatenation test completed - -test-lz4-multiple: lz4 datagen - @echo "\n ---- test multiple files ----" - @./datagen -s1 > tmp-tlm1 2> $(VOID) - @./datagen -s2 -g100K > tmp-tlm2 2> $(VOID) - @./datagen -s3 -g1M > tmp-tlm3 2> $(VOID) - $(LZ4) -f -m tmp-tlm* - ls -ls tmp-tlm* - @$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3 - $(LZ4) -df -m tmp-tlm*.lz4 - ls -ls tmp-tlm* - $(LZ4) -f -m tmp-tlm1 notHere tmp-tlm2; echo $$? - @$(RM) tmp-tlm* - -test-lz4-basic: lz4 datagen unlz4 lz4cat - @echo "\n ---- test lz4 basic compression/decompression ----" - ./datagen -g0 | $(LZ4) -v | $(LZ4) -t - ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t - ./datagen -g20KB > tmp-tlb-dg20k - $(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec - $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec - $(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec - $(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec - ./datagen | $(LZ4) | $(LZ4) -t - ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t - ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt - ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t - ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t - @echo "hello world" > tmp-tlb-hw - $(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4 - test ! -f tmp-tlb-hw # must fail (--rm) - test -f tmp-tlb-hw.lz4 - $(PRGDIR)/lz4cat tmp-tlb-hw.lz4 # must display hello world - test -f tmp-tlb-hw.lz4 - $(PRGDIR)/unlz4 --rm tmp-tlb-hw.lz4 tmp-tlb-hw - test -f tmp-tlb-hw - test ! -f tmp-tlb-hw.lz4 # must fail (--rm) - test ! -f tmp-tlb-hw.lz4.lz4 # must fail (unlz4) - $(PRGDIR)/lz4cat tmp-tlb-hw # pass-through mode - test -f tmp-tlb-hw - test ! -f tmp-tlb-hw.lz4 # must fail (lz4cat) - $(LZ4) tmp-tlb-hw tmp-tlb-hw.lz4 # creates tmp-tlb-hw.lz4 - $(PRGDIR)/lz4cat < tmp-tlb-hw.lz4 > tmp-tlb3 # checks lz4cat works with stdin (#285) - $(DIFF) -q tmp-tlb-hw tmp-tlb3 - $(PRGDIR)/lz4cat < tmp-tlb-hw > tmp-tlb2 # checks lz4cat works in pass-through mode - $(DIFF) -q tmp-tlb-hw tmp-tlb2 - cp tmp-tlb-hw ./-d - $(LZ4) --rm -- -d -d.lz4 # compresses ./d into ./-d.lz4 - test -f ./-d.lz4 - test ! -f ./-d - mv ./-d.lz4 ./-z - $(LZ4) -d --rm -- -z tmp-tlb4 # uncompresses ./-z into tmp-tlb4 - test ! -f ./-z - $(DIFF) -q tmp-tlb-hw tmp-tlb4 - $(LZ4) -f tmp-tlb-hw - cat tmp-tlb-hw >> tmp-tlb-hw.lz4 - $(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data - $(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum - @$(RM) tmp-tlb* - -test-lz4-dict: lz4 datagen - @echo "\n ---- test lz4 compression/decompression with dictionary ----" - ./datagen -g16KB > tmp-dict - ./datagen -g32KB > tmp-dict-sample-32k - < tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k - ./datagen -g128MB > tmp-dict-sample-128m - < tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m - touch tmp-dict-sample-0 - < tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0 - - < tmp-dict-sample-32k $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-32k - < tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0 - - @echo "\n ---- test lz4 dictionary loading ----" - ./datagen -g128KB > tmp-dict-data-128KB - set -e; \ - for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \ - ./datagen -g$$l > tmp-dict-$$l; \ - $(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \ - < tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \ - < tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \ - done - - @$(RM) tmp-dict* - -test-lz4-hugefile: lz4 datagen - @echo "\n ---- test huge files compression/decompression ----" - ./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt - ./datagen -g6GB | $(LZ4) -v5BD | $(LZ4) -qt - -test-lz4-testmode: lz4 datagen - @echo "\n ---- bench mode ----" - $(LZ4) -bi1 - @echo "\n ---- test mode ----" - ! ./datagen | $(LZ4) -t - ! ./datagen | $(LZ4) -tf - @echo "\n ---- pass-through mode ----" - ! ./datagen | $(LZ4) -d > $(VOID) - ./datagen | $(LZ4) -df > $(VOID) - @echo "Hello World !" > tmp-tlt1 - $(LZ4) -dcf tmp-tlt1 - @echo "from underground..." > tmp-tlt2 - $(LZ4) -dcfm tmp-tlt1 tmp-tlt2 - @echo "\n ---- non-existing source ----" - ! $(LZ4) file-does-not-exist - ! $(LZ4) -f file-does-not-exist - ! $(LZ4) -fm file1-dne file2-dne - @$(RM) tmp-tlt - -test-lz4-opt-parser: lz4 datagen - @echo "\n ---- test opt-parser ----" - ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t - ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t - ./datagen -g256K | $(LZ4) -12B4D | $(LZ4) -t - ./datagen -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t - ./datagen -g1M | $(LZ4) -12B5 | $(LZ4) -t - ./datagen -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t - ./datagen -g4M | $(LZ4) -11vq | $(LZ4) -qt - ./datagen -g8M | $(LZ4) -11B4 | $(LZ4) -t - ./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t - ./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t - -test-lz4: lz4 datagen test-lz4-basic test-lz4-opt-parser test-lz4-multiple \ - test-lz4-sparse test-lz4-frame-concatenation test-lz4-testmode \ - test-lz4-contentSize test-lz4-hugefile test-lz4-dict - @$(RM) tmp* - -test-lz4c: lz4c datagen - @echo "\n ---- test lz4c variant ----" - ./datagen -g256MB | $(LZ4)c -l -v | $(LZ4)c -t - -test-lz4c32: CFLAGS+=-m32 -test-lz4c32: test-lz4 - -test-interop-32-64: lz4 lz4c32 datagen - @echo "\n ---- test interoperability 32-bits -vs- 64 bits ----" - ./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4) -t - ./datagen -P10 | $(LZ4) -9B4 | $(LZ4)c32 -t - ./datagen | $(LZ4)c32 | $(LZ4) -t - ./datagen -g1M | $(LZ4) -3B5 | $(LZ4)c32 -t - ./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4) -qt - ./datagen -g1G -P90 | $(LZ4) | $(LZ4)c32 -t - ./datagen -g6GB | $(LZ4)c32 -vq9BD | $(LZ4) -qt - -test-lz4c32-basic: lz4c32 datagen - @echo "\n ---- test lz4c32 32-bits version ----" - ./datagen -g16KB | $(LZ4)c32 -9 | $(LZ4)c32 -t - ./datagen | $(LZ4)c32 | $(LZ4)c32 -t - ./datagen -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt - ./datagen -g6GB | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt - -test-platform: - @echo "\n ---- test lz4 $(QEMU_SYS) platform ----" - $(QEMU_SYS) ./datagen -g16KB | $(QEMU_SYS) $(LZ4) -9 | $(QEMU_SYS) $(LZ4) -t - $(QEMU_SYS) ./datagen | $(QEMU_SYS) $(LZ4) | $(QEMU_SYS) $(LZ4) -t - $(QEMU_SYS) ./datagen -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt -ifneq ($(QEMU_SYS),qemu-arm-static) - $(QEMU_SYS) ./datagen -g3GB | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt -endif - -test-fullbench: fullbench - ./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES) - -test-fullbench32: CFLAGS += -m32 -test-fullbench32: test-fullbench - -test-fuzzer: fuzzer - ./fuzzer $(FUZZER_TIME) - -test-fuzzer32: CFLAGS += -m32 -test-fuzzer32: test-fuzzer - -test-frametest: frametest - ./frametest $(FUZZER_TIME) - -test-frametest32: CFLAGS += -m32 -test-frametest32: test-frametest - -test-mem: lz4 datagen fuzzer frametest fullbench - @echo "\n ---- valgrind tests : memory analyzer ----" - valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) - ./datagen -g16KB > ftmdg16K - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID) - ./datagen -g16KB -s2 > ftmdg16K2 - ./datagen -g16KB -s3 > ftmdg16K3 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3 - ./datagen -g16MB > ftmdg16M - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2 - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M - valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2 - ./datagen -g256MB > ftmdg256M - valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID) - $(RM) ftm* - valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1 - valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256 - -test-mem32: lz4c32 datagen -# unfortunately, valgrind doesn't seem to work with non-native binary... - -endif diff --git a/library/lz4-mac/tests/README.md b/library/lz4-mac/tests/README.md deleted file mode 100755 index 75b7b9f5..00000000 --- a/library/lz4-mac/tests/README.md +++ /dev/null @@ -1,71 +0,0 @@ -Programs and scripts for automated testing of LZ4 -======================================================= - -This directory contains the following programs and scripts: -- `datagen` : Synthetic and parametrable data generator, for tests -- `frametest` : Test tool that checks lz4frame integrity on target platform -- `fullbench` : Precisely measure speed for each lz4 inner functions -- `fuzzer` : Test tool, to check lz4 integrity on target platform -- `test-lz4-speed.py` : script for testing lz4 speed difference between commits -- `test-lz4-versions.py` : compatibility test between lz4 versions stored on Github - - -#### `test-lz4-versions.py` - script for testing lz4 interoperability between versions - -This script creates `versionsTest` directory to which lz4 repository is cloned. -Then all taged (released) versions of lz4 are compiled. -In the following step interoperability between lz4 versions is checked. - - -#### `test-lz4-speed.py` - script for testing lz4 speed difference between commits - -This script creates `speedTest` directory to which lz4 repository is cloned. -Then it compiles all branches of lz4 and performs a speed benchmark for a given list of files (the `testFileNames` parameter). -After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits. -If a new commit is found it is compiled and a speed benchmark for this commit is performed. -The results of the speed benchmark are compared to the previous results. -If compression or decompression speed for one of lz4 levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted. -If second results are also lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter). - -Additional remarks: -- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel -- Using the script with virtual machines can lead to large variations of speed results -- The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75) -- The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning - - -The example usage with two test files, one e-mail address, and with an additional message: -``` -./test-lz4-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60 -``` - -To run the script in background please use: -``` -nohup ./test-lz4-speed.py testFileNames emails & -``` - -The full list of parameters: -``` -positional arguments: - testFileNames file names list for speed benchmark - emails list of e-mail addresses to send warnings - -optional arguments: - -h, --help show this help message and exit - --message MESSAGE attach an additional message to e-mail - --lowerLimit LOWERLIMIT - send email if speed is lower than given limit - --maxLoadAvg MAXLOADAVG - maximum load average to start testing - --lastCLevel LASTCLEVEL - last compression level for testing - --sleepTime SLEEPTIME - frequency of repository checking in seconds -``` - - -#### License - -All files in this directory are licensed under GPL-v2. -See [COPYING](COPYING) for details. -The text of the license is also included at the top of each source file. diff --git a/library/lz4-mac/tests/datagencli.c b/library/lz4-mac/tests/datagencli.c deleted file mode 100755 index c985197f..00000000 --- a/library/lz4-mac/tests/datagencli.c +++ /dev/null @@ -1,172 +0,0 @@ -/* - datagencli.c - compressible data command line generator - Copyright (C) Yann Collet 2012-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/************************************** -* Includes -**************************************/ -#include "util.h" /* U32 */ -#include /* fprintf, stderr */ -#include "datagen.h" /* RDG_generate */ -#include "lz4.h" /* LZ4_VERSION_STRING */ - - -/************************************** -* Constants -**************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define SIZE_DEFAULT (64 KB) -#define SEED_DEFAULT 0 -#define COMPRESSIBILITY_DEFAULT 50 - - -/************************************** -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; - - -/********************************************************* -* Command line -*********************************************************/ -static int usage(char* programName) -{ - DISPLAY( "Compressible data generator\n"); - DISPLAY( "Usage :\n"); - DISPLAY( " %s [size] [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); - DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT); - DISPLAY( " -h : display help and exit\n"); - DISPLAY( "Special values :\n"); - DISPLAY( " -P0 : generate incompressible noise\n"); - DISPLAY( " -P100 : generate sparse files\n"); - return 0; -} - - -int main(int argc, char** argv) -{ - int argNb; - double proba = (double)COMPRESSIBILITY_DEFAULT / 100; - double litProba = 0.0; - U64 size = SIZE_DEFAULT; - U32 seed = SEED_DEFAULT; - char* programName; - - /* Check command line */ - programName = argv[0]; - for(argNb=1; argNb='0') && (*argument<='9')) - { - size *= 10; - size += *argument - '0'; - argument++; - } - if (*argument=='K') { size <<= 10; argument++; } - if (*argument=='M') { size <<= 20; argument++; } - if (*argument=='G') { size <<= 30; argument++; } - if (*argument=='B') { argument++; } - break; - case 's': - argument++; - seed=0; - while ((*argument>='0') && (*argument<='9')) - { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - case 'P': - argument++; - proba=0.0; - while ((*argument>='0') && (*argument<='9')) - { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba>100.) proba=100.; - proba /= 100.; - break; - case 'L': /* hidden argument : Literal distribution probability */ - argument++; - litProba=0.; - while ((*argument>='0') && (*argument<='9')) - { - litProba *= 10; - litProba += *argument - '0'; - argument++; - } - if (litProba>100.) litProba=100.; - litProba /= 100.; - break; - case 'v': - displayLevel = 4; - argument++; - break; - default: - return usage(programName); - } - } - - } - } - - DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION_STRING); - DISPLAYLEVEL(3, "Seed = %u \n", seed); - if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100)); - - RDG_genOut(size, proba, litProba, seed); - DISPLAYLEVEL(1, "\n"); - - return 0; -} diff --git a/library/lz4-mac/tests/frametest.c b/library/lz4-mac/tests/frametest.c deleted file mode 100755 index 88d0afd4..00000000 --- a/library/lz4-mac/tests/frametest.c +++ /dev/null @@ -1,1066 +0,0 @@ -/* - frameTest - test tool for lz4frame - Copyright (C) Yann Collet 2014-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ - -/*-************************************ -* Compiler specific -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ -#endif - - -/*-************************************ -* Includes -**************************************/ -#include "util.h" /* U32 */ -#include /* malloc, free */ -#include /* fprintf */ -#include /* strcmp */ -#include /* clock_t, clock(), CLOCKS_PER_SEC */ -#include -#include "lz4frame_static.h" -#include "lz4.h" /* LZ4_VERSION_STRING */ -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" /* XXH64 */ - - -/* unoptimized version; solves endianess & alignment issues */ -static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) -{ - BYTE* dstPtr = (BYTE*)dstVoidPtr; - dstPtr[0] = (BYTE)value32; - dstPtr[1] = (BYTE)(value32 >> 8); - dstPtr[2] = (BYTE)(value32 >> 16); - dstPtr[3] = (BYTE)(value32 >> 24); -} - - -/*-************************************ -* Constants -**************************************/ -#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -static const U32 nbTestsDefault = 256 KB; -#define FUZ_COMPRESSIBILITY_DEFAULT 50 -static const U32 prime1 = 2654435761U; -static const U32 prime2 = 2246822519U; - - - -/*-************************************ -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ - if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \ - { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stdout); } } -static const clock_t refreshRate = CLOCKS_PER_SEC / 6; -static clock_t g_clockTime = 0; - - -/*-*************************************** -* Local Parameters -*****************************************/ -static U32 no_prompt = 0; -static U32 displayLevel = 2; -static U32 use_pause = 0; - - -/*-******************************************************* -* Fuzzer functions -*********************************************************/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) -#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) - -static clock_t FUZ_GetClockSpan(clock_t clockStart) -{ - return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ -} - - -#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -unsigned int FUZ_rand(unsigned int* src) -{ - U32 rand32 = *src; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZ_rotl32(rand32, 13); - *src = rand32; - return rand32 >> 5; -} - - -#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) -#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) -static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) -{ - BYTE* BBuffer = (BYTE*)buffer; - size_t pos = 0; - U32 P32 = (U32)(32768 * proba); - - /* First Byte */ - BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); - - while (pos < bufferSize) { - /* Select : Literal (noise) or copy (within 64K) */ - if (FUZ_RAND15BITS < P32) { - /* Copy (within 64K) */ - size_t const lengthRand = FUZ_RANDLENGTH + 4; - size_t const length = MIN(lengthRand, bufferSize - pos); - size_t const end = pos + length; - size_t const offsetRand = FUZ_RAND15BITS + 1; - size_t const offset = MIN(offsetRand, pos); - size_t match = pos - offset; - while (pos < end) BBuffer[pos++] = BBuffer[match++]; - } else { - /* Literal (noise) */ - size_t const lengthRand = FUZ_RANDLENGTH + 4; - size_t const length = MIN(lengthRand, bufferSize - pos); - size_t const end = pos + length; - while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); - } - } -} - - -static unsigned FUZ_highbit(U32 v32) -{ - unsigned nbBits = 0; - if (v32==0) return 0; - while (v32) v32 >>= 1, nbBits ++; - return nbBits; -} - - -/*-******************************************************* -* Tests -*********************************************************/ -#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) goto _output_error -#define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); } - -int basicTests(U32 seed, double compressibility) -{ -#define COMPRESSIBLE_NOISE_LENGTH (2 MB) - void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL); - void* const compressedBuffer = malloc(cBuffSize); - void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - U32 randState = seed; - size_t cSize, testSize; - LZ4F_decompressionContext_t dCtx = NULL; - LZ4F_compressionContext_t cctx = NULL; - U64 crcOrig; - int basicTests_error = 0; - LZ4F_preferences_t prefs; - memset(&prefs, 0, sizeof(prefs)); - - if (!CNBuffer || !compressedBuffer || !decodedBuffer) { - DISPLAY("allocation error, not enough memory to start fuzzer tests \n"); - goto _output_error; - } - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - - /* LZ4F_compressBound() : special case : srcSize == 0 */ - DISPLAYLEVEL(3, "LZ4F_compressBound(0) = "); - { size_t const cBound = LZ4F_compressBound(0, NULL); - if (cBound < 64 KB) goto _output_error; - DISPLAYLEVEL(3, " %u \n", (U32)cBound); - } - - /* Special case : null-content frame */ - testSize = 0; - DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : "); - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL)); - DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize); - - DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n"); - CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); - - DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n"); - { size_t avail_in = cSize; - LZ4F_frameInfo_t frame_info; - CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) ); - } - - DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); - CHECK( LZ4F_freeDecompressionContext(dCtx) ); - dCtx = NULL; - - /* test one-pass frame compression */ - testSize = COMPRESSIBLE_NOISE_LENGTH; - - DISPLAYLEVEL(3, "LZ4F_compressFrame, using fast level -3 : "); - { LZ4F_preferences_t fastCompressPrefs; - memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs)); - fastCompressPrefs.compressionLevel = -3; - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs)); - DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); - } - - DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : "); - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL)); - DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); - - DISPLAYLEVEL(3, "Decompression test : \n"); - { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - size_t compressedBufferSize = cSize; - - CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); - - DISPLAYLEVEL(3, "Single Pass decompression : "); - CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) ); - { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1); - if (crcDest != crcOrig) goto _output_error; } - DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize); - - DISPLAYLEVEL(3, "Reusing decompression context \n"); - { size_t const missingBytes = 4; - size_t iSize = compressedBufferSize - missingBytes; - const BYTE* cBuff = (const BYTE*) compressedBuffer; - BYTE* const ostart = (BYTE*)decodedBuffer; - BYTE* op = ostart; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH; - DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes); - CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL)); - if (decResult != missingBytes) { - DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult); - goto _output_error; - } - DISPLAYLEVEL(3, "indeed, requests %u bytes \n", (unsigned)decResult); - cBuff += iSize; - iSize = decResult; - op += oSize; - oSize = (size_t)(oend-op); - decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); - if (decResult != 0) goto _output_error; /* should finish now */ - op += oSize; - if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; } - { U64 const crcDest = XXH64(decodedBuffer, op-ostart, 1); - if (crcDest != crcOrig) goto _output_error; - } } - - { size_t oSize = 0; - size_t iSize = 0; - LZ4F_frameInfo_t fi; - const BYTE* ip = (BYTE*)compressedBuffer; - - DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); - CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) ); - //DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); - DISPLAYLEVEL(3, " OK \n"); - - DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : "); - { size_t nullSize = 0; - size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize); - if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) { - DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", - LZ4F_getErrorName(fiError)); - goto _output_error; - } - DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError)); - } - - DISPLAYLEVEL(3, "LZ4F_getFrameInfo on not enough input : "); - { size_t inputSize = 6; - size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &inputSize); - if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) { - DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError)); - goto _output_error; - } - DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError)); - } - - DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : "); - iSize = 15 - iSize; - CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) ); - DISPLAYLEVEL(3, " correctly decoded \n"); - ip += iSize; - } - - DISPLAYLEVEL(3, "Decode a buggy input : "); - assert(COMPRESSIBLE_NOISE_LENGTH > 64); - assert(cSize > 48); - memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */ - memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */ - { size_t dbSize = COMPRESSIBLE_NOISE_LENGTH; - size_t cbSize = cSize; - size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize, - compressedBuffer, &cbSize, - NULL); - if (!LZ4F_isError(decompressError)) goto _output_error; - DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError)); - } - memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */ - - DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n"); - LZ4F_resetDecompressionContext(dCtx); /* always successful */ - - DISPLAYLEVEL(3, "Byte after byte : "); - { BYTE* const ostart = (BYTE*)decodedBuffer; - BYTE* op = ostart; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - const BYTE* ip = (const BYTE*) compressedBuffer; - const BYTE* const iend = ip + cSize; - while (ip < iend) { - size_t oSize = oend-op; - size_t iSize = 1; - CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); - op += oSize; - ip += iSize; - } - { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; } - DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH); - } - } - - DISPLAYLEVEL(3, "Using 64 KB block : "); - prefs.frameInfo.blockSizeID = LZ4F_max64KB; - prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "without checksum : "); - prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Using 256 KB block : "); - prefs.frameInfo.blockSizeID = LZ4F_max256KB; - prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Decompression test : \n"); - { size_t const decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - unsigned const maxBits = FUZ_highbit((U32)decodedBufferSize); - BYTE* const ostart = (BYTE*)decodedBuffer; - BYTE* op = ostart; - BYTE* const oend = ostart + COMPRESSIBLE_NOISE_LENGTH; - const BYTE* ip = (const BYTE*)compressedBuffer; - const BYTE* const iend = (const BYTE*)compressedBuffer + cSize; - - DISPLAYLEVEL(3, "random segment sizes : "); - while (ip < iend) { - unsigned const nbBits = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); - op += oSize; - ip += iSize; - } - { size_t const decodedSize = (size_t)(op - ostart); - U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); - } - - CHECK( LZ4F_freeDecompressionContext(dCtx) ); - dCtx = NULL; - } - - DISPLAYLEVEL(3, "without checksum : "); - prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Using 1 MB block : "); - prefs.frameInfo.blockSizeID = LZ4F_max1MB; - prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "without frame checksum : "); - prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Using 4 MB block : "); - prefs.frameInfo.blockSizeID = LZ4F_max4MB; - prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); - DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity) - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) ); - DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); - } - - DISPLAYLEVEL(3, "without frame checksum : "); - prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; - { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); - DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity) - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) ); - DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); - } - - DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : "); - memset(&prefs, 0, sizeof(prefs)); - prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; - CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Decompress with block checksum : "); - { size_t iSize = cSize; - size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; - LZ4F_decompressionContext_t dctx; - CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); - CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) ); - if (decodedSize != testSize) goto _output_error; - if (iSize != cSize) goto _output_error; - { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1); - U64 const crcSrc = XXH64(CNBuffer, testSize, 1); - if (crcDest != crcSrc) goto _output_error; - } - DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); - - CHECK( LZ4F_freeDecompressionContext(dctx) ); - } - - /* frame content size tests */ - { size_t cErr; - BYTE* const ostart = (BYTE*)compressedBuffer; - BYTE* op = ostart; - CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); - - DISPLAYLEVEL(3, "compress without frameSize : "); - memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); - CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); - op += cErr; - CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); - op += cErr; - CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) ); - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); - - DISPLAYLEVEL(3, "compress with frameSize : "); - prefs.frameInfo.contentSize = testSize; - op = ostart; - CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); - op += cErr; - CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); - op += cErr; - CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) ); - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); - - DISPLAYLEVEL(3, "compress with wrong frameSize : "); - prefs.frameInfo.contentSize = testSize+1; - op = ostart; - CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); - op += cErr; - CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); - op += cErr; - cErr = LZ4F_compressEnd(cctx, op, testSize, NULL); - if (!LZ4F_isError(cErr)) goto _output_error; - DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr)); - - CHECK( LZ4F_freeCompressionContext(cctx) ); - cctx = NULL; - } - - /* dictID tests */ - { size_t cErr; - U32 const dictID = 0x99; - CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); - - DISPLAYLEVEL(3, "insert a dictID : "); - memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); - prefs.frameInfo.dictID = dictID; - CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); - DISPLAYLEVEL(3, "created frame header of size %i bytes \n", (int)cErr); - - DISPLAYLEVEL(3, "read a dictID : "); - CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); - memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); - CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) ); - if (prefs.frameInfo.dictID != dictID) goto _output_error; - DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID); - - CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL; - CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; - } - - - /* Dictionary compression test */ - { size_t const dictSize = 63 KB; - size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL); - size_t cSizeNoDict, cSizeWithDict; - LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); - if (cdict == NULL) goto _output_error; - DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); - CHECK_V(cSizeNoDict, - LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, - CNBuffer, dictSize, - NULL, NULL) ); - DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict); - - DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : "); - CHECK_V(cSizeWithDict, - LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, - CNBuffer, dictSize, - cdict, NULL) ); - DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", - (unsigned)dictSize, (unsigned)cSizeWithDict); - if (cSizeWithDict >= cSizeNoDict) goto _output_error; /* must be more efficient */ - crcOrig = XXH64(CNBuffer, dictSize, 0); - - DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : "); - { LZ4F_dctx* dctx; - size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; - size_t compressedSize = cSizeWithDict; - CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); - CHECK( LZ4F_decompress_usingDict(dctx, - decodedBuffer, &decodedSize, - compressedBuffer, &compressedSize, - CNBuffer, dictSize, - NULL) ); - if (compressedSize != cSizeWithDict) goto _output_error; - if (decodedSize != dictSize) goto _output_error; - { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); - if (crcDest != crcOrig) goto _output_error; } - DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); - CHECK( LZ4F_freeDecompressionContext(dctx) ); - } - - DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : "); - { size_t cSizeLevelMax; - LZ4F_preferences_t cParams; - memset(&cParams, 0, sizeof(cParams)); - cParams.compressionLevel = -3; - CHECK_V(cSizeLevelMax, - LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, - CNBuffer, dictSize, - cdict, &cParams) ); - DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax); - } - - DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : "); - { size_t cSizeLevelMax; - LZ4F_preferences_t cParams; - memset(&cParams, 0, sizeof(cParams)); - cParams.compressionLevel = LZ4F_compressionLevel_max(); - CHECK_V(cSizeLevelMax, - LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity, - CNBuffer, dictSize, - cdict, &cParams) ); - DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax); - } - - DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : "); - { size_t cSizeContiguous; - size_t const inSize = dictSize * 3; - size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL); - LZ4F_preferences_t cParams; - memset(&cParams, 0, sizeof(cParams)); - cParams.frameInfo.blockMode = LZ4F_blockLinked; - cParams.frameInfo.blockSizeID = LZ4F_max64KB; - CHECK_V(cSizeContiguous, - LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity, - CNBuffer, inSize, - cdict, &cParams) ); - DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", - (unsigned)inSize, (unsigned)cSizeContiguous); - - DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : "); - { LZ4F_dctx* dctx; - size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; - size_t compressedSize = cSizeContiguous; - CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); - CHECK( LZ4F_decompress_usingDict(dctx, - decodedBuffer, &decodedSize, - compressedBuffer, &compressedSize, - CNBuffer, dictSize, - NULL) ); - if (compressedSize != cSizeContiguous) goto _output_error; - if (decodedSize != inSize) goto _output_error; - crcOrig = XXH64(CNBuffer, inSize, 0); - { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); - if (crcDest != crcOrig) goto _output_error; } - DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); - CHECK( LZ4F_freeDecompressionContext(dctx) ); - } - } - - - DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : "); - { size_t cSizeIndep; - size_t const inSize = dictSize * 3; - size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL); - LZ4F_preferences_t cParams; - memset(&cParams, 0, sizeof(cParams)); - cParams.frameInfo.blockMode = LZ4F_blockIndependent; - cParams.frameInfo.blockSizeID = LZ4F_max64KB; - CHECK_V(cSizeIndep, - LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity, - CNBuffer, inSize, - cdict, &cParams) ); - DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", - (unsigned)inSize, (unsigned)cSizeIndep); - - DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : "); - { LZ4F_dctx* dctx; - size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; - size_t compressedSize = cSizeIndep; - CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); - CHECK( LZ4F_decompress_usingDict(dctx, - decodedBuffer, &decodedSize, - compressedBuffer, &compressedSize, - CNBuffer, dictSize, - NULL) ); - if (compressedSize != cSizeIndep) goto _output_error; - if (decodedSize != inSize) goto _output_error; - crcOrig = XXH64(CNBuffer, inSize, 0); - { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); - if (crcDest != crcOrig) goto _output_error; } - DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); - CHECK( LZ4F_freeDecompressionContext(dctx) ); - } - } - - LZ4F_freeCDict(cdict); - } - - - DISPLAYLEVEL(3, "Skippable frame test : \n"); - { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* iend = (BYTE*)compressedBuffer + cSize + 8; - - CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); - - /* generate skippable frame */ - FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START); - FUZ_writeLE32(ip+4, (U32)cSize); - - DISPLAYLEVEL(3, "random segment sizes : \n"); - while (ip < iend) { - unsigned nbBits = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); - op += oSize; - ip += iSize; - } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize); - - /* generate zero-size skippable frame */ - DISPLAYLEVEL(3, "zero-size skippable frame\n"); - ip = (BYTE*)compressedBuffer; - op = (BYTE*)decodedBuffer; - FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1); - FUZ_writeLE32(ip+4, 0); - iend = ip+8; - - while (ip < iend) { - unsigned const nbBits = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); - op += oSize; - ip += iSize; - } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); - - DISPLAYLEVEL(3, "Skippable frame header complete in first call \n"); - ip = (BYTE*)compressedBuffer; - op = (BYTE*)decodedBuffer; - FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2); - FUZ_writeLE32(ip+4, 10); - iend = ip+18; - while (ip < iend) { - size_t iSize = 10; - size_t oSize = 10; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); - op += oSize; - ip += iSize; - } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); - } - - DISPLAY("Basic tests completed \n"); -_end: - free(CNBuffer); - free(compressedBuffer); - free(decodedBuffer); - LZ4F_freeDecompressionContext(dCtx); dCtx = NULL; - LZ4F_freeCompressionContext(cctx); cctx = NULL; - return basicTests_error; - -_output_error: - basicTests_error = 1; - DISPLAY("Error detected ! \n"); - goto _end; -} - - -static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) -{ - int p=0; - const BYTE* b1=(const BYTE*)buff1; - const BYTE* b2=(const BYTE*)buff2; - if (nonContiguous) { - DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size); - return; - } - while (b1[p]==b2[p]) p++; - DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]); -} - - -int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) -{ - unsigned testResult = 0; - unsigned testNb = 0; - size_t const srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ - void* srcBuffer = NULL; - size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL); - void* compressedBuffer = NULL; - void* decodedBuffer = NULL; - U32 coreRand = seed; - LZ4F_decompressionContext_t dCtx = NULL; - LZ4F_compressionContext_t cCtx = NULL; - size_t result; - clock_t const startClock = clock(); - clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; -# undef CHECK -# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } - - /* Create buffers */ - result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); - result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); - CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); - srcBuffer = malloc(srcDataLength); - CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); - compressedBuffer = malloc(compressedBufferSize); - CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); - decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ - CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); - FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); - - /* jump to requested testNb */ - for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */ - - /* main fuzzer test loop */ - for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { - U32 randState = coreRand ^ prime1; - unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(srcDataLength-1)) - 1)) + 1; - size_t const srcSize = (FUZ_rand(&randState) & ((1<=oend, "LZ4F_compressFrameBound overflow"); - result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); - CHECK(LZ4F_isError(result), "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); - op += result; - cSize = op-(BYTE*)compressedBuffer; - DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); - } - - /* multi-segments decompression */ - { const BYTE* ip = (const BYTE*)compressedBuffer; - const BYTE* const iend = ip + cSize; - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = op + srcDataLength; - unsigned const suggestedBits = FUZ_highbit((U32)cSize); - unsigned const maxBits = MAX(3, suggestedBits); - unsigned const nonContiguousDst = FUZ_rand(&randState) % 3; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ - size_t totalOut = 0; - XXH64_state_t xxh64; - XXH64_reset(&xxh64, 1); - while (ip < iend) { - unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; - unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; - size_t const iSizeMax = (FUZ_rand(&randState) & ((1<='0') && (*argument<='9')) { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } - break; - - case 'T': - argument++; - nbTests = 0; duration = 0; - for (;;) { - switch(*argument) - { - case 'm': duration *= 60; argument++; continue; - case 's': - case 'n': argument++; continue; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; - } - break; - } - break; - - case 's': - argument++; - seed=0; - seedset=1; - while ((*argument>='0') && (*argument<='9')) { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - case 't': - argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } - break; - case 'P': /* compressibility % */ - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - default: - ; - return FUZ_usage(programName); - } - } - } - } - - /* Get Seed */ - DISPLAY("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); - - if (!seedset) { - time_t const t = time(NULL); - U32 const h = XXH32(&t, sizeof(t), 1); - seed = h % 10000; - } - DISPLAY("Seed = %u\n", seed); - if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); - - if (nbTests<=0) nbTests=1; - - if (testNb==0) result = basicTests(seed, ((double)proba) / 100); - if (result) return 1; - return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration); -} diff --git a/library/lz4-mac/tests/fullbench.c b/library/lz4-mac/tests/fullbench.c deleted file mode 100755 index f489392d..00000000 --- a/library/lz4-mac/tests/fullbench.c +++ /dev/null @@ -1,723 +0,0 @@ -/* - bench.c - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - - -// S_ISREG & gettimeofday() are not supported by MSVC -#if defined(_MSC_VER) || defined(_WIN32) -# define BMK_LEGACY_TIMER 1 -#endif - - -/************************************** -* Includes -**************************************/ -#include "platform.h" /* _CRT_SECURE_NO_WARNINGS, Large Files support */ -#include "util.h" /* U32, UTIL_getFileSize */ -#include /* malloc, free */ -#include /* fprintf, fopen, ftello */ -#include /* stat64 */ -#include /* stat64 */ -#include /* strcmp */ -#include /* clock_t, clock(), CLOCKS_PER_SEC */ - -#include "lz4.h" -#include "lz4hc.h" -#include "lz4frame.h" - -#include "xxhash.h" - - -/************************************** -* Constants -**************************************/ -#define PROGRAM_DESCRIPTION "LZ4 speed analyzer" -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s v%s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR - -#define NBLOOPS 6 -#define TIMELOOP (CLOCKS_PER_SEC * 25 / 10) - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define KNUTH 2654435761U -#define MAX_MEM (1920 MB) -#define DEFAULT_CHUNKSIZE (4 MB) - -#define ALL_COMPRESSORS 0 -#define ALL_DECOMPRESSORS 0 - - -/************************************** -* Local structures -**************************************/ -struct chunkParameters -{ - U32 id; - char* origBuffer; - char* compressedBuffer; - int origSize; - int compressedSize; -}; - - -/************************************** -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__) - - -/************************************** -* Benchmark Parameters -**************************************/ -static int g_chunkSize = DEFAULT_CHUNKSIZE; -static int g_nbIterations = NBLOOPS; -static int g_pause = 0; -static int g_compressionTest = 1; -static int g_compressionAlgo = ALL_COMPRESSORS; -static int g_decompressionTest = 1; -static int g_decompressionAlgo = ALL_DECOMPRESSORS; -static int g_noPrompt = 0; - -static void BMK_setBlocksize(int bsize) -{ - g_chunkSize = bsize; - DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10); -} - -static void BMK_setNbIterations(int nbLoops) -{ - g_nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", g_nbIterations); -} - -static void BMK_setPause(void) -{ - g_pause = 1; -} - - -/********************************************************* -* Private functions -*********************************************************/ -static clock_t BMK_GetClockSpan( clock_t clockStart ) -{ - return clock() - clockStart; /* works even if overflow; max span ~30 mn */ -} - - -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t step = 64 MB; - BYTE* testmem=NULL; - - requiredMem = (((requiredMem >> 26) + 1) << 26); - requiredMem += 2*step; - if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; - - while (!testmem) { - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - testmem = (BYTE*) malloc ((size_t)requiredMem); - } - free (testmem); - - /* keep some space available */ - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - - return (size_t)requiredMem; -} - - -/********************************************************* -* Benchmark function -*********************************************************/ -static LZ4_stream_t LZ4_stream; -static void local_LZ4_resetDictT(void) -{ - LZ4_resetStream(&LZ4_stream); -} - -static void local_LZ4_createStream(void) -{ - LZ4_resetStream(&LZ4_stream); -} - -static int local_LZ4_saveDict(const char* in, char* out, int inSize) -{ - (void)in; - return LZ4_saveDict(&LZ4_stream, out, inSize); -} - -static int local_LZ4_compress_default_large(const char* in, char* out, int inSize) -{ - return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)); -} - -static int local_LZ4_compress_default_small(const char* in, char* out, int inSize) -{ - return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1); -} - -static int local_LZ4_compress_fast0(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0); -} - -static int local_LZ4_compress_fast1(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 1); -} - -static int local_LZ4_compress_fast2(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 2); -} - -static int local_LZ4_compress_fast17(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 17); -} - -static int local_LZ4_compress_fast_extState0(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0); -} - -static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSize) -{ - return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0); -} - -#ifndef LZ4_DLL_IMPORT -/* declare hidden function */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); - -static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize) -{ - return LZ4_compress_forceExtDict(&LZ4_stream, in, out, inSize); -} -#endif - - -/* HC compression functions */ -LZ4_streamHC_t LZ4_streamHC; -static void local_LZ4_resetStreamHC(void) -{ - LZ4_resetStreamHC(&LZ4_streamHC, 0); -} - -static int local_LZ4_saveDictHC(const char* in, char* out, int inSize) -{ - (void)in; - return LZ4_saveDictHC(&LZ4_streamHC, out, inSize); -} - -static int local_LZ4_compress_HC(const char* in, char* out, int inSize) -{ - return LZ4_compress_HC(in, out, inSize, LZ4_compressBound(inSize), 9); -} - -static int local_LZ4_compress_HC_extStateHC(const char* in, char* out, int inSize) -{ - return LZ4_compress_HC_extStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize), 9); -} - -static int local_LZ4_compress_HC_continue(const char* in, char* out, int inSize) -{ - return LZ4_compress_HC_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)); -} - - -/* decompression functions */ -static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_fast(in, out, outSize); - return outSize; -} - -static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65536); - return outSize; -} - -static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_safe_usingDict(in, out, inSize, outSize, out - 65536, 65536); - return outSize; -} - -#ifndef LZ4_DLL_IMPORT -extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const char* dict, int dictSize); - -static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536); - return outSize; -} -#endif - -static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize) -{ - return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); -} - - -/* frame functions */ -static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) -{ - return (int)LZ4F_compressFrame(out, LZ4F_compressFrameBound(inSize, NULL), in, inSize, NULL); -} - -static LZ4F_decompressionContext_t g_dCtx; - -static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize) -{ - size_t srcSize = inSize; - size_t dstSize = outSize; - size_t result; - result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL); - if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame\n"); exit(8); } - if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); } - return (int)dstSize; -} - - -#define NB_COMPRESSION_ALGORITHMS 100 -#define NB_DECOMPRESSION_ALGORITHMS 100 -int fullSpeedBench(const char** fileNamesTable, int nbFiles) -{ - int fileIdx=0; - - /* Init */ - { size_t const errorCode = LZ4F_createDecompressionContext(&g_dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; } } - - /* Loop for each fileName */ - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - - /* Allocation */ - chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters)); - orig_buff = (char*) malloc(benchedSize); - nbChunks = (int) ((benchedSize + (g_chunkSize-1)) / g_chunkSize); - maxCompressedChunkSize = LZ4_compressBound(g_chunkSize); - compressedBuffSize = nbChunks * maxCompressedChunkSize; - compressed_buff = (char*)malloc((size_t)compressedBuffSize); - if(!chunkP || !orig_buff || !compressed_buff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return(12); - } - - /* Fill in src buffer */ - DISPLAY("Loading %s... \r", inFileName); - readSize = fread(orig_buff, 1, benchedSize, inFile); - fclose(inFile); - - if (readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return 13; - } - - /* Calculating input Checksum */ - crcOriginal = XXH32(orig_buff, benchedSize,0); - - - /* Bench */ - { int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; - size_t cSize=0; - double ratio=0.; - - DISPLAY("\r%79s\r", ""); - DISPLAY(" %s : \n", inFileName); - - /* Bench Compression Algorithms */ - for (cAlgNb=0; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++) { - const char* compressorName; - int (*compressionFunction)(const char*, char*, int); - void (*initFunction)(void) = NULL; - double bestTime = 100000000.; - - /* filter compressionAlgo only */ - if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue; - - /* Init data chunks */ - { int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressed_buff; - nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); - for (i=0; i g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - - switch(cAlgNb) - { - case 0 : DISPLAY("Compression functions : \n"); continue; - case 1 : compressionFunction = local_LZ4_compress_default_large; compressorName = "LZ4_compress_default"; break; - case 2 : compressionFunction = local_LZ4_compress_default_small; compressorName = "LZ4_compress_default(small dst)"; break; - case 3 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break; - case 4 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break; - case 5 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break; - case 6 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break; - case 7 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break; - case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break; - - case 10: compressionFunction = local_LZ4_compress_HC; compressorName = "LZ4_compress_HC"; break; - case 12: compressionFunction = local_LZ4_compress_HC_extStateHC; compressorName = "LZ4_compress_HC_extStateHC"; break; - case 14: compressionFunction = local_LZ4_compress_HC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compress_HC_continue"; break; -#ifndef LZ4_DLL_IMPORT - case 20: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; -#endif - case 30: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; - chunkP[0].origSize = (int)benchedSize; nbChunks=1; - break; - case 40: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; - if (chunkP[0].origSize < 8) { DISPLAY(" cannot bench %s with less then 8 bytes \n", compressorName); continue; } - LZ4_loadDict(&LZ4_stream, chunkP[0].origBuffer, chunkP[0].origSize); - break; - case 41: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC"; - if (chunkP[0].origSize < 8) { DISPLAY(" cannot bench %s with less then 8 bytes \n", compressorName); continue; } - LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize); - break; - default : - continue; /* unknown ID : just skip */ - } - - for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { - double averageTime; - clock_t clockTime; - - PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize); - { size_t i; for (i=0; i%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); - } - - if (ratio<100.) - DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); - else - DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000); - } - - /* Prepare layout for decompression */ - /* Init data chunks */ - { int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressed_buff; - - nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); - for (i=0; i g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - for (chunkNb=0; chunkNb\r", loopNb, dName, (int)benchedSize); - - nb_loops = 0; - clockTime = clock(); - while(clock() == clockTime); - clockTime = clock(); - while(BMK_GetClockSpan(clockTime) < TIMELOOP) { - for (chunkNb=0; chunkNb %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000); - - /* CRC Checking */ - crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); - if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } - } - - DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000); - } - } - free(orig_buff); - free(compressed_buff); - free(chunkP); - } - - LZ4F_freeDecompressionContext(g_dCtx); - if (g_pause) { printf("press enter...\n"); (void)getchar(); } - - return 0; -} - - -static int usage(const char* exename) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); - DISPLAY( "Arguments :\n"); - DISPLAY( " -c : compression tests only\n"); - DISPLAY( " -d : decompression tests only\n"); - DISPLAY( " -H/-h : Help (this text + advanced options)\n"); - return 0; -} - -static int usage_advanced(void) -{ - DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS); - DISPLAY( " -d# : test only decompression function # [1-%i]\n", NB_DECOMPRESSION_ALGORITHMS); - DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); - DISPLAY( " -B# : Block size [4-7](default : 7)\n"); - return 0; -} - -static int badusage(const char* exename) -{ - DISPLAY("Wrong parameters\n"); - usage(exename); - return 0; -} - -int main(int argc, const char** argv) -{ - int i, - filenamesStart=2; - const char* exename = argv[0]; - const char* input_filename=0; - - // Welcome message - DISPLAY(WELCOME_MESSAGE); - - if (argc<2) { badusage(exename); return 1; } - - for(i=1; i= '0') && (argument[1]<= '9')) { - g_compressionAlgo *= 10; - g_compressionAlgo += argument[1] - '0'; - argument++; - } - break; - - // Select decompression algorithm only - case 'd': - g_compressionTest = 0; - while ((argument[1]>= '0') && (argument[1]<= '9')) { - g_decompressionAlgo *= 10; - g_decompressionAlgo += argument[1] - '0'; - argument++; - } - break; - - // Display help on usage - case 'h' : - case 'H': usage(exename); usage_advanced(); return 0; - - // Modify Block Properties - case 'B': - while (argument[1]!=0) - switch(argument[1]) - { - case '4': - case '5': - case '6': - case '7': - { int B = argument[1] - '0'; - int S = 1 << (8 + 2*B); - BMK_setBlocksize(S); - argument++; - break; - } - case 'D': argument++; break; - default : goto _exit_blockProperties; - } -_exit_blockProperties: - break; - - // Modify Nb Iterations - case 'i': - if ((argument[1] >='0') && (argument[1] <='9')) { - int iters = argument[1] - '0'; - BMK_setNbIterations(iters); - argument++; - } - break; - - // Pause at the end (hidden option) - case 'p': BMK_setPause(); break; - - // Unknown command - default : badusage(exename); return 1; - } - } - continue; - } - - // first provided filename is input - if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } - - } - - // No input filename ==> Error - if(!input_filename) { badusage(exename); return 1; } - - return fullSpeedBench(argv+filenamesStart, argc-filenamesStart); - -} diff --git a/library/lz4-mac/tests/fuzzer.c b/library/lz4-mac/tests/fuzzer.c deleted file mode 100755 index 2e22912d..00000000 --- a/library/lz4-mac/tests/fuzzer.c +++ /dev/null @@ -1,1180 +0,0 @@ -/* - fuzzer.c - Fuzzer test tool for LZ4 - Copyright (C) Yann Collet 2012-2017 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repo : https://github.com/lz4/lz4 -*/ - -/*-************************************ -* Compiler options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ -# pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ -#endif - -#define LZ4_DISABLE_DEPRECATE_WARNINGS - -/*-************************************ -* Dependencies -**************************************/ -#include "platform.h" /* _CRT_SECURE_NO_WARNINGS */ -#include "util.h" /* U32 */ -#include -#include /* fgets, sscanf */ -#include /* strcmp */ -#include /* clock_t, clock, CLOCKS_PER_SEC */ -#define LZ4_HC_STATIC_LINKING_ONLY -#include "lz4hc.h" -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" - - -/*-************************************ -* Basic Types -**************************************/ -#if !defined(__cplusplus) && !(defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */ -#endif - - -/*-************************************ -* Constants -**************************************/ -#define NB_ATTEMPTS (1<<16) -#define COMPRESSIBLE_NOISE_LENGTH (1 << 21) -#define FUZ_MAX_BLOCK_SIZE (1 << 17) -#define FUZ_MAX_DICT_SIZE (1 << 15) -#define FUZ_COMPRESSIBILITY_DEFAULT 60 -#define PRIME1 2654435761U -#define PRIME2 2246822519U -#define PRIME3 3266489917U - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - - -/*-*************************************** -* Macros -*****************************************/ -#define DISPLAY(...) fprintf(stdout, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static int g_displayLevel = 2; - -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - - -/*-******************************************************* -* Fuzzer functions -*********************************************************/ -static clock_t FUZ_GetClockSpan(clock_t clockStart) -{ - return clock() - clockStart; /* works even if overflow; max span ~ 30mn */ -} - -static void FUZ_displayUpdate(unsigned testNb) -{ - static clock_t g_time = 0; - static const clock_t g_refreshRate = CLOCKS_PER_SEC / 5; - if ((FUZ_GetClockSpan(g_time) > g_refreshRate) || (g_displayLevel>=4)) { - g_time = clock(); - DISPLAY("\r%5u ", testNb); - fflush(stdout); - } -} - -static U32 FUZ_rotl32(U32 u32, U32 nbBits) -{ - return ((u32 << nbBits) | (u32 >> (32 - nbBits))); -} - -static U32 FUZ_rand(U32* src) -{ - U32 rand32 = *src; - rand32 *= PRIME1; - rand32 ^= PRIME2; - rand32 = FUZ_rotl32(rand32, 13); - *src = rand32; - return rand32; -} - - -#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767) -#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) -static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) -{ - BYTE* const BBuffer = (BYTE*)buffer; - size_t pos = 0; - U32 const P32 = (U32)(32768 * proba); - - /* First Bytes */ - while (pos < 20) - BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); - - while (pos < bufferSize) { - /* Select : Literal (noise) or copy (within 64K) */ - if (FUZ_RAND15BITS < P32) { - /* Copy (within 64K) */ - size_t const length = FUZ_RANDLENGTH + 4; - size_t const d = MIN(pos+length, bufferSize); - size_t match; - size_t offset = FUZ_RAND15BITS + 1; - while (offset > pos) offset >>= 1; - match = pos - offset; - while (pos < d) BBuffer[pos++] = BBuffer[match++]; - } else { - /* Literal (noise) */ - size_t const length = FUZ_RANDLENGTH; - size_t const d = MIN(pos+length, bufferSize); - while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); - } - } -} - - -#define MAX_NB_BUFF_I134 150 -#define BLOCKSIZE_I134 (32 MB) -/*! FUZ_AddressOverflow() : -* Aggressively pushes memory allocation limits, -* and generates patterns which create address space overflow. -* only possible in 32-bits mode */ -static int FUZ_AddressOverflow(void) -{ - char* buffers[MAX_NB_BUFF_I134+1]; - int nbBuff=0; - int highAddress = 0; - - DISPLAY("Overflow tests : "); - - /* Only possible in 32-bits */ - if (sizeof(void*)==8) { - DISPLAY("64 bits mode : no overflow \n"); - fflush(stdout); - return 0; - } - - buffers[0] = (char*)malloc(BLOCKSIZE_I134); - buffers[1] = (char*)malloc(BLOCKSIZE_I134); - if ((!buffers[0]) || (!buffers[1])) { - free(buffers[0]); free(buffers[1]); - DISPLAY("not enough memory for tests \n"); - return 0; - } - - for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) { - DISPLAY("%3i \b\b\b\b", nbBuff); fflush(stdout); - buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134); - if (buffers[nbBuff]==NULL) goto _endOfTests; - - if (((uintptr_t)buffers[nbBuff] > (uintptr_t)0x80000000) && (!highAddress)) { - DISPLAY("high address detected : "); - fflush(stdout); - highAddress=1; - } - - { size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512); - unsigned const nbOf255 = (unsigned)((sizeToGenerateOverflow / 255) + 1); - char* const input = buffers[nbBuff-1]; - char* output = buffers[nbBuff]; - int r; - input[0] = (char)0xF0; /* Literal length overflow */ - input[1] = (char)0xFF; - input[2] = (char)0xFF; - input[3] = (char)0xFF; - { unsigned u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; } - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; } - input[0] = (char)0x1F; /* Match length overflow */ - input[1] = (char)0x01; - input[2] = (char)0x01; - input[3] = (char)0x00; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; } - - output = buffers[nbBuff-2]; /* Reverse in/out pointer order */ - input[0] = (char)0xF0; /* Literal length overflow */ - input[1] = (char)0xFF; - input[2] = (char)0xFF; - input[3] = (char)0xFF; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - input[0] = (char)0x1F; /* Match length overflow */ - input[1] = (char)0x01; - input[2] = (char)0x01; - input[3] = (char)0x00; - r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); - if (r>0) goto _overflowError; - } - } - - nbBuff++; -_endOfTests: - { int i; for (i=0 ; i=4 ? printf("%2u\b\b", testNb), fflush(stdout) : 0; } - - - /* init */ - if(!CNBuffer || !compressedBuffer || !decodedBuffer) { - DISPLAY("Not enough memory to start fuzzer tests"); - goto _output_error; - } - memset(&LZ4dict, 0, sizeof(LZ4dict)); - { U32 randState = coreRandState ^ PRIME3; - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - } - - /* move to startCycle */ - for (cycleNb = 0; cycleNb < startCycle; cycleNb++) - (void) FUZ_rand(&coreRandState); /* sync coreRandState */ - - /* Main test loop */ - for (cycleNb = startCycle; - (cycleNb < nbCycles) || (FUZ_GetClockSpan(clockStart) < clockDuration); - cycleNb++) { - U32 testNb = 0; - U32 randState = FUZ_rand(&coreRandState) ^ PRIME3; - int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1; - int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - int const dictSize = MIN(dictSizeRand, blockStart); - int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); - char* const block = ((char*)CNBuffer) + blockStart; - const char* dict = block - dictSize; - int compressedSize, HCcompressedSize; - int blockContinueCompressedSize; - U32 const crcOrig = XXH32(block, blockSize, 0); - U32 crcCheck; - int ret; - - FUZ_displayUpdate(cycleNb); - - /* Compression tests */ - - /* Test compression destSize */ - FUZ_DISPLAYTEST; - { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char endCheck = FUZ_rand(&randState) & 255; - compressedBuffer[targetSize] = endCheck; - ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); - FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); - FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !"); - FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); - if (targetSize>0) { - /* check correctness */ - U32 const crcBase = XXH32(block, srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; - FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); - FUZ_DISPLAYTEST; - compressedSize = ret; - decodedBuffer[srcSize] = canary; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); - FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); - FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); - FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } - - DISPLAYLEVEL(5, " OK \n"); - } - else - DISPLAYLEVEL(5, " \n"); - } - - /* Test compression HC destSize */ - FUZ_DISPLAYTEST; - { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char const endCheck = FUZ_rand(&randState) & 255; - void* ctx = LZ4_createHC(block); - FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); - compressedBuffer[targetSize] = endCheck; - ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); - LZ4_freeHC(ctx); - FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); - FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !"); - FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); - if (targetSize>0) { - /* check correctness */ - U32 const crcBase = XXH32(block, srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; - FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed"); - FUZ_DISPLAYTEST; - compressedSize = ret; - decodedBuffer[srcSize] = canary; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); - FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); - FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); - FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } - - DISPLAYLEVEL(5, " OK \n"); - } - else - DISPLAYLEVEL(5, " \n"); - } - - /* Test compression HC */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); - FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed"); - HCcompressedSize = ret; - - /* Test compression HC using external state */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); - FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed"); - - /* Test compression using external state */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8); - FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed"); - - /* Test compression */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed"); - compressedSize = ret; - - /* Decompression tests */ - - /* Test decoding with output size being exactly what's necessary => must work */ - FUZ_DISPLAYTEST; - ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); - FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); - - /* Test decoding with one byte missing => must fail */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); - - /* Test decoding with one byte too much => must fail */ - FUZ_DISPLAYTEST; - ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); - - /* Test decoding with output size exactly what's necessary => must work */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); - - // Test decoding with more than enough output size => must work - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - decodedBuffer[blockSize+1] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); - //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? - FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); - - // Test decoding with output size being one byte too short => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size"); - - // Test decoding with output size being 10 bytes too short => must fail - FUZ_DISPLAYTEST; - if (blockSize>10) - { - decodedBuffer[blockSize-10] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size"); - } - - // Test decoding with input size being one byte too short => must fail - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); - - // Test decoding with input size being one byte too large => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - - // Test partial decoding with target output size being max/2 => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); - - // Test partial decoding with target output size being just below max => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); - - /* Test Compression with limited output size */ - - /* Test compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space"); - - /* Test compression with output size being exactly what's necessary and external state (should work) */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1); - FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space"); - - /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); - FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space"); - - /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); - FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space"); - - /* Test compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; - { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; - if (missingBytes >= compressedSize) missingBytes = compressedSize-1; - missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ - compressedBuffer[compressedSize-missingBytes] = 0; - ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes); - FUZ_CHECKTEST(ret, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes) - } - - /* Test HC compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; - { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; - if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; - missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ - compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel); - FUZ_CHECKTEST(ret, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes) - } - - - /*-******************/ - /* Dictionary tests */ - /*-******************/ - - /* Compress using dictionary */ - FUZ_DISPLAYTEST; - { LZ4_stream_t LZ4_stream; - LZ4_resetStream(&LZ4_stream); - LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1); /* Just to fill hash tables */ - blockContinueCompressedSize = LZ4_compress_fast_continue (&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed"); - } - - /* Decompress with dictionary as prefix */ - FUZ_DISPLAYTEST; - memcpy(decodedBuffer, dict, dictSize); - ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - if (crcCheck!=crcOrig) { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); - - } - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - - /* Compress using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_loadDict(&LZ4dict, dict, dictSize); - blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed"); - - FUZ_DISPLAYTEST; - LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); - FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); - - FUZ_DISPLAYTEST; - LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer"); - - /* Decompress with dictionary as external */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - - FUZ_DISPLAYTEST; - { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; - if ((U32)blockSize > missingBytes) { - decodedBuffer[blockSize-missingBytes] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); - FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); - } } - - /* Compress HC using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed"); - - FUZ_DISPLAYTEST; - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); - - FUZ_DISPLAYTEST; - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - - /* Compress HC continue destSize */ - FUZ_DISPLAYTEST; - { int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5; - int consumedSize = blockSize; - FUZ_DISPLAYTEST; - LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); - LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); - blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace); - DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed"); - FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow"); - FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow"); - - FUZ_DISPLAYTEST; - decodedBuffer[consumedSize] = 0; - ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize); - FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") - { U32 const crcSrc = XXH32(block, consumedSize, 0); - U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0); - if (crcSrc!=crcDst) - FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data"); - } - } - - /* ***** End of tests *** */ - /* Fill stats */ - bytes += blockSize; - cbytes += compressedSize; - hcbytes += HCcompressedSize; - ccbytes += blockContinueCompressedSize; - } - - if (nbCycles<=1) nbCycles = cycleNb; /* end by time */ - bytes += !bytes; /* avoid division by 0 */ - printf("\r%7u /%7u - ", cycleNb, nbCycles); - printf("all tests completed successfully \n"); - printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); - printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); - printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); - - /* release memory */ - { -_exit: - free(CNBuffer); - free(compressedBuffer); - free(decodedBuffer); - free(stateLZ4); - free(stateLZ4HC); - return result; - -_output_error: - result = 1; - goto _exit; - } -} - - -#define testInputSize (192 KB) -#define testCompressedSize (128 KB) -#define ringBufferSize (8 KB) - -static void FUZ_unitTests(int compressionLevel) -{ - const unsigned testNb = 0; - const unsigned seed = 0; - const unsigned cycleNb= 0; - char testInput[testInputSize]; - char testCompressed[testCompressedSize]; - char testVerify[testInputSize]; - char ringBuffer[ringBufferSize]; - U32 randState = 1; - - /* Init */ - FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); - - /* 32-bits address space overflow test */ - FUZ_AddressOverflow(); - - /* LZ4 streaming tests */ - { LZ4_stream_t* statePtr; - LZ4_stream_t streamingState; - U64 crcOrig; - int result; - - /* Allocation test */ - statePtr = LZ4_createStream(); - FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed"); - LZ4_freeStream(statePtr); - - /* simple compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStream(&streamingState); - result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); - FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!"); - - result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } - - /* ring buffer test */ - { XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - LZ4_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; - const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; - if (dNext + messageSize > dBufferSize) dNext = 0; - } - } - } - - /* LZ4 HC streaming tests */ - { LZ4_streamHC_t* sp; - LZ4_streamHC_t sHC; - U64 crcOrig; - int result; - - /* Allocation test */ - sp = LZ4_createStreamHC(); - FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); - LZ4_freeStreamHC(sp); - - /* simple HC compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); - result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } - - /* simple dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 64 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); } - - /* multiple HC compression test with dictionary */ - { int result1, result2; - int segSize = testCompressedSize / 2; - crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, segSize); - result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); - FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); - result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); - FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); - result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } - } - - /* remote dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 32 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); } - - /* multiple HC compression with ext. dictionary */ - { XXH64_state_t crcOrigState; - XXH64_state_t crcNewState; - const char* dict = testInput + 3; - int dictSize = (FUZ_rand(&randState) & 8191); - char* dst = testVerify; - - size_t segStart = dictSize + 7; - int segSize = (FUZ_rand(&randState) & 8191); - int segNb = 1; - - LZ4_resetStreamHC(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, dict, dictSize); - - XXH64_reset(&crcOrigState, 0); - XXH64_reset(&crcNewState, 0); - - while (segStart + segSize < testInputSize) { - XXH64_update(&crcOrigState, testInput + segStart, segSize); - crcOrig = XXH64_digest(&crcOrigState); - result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - - result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); - XXH64_update(&crcNewState, dst, segSize); - { U64 const crcNew = XXH64_digest(&crcNewState); - if (crcOrig != crcNew) FUZ_findDiff(dst, testInput+segStart); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); - } - - dict = dst; - //dict = testInput + segStart; - dictSize = segSize; - - dst += segSize + 1; - segNb ++; - - segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; - segSize = (FUZ_rand(&randState) & 8191); - } - } - - /* ring buffer test */ - { XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - LZ4_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; - const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; - if (dNext + messageSize > dBufferSize) dNext = 0; - } - } - - /* small decoder-side ring buffer test */ - { XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - LZ4_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 12; - const U32 maxMessageSizeMask = (1< dBufferSize) dNext = 0; - - while (totalMessageSize < 9 MB) { - XXH64_update(&xxhOrig, testInput + iNext, messageSize); - crcOrig = XXH64_digest(&xxhOrig); - - result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - - result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); - - XXH64_update(&xxhNew, testVerify + dNext, messageSize); - { U64 const crcNew = XXH64_digest(&xxhNew); - if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test"); - } - /* prepare next message */ - dNext += messageSize; - totalMessageSize += messageSize; - messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; - iNext = (FUZ_rand(&randState) & 65535); - if (dNext > dBufferSize) dNext = 0; - } - } - } - - printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel); - return; -_output_error: - exit(1); -} - - -static int FUZ_usage(const char* programName) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); - DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); - DISPLAY( " -s# : Select seed (default:prompt user)\n"); - DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); - DISPLAY( " -v : verbose\n"); - DISPLAY( " -p : pause at the end\n"); - DISPLAY( " -h : display help and exit\n"); - return 0; -} - - -int main(int argc, const char** argv) -{ - U32 seed = 0; - int seedset = 0; - int argNb; - int nbTests = NB_ATTEMPTS; - int testNb = 0; - int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int use_pause = 0; - const char* programName = argv[0]; - U32 duration = 0; - - /* Check command line */ - for(argNb=1; argNb='0') && (*argument<='9')) { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } - break; - - case 'T': - argument++; - nbTests = 0; duration = 0; - for (;;) { - switch(*argument) - { - case 'm': duration *= 60; argument++; continue; - case 's': - case 'n': argument++; continue; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; - } - break; - } - break; - - case 's': - argument++; - seed=0; seedset=1; - while ((*argument>='0') && (*argument<='9')) { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - - case 't': /* select starting test nb */ - argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } - break; - - case 'P': /* change probability */ - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - default: ; - } - } - } - } - - printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_versionString()); - - if (!seedset) { - time_t const t = time(NULL); - U32 const h = XXH32(&t, sizeof(t), 1); - seed = h % 10000; - } - printf("Seed = %u\n", seed); - - if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - - if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); } - - if (nbTests<=0) nbTests=1; - - { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); - if (use_pause) { - DISPLAY("press enter ... \n"); - (void)getchar(); - } - return result; - } -} diff --git a/library/lz4-mac/tests/test-lz4-speed.py b/library/lz4-mac/tests/test-lz4-speed.py deleted file mode 100755 index ca8f0101..00000000 --- a/library/lz4-mac/tests/test-lz4-speed.py +++ /dev/null @@ -1,351 +0,0 @@ -#! /usr/bin/env python3 - -# -# Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. -# - -# Limitations: -# - doesn't support filenames with spaces -# - dir1/lz4 and dir2/lz4 will be merged in a single results file - -import argparse -import os -import string -import subprocess -import time -import traceback -import hashlib - -script_version = 'v1.7.2 (2016-11-08)' -default_repo_url = 'https://github.com/lz4/lz4.git' -working_dir_name = 'speedTest' -working_path = os.getcwd() + '/' + working_dir_name # /path/to/lz4/tests/speedTest -clone_path = working_path + '/' + 'lz4' # /path/to/lz4/tests/speedTest/lz4 -email_header = 'lz4_speedTest' -pid = str(os.getpid()) -verbose = False -clang_version = "unknown" -gcc_version = "unknown" -args = None - - -def hashfile(hasher, fname, blocksize=65536): - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(blocksize), b""): - hasher.update(chunk) - return hasher.hexdigest() - - -def log(text): - print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text) - - -def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True): - if print_command: - log("> " + command) - popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd) - stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout) - stderr_lines = stderr_lines.decode("utf-8") - stdout_lines = stdout_lines.decode("utf-8") - if print_output: - if stdout_lines: - print(stdout_lines) - if stderr_lines: - print(stderr_lines) - if popen.returncode is not None and popen.returncode != 0: - if stderr_lines and not print_output and print_error: - print(stderr_lines) - raise RuntimeError(stdout_lines + stderr_lines) - return (stdout_lines + stderr_lines).splitlines() -execute.cwd = None - - -def does_command_exist(command): - try: - execute(command, verbose, False, False) - except Exception: - return False - return True - - -def send_email(emails, topic, text, have_mutt, have_mail): - logFileName = working_path + '/' + 'tmpEmailContent' - with open(logFileName, "w") as myfile: - myfile.writelines(text) - myfile.close() - if have_mutt: - execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose) - elif have_mail: - execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose) - else: - log("e-mail cannot be sent (mail or mutt not found)") - - -def send_email_with_attachments(branch, commit, last_commit, args, text, results_files, - logFileName, have_mutt, have_mail): - with open(logFileName, "w") as myfile: - myfile.writelines(text) - myfile.close() - email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \ - % (email_header, pid, branch, commit, last_commit, - args.lowerLimit, args.ratioLimit) - if have_mutt: - execute('mutt -s "' + email_topic + '" ' + args.emails + ' -a ' + results_files - + ' < ' + logFileName) - elif have_mail: - execute('mail -s "' + email_topic + '" ' + args.emails + ' < ' + logFileName) - else: - log("e-mail cannot be sent (mail or mutt not found)") - - -def git_get_branches(): - execute('git fetch -p', verbose) - branches = execute('git branch -rl', verbose) - output = [] - for line in branches: - if ("HEAD" not in line) and ("coverity_scan" not in line) and ("gh-pages" not in line): - output.append(line.strip()) - return output - - -def git_get_changes(branch, commit, last_commit): - fmt = '--format="%h: (%an) %s, %ar"' - if last_commit is None: - commits = execute('git log -n 10 %s %s' % (fmt, commit)) - else: - commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit)) - return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits) - - -def get_last_results(resultsFileName): - if not os.path.isfile(resultsFileName): - return None, None, None, None - commit = None - csize = [] - cspeed = [] - dspeed = [] - with open(resultsFileName, 'r') as f: - for line in f: - words = line.split() - if len(words) <= 4: # branch + commit + compilerVer + md5 - commit = words[1] - csize = [] - cspeed = [] - dspeed = [] - if (len(words) == 8) or (len(words) == 9): # results: "filename" or "XX files" - csize.append(int(words[1])) - cspeed.append(float(words[3])) - dspeed.append(float(words[5])) - return commit, csize, cspeed, dspeed - - -def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, - testFilePath, fileName, last_csize, last_cspeed, last_dspeed): - sleepTime = 30 - while os.getloadavg()[0] > args.maxLoadAvg: - log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" - % (os.getloadavg()[0], args.maxLoadAvg, sleepTime)) - time.sleep(sleepTime) - start_load = str(os.getloadavg()) - result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True) - end_load = str(os.getloadavg()) - linesExpected = args.lastCLevel + 1 - if len(result) != linesExpected: - raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result))) - with open(resultsFileName, "a") as myfile: - myfile.write('%s %s %s md5=%s\n' % (branch, commit, compilerVersion, md5sum)) - myfile.write('\n'.join(result) + '\n') - myfile.close() - if (last_cspeed == None): - log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName)) - return "" - commit, csize, cspeed, dspeed = get_last_results(resultsFileName) - text = "" - for i in range(0, min(len(cspeed), len(last_cspeed))): - print("%s:%s -%d cSpeed=%6.2f cLast=%6.2f cDiff=%1.4f dSpeed=%6.2f dLast=%6.2f dDiff=%1.4f ratioDiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], float(last_csize[i])/csize[i], fileName)) - if (cspeed[i]/last_cspeed[i] < args.lowerLimit): - text += "WARNING: %s -%d cSpeed=%.2f cLast=%.2f cDiff=%.4f %s\n" % (executableName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName) - if (dspeed[i]/last_dspeed[i] < args.lowerLimit): - text += "WARNING: %s -%d dSpeed=%.2f dLast=%.2f dDiff=%.4f %s\n" % (executableName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName) - if (float(last_csize[i])/csize[i] < args.ratioLimit): - text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName) - if text: - text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s\n%s last_commit=%s md5=%s\n" % (args.maxLoadAvg, start_load, end_load, compilerVersion, last_commit, md5sum)) + text - return text - - -def update_config_file(branch, commit): - last_commit = None - commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt" - if os.path.isfile(commitFileName): - with open(commitFileName, 'r') as infile: - last_commit = infile.read() - with open(commitFileName, 'w') as outfile: - outfile.write(commit) - return last_commit - - -def double_check(branch, commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName): - last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName) - if not args.dry_run: - text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed) - if text: - log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit)) - text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed) - return text - - -def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail): - local_branch = branch.split('/')[1] - version = local_branch.rpartition('-')[2] + '_' + commit - if not args.dry_run: - execute('make -C programs clean lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DLZ4_GIT_COMMIT=%s" && ' % version + - 'mv programs/lz4 programs/lz4_clang && ' + - 'make -C programs clean lz4 lz4c32 MOREFLAGS="-DLZ4_GIT_COMMIT=%s"' % version) - md5_lz4 = hashfile(hashlib.md5(), clone_path + '/programs/lz4') - md5_lz4c32 = hashfile(hashlib.md5(), clone_path + '/programs/lz4c32') - md5_lz4_clang = hashfile(hashlib.md5(), clone_path + '/programs/lz4_clang') - print("md5(lz4)=%s\nmd5(lz4c32)=%s\nmd5(lz4_clang)=%s" % (md5_lz4, md5_lz4c32, md5_lz4_clang)) - print("gcc_version=%s clang_version=%s" % (gcc_version, clang_version)) - - logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" - text_to_send = [] - results_files = "" - - for filePath in testFilePaths: - fileName = filePath.rpartition('/')[2] - resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - text = double_check(branch, commit, args, 'lz4', md5_lz4, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName) - if text: - text_to_send.append(text) - results_files += resultsFileName + " " - resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - text = double_check(branch, commit, args, 'lz4c32', md5_lz4c32, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName) - if text: - text_to_send.append(text) - results_files += resultsFileName + " " - resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" - text = double_check(branch, commit, args, 'lz4_clang', md5_lz4_clang, 'clang_version='+clang_version, resultsFileName, filePath, fileName) - if text: - text_to_send.append(text) - results_files += resultsFileName + " " - if text_to_send: - send_email_with_attachments(branch, commit, last_commit, args, text_to_send, results_files, logFileName, have_mutt, have_mail) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('testFileNames', help='file or directory names list for speed benchmark') - parser.add_argument('emails', help='list of e-mail addresses to send warnings') - parser.add_argument('--message', '-m', help='attach an additional message to e-mail', default="") - parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url) - parser.add_argument('--lowerLimit', '-l', type=float, help='send email if speed is lower than given limit', default=0.98) - parser.add_argument('--ratioLimit', '-r', type=float, help='send email if ratio is lower than given limit', default=0.999) - parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75) - parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5) - parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300) - parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=1800) - parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False) - parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False) - args = parser.parse_args() - verbose = args.verbose - - # check if test files are accessible - testFileNames = args.testFileNames.split() - testFilePaths = [] - for fileName in testFileNames: - fileName = os.path.expanduser(fileName) - if os.path.isfile(fileName) or os.path.isdir(fileName): - testFilePaths.append(os.path.abspath(fileName)) - else: - log("ERROR: File/directory not found: " + fileName) - exit(1) - - # check availability of e-mail senders - have_mutt = does_command_exist("mutt -h") - have_mail = does_command_exist("mail -V") - if not have_mutt and not have_mail: - log("ERROR: e-mail senders 'mail' or 'mutt' not found") - exit(1) - - clang_version = execute("clang -v 2>&1 | grep 'clang version' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0]; - gcc_version = execute("gcc -dumpversion", verbose)[0]; - - if verbose: - print("PARAMETERS:\nrepoURL=%s" % args.repoURL) - print("working_path=%s" % working_path) - print("clone_path=%s" % clone_path) - print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)) - print("message=%s" % args.message) - print("emails=%s" % args.emails) - print("maxLoadAvg=%s" % args.maxLoadAvg) - print("lowerLimit=%s" % args.lowerLimit) - print("ratioLimit=%s" % args.ratioLimit) - print("lastCLevel=%s" % args.lastCLevel) - print("sleepTime=%s" % args.sleepTime) - print("timeout=%s" % args.timeout) - print("dry_run=%s" % args.dry_run) - print("verbose=%s" % args.verbose) - print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail)) - - # clone lz4 repo if needed - if not os.path.isdir(working_path): - os.mkdir(working_path) - if not os.path.isdir(clone_path): - execute.cwd = working_path - execute('git clone ' + args.repoURL) - if not os.path.isdir(clone_path): - log("ERROR: lz4 clone not found: " + clone_path) - exit(1) - execute.cwd = clone_path - - # check if speedTest.pid already exists - pidfile = "./speedTest.pid" - if os.path.isfile(pidfile): - log("ERROR: %s already exists, exiting" % pidfile) - exit(1) - - send_email(args.emails, '[%s:%s] test-lz4-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail) - with open(pidfile, 'w') as the_file: - the_file.write(pid) - - branch = "" - commit = "" - first_time = True - while True: - try: - if first_time: - first_time = False - else: - if verbose: - log("sleep for %s seconds" % args.sleepTime) - time.sleep(args.sleepTime) - loadavg = os.getloadavg()[0] - if (loadavg <= args.maxLoadAvg): - branches = git_get_branches() - for branch in branches: - commit = execute('git show -s --format=%h ' + branch, verbose)[0] - last_commit = update_config_file(branch, commit) - if commit == last_commit: - log("skipping branch %s: head %s already processed" % (branch, commit)) - else: - log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit)) - execute('git checkout -- . && git checkout ' + branch) - print(git_get_changes(branch, commit, last_commit)) - test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail) - else: - log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg)) - except Exception as e: - stack = traceback.format_exc() - email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit) - send_email(args.emails, email_topic, stack, have_mutt, have_mail) - print(stack) - except KeyboardInterrupt: - os.unlink(pidfile) - send_email(args.emails, '[%s:%s] test-lz4-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail) - exit(0) diff --git a/library/lz4-mac/tests/test-lz4-versions.py b/library/lz4-mac/tests/test-lz4-versions.py deleted file mode 100755 index d7fd199e..00000000 --- a/library/lz4-mac/tests/test-lz4-versions.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 -"""Test LZ4 interoperability between versions""" - -# -# Copyright (C) 2011-present, Takayuki Matsuoka -# All rights reserved. -# GPL v2 License -# - -import glob -import subprocess -import filecmp -import os -import shutil -import sys -import hashlib - -repo_url = 'https://github.com/lz4/lz4.git' -tmp_dir_name = 'tests/versionsTest' -make_cmd = 'make' -git_cmd = 'git' -test_dat_src = 'README.md' -test_dat = 'test_dat' -head = 'v999' - -def proc(cmd_args, pipe=True, dummy=False): - if dummy: - return - if pipe: - subproc = subprocess.Popen(cmd_args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - else: - subproc = subprocess.Popen(cmd_args) - return subproc.communicate() - -def make(args, pipe=True): - return proc([make_cmd] + args, pipe) - -def git(args, pipe=True): - return proc([git_cmd] + args, pipe) - -def get_git_tags(): - stdout, stderr = git(['tag', '-l', 'r[0-9][0-9][0-9]']) - tags = stdout.decode('utf-8').split() - stdout, stderr = git(['tag', '-l', 'v[1-9].[0-9].[0-9]']) - tags += stdout.decode('utf-8').split() - return tags - -# https://stackoverflow.com/a/19711609/2132223 -def sha1_of_file(filepath): - with open(filepath, 'rb') as f: - return hashlib.sha1(f.read()).hexdigest() - -if __name__ == '__main__': - error_code = 0 - base_dir = os.getcwd() + '/..' # /path/to/lz4 - tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/lz4/tests/versionsTest - clone_dir = tmp_dir + '/' + 'lz4' # /path/to/lz4/tests/versionsTest/lz4 - programs_dir = base_dir + '/programs' # /path/to/lz4/programs - os.makedirs(tmp_dir, exist_ok=True) - - # since Travis clones limited depth, we should clone full repository - if not os.path.isdir(clone_dir): - git(['clone', repo_url, clone_dir]) - - shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat) - - # Retrieve all release tags - print('Retrieve all release tags :') - os.chdir(clone_dir) - tags = [head] + get_git_tags() - print(tags); - - # Build all release lz4c and lz4c32 - for tag in tags: - os.chdir(base_dir) - dst_lz4c = '{}/lz4c.{}' .format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c. - dst_lz4c32 = '{}/lz4c32.{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c32. - if not os.path.isfile(dst_lz4c) or not os.path.isfile(dst_lz4c32) or tag == head: - if tag != head: - r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/ - os.makedirs(r_dir, exist_ok=True) - os.chdir(clone_dir) - git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) - os.chdir(r_dir + '/programs') # /path/to/lz4/lz4test//programs - else: - os.chdir(programs_dir) - make(['clean', 'lz4c'], False) - shutil.copy2('lz4c', dst_lz4c) - make(['clean', 'lz4c32'], False) - shutil.copy2('lz4c32', dst_lz4c32) - - # Compress test.dat by all released lz4c and lz4c32 - print('Compress test.dat by all released lz4c and lz4c32') - os.chdir(tmp_dir) - for lz4 in glob.glob("*.lz4"): - os.remove(lz4) - for tag in tags: - proc(['./lz4c.' + tag, '-1fz', test_dat, test_dat + '_1_64_' + tag + '.lz4']) - proc(['./lz4c.' + tag, '-9fz', test_dat, test_dat + '_9_64_' + tag + '.lz4']) - proc(['./lz4c32.' + tag, '-1fz', test_dat, test_dat + '_1_32_' + tag + '.lz4']) - proc(['./lz4c32.' + tag, '-9fz', test_dat, test_dat + '_9_32_' + tag + '.lz4']) - - print('Full list of compressed files') - lz4s = sorted(glob.glob('*.lz4')) - for lz4 in lz4s: - print(lz4 + ' : ' + repr(os.path.getsize(lz4))) - - # Remove duplicated .lz4 files - print('') - print('Duplicated files') - lz4s = sorted(glob.glob('*.lz4')) - for i, lz4 in enumerate(lz4s): - if not os.path.isfile(lz4): - continue - for j in range(i+1, len(lz4s)): - lz4t = lz4s[j] - if not os.path.isfile(lz4t): - continue - if filecmp.cmp(lz4, lz4t): - os.remove(lz4t) - print('{} == {}'.format(lz4, lz4t)) - - print('Enumerate only different compressed files') - lz4s = sorted(glob.glob('*.lz4')) - for lz4 in lz4s: - print(lz4 + ' : ' + repr(os.path.getsize(lz4)) + ', ' + sha1_of_file(lz4)) - - # Decompress remained .lz4 files by all released lz4c and lz4c32 - print('Decompression tests and verifications') - lz4s = sorted(glob.glob('*.lz4')) - for dec in glob.glob("*.dec"): - os.remove(dec) - for lz4 in lz4s: - print(lz4, end=" ") - for tag in tags: - print(tag, end=" ") - proc(['./lz4c.' + tag, '-df', lz4, lz4 + '_d64_' + tag + '.dec']) - proc(['./lz4c32.' + tag, '-df', lz4, lz4 + '_d32_' + tag + '.dec']) - print(' OK') # well, here, decompression has worked; but file is not yet verified - - # Compare all '.dec' files with test_dat - decs = glob.glob('*.dec') - for dec in decs: - if not filecmp.cmp(dec, test_dat): - print('ERR : ' + dec) - error_code = 1 - else: - print('OK : ' + dec) - os.remove(dec) - - if error_code != 0: - print('ERROR') - - sys.exit(error_code) diff --git a/library/lz4.exec b/library/lz4.exec new file mode 100755 index 00000000..745dfb0e Binary files /dev/null and b/library/lz4.exec differ diff --git a/package.json b/package.json index edaf20b9..e8bf9775 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "config/Symphony.config", "library/libsymphonysearch.dylib", "library/indexvalidator.exec", - "library/lz4-mac" + "library/lz4.exec" ], "appId": "symphony-electron-desktop", "mac": {