diff --git a/Makefile b/Makefile index 7a6d461..b3d1b72 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ # args MT := 1 STATIC_CPP := 0 +STATIC_C := 0 # used clang? CL := 0 # build with -m32? @@ -71,7 +72,11 @@ else endif ifeq ($(MINS),0) else - TINY_LINK += -Wl,--gc-sections,--as-needed + TINY_LINK += -s -Wl,--gc-sections,--as-needed +endif +ifeq ($(STATIC_C),0) +else + TINY_LINK += -static endif ifeq ($(CL),1) CXX := clang++ diff --git a/README.md b/README.md index cb12574..a080d56 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# [tinyuz](https://github.com/sisong/tinyuz) -[![release](https://img.shields.io/badge/release-v1.0.0-blue.svg)](https://github.com/sisong/tinyuz/releases) +# [tinyuz] +[![release](https://img.shields.io/badge/release-v1.1.0-blue.svg)](https://github.com/sisong/tinyuz/releases) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/tinyuz/blob/master/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/tinyuz/pulls) [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/tinyuz?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/tinyuz/issues) @@ -8,17 +8,27 @@ **tinyuz** is a lossless compression algorithm, designed for embedded systems,MCU, NB-IoT, etc., with better compression ratios. Which is characterized by a very small decompress code(ROM or flash occupancy); -The stream decompresser compiled by Mbed Studio is 856 bytes(can define to 758 bytes), -and the memory decompresser is 424 bytes(can define to unsafe mode 298 bytes). +The stream decompresser compiled by Mbed Studio is 626 bytes(can define to 468 bytes), +and the memory decompresser is 424 bytes(can define to 298 bytes). At the same time, the stream decompress memory(RAM occupancy) can also be very small, RAM size = dictionary size(>=1Byte, specified when compress) + cache size(>=2Byte, specified when decompress). Tip: The smaller the dictionary, the lower the compression ratio; while the smaller cache only affects the decompress speed. -( other decompresser compiled by Mbed Studio: zlib v1.2.12 stream decompresser >~10k; lzma v22.01 stream decompresser ~6k; minilzo v2.10 memory decompresser 868 bytes(unsafe mode 628 bytes). ) +( other decompresser compiled by Mbed Studio: [zlib] v1.3.1 stream decompresser >~10k; [lzma] v22.01 stream decompresser ~6k; [miniLZO] v2.10 memory decompresser 868 bytes(unsafe mode 628 bytes). ) Large data are supported, both compress and decompress support streaming. The compress and decompress speed is related to the characteristics of the input data and parameter settings; On modern CPUs, compress speed is slower by about 0.4MB/S--2MB/S, and decompress speed is faster by about 180MB/S--300MB/S. +[tinyuz]: https://github.com/sisong/tinyuz +[HPatchLite]: https://github.com/sisong/HPatchLite +[zlib]: https://github.com/madler/zlib +[lzma]: https://www.7-zip.org/sdk.html +[QuickLZ]: http://www.quicklz.com/order.html +[tamp]: https://github.com/BrianPugh/tamp +[heatshrink]: https://github.com/atomicobject/heatshrink +[FastLZ]: https://github.com/ariya/fastlz +[miniLZO]: http://www.oberhumer.com/opensource/lzo + --- ## Releases/Binaries [Download from latest release](https://github.com/sisong/tinyuz/releases) : Command line app for Windows, Linux, MacOS. @@ -76,35 +86,45 @@ can also decompress at once in memory: tuz_TResult tuz_decompress_mem(const tuz_byte* in_code,tuz_size_t code_size,tuz_byte* out_data,tuz_size_t* data_size); ``` +--- +## Porting decompress algorithm to embedded devices: +* Add or copy the entire directory `tinyuz/decompress/` to your project; +* Add a reference to the `tuz_dec.h` header file where the decompress algorithm is needed, and call the decompress functions declared in this file. + --- ## test compression ratio: ratio: compressedSize/uncompressedSize -zlib v1.2.11: test with compress level 9, windowBits -15 -tinyuz v0.9.2: test with multiple different dictSizes 32MB,1MB,32KB,5KB,1KB,255B,79B,24B - ('tuz -32m' means: tinyuz -c-32m) +[tinyuz] v1.1.0: test with multiple different dictSize 1MB,32KB,4KB,1KB,255B +('tuz -32k' means: tinyuz -c-32k) +[zlib] v1.3.1 test with compress level 9, windowBits -15(i.e. dictSize 32KB) +[QuickLZ] v1.5.0 test compress with default setting QLZ_COMPRESSION_LEVEL=3, QLZ_STREAMING_BUFFER=1048576 +[tamp] v1.7.0 test compress with windowBits 15 & 12(i.e. dictSize 32KB & 4KB) +[heatshrink] v0.4.1 test compress with windowBits 12(i.e. dictSize 4KB), lookahead_sz2=6 +[FastLZ] v0.5.0 test with compress level 2 +[miniLZO] v2.10 test with `lzo1x_1_compress` function, wrkmem used default `LZO1X_1_MEM_COMPRESS=16k*sizeof(void*)` size "aMCU.bin" is a firmware file of MCU device; -"aMCU.bin.diff" is a uncompressed differential file between two versions of firmware files (created by [HPatchLite](https://github.com/sisong/HPatchLite)); +"aMCU.bin.diff" is a uncompressed differential file between two versions of firmware files (created by [HPatchLite]); "A10.jpg"--"world95.txt" download from http://www.maximumcompression.com/data/files/index.html "enwik8" download from https://data.deepai.org/enwik8.zip "silesia.tar" download from https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia -||zlib -9|tuz -32m|tuz -1m|tuz -32k|tuz -5k|tuz -1k|tuz -255|tuz -79|tuz -24| -|:----|----:|----:|----:|----:|----:|----:|----:|----:|----:| -|aMCU.bin|46.54%|45.80%|45.80%|45.98%|49.16%|54.29%|60.61%|68.03%|77.95%| -|aMCU.bin.diff|5.29%|5.75%|5.75%|5.75%|5.95%|6.35%|6.89%|7.85%|9.54%| -|A10.jpg|99.88%|99.99%|99.99%|99.99%|99.99%|99.99%|99.99%|99.99%|99.99%| -|AcroRd32.exe|44.88%|42.01%|42.12%|43.80%|46.53%|51.48%|58.29%|67.57%|78.81%| -|english.dic|25.83%|28.62%|28.65%|29.20%|29.98%|31.25%|33.49%|36.53%|39.93%| -|FlashMX.pdf|84.76%|86.08%|85.34%|85.81%|87.34%|88.31%|89.90%|92.05%|96.83%| -|FP.LOG|6.46%|4.95%|5.26%|7.36%|9.99%|12.67%|19.27%|99.25%|100.00%| -|MSO97.DLL|57.94%|53.54%|54.12%|56.96%|59.80%|64.38%|70.62%|78.36%|87.73%| -|ohs.doc|24.05%|20.65%|21.03%|24.50%|26.85%|31.08%|37.50%|69.31%|82.85%| -|rafale.bmp|30.23%|30.30%|30.40%|32.66%|35.51%|40.81%|43.52%|47.70%|54.42%| -|vcfiu.hlp|20.41%|17.71%|17.79%|20.39%|24.24%|27.46%|32.39%|49.01%|69.64%| -|world95.txt|28.87%|22.88%|23.44%|30.79%|47.15%|54.96%|65.23%|78.53%|97.20%| -|enwik8|36.45%|30.09%|33.22%|38.36%|43.96%|51.53%|63.38%|79.63%|96.78%| -|silesia.tar|31.98%|28.41%|29.66%|33.27%|38.21%|44.45%|52.58%|63.62%|78.49%| +|test file|tuz 1m|tuz 32k|tuz 4k|tuz 1k|tuz 255|zlib 32k|QuickLZ|tamp 32k|tamp 4k|heatshrink 4k|FastLZ|miniLZO +|:----|----:|----:|----:|----:|----:|----:|----:|----:|----:|----:|----:|----:| +aMCU.bin|45.80%|45.98%|49.64%|54.29%|60.61%|46.54%|54.49%|57.87%|56.11%|58.52%|62.21%|61.33% +aMCU.bin.diff|5.75%|5.75%|5.99%|6.35%|6.89%|5.29%|9.52%|18.91%|16.78%|9.26%|12.50%|14.21% +A10.jpg|99.99%|99.99%|99.99%|99.99%|99.99%|99.88%|100.00%|107.79%|108.27%|112.16%|102.91%|100.38% +AcroRd32.exe|42.12%|43.80%|46.99%|51.48%|58.29%|44.88%|52.07%|55.86%|54.17%|56.15%|61.22%|61.44% +english.dic|28.65%|29.20%|30.10%|31.25%|33.49%|25.83%|35.50%|39.81%|36.64%|36.82%|40.86%|43.82% +FlashMX.pdf|85.34%|85.81%|87.46%|88.31%|89.90%|84.76%|100.00%|92.92%|93.40%|96.60%|89.57%|91.73% +FP.LOG|5.26%|7.36%|10.34%|12.67%|19.27%|6.46%|8.59%|20.95%|21.51%|14.12%|11.97%|13.01% +MSO97.DLL|54.12%|56.96%|60.23%|64.38%|70.62%|57.94%|65.65%|67.75%|65.78%|70.49%|74.80%|75.57% +ohs.doc|21.03%|24.50%|27.14%|31.08%|37.50%|24.05%|25.72%|38.34%|38.15%|33.51%|28.31%|30.41% +rafale.bmp|30.40%|32.66%|35.80%|40.81%|43.52%|30.23%|42.06%|37.38%|39.72%|40.30%|52.63%|55.41% +vcfiu.hlp|17.79%|20.39%|24.51%|27.46%|32.39%|20.41%|24.88%|32.77%|33.87%|30.42%|32.36%|34.10% +world95.txt|23.44%|30.79%|48.07%|54.96%|65.23%|28.87%|35.17%|38.18%|51.30%|54.32%|52.04%|51.56% +enwik8|33.22%|38.36%|44.81%|51.53%|63.38%|36.45%|44.79%|44.48%|48.04%|50.41%|54.52%|55.79% +silesia.tar|29.66%|33.27%|38.99%|44.45%|52.58%|31.98%|38.60%|42.77%|45.09%|44.27%|47.25%|47.50% --- ## Contact diff --git a/README_cn.md b/README_cn.md index 9bf0557..3e0f175 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@ -# [tinyuz](https://github.com/sisong/tinyuz) -[![release](https://img.shields.io/badge/release-v1.0.0-blue.svg)](https://github.com/sisong/tinyuz/releases) +# [tinyuz] +[![release](https://img.shields.io/badge/release-v1.1.0-blue.svg)](https://github.com/sisong/tinyuz/releases) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/tinyuz/blob/master/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/tinyuz/pulls) [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/tinyuz?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/tinyuz/issues) @@ -8,14 +8,24 @@ **tinyuz** 是一个无损压缩算法，为超小型嵌入式设备(MCU、NB-IoT等)设计，保持还不错的压缩率。 特色是编译后的解压缩代码(ROM或flash占用)非常的小； -流模式解压用 Mbed Studio 编译后为 856 字节(可以调整宏定义后到 758 字节)； -而内存模式解压为 424 字节(可以调整宏定义后到非安全模式 298 字节)。 -（其他解压器用 Mbed Studio 编译后大小参考: zlib v1.2.12 流模式解压 约大于10k; lzma v22.01 流模式解压 约6k; minilzo v2.10 内存模式解压为 868 字节(非安全模式 628 字节)。） +流模式解压用 Mbed Studio 编译后为 626 字节(可以调整宏定义后可以到 468 字节)； +而内存模式解压为 424 字节(可以调整宏定义后可以到 298 字节)。 +（其他解压器用 Mbed Studio 编译后大小参考: [zlib] v1.3.1 流模式解压 约大于10k; [lzma] v22.01 流模式解压 约6k; [miniLZO] v2.10 内存模式解压为 868 字节(非安全模式 628 字节)。） 同时，流模式解压时内存(RAM占用)也可以非常的小，RAM大小= 字典大小(>=1Byte，压缩时指定) + 缓冲区大小(>=2Byte，解压缩时指定)。 提示：字典越小压缩率越低，而缓冲区较小时只影响解压缩速度。 支持处理巨大的数据，压缩和解压缩时都支持流式处理。 压缩和解压缩速度与数据特性和参数设置有关；在现代 CPU 上，压缩时比较慢约 0.4MB/S--2MB/S，解压缩较快约 180MB/S--300MB/S。 +[tinyuz]: https://github.com/sisong/tinyuz +[HPatchLite]: https://github.com/sisong/HPatchLite +[zlib]: https://github.com/madler/zlib +[lzma]: https://www.7-zip.org/sdk.html +[QuickLZ]: http://www.quicklz.com/order.html +[tamp]: https://github.com/BrianPugh/tamp +[heatshrink]: https://github.com/atomicobject/heatshrink +[FastLZ]: https://github.com/ariya/fastlz +[miniLZO]: http://www.oberhumer.com/opensource/lzo + --- ## 二进制发布包 [从 release 下载](https://github.com/sisong/tinyuz/releases) : 分别运行在 Windows、Linux、MacOS操作系统的命令行程序。 @@ -73,35 +83,45 @@ tuz_TResult tuz_TStream_decompress_partial(tuz_TStream* self,tuz_byte* out_data, tuz_TResult tuz_decompress_mem(const tuz_byte* in_code,tuz_size_t code_size,tuz_byte* out_data,tuz_size_t* data_size); ``` +--- +## 移植解压算法到嵌入式设备: +* 将 `tinyuz/decompress/` 整个目录添加或拷贝到你的项目工程中； +* 在需要使用解压缩算法的地方添加 `tuz_dec.h` 头文件的引用，并调用该文件中声明的解压缩函数。 + --- ## 压缩率测试: 压缩率: 压缩后大小/压缩前大小 -zlib v1.2.11: 测试时设置压缩水平为9, 窗口比特大小设置为-15 -tinyuz v0.9.2: 测试时设置多个不同的字典大小 32MB,1MB,32KB,5KB,1KB,255B,79B,24B - (表中'tuz -32m' 表示: tinyuz -c-32m) +[tinyuz] v1.1.0 测试时字典大小分别设置为 1MB,32KB,4KB,1KB,255B +(表中'tuz -32k' 表示: tinyuz -c-32k) +[zlib] v1.3.1 测试时设置压缩水平为9, 窗口比特大小设置为-15(即字典大小32KB) +[QuickLZ] v1.5.0 测试时使用默认设置QLZ_COMPRESSION_LEVEL=3, QLZ_STREAMING_BUFFER=1048576 +[tamp] v1.7.0 测试时窗口比特大小分别设置为15和12(即字典大小32KB和4KB) +[heatshrink] v0.4.1 测试时窗口比特大小设置为12(即字典大小4KB), lookahead_sz2=6 +[FastLZ] v0.5.0 测试时设置压缩水平为2 +[miniLZO] v2.10 测试时`lzo1x_1_compress`函数, wrkmem使用了默认的`LZO1X_1_MEM_COMPRESS=16k*sizeof(void*)`大小 "aMCU.bin" 是一个MCU设备的固件文件; -"aMCU.bin.diff" 是用两个不同版本的固件文件来创建的一个未压缩的补丁文件(用 [HPatchLite](https://github.com/sisong/HPatchLite) 所创建); +"aMCU.bin.diff" 是用两个不同版本的固件文件来创建的一个未压缩的补丁文件(用 [HPatchLite] 所创建); "A10.jpg"--"world95.txt" 从 http://www.maximumcompression.com/data/files/index.html 下载 "enwik8" 从 https://data.deepai.org/enwik8.zip 下载 "silesia.tar" 从 https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia 下载 -||zlib -9|tuz -32m|tuz -1m|tuz -32k|tuz -5k|tuz -1k|tuz -255|tuz -79|tuz -24| -|:----|----:|----:|----:|----:|----:|----:|----:|----:|----:| -|aMCU.bin|46.54%|45.80%|45.80%|45.98%|49.16%|54.29%|60.61%|68.03%|77.95%| -|aMCU.bin.diff|5.29%|5.75%|5.75%|5.75%|5.95%|6.35%|6.89%|7.85%|9.54%| -|A10.jpg|99.88%|99.99%|99.99%|99.99%|99.99%|99.99%|99.99%|99.99%|99.99%| -|AcroRd32.exe|44.88%|42.01%|42.12%|43.80%|46.53%|51.48%|58.29%|67.57%|78.81%| -|english.dic|25.83%|28.62%|28.65%|29.20%|29.98%|31.25%|33.49%|36.53%|39.93%| -|FlashMX.pdf|84.76%|86.08%|85.34%|85.81%|87.34%|88.31%|89.90%|92.05%|96.83%| -|FP.LOG|6.46%|4.95%|5.26%|7.36%|9.99%|12.67%|19.27%|99.25%|100.00%| -|MSO97.DLL|57.94%|53.54%|54.12%|56.96%|59.80%|64.38%|70.62%|78.36%|87.73%| -|ohs.doc|24.05%|20.65%|21.03%|24.50%|26.85%|31.08%|37.50%|69.31%|82.85%| -|rafale.bmp|30.23%|30.30%|30.40%|32.66%|35.51%|40.81%|43.52%|47.70%|54.42%| -|vcfiu.hlp|20.41%|17.71%|17.79%|20.39%|24.24%|27.46%|32.39%|49.01%|69.64%| -|world95.txt|28.87%|22.88%|23.44%|30.79%|47.15%|54.96%|65.23%|78.53%|97.20%| -|enwik8|36.45%|30.09%|33.22%|38.36%|43.96%|51.53%|63.38%|79.63%|96.78%| -|silesia.tar|31.98%|28.41%|29.66%|33.27%|38.21%|44.45%|52.58%|63.62%|78.49%| +|test file|tuz 1m|tuz 32k|tuz 4k|tuz 1k|tuz 255|zlib 32k|QuickLZ|tamp 32k|tamp 4k|heatshrink 4k|FastLZ|miniLZO +|:----|----:|----:|----:|----:|----:|----:|----:|----:|----:|----:|----:|----:| +aMCU.bin|45.80%|45.98%|49.64%|54.29%|60.61%|46.54%|54.49%|57.87%|56.11%|58.52%|62.21%|61.33% +aMCU.bin.diff|5.75%|5.75%|5.99%|6.35%|6.89%|5.29%|9.52%|18.91%|16.78%|9.26%|12.50%|14.21% +A10.jpg|99.99%|99.99%|99.99%|99.99%|99.99%|99.88%|100.00%|107.79%|108.27%|112.16%|102.91%|100.38% +AcroRd32.exe|42.12%|43.80%|46.99%|51.48%|58.29%|44.88%|52.07%|55.86%|54.17%|56.15%|61.22%|61.44% +english.dic|28.65%|29.20%|30.10%|31.25%|33.49%|25.83%|35.50%|39.81%|36.64%|36.82%|40.86%|43.82% +FlashMX.pdf|85.34%|85.81%|87.46%|88.31%|89.90%|84.76%|100.00%|92.92%|93.40%|96.60%|89.57%|91.73% +FP.LOG|5.26%|7.36%|10.34%|12.67%|19.27%|6.46%|8.59%|20.95%|21.51%|14.12%|11.97%|13.01% +MSO97.DLL|54.12%|56.96%|60.23%|64.38%|70.62%|57.94%|65.65%|67.75%|65.78%|70.49%|74.80%|75.57% +ohs.doc|21.03%|24.50%|27.14%|31.08%|37.50%|24.05%|25.72%|38.34%|38.15%|33.51%|28.31%|30.41% +rafale.bmp|30.40%|32.66%|35.80%|40.81%|43.52%|30.23%|42.06%|37.38%|39.72%|40.30%|52.63%|55.41% +vcfiu.hlp|17.79%|20.39%|24.51%|27.46%|32.39%|20.41%|24.88%|32.77%|33.87%|30.42%|32.36%|34.10% +world95.txt|23.44%|30.79%|48.07%|54.96%|65.23%|28.87%|35.17%|38.18%|51.30%|54.32%|52.04%|51.56% +enwik8|33.22%|38.36%|44.81%|51.53%|63.38%|36.45%|44.79%|44.48%|48.04%|50.41%|54.52%|55.79% +silesia.tar|29.66%|33.27%|38.99%|44.45%|52.58%|31.98%|38.60%|42.77%|45.09%|44.27%|47.25%|47.50% --- ## 联系 diff --git a/builds/codeblocks/tinyuz.cbp b/builds/codeblocks/tinyuz.cbp deleted file mode 100644 index 9f53211..0000000 --- a/builds/codeblocks/tinyuz.cbp +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - diff --git a/builds/vc/libtinyuz.vcxproj b/builds/vc/libtinyuz.vcxproj index 5285ca5..3dec80c 100644 --- a/builds/vc/libtinyuz.vcxproj +++ b/builds/vc/libtinyuz.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release + ARM64 + Release Win32 @@ -47,7 +55,6 @@ v4.0 ManagedCProj tinyuz - 10.0 @@ -64,6 +71,13 @@ Unicode v143 + + StaticLibrary + true + false + Unicode + v143 + StaticLibrary false @@ -80,6 +94,14 @@ true v143 + + StaticLibrary + false + false + Unicode + true + v143 + @@ -89,12 +111,18 @@ + + + + + + true @@ -102,12 +130,18 @@ true + + true + false false + + false + Level3 @@ -140,6 +174,22 @@ + + + Level3 + Disabled + WIN32;_DEBUG;%(PreprocessorDefinitions) + NotUsing + MultiThreadedDebug + + + + + true + + + + Level3 @@ -172,6 +222,22 @@ + + + Level3 + WIN32;NDEBUG;%(PreprocessorDefinitions) + NotUsing + true + MultiThreaded + + + + + true + + + + diff --git a/builds/vc/speedTest.vcxproj b/builds/vc/speedTest.vcxproj index 4f97e00..73d508d 100644 --- a/builds/vc/speedTest.vcxproj +++ b/builds/vc/speedTest.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release + ARM64 + Release Win32 @@ -33,7 +41,6 @@ {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF} Win32Proj speedTest - 10.0 @@ -48,6 +55,12 @@ Unicode v143 + + Application + true + Unicode + v143 + Application false @@ -62,6 +75,13 @@ Unicode v143 + + Application + false + true + Unicode + v143 + @@ -71,12 +91,18 @@ + + + + + + true @@ -86,6 +112,10 @@ true ..\..\..\zlib;$(IncludePath) + + true + ..\..\..\zlib;$(IncludePath) + false ..\..\..\zlib;$(IncludePath) @@ -94,6 +124,10 @@ false ..\..\..\zlib;$(IncludePath) + + false + ..\..\..\zlib;$(IncludePath) + NotUsing @@ -126,6 +160,22 @@ true + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;%(PreprocessorDefinitions) + MultiThreadedDebug + + + + + + + true + + Level3 @@ -144,6 +194,7 @@ true true true + UseLinkTimeCodeGeneration @@ -164,6 +215,28 @@ true true true + UseLinkTimeCodeGeneration + + + + + Level3 + NotUsing + MaxSpeed + true + true + WIN32;NDEBUG;%(PreprocessorDefinitions) + MultiThreaded + + + + + + + true + true + true + UseLinkTimeCodeGeneration diff --git a/builds/vc/tinyuz.sln b/builds/vc/tinyuz.sln index d735acd..5ddcbf8 100644 --- a/builds/vc/tinyuz.sln +++ b/builds/vc/tinyuz.sln @@ -17,48 +17,70 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tinyuz", "tinyuz.vcxproj", EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|ARM64 = Debug|ARM64 Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 + Release|ARM64 = Release|ARM64 Release|Win32 = Release|Win32 Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Debug|ARM64.Build.0 = Debug|ARM64 {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Debug|Win32.ActiveCfg = Debug|Win32 {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Debug|Win32.Build.0 = Debug|Win32 {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Debug|x64.ActiveCfg = Debug|x64 {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Debug|x64.Build.0 = Debug|x64 + {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Release|ARM64.ActiveCfg = Release|ARM64 + {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Release|ARM64.Build.0 = Release|ARM64 {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Release|Win32.ActiveCfg = Release|Win32 {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Release|Win32.Build.0 = Release|Win32 {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Release|x64.ActiveCfg = Release|x64 {6609E1CF-8DAE-41E9-BC20-1574E49EC91B}.Release|x64.Build.0 = Release|x64 + {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Debug|ARM64.Build.0 = Debug|ARM64 {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Debug|Win32.ActiveCfg = Debug|Win32 {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Debug|Win32.Build.0 = Debug|Win32 {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Debug|x64.ActiveCfg = Debug|x64 {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Debug|x64.Build.0 = Debug|x64 + {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Release|ARM64.ActiveCfg = Release|ARM64 + {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Release|ARM64.Build.0 = Release|ARM64 {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Release|Win32.ActiveCfg = Release|Win32 {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Release|Win32.Build.0 = Release|Win32 {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Release|x64.ActiveCfg = Release|x64 {EF8D668D-DF30-4CC6-AFAC-929D951BC8E7}.Release|x64.Build.0 = Release|x64 + {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Debug|ARM64.Build.0 = Debug|ARM64 {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Debug|Win32.ActiveCfg = Debug|Win32 {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Debug|Win32.Build.0 = Debug|Win32 {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Debug|x64.ActiveCfg = Debug|x64 {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Debug|x64.Build.0 = Debug|x64 + {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Release|ARM64.ActiveCfg = Release|ARM64 + {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Release|ARM64.Build.0 = Release|ARM64 {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Release|Win32.ActiveCfg = Release|Win32 {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Release|Win32.Build.0 = Release|Win32 {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Release|x64.ActiveCfg = Release|x64 {2668A815-C1D0-48AD-A4AE-FE1ADB90B1FF}.Release|x64.Build.0 = Release|x64 + {35ED7715-6B17-4225-8171-C954968D8F8C}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {35ED7715-6B17-4225-8171-C954968D8F8C}.Debug|ARM64.Build.0 = Debug|ARM64 {35ED7715-6B17-4225-8171-C954968D8F8C}.Debug|Win32.ActiveCfg = Debug|Win32 {35ED7715-6B17-4225-8171-C954968D8F8C}.Debug|Win32.Build.0 = Debug|Win32 {35ED7715-6B17-4225-8171-C954968D8F8C}.Debug|x64.ActiveCfg = Debug|x64 {35ED7715-6B17-4225-8171-C954968D8F8C}.Debug|x64.Build.0 = Debug|x64 + {35ED7715-6B17-4225-8171-C954968D8F8C}.Release|ARM64.ActiveCfg = Release|ARM64 + {35ED7715-6B17-4225-8171-C954968D8F8C}.Release|ARM64.Build.0 = Release|ARM64 {35ED7715-6B17-4225-8171-C954968D8F8C}.Release|Win32.ActiveCfg = Release|Win32 {35ED7715-6B17-4225-8171-C954968D8F8C}.Release|Win32.Build.0 = Release|Win32 {35ED7715-6B17-4225-8171-C954968D8F8C}.Release|x64.ActiveCfg = Release|x64 {35ED7715-6B17-4225-8171-C954968D8F8C}.Release|x64.Build.0 = Release|x64 + {981B8BF9-84DD-4DF9-990B-0428808E770E}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {981B8BF9-84DD-4DF9-990B-0428808E770E}.Debug|ARM64.Build.0 = Debug|ARM64 {981B8BF9-84DD-4DF9-990B-0428808E770E}.Debug|Win32.ActiveCfg = Debug|Win32 {981B8BF9-84DD-4DF9-990B-0428808E770E}.Debug|Win32.Build.0 = Debug|Win32 {981B8BF9-84DD-4DF9-990B-0428808E770E}.Debug|x64.ActiveCfg = Debug|x64 {981B8BF9-84DD-4DF9-990B-0428808E770E}.Debug|x64.Build.0 = Debug|x64 + {981B8BF9-84DD-4DF9-990B-0428808E770E}.Release|ARM64.ActiveCfg = Release|ARM64 + {981B8BF9-84DD-4DF9-990B-0428808E770E}.Release|ARM64.Build.0 = Release|ARM64 {981B8BF9-84DD-4DF9-990B-0428808E770E}.Release|Win32.ActiveCfg = Release|Win32 {981B8BF9-84DD-4DF9-990B-0428808E770E}.Release|Win32.Build.0 = Release|Win32 {981B8BF9-84DD-4DF9-990B-0428808E770E}.Release|x64.ActiveCfg = Release|x64 diff --git a/builds/vc/tinyuz.vcxproj b/builds/vc/tinyuz.vcxproj index 1109c03..b138af9 100644 --- a/builds/vc/tinyuz.vcxproj +++ b/builds/vc/tinyuz.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release + ARM64 + Release Win32 @@ -21,7 +29,6 @@ {981B8BF9-84DD-4DF9-990B-0428808E770E} tinyuz - 10.0 @@ -36,6 +43,12 @@ Unicode v143 + + Application + true + Unicode + v143 + Application false @@ -50,6 +63,13 @@ Unicode v143 + + Application + false + true + Unicode + v143 + @@ -59,12 +79,18 @@ + + + + + + @@ -93,6 +119,19 @@ ..\..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + + + Level3 + Disabled + MultiThreadedDebug + + + true + + + ..\..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + + Level3 @@ -129,6 +168,24 @@ ..\..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + + + Level3 + MaxSpeed + true + true + MultiThreaded + + + false + true + true + UseLinkTimeCodeGeneration + + + ..\..\..\HDiffPatch\builds\vc\longPathAware.exe.manifest %(AdditionalManifestFiles) + + {6609e1cf-8dae-41e9-bc20-1574e49ec91b} diff --git a/builds/vc/unitTest.vcxproj b/builds/vc/unitTest.vcxproj index e5af9a9..38e78b5 100644 --- a/builds/vc/unitTest.vcxproj +++ b/builds/vc/unitTest.vcxproj @@ -1,6 +1,10 @@  + + Debug + ARM64 + Debug Win32 @@ -9,6 +13,10 @@ Debug x64 + + Release + ARM64 + Release Win32 @@ -31,7 +39,6 @@ v4.0 ManagedCProj unitTest - 10.0 @@ -48,6 +55,13 @@ Unicode v143 + + Application + true + false + Unicode + v143 + Application false @@ -64,6 +78,14 @@ true v143 + + Application + false + false + Unicode + true + v143 + @@ -73,12 +95,18 @@ + + + + + + true @@ -86,12 +114,18 @@ true + + true + false false + + false + Level3 @@ -124,6 +158,22 @@ + + + Level3 + Disabled + WIN32;_DEBUG;%(PreprocessorDefinitions) + NotUsing + + + MultiThreadedDebug + + + true + + + + Level3 @@ -154,6 +204,21 @@ + + + Level3 + WIN32;NDEBUG;%(PreprocessorDefinitions) + NotUsing + + + MultiThreaded + + + true + + + + diff --git a/compress/tuz_enc_private/tuz_enc_code.cpp b/compress/tuz_enc_private/tuz_enc_code.cpp index 6f27c43..a5c1967 100644 --- a/compress/tuz_enc_private/tuz_enc_code.cpp +++ b/compress/tuz_enc_private/tuz_enc_code.cpp @@ -160,13 +160,13 @@ size_t TTuzCode::_getSavedDictLenBit(size_t len)const{ } void TTuzCode::outDict(size_t match_len,size_t dict_pos){ outType(tuz_codeType_dict); - size_t saved_dict_pos=dict_pos+1; //0 for ctrl + const size_t saved_dict_pos=dict_pos+1; //0 for ctrl if (saved_dict_pos>_dict_size_max) _dict_size_max=saved_dict_pos; const size_t isSamePos=(_dictPos_back==saved_dict_pos)?1:0; const size_t isSavedSamePos=(isSamePos&&_isHaveData_back)?1:0; size_t len=match_len-tuz_kMinDictMatchLen; if (!isSavedSamePos){ - if (saved_dict_pos>tuz_kBigPosForLen) { checkv(match_len>=3); --len; } + if (saved_dict_pos>tuz_kBigPosForLen) { checkv(match_len>=tuz_kMinDictMatchLen+1); --len; } //if (saved_dict_pos>((1<<17)+(1<<15)+(1<<13)+(1<<11)+(1<<9)+(1<<7)-1)) { checkv(match_len>=4); --len; } } outDictLen(len); @@ -200,7 +200,7 @@ void TTuzCode::outDict(size_t match_len,size_t dict_pos){ mdict_offs=dict_pos; #endif _isHaveData_back=false; - _dictPos_back=dict_pos+1; + _dictPos_back=saved_dict_pos; } void TTuzCode::outCtrl_streamEnd(){ diff --git a/compress/tuz_enc_private/tuz_enc_match.cpp b/compress/tuz_enc_private/tuz_enc_match.cpp index d60e45a..9ad01bb 100644 --- a/compress/tuz_enc_private/tuz_enc_match.cpp +++ b/compress/tuz_enc_private/tuz_enc_match.cpp @@ -6,14 +6,11 @@ #include "tuz_enc_match.h" #include "tuz_enc_code.h" namespace _tuz_private{ - - typedef TMatch::TInt TInt; - typedef TMatch::TUInt TUInt; - typedef TMatch::TLCPInt TLCPInt; + const TUInt kNullCostValue=~(TUInt)0; #define _k_kBMatchStep 512 // skip by a big match - static TInt _sstr_eqLen(const tuz_byte* ss_end,size_t maxSaveLength, + static inline TInt _sstr_eqLen(const tuz_byte* ss_end,size_t maxSaveLength, const tuz_byte* matchedString,const tuz_byte* curString){ TInt eqLen=0; while ((curString& cost){ } } -void TMatch::_getCostByLiteralLen(const tuz_byte* cur0,std::vector& cost){ + + + #define _MatchedAtBy_byLiteralLen(isSamePos){ \ + const size_t saveDictCost=coder.getSavedDictPosBit(dict_pos,isHaveData); \ + const size_t curSaveCost=cost[curi-1]+saveDictCost; \ + assert(curSaveCost(match_len,dict_pos); \ + const size_t ni=curi+match_len-1; \ + const size_t cost_ni=cost[ni]; \ + if (dictCost& cost){ + const size_t back_pos=dictPos[curi-1]; + const size_t isHaveData=((matchLen[curi-1]==0)&&(curi>1))?1:0; + + if (isHaveData&&(back_pos==dict_pos)){ //same pos match + _MatchedAtBy_byLiteralLen(true); + }else{ + _MatchedAtBy_byLiteralLen(false); + } + } + +void TMatch::_getCostByLiteralLen(std::vector& cost){ size_t costSize=cost.size(); size_t unmatched_len=0; size_t unmatched_fill0_i=0; @@ -193,37 +216,45 @@ void TMatch::_getCostByLiteralLen(const tuz_byte* cur0,std::vector& cost) unmatched_fill0_i=i; } ++i; - } + } } -void TMatch::_getCost(const tuz_byte* cur0){ - std::vector cost; - size_t costSize=(sstring.src_end-cur0)+1; + + static void _restoreMatch(size_t costSize,TPosInt* dictPos,TLCPInt* matchLen){ + size_t i_inc=costSize; + while (i_inc>0) { + size_t i=i_inc-1; + TLCPInt mlen=matchLen[i]; + if (mlen>0){ + dictPos[i-mlen+1]=dictPos[i]; + matchLen[i-mlen+1]=mlen; + i_inc-=mlen; + }else{ + --i_inc; + } + } + } +void TMatch::_initCost(std::vector& cost,size_t costSize){ + cost.clear(); + matchLen.clear(); cost.resize(costSize,kNullCostValue); - dictPos.resize(costSize); matchLen.resize(costSize,0); + dictPos.clear(); + dictPos.resize(costSize,~(size_t)0); cost[0]=0; dictPos[0]=0; matchLen[0]=0; - - _getCostByMatch(cur0,cost); - - size_t i_inc=costSize; - while (i_inc>0) { - size_t i=i_inc-1; - TLCPInt mlen=matchLen[i]; - if (mlen>0){ - dictPos[i-mlen+1]=dictPos[i]; - matchLen[i-mlen+1]=mlen; - if (props.isNeedLiteralLine) - cost[i-mlen+1]=cost[i]; - i_inc-=mlen; - }else{ - --i_inc; - } - } +} +void TMatch::_getCost(const tuz_byte* cur0){ + std::vector cost; + const size_t costSize=(sstring.src_end-cur0)+1; + _initCost(cost,costSize); + _getCostByMatch(cur0,cost); + sstring.clearMem(); + _restoreMatch(costSize,dictPos.data(),matchLen.data()); + if (props.isNeedLiteralLine) - _getCostByLiteralLen(cur0,cost); + _getCostByLiteralLen(cost); } bool TMatch::match(const tuz_byte** out_matched,size_t* out_match_len, @@ -241,4 +272,4 @@ bool TMatch::match(const tuz_byte** out_matched,size_t* out_match_len, } } -} +}//namespace _tuz_private diff --git a/compress/tuz_enc_private/tuz_enc_match.h b/compress/tuz_enc_private/tuz_enc_match.h index d4e2ede..76618f3 100644 --- a/compress/tuz_enc_private/tuz_enc_match.h +++ b/compress/tuz_enc_private/tuz_enc_match.h @@ -7,8 +7,18 @@ #define _tuz_enc_match_h #include "tuz_sstring.h" namespace _tuz_private{ + struct TTuzCode; + typedef TSuffixString::TInt TInt; + typedef uint32_t TUInt; + typedef TSuffixString::TLCPInt TLCPInt; + #if (tuz_kMaxOfDictSize>=(1<<16)) + typedef TUInt TPosInt; + #else + typedef TLCPInt TPosInt; + #endif + struct TMatch{ explicit TMatch(const tuz_byte* data,const tuz_byte* data_end, const TTuzCode& _coder,const tuz_TCompressProps& _props) @@ -16,25 +26,20 @@ namespace _tuz_private{ props(_props){ } bool match(const tuz_byte** out_matched,size_t* out_match_len, const tuz_byte* cur); - typedef TSuffixString::TInt TInt; - typedef uint32_t TUInt; - typedef TSuffixString::TLCPInt TLCPInt; private: TSuffixString sstring; const TTuzCode& coder; const tuz_TCompressProps& props; - #if (tuz_kMaxOfDictSize>=(1<<16)) - typedef TUInt TPosInt; - #else - typedef TLCPInt TPosInt; - #endif std::vector dictPos; std::vector matchLen; void _cost_match(const TInt curString,const size_t curi, size_t* out_maxMatchLen,std::vector& cost); + void _initCost(std::vector& cost,size_t costSize); void _getCost(const tuz_byte* cur0); void _getCostByMatch(const tuz_byte* cur0,std::vector& cost); - void _getCostByLiteralLen(const tuz_byte* cur0,std::vector& cost); + void _cost_match_byLiteralLen(const size_t match_len,const size_t dict_pos, + const size_t curi,std::vector& cost); + void _getCostByLiteralLen(std::vector& cost); }; } diff --git a/compress/tuz_enc_private/tuz_sstring.cpp b/compress/tuz_enc_private/tuz_sstring.cpp index b96bd34..d516cdd 100644 --- a/compress/tuz_enc_private/tuz_sstring.cpp +++ b/compress/tuz_enc_private/tuz_sstring.cpp @@ -44,4 +44,15 @@ void TSuffixString::_init(TLCPInt maxLCPValue){ _LCP_create_withR(src,(TInt)SA.size(),SA.data(),R.data(),LCP.data(),maxLCPValue); } + template + static void _clearVector(std::vector& v){ + std::vector _tmp; + v.swap(_tmp); + } +void TSuffixString::clearMem(){ + _clearVector(SA); + _clearVector(R); + _clearVector(LCP); +} + } diff --git a/compress/tuz_enc_private/tuz_sstring.h b/compress/tuz_enc_private/tuz_sstring.h index 70a3753..40b5e26 100644 --- a/compress/tuz_enc_private/tuz_sstring.h +++ b/compress/tuz_enc_private/tuz_sstring.h @@ -21,6 +21,7 @@ namespace _tuz_private{ std::vector LCP; // lcp(i,i+1), longest common prefix between adjacent sstring size_t threadNum; inline size_t size()const { return src_end-src; } + void clearMem(); private: void _init(TLCPInt maxLCPValue); }; diff --git a/decompress/tuz_dec.c b/decompress/tuz_dec.c index 005505f..32ed962 100644 --- a/decompress/tuz_dec.c +++ b/decompress/tuz_dec.c @@ -1,29 +1,15 @@ // tuz_dec.c /* The MIT License (MIT) - Copyright (c) 2012-2022 HouSisong All Rights Reserved. + Copyright (c) 2012-2025 HouSisong All Rights Reserved. */ #include "tuz_dec.h" #include "tuz_types_private.h" -#if _IS_USED_C_MEMCPY -# include // memcpy -#endif #if (_IS_RUN_MEM_SAFE_CHECK) # define __RUN_MEM_SAFE_CHECK #endif -#define _memmove_order memmove_order -#if (!(_IS_USED_C_MEMCPY)) -# define _memcpy memmove_order -#else -# define _memcpy memcpy - tuz_force_inline -#endif - static void memmove_order(tuz_byte* dst,const tuz_byte* src,tuz_size_t len){ - while (len--) *dst++=*src++; - } - //low to high bitmap: xx?xx?xx?xx? ... #define _def_unpack_len(self,readBit,_read_lowbits){ \ tuz_length_t v=0; \ @@ -120,63 +106,6 @@ static tuz_force_inline tuz_size_t _cache_unpack_dict_pos(tuz_TStream* self){ return ((result&((1<<7)-1))|(_cache_unpack_pos_len(self)<<7))+(1<<7); } - -static void _update_dict(tuz_TStream *self,const tuz_byte* out_data,const tuz_byte* cur_out_data) { - // [ dict buf ]|[ out buf ] - // |dict_cur dict_size|out_data cur_out_data] - // [ out buf ] - // [ out buf ] - const tuz_size_t out_len=(tuz_size_t)(cur_out_data-out_data); - const tuz_size_t dict_size=self->_dict.dict_size; - tuz_byte* dict=self->_dict.dict_buf; - if (self->_state.dictType_pos_inc>=out_len){ - self->_state.dictType_pos_inc-=out_len; - }else{ - self->_state.dictType_pos-=(tuz_size_t)(out_len-self->_state.dictType_pos_inc); - self->_state.dictType_pos_inc=0; - } - if (out_len>=dict_size){ - _memcpy(dict,cur_out_data-dict_size,dict_size); - self->_dict.dict_cur=0; - }else{ - tuz_size_t dict_cur=self->_dict.dict_cur; - const tuz_size_t sub_len=dict_size-dict_cur; - if (out_len<=sub_len){ - _memcpy(dict+dict_cur,out_data,out_len); - }else{ - _memcpy(dict+dict_cur,out_data,sub_len); - _memcpy(dict,out_data+sub_len,out_len-sub_len); - } - self->_dict.dict_cur=(out_len<=sub_len)? - (out_len+dict_cur):(out_len-sub_len); - } -} - -static tuz_size_t _copy_from_dict(tuz_TStream *self,tuz_byte* cur_out_data,tuz_size_t dsize) { - // [ dict buf ] - // dict_cur+dictType_pos| <-- len --> | - // dict_cur+dictType_pos| <-- len --> | - // dict_cur+dictType_pos| <-- len --> | - tuz_size_t len,pos; - len=self->_dict.dict_size-self->_state.dictType_pos; - len=(len_state.dictType_len_state.dictType_len:len; - pos=self->_dict.dict_size-self->_dict.dict_cur; - if (self->_state.dictType_pos_dict.dict_cur+self->_state.dictType_pos; - else - pos=self->_state.dictType_pos-pos; - if (len<=(self->_dict.dict_size-pos)){ - _memcpy(cur_out_data,self->_dict.dict_buf+pos,len); - }else{ - tuz_size_t part_len=self->_dict.dict_size-pos; - _memcpy(cur_out_data,self->_dict.dict_buf+pos,part_len); - _memcpy(cur_out_data+part_len,self->_dict.dict_buf,len-part_len); - } - return len; -} - - tuz_size_t tuz_TStream_read_dict_size(tuz_TInputStreamHandle inputStream,tuz_TInputStream_read read_code){ tuz_size_t v=tuz_kDictSizeSavedBytes; tuz_byte saved[tuz_kDictSizeSavedBytes]; @@ -218,9 +147,8 @@ tuz_TResult tuz_TStream_open(tuz_TStream* self,tuz_TInputStreamHandle inputStrea self->_dict.dict_buf=dict_and_cache; self->_state.dictType_pos=0; - self->_state.dictType_pos_inc=0; self->_state.dict_pos_back=1; - self->_state.dictType_len=0; + self->_state.dictType_len=0; #if tuz_isNeedLiteralLine self->_state.literalType_len=0; #endif @@ -230,34 +158,38 @@ tuz_TResult tuz_TStream_open(tuz_TStream* self,tuz_TInputStreamHandle inputStrea return tuz_OK; } -tuz_TResult tuz_TStream_decompress_partial(tuz_TStream* self,tuz_byte* out_data,tuz_size_t* data_size){ - tuz_byte* cur_out_data=out_data; - tuz_size_t dsize=*data_size; + +static tuz_force_inline tuz_byte _dict_read_byte(tuz_TStream* self){ + // [ dict buf ] + // dict_cur+dictType_pos| + // dict_cur+dictType_pos| + tuz_size_t index_pos=self->_dict.dict_cur+self->_state.dictType_pos; + index_pos-=(self->_state.dictType_pos<(tuz_size_t)(self->_dict.dict_size-self->_dict.dict_cur))?0:self->_dict.dict_size; + return self->_dict.dict_buf[index_pos]; +} + +static tuz_try_inline void _dict_write_byte(tuz_TStream* self,tuz_byte bdata){ + // [ dict buf ] + // |dict_cur dict_cur+dict_size| + tuz_size_t index_pos=self->_dict.dict_cur; + self->_dict.dict_buf[index_pos++]=bdata; + self->_dict.dict_cur=(index_pos_dict.dict_size)?index_pos:0; +} + +tuz_TResult tuz_TStream_decompress_partial(tuz_TStream* self,tuz_byte* cur_out_data,tuz_size_t* data_size){ + tuz_byte* const out_data_end=cur_out_data+(*data_size); #ifdef __RUN_MEM_SAFE_CHECK - const tuz_BOOL isNeedOut=(dsize>0); + const tuz_BOOL isNeedOut=(cur_out_data_state.dictType_len){ //copy from dict or out_data - copyDict_process: - if (dsize){ - // [ dict buf ]|[ out buf | ] - // |dict_cur dict_size|out_data cur_out_data| <-- dsize -->| - // dictType_pos| <-- dictType_len --> | - // dictType_pos| <-- dictType_len --> | - // dictType_pos| <- dictType_len -> | - tuz_size_t len; - if (self->_state.dictType_pos_dict.dict_size){ - len=_copy_from_dict(self,cur_out_data,dsize); - self->_state.dictType_pos+=len; - }else{ - len=(self->_state.dictType_len_state.dictType_len:dsize; - _memmove_order(cur_out_data,out_data+self->_state.dictType_pos_inc,len); - self->_state.dictType_pos_inc+=len; - } - self->_state.dictType_len-=len; - cur_out_data+=len; - dsize-=len; + //copyDict_process: + if (cur_out_data_state.dictType_len--; goto copyDict_cmp_process; }else{ break; @@ -265,15 +197,14 @@ tuz_TResult tuz_TStream_decompress_partial(tuz_TStream* self,tuz_byte* out_data, } #if tuz_isNeedLiteralLine - copyLiteral_cmp_process: + copyLiteral_cmp_process: if (self->_state.literalType_len){ - copyLiteral_process: - if (dsize){ - tuz_length_t cpyLen=(self->_state.literalType_len_state.literalType_len:dsize; - self->_state.literalType_len-=cpyLen; - dsize-=cpyLen; - while (cpyLen--) - *cur_out_data++=_cache_read_1byte(&self->_code_cache); + //copyLiteral_process: + if (cur_out_data_code_cache); + _dict_write_byte(self,bdata); + *cur_out_data++=bdata; + self->_state.literalType_len--; goto copyLiteral_cmp_process; }else{ break; @@ -295,27 +226,20 @@ tuz_TResult tuz_TStream_decompress_partial(tuz_TStream* self,tuz_byte* out_data, self->_state.isHaveData_back=tuz_FALSE; if (saved_dict_pos){ - const tuz_size_t outed_size=(tuz_size_t)(cur_out_data-out_data); self->_state.dict_pos_back=saved_dict_pos; self->_state.dictType_len=saved_len+tuz_kMinDictMatchLen; saved_dict_pos=(self->_dict.dict_size-saved_dict_pos); #ifdef __RUN_MEM_SAFE_CHECK if (saved_dict_pos>=self->_dict.dict_size) return tuz_DICT_POS_ERROR; #endif - if (outed_size_dict.dict_size-saved_dict_pos){ - self->_state.dictType_pos=outed_size+saved_dict_pos; - self->_state.dictType_pos_inc=0; - }else{ - self->_state.dictType_pos=self->_dict.dict_size; - self->_state.dictType_pos_inc=outed_size+saved_dict_pos-self->_dict.dict_size; - } - goto copyDict_process; + self->_state.dictType_pos=saved_dict_pos; + continue; // goto copyDict_process; }else{ // ctrlType #if tuz_isNeedLiteralLine if (tuz_ctrlType_literalLine==saved_len){ self->_state.isHaveData_back=tuz_TRUE; self->_state.literalType_len=_cache_unpack_pos_len(self)+tuz_kMinLiteralLen; - goto copyLiteral_process; + continue; // goto copyLiteral_process; } #endif @@ -324,7 +248,7 @@ tuz_TResult tuz_TStream_decompress_partial(tuz_TStream* self,tuz_byte* out_data, if (tuz_ctrlType_clipEnd==saved_len){ //clip end goto type_process; }else if (tuz_ctrlType_streamEnd==saved_len){ //stream end - *data_size=(tuz_size_t)(cur_out_data-out_data); + (*data_size)-=(tuz_size_t)(out_data_end-cur_out_data); return tuz_STREAM_END; }else{ return _cache_success_finish(&self->_code_cache)? @@ -332,10 +256,11 @@ tuz_TResult tuz_TStream_decompress_partial(tuz_TStream* self,tuz_byte* out_data, } } }else{ - if (dsize){ + if (cur_out_data_code_cache); + _dict_write_byte(self,bdata); + *cur_out_data++=bdata; self->_state.isHaveData_back=tuz_TRUE; - *cur_out_data++=_cache_read_1byte(&self->_code_cache); - --dsize; goto type_process; }else{ _cache_push_1bit(self,tuz_codeType_data); @@ -347,9 +272,7 @@ tuz_TResult tuz_TStream_decompress_partial(tuz_TStream* self,tuz_byte* out_data, //return_process: { - assert(dsize==0); - if (out_data!=cur_out_data) - _update_dict(self,out_data,cur_out_data); + assert(cur_out_data==out_data_end); if (!_cache_success_finish(&self->_code_cache)) return tuz_READ_CODE_ERROR; @@ -361,7 +284,6 @@ tuz_TResult tuz_TStream_decompress_partial(tuz_TStream* self,tuz_byte* out_data, } } - //--------------------------------------------------------------------------------------------- typedef struct _mem_TStream{ diff --git a/decompress/tuz_dec.h b/decompress/tuz_dec.h index ae8feb0..ddc3e6f 100644 --- a/decompress/tuz_dec.h +++ b/decompress/tuz_dec.h @@ -1,7 +1,7 @@ // tuz_dec.h /* The MIT License (MIT) - Copyright (c) 2012-2022 HouSisong All Rights Reserved. + Copyright (c) 2012-2025 HouSisong All Rights Reserved. */ #ifndef _tuz_dec_h #define _tuz_dec_h @@ -28,8 +28,8 @@ typedef enum tuz_TResult{ //----------------------------------------------------------------------------------------------------------------- -// decompress step by step: compiled by Mbed Studio is 856 bytes -// if set tuz_isNeedLiteralLine=0 & _IS_RUN_MEM_SAFE_CHECK=0, compiled by Mbed Studio is 758 bytes +// decompress step by step: compiled by Mbed Studio is 626 bytes +// if set tuz_isNeedLiteralLine=0 & _IS_RUN_MEM_SAFE_CHECK=0, compiled by Mbed Studio is 468 bytes typedef struct tuz_TStream{ _tuz_TInputCache _code_cache; diff --git a/decompress/tuz_types.h b/decompress/tuz_types.h index 1c069c9..191f126 100644 --- a/decompress/tuz_types.h +++ b/decompress/tuz_types.h @@ -11,7 +11,7 @@ #endif #ifndef tuz_isNeedLiteralLine // optimize incompressible data for improve compression ratio -//if tuz_isNeedLiteralLine==0 when decompress, must also be set to 0 when compress, can reduce 80 bytes +//if tuz_isNeedLiteralLine==0 when decompress, must also be set to 0 when compress, can reduce 54 bytes # define tuz_isNeedLiteralLine 1 #endif @@ -26,9 +26,6 @@ # define tuz_FALSE hpi_FALSE # define tuz_TRUE hpi_TRUE # define tuz_size_t hpi_size_t //memory size type -# define tuz_inline hpi_inline -# define tuz_force_inline hpi_force_inline -# define tuz_try_inline hpi_try_inline # define tuz_TInputStreamHandle hpi_TInputStreamHandle # define tuz_TInputStream_read hpi_TInputStream_read #endif @@ -100,7 +97,6 @@ extern "C" { #endif #ifndef tuz_try_inline -//# define tuz_try_inline tuz_inline # define tuz_try_inline #endif @@ -108,10 +104,6 @@ extern "C" { # define _IS_RUN_MEM_SAFE_CHECK 1 #endif -#ifndef _IS_USED_C_MEMCPY // use memcpy() in ? -# define _IS_USED_C_MEMCPY 1 -#endif - #ifndef tuz_kMaxOfDictSize # define tuz_kMaxOfDictSize __tuz_kMaxOfDictSize_MAX //# define tuz_kMaxOfDictSize ((1<<24)-1) //3 bytes @@ -161,7 +153,6 @@ typedef struct _tuz_TDict{ } _tuz_TDict; typedef struct _tuz_TState{ tuz_size_t dictType_pos; - tuz_size_t dictType_pos_inc; tuz_size_t dict_pos_back; tuz_length_t dictType_len; #if tuz_isNeedLiteralLine diff --git a/speed_test.cpp b/speed_test.cpp index 7e3d0e2..5f01235 100644 --- a/speed_test.cpp +++ b/speed_test.cpp @@ -2,7 +2,7 @@ // for tinyuz /* The MIT License (MIT) - Copyright (c) 2012-2022 HouSisong All Rights Reserved. + Copyright (c) 2012-2025 HouSisong All Rights Reserved. */ #include #include @@ -14,6 +14,10 @@ #include "decompress/tuz_dec.h" #include "compress/tuz_enc.h" #include "zlib.h" +#define _IS_NEED_TEST_OTHERS 0 +#if (_IS_NEED_TEST_OTHERS) +#include "test_others.h" +#endif #ifdef min # undef min @@ -22,8 +26,10 @@ std::string TEST_FILE_DIR; tuz_BOOL isDictSizeTest=0; -static const tuz_size_t tDictSizes[]={24,79,255,1<<10,5<<10,32<<10,1<<20,32<<20}; -static const char* tDictSizes_tag[]={"24","79","255","1k","5k","32k","1m","32m"}; +int threadTest=1; +static const tuz_size_t tDictSizes[]={255,1<<10,4<<10,32<<10,1<<20}; +static const char* tDictSizes_tag[]={"255","1k","4k","32k","1m"}; +static std::string tTestPrograms; void readFile(std::vector& data,const char* fileName){ FILE * file=fopen(fileName, "rb"); @@ -71,8 +77,8 @@ double testDecodeProc(T_decompress proc_decompress,unsigned char* out_data,unsig do { for (int i=0; i<10; ++i){ bool ret=proc_decompress(out_data,out_data_end,zip_code,zip_code_end); - ++testDecompressCount; if (!ret) throw "error result!"; + ++testDecompressCount; if (isDictSizeTest) break; } time2=clock_s(); @@ -89,6 +95,7 @@ double testEncodeProc(T_compress proc_compress,std::vector& compr double time1=clock_s(); do{ dstCodeSize=proc_compress(&compressedCode[0],&compressedCode[0]+compressedCode.size(),src,src_end); + if (dstCodeSize<=0) throw "error result!"; ++testCompressCount; }while ((clock_s()-time1)& compr return compressTime_s; } -TTestResult testProc(const char* srcFileName,T_compress proc_compress,const char* proc_compress_Name, +TTestResult testProc(const char* srcFileName,T_compress proc_compress, T_decompress proc_decompress,const char* proc_decompress_Name){ std::string testFilePath=TEST_FILE_DIR; testFilePath.append(srcFileName); std::vector oldData; readFile(oldData,testFilePath.c_str()); @@ -132,15 +139,15 @@ TTestResult testProc(const char* srcFileName,T_compress proc_compress,const char static void outResult(const TTestResult& rt){ const bool isSimpleView=isDictSizeTest; + static tuz_BOOL isOutedTag=0; if (isSimpleView){ - static tuz_BOOL isOutedTag=0; if (!isOutedTag){ isOutedTag=1; - printf("|zlib -9"); - for (int i=(sizeof(tDictSizes)/sizeof(tDictSizes[0])-1);i>=0; --i) { - std::string tag=std::string("tuz -")+tDictSizes_tag[i]; - printf("|%s",tag.c_str()); - } + std::string str="test file"; + for (int i=(sizeof(tDictSizes)/sizeof(tDictSizes[0])-1);i>=0; --i) + str+=std::string("|tuz ")+tDictSizes_tag[i]; + str+=tTestPrograms; + printf(str.c_str()); } static std::string srcFileName_back; @@ -150,16 +157,20 @@ static void outResult(const TTestResult& rt){ } printf("|%.2f%%",rt.zipSize*100.0/rt.srcSize); }else{//default view + if (!isOutedTag){ + isOutedTag=1; + printf("test file|size|program|C M/S|D M/S|C size|C ratio\n"); + } printf("%s|%d|%s|",rt.srcFileName.c_str(),rt.srcSize,rt.procName.c_str()); printf("%.2f|%.0f|",rt.srcSize/rt.compressTime_s/1024/1024,rt.srcSize/rt.decompressTime_s/1024/1024); - printf("%.2f|%.0f|",rt.zipSize/rt.compressTime_s/1024/1024,rt.zipSize/rt.decompressTime_s/1024/1024); + //printf("%.2f|%.0f|",rt.zipSize/rt.compressTime_s/1024/1024,rt.zipSize/rt.decompressTime_s/1024/1024); printf("%d|%.2f%%\n",rt.zipSize,rt.zipSize*100.0/rt.srcSize); } } //// -int zlib_windowBits = 0; +int zlib_windowBits = -15; int zlib_level = 9; int zlib_compress(unsigned char* out_data,unsigned char* out_data_end, @@ -248,7 +259,7 @@ int _test_tuz_compress(unsigned char* out_data,unsigned char* out_data_end, mem_as_hStreamInput(&in_stream,src,src_end); tuz_TCompressProps props=tuz_kDefaultCompressProps; props.dictSize=_tuz_kDictSize; - props.threadNum=8; + props.threadNum=threadTest; //props.maxSaveLength=255; hpatch_StreamPos_t codeSize=tuz_compress(&out_stream,&in_stream,&props); return (int)codeSize; @@ -318,23 +329,32 @@ bool _test_tuz_decompress_mem(unsigned char* out_data,unsigned char* out_data_en } static void testFile(const char* srcFileName){ - //zlib_level=6; - //outResult(testProc(srcFileName, zlib_compress, "", zlib_decompress, " zlib -6")); - zlib_level=9; - + tTestPrograms="|zlib 32k"; + tTestPrograms+=(_IS_NEED_TEST_OTHERS?"|QuickLZ|tamp 32k|tamp 4k|heatshrink 4k|FastLZ|miniLZO":""); if (!isDictSizeTest) { - outResult(testProc(srcFileName,zlib_compress ,"",zlib_decompress ," zlib -9")); - outResult(testProc(srcFileName,_test_tuz_compress,"",_test_tuz_decompress_stream,"tinyuz_stream")); - outResult(testProc(srcFileName,_test_tuz_compress,"",_test_tuz_decompress_mem ," tinyuz_mem")); + outResult(testProc(srcFileName,_test_tuz_compress ,_test_tuz_decompress_stream ," tinyuz_stream")); + outResult(testProc(srcFileName,_test_tuz_compress ,_test_tuz_decompress_mem ," tinyuz_mem")); }else{ - zlib_windowBits=-15; - outResult(testProc(srcFileName,zlib_compress ,"",zlib_decompress ,"zlib -9")); for (int i=(sizeof(tDictSizes)/sizeof(tDictSizes[0])-1);i>=0; --i) { - std::string tag=std::string("tinyuz -c-")+tDictSizes_tag[i]; _tuz_kDictSize=tDictSizes[i]; - outResult(testProc(srcFileName,_test_tuz_compress,"",_test_tuz_decompress_stream,tag.c_str())); + outResult(testProc(srcFileName,_test_tuz_compress,_test_tuz_decompress_stream,(std::string("tuz ")+tDictSizes_tag[i]).c_str())); } } + + outResult(testProc(srcFileName,zlib_compress ,zlib_decompress ," zlib 32k")); + #if (_IS_NEED_TEST_OTHERS) + outResult(testProc(srcFileName,quicklz_compress ,quicklz_decompress ," QuickLZ")); + tamp_windowBits=15; outResult(testProc(srcFileName,tamp_compress ,tamp_decompress ," tamp 32k")); + tamp_windowBits=12; outResult(testProc(srcFileName,tamp_compress ,tamp_decompress ," tamp 4k")); +//tamp_windowBits=10; outResult(testProc(srcFileName,tamp_compress ,tamp_decompress ," tamp 1k")); +//tamp_windowBits=8; outResult(testProc(srcFileName,tamp_compress ,tamp_decompress ," tamp 256")); +//hs_windowBits=15;outResult(testProc(srcFileName,heatshrink_compress,heatshrink_decompress,"heatshrink 32k"));// test fail when hs_windowBits=15 +hs_windowBits=12;outResult(testProc(srcFileName,heatshrink_compress,heatshrink_decompress," heatshrink 4k")); +//hs_windowBits=10;outResult(testProc(srcFileName,heatshrink_compress,heatshrink_decompress," heatshrink 1k")); +//hs_windowBits=8; outResult(testProc(srcFileName,heatshrink_compress,heatshrink_decompress,"heatshrink 256")); + outResult(testProc(srcFileName,_fastlz_compress ,_fastlz_decompress ," FastLZ")); + outResult(testProc(srcFileName,minilzo_compress ,minilzo_decompress ," miniLZO")); + #endif } int main(int argc, const char * argv[]){ @@ -342,7 +362,8 @@ int main(int argc, const char * argv[]){ std::cout << "speed_test \"testFile\"\n"; return -1; } - isDictSizeTest=0; + isDictSizeTest=1; + threadTest=16; const int testDictBit=15; zlib_windowBits=-testDictBit; _tuz_kDictSize = (1 << testDictBit); @@ -380,7 +401,7 @@ int main(int argc, const char * argv[]){ //testFile("enwik9"); //*/ - std::cout << "done!\n"; + std::cout << "\ndone!\n"; return 0; } diff --git a/test_others.h b/test_others.h new file mode 100644 index 0000000..d161f2e --- /dev/null +++ b/test_others.h @@ -0,0 +1,207 @@ +//test_others.h +#ifndef tinyuz_test_others_h +#define tinyuz_test_others_h + + +#include "../minilzo/minilzo.h" +// GNU GENERAL PUBLIC LICENSE Version 2 + +#define HEAP_ALLOC(var,size) \ + lzo_align_t __LZO_MMODEL var [ ((size) + (sizeof(lzo_align_t) - 1)) / sizeof(lzo_align_t) ] + +int minilzo_compress(unsigned char* out_data,unsigned char* out_data_end, + const unsigned char* src,const unsigned char* src_end){ + HEAP_ALLOC(wrkmem, LZO1X_1_MEM_COMPRESS); + lzo_uint dst_len=out_data_end-out_data; + int ret=lzo1x_1_compress(src,src_end-src,out_data,&dst_len,wrkmem); + if (ret!=LZO_E_OK) + return 0; + return dst_len; +} + +bool minilzo_decompress(unsigned char* out_data,unsigned char* out_data_end, + const unsigned char* zip_code,const unsigned char* zip_code_end){ + lzo_uint dst_len=out_data_end-out_data; + int ret=lzo1x_decompress_safe(zip_code,zip_code_end-zip_code,out_data,&dst_len,0); + return (ret==LZO_E_OK); +} + + + +#include "../quicklz/src/quicklz.h" +// GNU GENERAL PUBLIC LICENSE Version 2 + +int quicklz_compress(unsigned char* out_data,unsigned char* out_data_end, + const unsigned char* src,const unsigned char* src_end){ + qlz_state_compress *state_compress = (qlz_state_compress *)malloc(sizeof(qlz_state_compress)); + int c=qlz_compress(src,(char*)out_data,src_end-src, state_compress); + free(state_compress); + return c; +} + +bool quicklz_decompress(unsigned char* out_data,unsigned char* out_data_end, + const unsigned char* zip_code,const unsigned char* zip_code_end){ + qlz_state_decompress *state_decompress = (qlz_state_decompress *)malloc(sizeof(qlz_state_decompress)); + if (zip_code_end-zip_code<9) return false; + int dc = qlz_size_decompressed((const char*)zip_code); + int c = qlz_size_compressed((const char*)zip_code); + if (out_data_end-out_datatamp_decompressor_decompress(&d,out_data,out_data_end-out_data, + &output_size,zip_code,zip_code_end-zip_code,&input_size)) + return false; + free(buf); + return true; +} + +#endif //tinyuz_test_others_h \ No newline at end of file