Merge pull request #56 from Cyan4973/dev

HC modes
facebook · Oct 30, 2015 · fdc4a01 · fdc4a01
2 parents 747e1eb + 092a4f1
commit fdc4a01
Show file tree

Hide file tree

Showing 39 changed files with 6,783 additions and 560 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -16,8 +16,9 @@ env:
   - ZSTD_TRAVIS_CI_ENV=armtest  
   - ZSTD_TRAVIS_CI_ENV=test  
   - ZSTD_TRAVIS_CI_ENV="-C programs test32"  
-  - ZSTD_TRAVIS_CI_ENV=sanitize
-  - ZSTD_TRAVIS_CI_ENV="-C programs memtest"  
+  - ZSTD_TRAVIS_CI_ENV=usan
+  - ZSTD_TRAVIS_CI_ENV=asan
+  - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest"  
 
 matrix:
   fast_finish: true
diff --git a/Makefile b/Makefile
@@ -32,7 +32,7 @@
 # ################################################################
 
 # Version number
-export VERSION := 0.2.2
+export VERSION := 0.3.0
 
 PRGDIR  = programs
 ZSTDDIR = lib
@@ -83,7 +83,13 @@ armtest: clean
 	$(MAKE) -C $(ZSTDDIR) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
 	$(MAKE) -C $(PRGDIR) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror"
 
-sanitize: clean
+usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
 
+asan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
+
+uasan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
+
 endif
diff --git a/NEWS b/NEWS
@@ -1,3 +1,6 @@
+v0.3
+HC mode : compression levels 2-26
+
 v0.2.2
 Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
 

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
- **Zstd**, short for Zstandard, is a new lossless compression algorithm, which provides both good compression ratio _and_ speed for your standard compression needs. "Standard" translates into everyday situations which neither look for highest possible ratio nor extreme speed.
+ **Zstd**, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level compression ratio.
 
 It is provided as a BSD-license package, hosted on Github.
 
@@ -7,42 +7,40 @@ It is provided as a BSD-license package, hosted on Github.
 |master      | [![Build Status](https://travis-ci.org/Cyan4973/zstd.svg?branch=master)](https://travis-ci.org/Cyan4973/zstd) |
 |dev         | [![Build Status](https://travis-ci.org/Cyan4973/zstd.svg?branch=dev)](https://travis-ci.org/Cyan4973/zstd) |
 
-For a taste of its performance, here are a few benchmark numbers, completed on a Core i7-5600U @ 2.6 GHz, using [fsbench 0.14.3](http://encode.ru/threads/1371-Filesystem-benchmark?p=34029&viewfull=1#post34029), an open-source benchmark program by m^2.
-
-|Name            | Ratio | C.speed | D.speed |
-|----------------|-------|--------:|--------:|
-|                |       |   MB/s  |  MB/s   |
-| [zlib 1.2.8] -6| 3.099 |    21   |   320   |
-| **zstd 0.2**   |**2.871**|**255**| **670** |
-| [zlib 1.2.8] -1| 2.730 |    70   |   300   | 
-| [LZ4] HC r131  | 2.720 |    25   |  2100   |
-| QuickLZ 1.5.1b6| 2.237 |   370   |   415   |
-| LZO 2.06       | 2.106 |   400   |   580   |
-| Snappy 1.1.0   | 2.091 |   330   |  1100   |
-| [LZ4] r131     | 2.101 |   450   |  2100   |
-| LZF 3.6        | 2.077 |   200   |   560   |
+For a taste of its performance, here are a few benchmark numbers from a number of compression codecs suitable for real-time. The test was completed on a Core i7-5600U @ 2.6 GHz, using [fsbench 0.14.3](http://encode.ru/threads/1371-Filesystem-benchmark?p=34029&viewfull=1#post34029), an open-source benchmark program by m^2.
+
+|Name             | Ratio | C.speed | D.speed |
+|-----------------|-------|--------:|--------:|
+|                 |       |   MB/s  |  MB/s   |
+| **zstd 0.3**    |**2.858**|**280**| **670** |
+| [zlib 1.2.8] -1 | 2.730 |    70   |   300   | 
+| QuickLZ 1.5.1b6 | 2.237 |   370   |   415   |
+| LZO 2.06        | 2.106 |   400   |   580   |
+| [LZ4] r131      | 2.101 |   450   |  2100   |
+| Snappy 1.1.0    | 2.091 |   330   |  1100   |
+| LZF 3.6         | 2.077 |   200   |   560   |
 
 [zlib 1.2.8]:http://www.zlib.net/
 [LZ4]:http://www.lz4.org/
 
-An interesting feature of zstd is that it can qualify as both a reasonably strong compressor and a fast one.
-
-Zstd delivers very high decompression speed, at more than >600 MB/s per core.
-Obviously, your exact mileage will vary depending on your target system.
-
-Zstd compression speed will be configurable to fit different situations.
-The first available version is the fast one, at ~250 MB/s per core, which is suitable for a few real-time scenarios.
-But similar to [LZ4], zstd can offer derivatives trading compression time for compression ratio, keeping decompression properties intact. "Offline compression", where compression time is of little importance because the content is only compressed once and decompressed many times, will likely prefer this setup.
+Zstd can also offer stronger compression ratio at the cost of compression speed, but preserving its decompression speed. In the following test, a few compressors suitable for this scenario are selected (they offer very asymetric performance, useful when compression time has little importance). The test was completed on a Core i7-5600U @ 2.6 GHz, using [benchmark 0.6.1](http://encode.ru/threads/1266-In-memory-benchmark-with-fastest-LZSS-(QuickLZ-Snappy)-compressors?p=45217&viewfull=1#post45217), an open-source benchmark program by inikep.
 
-Note that high compression derivatives still have to be developed.
-It's a complex area which will require time and benefit from contributions.
+|Name             | Ratio | C.speed | D.speed |
+|-----------------|-------|--------:|--------:|
+|                 |       |   MB/s  |  MB/s   |
+| brotli -9       | 3.729 |     4   |   340   |
+| **zstd 0.3 -9** |**3.447**|**30** | **640** |
+| [zlib 1.2.8] -9 | 3.133 |    10   |   300   | 
+| LZO 2.06 -999   | 2.790 |     1   |   560   |
+| [LZ4] r131 -9   | 2.720 |    25   |  2100   |
 
+[lzma]:http://www.7-zip.org/
 
-Another property zstd is developed for is configurable memory requirement, with the objective to fit into low-memory configurations, or servers handling many connections in parallel.
+Zstd compression speed is highly configurable, by small increment, to fit different situations. Its memory requirement can also be configured to fit into low-memory hardware configurations, or servers handling multiple connections/contexts in parallel.
 
 Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entrop library](https://github.com/Cyan4973/FiniteStateEntropy).
 
-Zstd has not yet reached "stable" status. Specifically, it doesn't guarantee yet that its current compressed format will remain stable and supported in future versions. It may still change to adapt further optimizations still being investigated. However, the library starts to be pretty robust, able to withstand hazards situations, including invalid input. The library reliability has been tested using [Fuzz Testing](https://en.wikipedia.org/wiki/Fuzz_testing), using both [internal tools](programs/fuzzer.c) and [external ones](http://lcamtuf.coredump.cx/afl). Therefore, you can now safely test zstd, even within production environments.
+Zstd has not yet reached "stable" status. Specifically, it doesn't guarantee yet that its current compressed format will remain stable and supported in future versions. It may still change to adapt further optimizations still being investigated. That being said, the library is now pretty robust, able to withstand hazards situations, including invalid input. The library reliability has been tested using [Fuzz Testing](https://en.wikipedia.org/wiki/Fuzz_testing), with both [internal tools](programs/fuzzer.c) and [external ones](http://lcamtuf.coredump.cx/afl). Therefore, it's now safe to test Zstandard even within production environments.
 
 "Stable Format" is projected sometimes early 2016.
 

diff --git a/lib/Makefile b/lib/Makefile
@@ -40,7 +40,7 @@ VERSION?= $(LIBVER)
 
 DESTDIR?=
 PREFIX ?= /usr/local
-CPPFLAGS= -I./legacy -DZSTD_LEGACY_SUPPORT=1
+CPPFLAGS= -I. -I./legacy -DZSTD_LEGACY_SUPPORT=1
 CFLAGS ?= -O3
 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)

diff --git a/lib/bitstream.h b/lib/bitstream.h
@@ -150,7 +150,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
 MEM_STATIC unsigned BIT_highbit32 (register U32 val)
 {
 #   if defined(_MSC_VER)   /* Visual */
-    unsigned long r;
+    unsigned long r=0;
     _BitScanReverse ( &r, val );
     return (unsigned) r;
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */

diff --git a/lib/error.h b/lib/error.h
@@ -39,6 +39,12 @@ extern "C" {
 #endif
 
 
+/******************************************
+*  Includes
+******************************************/
+#include <stddef.h>    /* size_t, ptrdiff_t */
+
+
 /******************************************
 *  Compiler-specific
 ******************************************/
@@ -62,6 +68,7 @@ extern "C" {
 
 #define ERROR_LIST(ITEM) \
         ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
+        ITEM(PREFIX(memory_allocation)) \
         ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
         ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \
         ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \

diff --git a/lib/fse.h b/lib/fse.h
@@ -32,7 +32,8 @@
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
    - Public forum : https://groups.google.com/forum/#!forum/lz4c
 ****************************************************************** */
-#pragma once
+#ifndef FSE_H
+#define FSE_H
 
 #if defined (__cplusplus)
 extern "C" {
@@ -290,3 +291,5 @@ If there is an error, the function will return an error code, which can be teste
 #if defined (__cplusplus)
 }
 #endif
+
+#endif  /* FSE_H */
diff --git a/lib/fse_static.h b/lib/fse_static.h
@@ -32,7 +32,8 @@
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
    - Public forum : https://groups.google.com/forum/#!forum/lz4c
 ****************************************************************** */
-#pragma once
+#ifndef FSE_STATIC_H
+#define FSE_STATIC_H
 
 #if defined (__cplusplus)
 extern "C" {
@@ -305,3 +306,5 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
 #if defined (__cplusplus)
 }
 #endif
+
+#endif  /* FSE_STATIC_H */
diff --git a/lib/huff0.c b/lib/huff0.c
@@ -68,6 +68,18 @@
 #include "fse.h"        /* header compression */
 
 
+/****************************************************************
+*  Constants
+****************************************************************/
+#define HUF_ABSOLUTEMAX_TABLELOG  16   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
+#define HUF_MAX_TABLELOG  12           /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
+#define HUF_DEFAULT_TABLELOG  HUF_MAX_TABLELOG   /* tableLog by default, when not specified */
+#define HUF_MAX_SYMBOL_VALUE 255
+#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG)
+#  error "HUF_MAX_TABLELOG is too large !"
+#endif
+
+
 /****************************************************************
 *  Error Management
 ****************************************************************/
@@ -85,14 +97,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
 /*********************************************************
 *  Huff0 : Huffman block compression
 *********************************************************/
-#define HUF_ABSOLUTEMAX_TABLELOG  16   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
-#define HUF_MAX_TABLELOG  12           /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
-#define HUF_DEFAULT_TABLELOG  HUF_MAX_TABLELOG   /* tableLog by default, when not specified */
-#define HUF_MAX_SYMBOL_VALUE 255
-#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG)
-#  error "HUF_MAX_TABLELOG is too large !"
-#endif
-
 typedef struct HUF_CElt_s {
   U16  val;
   BYTE nbBits;
@@ -1017,29 +1021,24 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
         rankStart[0] = 0;   /* forget 0w symbols; this is beginning of weight(1) */
     }
 
-	/* Build rankVal */
+    /* Build rankVal */
     {
         const U32 minBits = tableLog+1 - maxW;
         U32 nextRankVal = 0;
         U32 w, consumed;
         const int rescale = (memLog-tableLog) - 1;   /* tableLog <= memLog */
-        U32* rankVal0 = rankVal[0];
         for (w=1; w<=maxW; w++)
         {
             U32 current = nextRankVal;
             nextRankVal += rankStats[w] << (w+rescale);
-            rankVal0[w] = current;
-        }
-        for (consumed = minBits; consumed <= memLog - minBits; consumed++)
-        {
-            U32* rankValPtr = rankVal[consumed];
-            for (w = 1; w <= maxW; w++)
-            {
-                rankValPtr[w] = rankVal0[w] >> consumed;
-            }
+            rankVal[0][w] = current;
         }
+        for (consumed=minBits; consumed <= memLog-minBits; consumed++)
+            for (w=1; w<=maxW; w++)
+                rankVal[consumed][w] = rankVal[0][w] >> consumed;
     }
 
+
     HUF_fillDTableX4(dt, memLog,
                    sortedSymbol, sizeOfSort,
                    rankStart0, rankVal, maxW,
@@ -1389,34 +1388,28 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
         rankStart[0] = 0;   /* forget 0w symbols; this is beginning of weight(1) */
     }
 
-	/* Build rankVal */
+    /* Build rankVal */
     {
         const U32 minBits = tableLog+1 - maxW;
         U32 nextRankVal = 0;
         U32 w, consumed;
         const int rescale = (memLog-tableLog) - 1;   /* tableLog <= memLog */
-        U32* rankVal0 = rankVal[0];
         for (w=1; w<=maxW; w++)
         {
             U32 current = nextRankVal;
             nextRankVal += rankStats[w] << (w+rescale);
-            rankVal0[w] = current;
-        }
-        for (consumed = minBits; consumed <= memLog - minBits; consumed++)
-        {
-            U32* rankValPtr = rankVal[consumed];
-            for (w = 1; w <= maxW; w++)
-            {
-                rankValPtr[w] = rankVal0[w] >> consumed;
-            }
+            rankVal[0][w] = current;
         }
+        for (consumed=minBits; consumed <= memLog-minBits; consumed++)
+            for (w=1; w<=maxW; w++)
+                rankVal[consumed][w] = rankVal[0][w] >> consumed;
     }
 
 
     /* fill tables */
     {
         HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1);
-        HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + (1<<(memLog-1)));
+        HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(memLog-1)));
         HUF_DSeqX6 DSeq;
         HUF_DDescX6 DDesc;
         DSeq.sequence = 0;
@@ -1476,7 +1469,7 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr
 static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
 {
     const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
-    const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + (1<<(dtLog-1)));
+    const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
     BYTE* const pStart = p;
 
     /* up to 16 symbols at a time */
@@ -1558,7 +1551,7 @@ size_t HUF_decompress4X6_usingDTable(
 
         const U32 dtLog = DTable[0];
         const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1);
-        const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + (1<<(dtLog-1)));
+        const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1)));
         size_t errorCode;
 
         /* Init */

diff --git a/lib/huff0.h b/lib/huff0.h
@@ -32,7 +32,8 @@
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
    - Public forum : https://groups.google.com/forum/#!forum/lz4c
 ****************************************************************** */
-#pragma once
+#ifndef HUFF0_H
+#define HUFF0_H
 
 #if defined (__cplusplus)
 extern "C" {
@@ -91,3 +92,5 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
 #if defined (__cplusplus)
 }
 #endif
+
+#endif   /* HUFF0_H */
diff --git a/lib/huff0_static.h b/lib/huff0_static.h
@@ -32,7 +32,8 @@
    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
    - Public forum : https://groups.google.com/forum/#!forum/lz4c
 ****************************************************************** */
-#pragma once
+#ifndef HUFF0_STATIC_H
+#define HUFF0_STATIC_H
 
 #if defined (__cplusplus)
 extern "C" {
@@ -74,3 +75,5 @@ size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cS
 #if defined (__cplusplus)
 }
 #endif
+
+#endif /* HUFF0_STATIC_H */