Compress-Stream-Zstd
view release on metacpan or search on metacpan
ext/zstd/lib/legacy/zstd_v06.c view on Meta::CPAN
/* The prototypes defined within this file are considered experimental.
* They should not be used in the context DLL as they may change in the future.
* Prefer static linking if you need them, to control breaking version changes issues.
*/
#if defined (__cplusplus)
extern "C" {
#endif
/*- Advanced Decompression functions -*/
/*! ZSTDv06_decompress_usingPreparedDCtx() :
* Same as ZSTDv06_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTDv06_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
ZSTDLIBv06_API size_t ZSTDv06_decompress_usingPreparedDCtx(
ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
#define ZSTDv06_FRAMEHEADERSIZE_MAX 13 /* for static allocation */
static const size_t ZSTDv06_frameHeaderSize_min = 5;
static const size_t ZSTDv06_frameHeaderSize_max = ZSTDv06_FRAMEHEADERSIZE_MAX;
ZSTDLIBv06_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
/*
Streaming decompression, direct mode (bufferless)
A ZSTDv06_DCtx object is required to track streaming operations.
Use ZSTDv06_createDCtx() / ZSTDv06_freeDCtx() to manage it.
A ZSTDv06_DCtx object can be re-used multiple times.
First optional operation is to retrieve frame parameters, using ZSTDv06_getFrameParams(), which doesn't consume the input.
It can provide the minimum size of rolling buffer required to properly decompress data,
and optionally the final size of uncompressed content.
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
Frame parameters are extracted from the beginning of compressed frame.
The amount of data to read is variable, from ZSTDv06_frameHeaderSize_min to ZSTDv06_frameHeaderSize_max (so if `srcSize` >= ZSTDv06_frameHeaderSize_max, it will always work)
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
Result : 0 when successful, it means the ZSTDv06_frameParams structure has been filled.
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
errorCode, which can be tested using ZSTDv06_isError()
Start decompression, with ZSTDv06_decompressBegin() or ZSTDv06_decompressBegin_usingDict().
Alternatively, you can copy a prepared context, using ZSTDv06_copyDCtx().
Then use ZSTDv06_nextSrcSizeToDecompress() and ZSTDv06_decompressContinue() alternatively.
ZSTDv06_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTDv06_decompressContinue().
ZSTDv06_decompressContinue() requires this exact amount of bytes, or it will fail.
ZSTDv06_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
@result of ZSTDv06_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity)
It can be zero, which is not an error; it just means ZSTDv06_decompressContinue() has decoded some header.
A frame is fully decoded when ZSTDv06_nextSrcSizeToDecompress() returns zero.
Context can then be reset to start a new decompression.
*/
/* **************************************
* Block functions
****************************************/
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
A few rules to respect :
- Uncompressed block size must be <= ZSTDv06_BLOCKSIZE_MAX (128 KB)
- Compressing or decompressing requires a context structure
+ Use ZSTDv06_createCCtx() and ZSTDv06_createDCtx()
- It is necessary to init context before starting
+ compression : ZSTDv06_compressBegin()
+ decompression : ZSTDv06_decompressBegin()
+ variants _usingDict() are also allowed
+ copyCCtx() and copyDCtx() work too
- When a block is considered not compressible enough, ZSTDv06_compressBlock() result will be zero.
In which case, nothing is produced into `dst`.
+ User must test for such outcome and deal directly with uncompressed data
+ ZSTDv06_decompressBlock() doesn't accept uncompressed data as input !!
*/
#define ZSTDv06_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
ZSTDLIBv06_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDv06_STATIC_H */
/*
zstd_internal - common functions to include
Header File for include
Copyright (C) 2014-2016, Yann Collet.
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
ext/zstd/lib/legacy/zstd_v06.c view on Meta::CPAN
/* single stream variants */
size_t HUFv06_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUFv06_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUFv06_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUFv06_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
/* **************************************************************
* Constants
****************************************************************/
#define HUFv06_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUFv06_MAX_TABLELOG. Beyond that value, code does not work */
#define HUFv06_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUFv06_ABSOLUTEMAX_TABLELOG */
#define HUFv06_DEFAULT_TABLELOG HUFv06_MAX_TABLELOG /* tableLog by default, when not specified */
#define HUFv06_MAX_SYMBOL_VALUE 255
#if (HUFv06_MAX_TABLELOG > HUFv06_ABSOLUTEMAX_TABLELOG)
# error "HUFv06_MAX_TABLELOG is too large !"
#endif
/*! HUFv06_readStats() :
Read compact Huffman tree, saved by HUFv06_writeCTable().
`huffWeight` is destination buffer.
@return : size read from `src`
*/
MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize)
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
size_t iSize;
size_t oSize;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
if (iSize >= (242)) { /* RLE */
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
oSize = l[iSize-242];
memset(huffWeight, 1, hwSize);
iSize = 0;
}
else { /* Incompressible */
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
{ U32 n;
for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
} } } }
else { /* header compressed with FSE (normal case) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSEv06_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
if (FSEv06_isError(oSize)) return oSize;
}
/* collect weight stats */
memset(rankStats, 0, (HUFv06_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32));
weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) {
if (huffWeight[n] >= HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BITv06_highbit32(weightTotal) + 1;
if (tableLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
*tableLogPtr = tableLog;
/* determine last weight */
{ U32 const total = 1 << tableLog;
U32 const rest = total - weightTotal;
U32 const verif = 1 << BITv06_highbit32(rest);
U32 const lastWeight = BITv06_highbit32(rest) + 1;
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
huffWeight[oSize] = (BYTE)lastWeight;
rankStats[lastWeight]++;
} }
/* check tree construction validity */
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
/* results */
*nbSymbolsPtr = (U32)(oSize+1);
return iSize+1;
}
#if defined (__cplusplus)
}
#endif
#endif /* HUFv06_STATIC_H */
/* ******************************************************************
Huffman decoder, part of New Generation Entropy library
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
ext/zstd/lib/legacy/zstd_v06.c view on Meta::CPAN
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BITv06_initDStream(&bitD1, istart1, length1);
if (HUFv06_isError(errorCode)) return errorCode;
errorCode = BITv06_initDStream(&bitD2, istart2, length2);
if (HUFv06_isError(errorCode)) return errorCode;
errorCode = BITv06_initDStream(&bitD3, istart3, length3);
if (HUFv06_isError(errorCode)) return errorCode;
errorCode = BITv06_initDStream(&bitD4, istart4, length4);
if (HUFv06_isError(errorCode)) return errorCode;
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BITv06_reloadDStream(&bitD1) | BITv06_reloadDStream(&bitD2) | BITv06_reloadDStream(&bitD3) | BITv06_reloadDStream(&bitD4);
for ( ; (endSignal==BITv06_DStream_unfinished) && (op4<(oend-7)) ; ) {
HUFv06_DECODE_SYMBOLX2_2(op1, &bitD1);
HUFv06_DECODE_SYMBOLX2_2(op2, &bitD2);
HUFv06_DECODE_SYMBOLX2_2(op3, &bitD3);
HUFv06_DECODE_SYMBOLX2_2(op4, &bitD4);
HUFv06_DECODE_SYMBOLX2_1(op1, &bitD1);
HUFv06_DECODE_SYMBOLX2_1(op2, &bitD2);
HUFv06_DECODE_SYMBOLX2_1(op3, &bitD3);
HUFv06_DECODE_SYMBOLX2_1(op4, &bitD4);
HUFv06_DECODE_SYMBOLX2_2(op1, &bitD1);
HUFv06_DECODE_SYMBOLX2_2(op2, &bitD2);
HUFv06_DECODE_SYMBOLX2_2(op3, &bitD3);
HUFv06_DECODE_SYMBOLX2_2(op4, &bitD4);
HUFv06_DECODE_SYMBOLX2_0(op1, &bitD1);
HUFv06_DECODE_SYMBOLX2_0(op2, &bitD2);
HUFv06_DECODE_SYMBOLX2_0(op3, &bitD3);
HUFv06_DECODE_SYMBOLX2_0(op4, &bitD4);
endSignal = BITv06_reloadDStream(&bitD1) | BITv06_reloadDStream(&bitD2) | BITv06_reloadDStream(&bitD3) | BITv06_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUFv06_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
HUFv06_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
HUFv06_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
HUFv06_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
/* check */
endSignal = BITv06_endOfDStream(&bitD1) & BITv06_endOfDStream(&bitD2) & BITv06_endOfDStream(&bitD3) & BITv06_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
}
size_t HUFv06_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUFv06_CREATE_STATIC_DTABLEX2(DTable, HUFv06_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t const errorCode = HUFv06_readDTableX2 (DTable, cSrc, cSrcSize);
if (HUFv06_isError(errorCode)) return errorCode;
if (errorCode >= cSrcSize) return ERROR(srcSize_wrong);
ip += errorCode;
cSrcSize -= errorCode;
return HUFv06_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
/* *************************/
/* double-symbols decoding */
/* *************************/
static void HUFv06_fillDTableX4Level2(HUFv06_DEltX4* DTable, U32 sizeLog, const U32 consumed,
const U32* rankValOrigin, const int minWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
U32 nbBitsBaseline, U16 baseSeq)
{
HUFv06_DEltX4 DElt;
U32 rankVal[HUFv06_ABSOLUTEMAX_TABLELOG + 1];
/* get pre-calculated rankVal */
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill skipped values */
if (minWeight>1) {
U32 i, skipSize = rankVal[minWeight];
MEM_writeLE16(&(DElt.sequence), baseSeq);
DElt.nbBits = (BYTE)(consumed);
DElt.length = 1;
for (i = 0; i < skipSize; i++)
DTable[i] = DElt;
}
/* fill DTable */
{ U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */
const U32 symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight;
const U32 nbBits = nbBitsBaseline - weight;
const U32 length = 1 << (sizeLog-nbBits);
const U32 start = rankVal[weight];
U32 i = start;
const U32 end = start + length;
MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
DElt.nbBits = (BYTE)(nbBits + consumed);
DElt.length = 2;
do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */
ext/zstd/lib/legacy/zstd_v06.c view on Meta::CPAN
ptr += HUFv06_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUFv06_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUFv06_MAX_TABLELOG<=12)) \
ptr += HUFv06_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUFv06_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
ptr += HUFv06_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
static inline size_t HUFv06_decodeStreamX4(BYTE* p, BITv06_DStream_t* bitDPtr, BYTE* const pEnd, const HUFv06_DEltX4* const dt, const U32 dtLog)
{
BYTE* const pStart = p;
/* up to 8 symbols at a time */
while ((BITv06_reloadDStream(bitDPtr) == BITv06_DStream_unfinished) && (p < pEnd-7)) {
HUFv06_DECODE_SYMBOLX4_2(p, bitDPtr);
HUFv06_DECODE_SYMBOLX4_1(p, bitDPtr);
HUFv06_DECODE_SYMBOLX4_2(p, bitDPtr);
HUFv06_DECODE_SYMBOLX4_0(p, bitDPtr);
}
/* closer to the end */
while ((BITv06_reloadDStream(bitDPtr) == BITv06_DStream_unfinished) && (p <= pEnd-2))
HUFv06_DECODE_SYMBOLX4_0(p, bitDPtr);
while (p <= pEnd-2)
HUFv06_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
if (p < pEnd)
p += HUFv06_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
return p-pStart;
}
size_t HUFv06_decompress1X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const void* const dtPtr = DTable;
const HUFv06_DEltX4* const dt = ((const HUFv06_DEltX4*)dtPtr) +1;
/* Init */
BITv06_DStream_t bitD;
{ size_t const errorCode = BITv06_initDStream(&bitD, istart, cSrcSize);
if (HUFv06_isError(errorCode)) return errorCode; }
/* decode */
HUFv06_decodeStreamX4(ostart, &bitD, oend, dt, dtLog);
/* check */
if (!BITv06_endOfDStream(&bitD)) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
size_t HUFv06_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUFv06_CREATE_STATIC_DTABLEX4(DTable, HUFv06_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t const hSize = HUFv06_readDTableX4 (DTable, cSrc, cSrcSize);
if (HUFv06_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUFv06_decompress1X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
size_t HUFv06_decompress4X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
{
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable;
const HUFv06_DEltX4* const dt = ((const HUFv06_DEltX4*)dtPtr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
/* Init */
BITv06_DStream_t bitD1;
BITv06_DStream_t bitD2;
BITv06_DStream_t bitD3;
BITv06_DStream_t bitD4;
const size_t length1 = MEM_readLE16(istart);
const size_t length2 = MEM_readLE16(istart+2);
const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BITv06_initDStream(&bitD1, istart1, length1);
if (HUFv06_isError(errorCode)) return errorCode;
errorCode = BITv06_initDStream(&bitD2, istart2, length2);
if (HUFv06_isError(errorCode)) return errorCode;
errorCode = BITv06_initDStream(&bitD3, istart3, length3);
if (HUFv06_isError(errorCode)) return errorCode;
errorCode = BITv06_initDStream(&bitD4, istart4, length4);
if (HUFv06_isError(errorCode)) return errorCode;
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BITv06_reloadDStream(&bitD1) | BITv06_reloadDStream(&bitD2) | BITv06_reloadDStream(&bitD3) | BITv06_reloadDStream(&bitD4);
for ( ; (endSignal==BITv06_DStream_unfinished) && (op4<(oend-7)) ; ) {
HUFv06_DECODE_SYMBOLX4_2(op1, &bitD1);
HUFv06_DECODE_SYMBOLX4_2(op2, &bitD2);
HUFv06_DECODE_SYMBOLX4_2(op3, &bitD3);
HUFv06_DECODE_SYMBOLX4_2(op4, &bitD4);
HUFv06_DECODE_SYMBOLX4_1(op1, &bitD1);
HUFv06_DECODE_SYMBOLX4_1(op2, &bitD2);
HUFv06_DECODE_SYMBOLX4_1(op3, &bitD3);
HUFv06_DECODE_SYMBOLX4_1(op4, &bitD4);
HUFv06_DECODE_SYMBOLX4_2(op1, &bitD1);
HUFv06_DECODE_SYMBOLX4_2(op2, &bitD2);
HUFv06_DECODE_SYMBOLX4_2(op3, &bitD3);
HUFv06_DECODE_SYMBOLX4_2(op4, &bitD4);
HUFv06_DECODE_SYMBOLX4_0(op1, &bitD1);
HUFv06_DECODE_SYMBOLX4_0(op2, &bitD2);
HUFv06_DECODE_SYMBOLX4_0(op3, &bitD3);
HUFv06_DECODE_SYMBOLX4_0(op4, &bitD4);
endSignal = BITv06_reloadDStream(&bitD1) | BITv06_reloadDStream(&bitD2) | BITv06_reloadDStream(&bitD3) | BITv06_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUFv06_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
HUFv06_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
HUFv06_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
HUFv06_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
/* check */
endSignal = BITv06_endOfDStream(&bitD1) & BITv06_endOfDStream(&bitD2) & BITv06_endOfDStream(&bitD3) & BITv06_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
}
size_t HUFv06_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUFv06_CREATE_STATIC_DTABLEX4(DTable, HUFv06_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUFv06_readDTableX4 (DTable, cSrc, cSrcSize);
if (HUFv06_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUFv06_decompress4X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
/* ********************************/
/* Generic decompression selector */
/* ********************************/
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
{
/* single, double, quad */
{{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */
{{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */
{{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */
{{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */
{{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */
{{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */
{{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */
{{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */
{{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */
{{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */
{{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */
{{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */
{{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */
{{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */
{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
};
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
static const decompressionAlgo decompress[3] = { HUFv06_decompress4X2, HUFv06_decompress4X4, NULL };
U32 Dtime[3]; /* decompression time estimation */
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
ext/zstd/lib/legacy/zstd_v06.c view on Meta::CPAN
dctx->base = dst;
dctx->previousDstEnd = dst;
}
}
static size_t ZSTDv06_decompressBlock_internal(ZSTDv06_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src;
if (srcSize >= ZSTDv06_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
/* Decode literals sub-block */
{ size_t const litCSize = ZSTDv06_decodeLiteralsBlock(dctx, src, srcSize);
if (ZSTDv06_isError(litCSize)) return litCSize;
ip += litCSize;
srcSize -= litCSize;
}
return ZSTDv06_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
}
size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
ZSTDv06_checkContinuity(dctx, dst);
return ZSTDv06_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
}
/*! ZSTDv06_decompressFrame() :
* `dctx` must be properly initialized */
static size_t ZSTDv06_decompressFrame(ZSTDv06_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* const iend = ip + srcSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + dstCapacity;
size_t remainingSize = srcSize;
blockProperties_t blockProperties = { bt_compressed, 0 };
/* check */
if (srcSize < ZSTDv06_frameHeaderSize_min+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
/* Frame Header */
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize;
if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
if (ZSTDv06_decodeFrameHeader(dctx, src, frameHeaderSize)) return ERROR(corruption_detected);
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
}
/* Loop on each block */
while (1) {
size_t decodedSize=0;
size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, iend-ip, &blockProperties);
if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
ip += ZSTDv06_blockHeaderSize;
remainingSize -= ZSTDv06_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
switch(blockProperties.blockType)
{
case bt_compressed:
decodedSize = ZSTDv06_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize);
break;
case bt_raw :
decodedSize = ZSTDv06_copyRawBlock(op, oend-op, ip, cBlockSize);
break;
case bt_rle :
return ERROR(GENERIC); /* not yet supported */
break;
case bt_end :
/* end of frame */
if (remainingSize) return ERROR(srcSize_wrong);
break;
default:
return ERROR(GENERIC); /* impossible */
}
if (cBlockSize == 0) break; /* bt_end */
if (ZSTDv06_isError(decodedSize)) return decodedSize;
op += decodedSize;
ip += cBlockSize;
remainingSize -= cBlockSize;
}
return op-ostart;
}
size_t ZSTDv06_decompress_usingPreparedDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* refDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
ZSTDv06_copyDCtx(dctx, refDCtx);
ZSTDv06_checkContinuity(dctx, dst);
return ZSTDv06_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
}
size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize)
{
ZSTDv06_decompressBegin_usingDict(dctx, dict, dictSize);
ZSTDv06_checkContinuity(dctx, dst);
return ZSTDv06_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
}
size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
return ZSTDv06_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
}
size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
#if defined(ZSTDv06_HEAPMODE) && (ZSTDv06_HEAPMODE==1)
size_t regenSize;
ZSTDv06_DCtx* dctx = ZSTDv06_createDCtx();
if (dctx==NULL) return ERROR(memory_allocation);
regenSize = ZSTDv06_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
ZSTDv06_freeDCtx(dctx);
return regenSize;
#else /* stack mode */
ZSTDv06_DCtx dctx;
return ZSTDv06_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
#endif
}
/* ZSTD_errorFrameSizeInfoLegacy() :
assumes `cSize` and `dBound` are _not_ NULL */
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
{
*cSize = ret;
*dBound = ZSTD_CONTENTSIZE_ERROR;
}
void ZSTDv06_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
{
ext/zstd/lib/legacy/zstd_v06.c view on Meta::CPAN
size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize)
{
{ size_t const errorCode = ZSTDv06_decompressBegin(dctx);
if (ZSTDv06_isError(errorCode)) return errorCode; }
if (dict && dictSize) {
size_t const errorCode = ZSTDv06_decompress_insertDictionary(dctx, dict, dictSize);
if (ZSTDv06_isError(errorCode)) return ERROR(dictionary_corrupted);
}
return 0;
}
/*
Buffered version of Zstd compression library
Copyright (C) 2015-2016, Yann Collet.
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd homepage : https://facebook.github.io/zstd/
*/
/*-***************************************************************************
* Streaming decompression howto
*
* A ZBUFFv06_DCtx object is required to track streaming operations.
* Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources.
* Use ZBUFFv06_decompressInit() to start a new decompression operation,
* or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary.
* Note that ZBUFFv06_DCtx objects can be re-init multiple times.
*
* Use ZBUFFv06_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
* or 0 when a frame is completely decoded,
* or an error code, which can be tested using ZBUFFv06_isError().
*
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize()
* output : ZBUFFv06_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
* input : ZBUFFv06_recommendedDInSize == 128KB + 3;
* just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFFv06_dStage;
/* *** Resource management *** */
struct ZBUFFv06_DCtx_s {
ZSTDv06_DCtx* zd;
ZSTDv06_frameParams fParams;
ZBUFFv06_dStage stage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t blockSize;
BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
size_t lhSize;
}; /* typedef'd to ZBUFFv06_DCtx within "zstd_buffered.h" */
ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void)
{
ZBUFFv06_DCtx* zbd = (ZBUFFv06_DCtx*)malloc(sizeof(ZBUFFv06_DCtx));
if (zbd==NULL) return NULL;
memset(zbd, 0, sizeof(*zbd));
zbd->zd = ZSTDv06_createDCtx();
zbd->stage = ZBUFFds_init;
return zbd;
}
size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* zbd)
{
if (zbd==NULL) return 0; /* support free on null */
ZSTDv06_freeDCtx(zbd->zd);
free(zbd->inBuff);
free(zbd->outBuff);
free(zbd);
return 0;
}
/* *** Initialization *** */
size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* zbd, const void* dict, size_t dictSize)
{
zbd->stage = ZBUFFds_loadHeader;
zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
return ZSTDv06_decompressBegin_usingDict(zbd->zd, dict, dictSize);
}
size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
{
return ZBUFFv06_decompressInitDictionary(zbd, NULL, 0);
}
ext/zstd/lib/legacy/zstd_v06.c view on Meta::CPAN
U32 notDone = 1;
while (notDone) {
switch(zbd->stage)
{
case ZBUFFds_init :
return ERROR(init_missing);
case ZBUFFds_loadHeader :
{ size_t const hSize = ZSTDv06_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
if (hSize != 0) {
size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
if (ZSTDv06_isError(hSize)) return hSize;
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
if (ip != NULL)
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
zbd->lhSize += iend-ip;
*dstCapacityPtr = 0;
return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
}
memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
break;
} }
/* Consume header */
{ size_t const h1Size = ZSTDv06_nextSrcSizeToDecompress(zbd->zd); /* == ZSTDv06_frameHeaderSize_min */
size_t const h1Result = ZSTDv06_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
if (ZSTDv06_isError(h1Result)) return h1Result;
if (h1Size < zbd->lhSize) { /* long header */
size_t const h2Size = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
size_t const h2Result = ZSTDv06_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
if (ZSTDv06_isError(h2Result)) return h2Result;
} }
/* Frame header instruct buffer sizes */
{ size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTDv06_BLOCKSIZE_MAX);
zbd->blockSize = blockSize;
if (zbd->inBuffSize < blockSize) {
free(zbd->inBuff);
zbd->inBuffSize = blockSize;
zbd->inBuff = (char*)malloc(blockSize);
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
}
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize + WILDCOPY_OVERLENGTH * 2;
if (zbd->outBuffSize < neededOutSize) {
free(zbd->outBuff);
zbd->outBuffSize = neededOutSize;
zbd->outBuff = (char*)malloc(neededOutSize);
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
} } }
zbd->stage = ZBUFFds_read;
/* fall-through */
case ZBUFFds_read:
{ size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
if (neededInSize==0) { /* end of frame */
zbd->stage = ZBUFFds_init;
notDone = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
size_t const decodedSize = ZSTDv06_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
ip, neededInSize);
if (ZSTDv06_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize) break; /* this was just a header */
zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush;
break;
}
if (ip==iend) { notDone = 0; break; } /* no more input */
zbd->stage = ZBUFFds_load;
}
/* fall-through */
case ZBUFFds_load:
{ size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFFv06_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
ip += loadedSize;
zbd->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ size_t const decodedSize = ZSTDv06_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
zbd->inBuff, neededInSize);
if (ZSTDv06_isError(decodedSize)) return decodedSize;
zbd->inPos = 0; /* input is consumed */
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush;
/* break; */ /* ZBUFFds_flush follows */
}
}
/* fall-through */
case ZBUFFds_flush:
{ size_t const toFlushSize = zbd->outEnd - zbd->outStart;
size_t const flushedSize = ZBUFFv06_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
op += flushedSize;
zbd->outStart += flushedSize;
if (flushedSize == toFlushSize) {
zbd->stage = ZBUFFds_read;
if (zbd->outStart + zbd->blockSize > zbd->outBuffSize)
zbd->outStart = zbd->outEnd = 0;
break;
}
/* cannot flush everything */
notDone = 0;
break;
}
default: return ERROR(GENERIC); /* impossible */
} }
/* result */
*srcSizePtr = ip-istart;
*dstCapacityPtr = op-ostart;
{ size_t nextSrcSizeHint = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
if (nextSrcSizeHint > ZSTDv06_blockHeaderSize) nextSrcSizeHint+= ZSTDv06_blockHeaderSize; /* get following block header too */
nextSrcSizeHint -= zbd->inPos; /* already loaded*/
return nextSrcSizeHint;
}
}
/* *************************************
* Tool functions
***************************************/
size_t ZBUFFv06_recommendedDInSize(void) { return ZSTDv06_BLOCKSIZE_MAX + ZSTDv06_blockHeaderSize /* block header size*/ ; }
size_t ZBUFFv06_recommendedDOutSize(void) { return ZSTDv06_BLOCKSIZE_MAX; }
( run in 0.927 second using v1.01-cache-2.11-cpan-39bf76dae61 )