Compress-Stream-Zstd
view release on metacpan or search on metacpan
ext/zstd/lib/legacy/zstd_v05.c view on Meta::CPAN
* Prefer static linking if you need them, to control breaking version changes issues.
*/
#if defined (__cplusplus)
extern "C" {
#endif
/*-*************************************
* Types
***************************************/
#define ZSTDv05_WINDOWLOG_ABSOLUTEMIN 11
/*-*************************************
* Advanced functions
***************************************/
/*- Advanced Decompression functions -*/
/*! ZSTDv05_decompress_usingPreparedDCtx() :
* Same as ZSTDv05_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTDv05_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the decompression operation */
size_t ZSTDv05_decompress_usingPreparedDCtx(
ZSTDv05_DCtx* dctx, const ZSTDv05_DCtx* preparedDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/* **************************************
* Streaming functions (direct mode)
****************************************/
size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx);
/*
Streaming decompression, direct mode (bufferless)
A ZSTDv05_DCtx object is required to track streaming operations.
Use ZSTDv05_createDCtx() / ZSTDv05_freeDCtx() to manage it.
A ZSTDv05_DCtx object can be re-used multiple times.
First typical operation is to retrieve frame parameters, using ZSTDv05_getFrameParams().
This operation is independent, and just needs enough input data to properly decode the frame header.
Objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding.
Result : 0 when successful, it means the ZSTDv05_parameters structure has been filled.
>0 : means there is not enough data into src. Provides the expected size to successfully decode header.
errorCode, which can be tested using ZSTDv05_isError()
Start decompression, with ZSTDv05_decompressBegin() or ZSTDv05_decompressBegin_usingDict()
Alternatively, you can copy a prepared context, using ZSTDv05_copyDCtx()
Then use ZSTDv05_nextSrcSizeToDecompress() and ZSTDv05_decompressContinue() alternatively.
ZSTDv05_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTDv05_decompressContinue().
ZSTDv05_decompressContinue() requires this exact amount of bytes, or it will fail.
ZSTDv05_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
@result of ZSTDv05_decompressContinue() is the number of bytes regenerated within 'dst'.
It can be zero, which is not an error; it just means ZSTDv05_decompressContinue() has decoded some header.
A frame is fully decoded when ZSTDv05_nextSrcSizeToDecompress() returns zero.
Context can then be reset to start a new decompression.
*/
/* **************************************
* Block functions
****************************************/
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
User will have to take in charge required information to regenerate data, such as block sizes.
A few rules to respect :
- Uncompressed block size must be <= 128 KB
- Compressing or decompressing requires a context structure
+ Use ZSTDv05_createCCtx() and ZSTDv05_createDCtx()
- It is necessary to init context before starting
+ compression : ZSTDv05_compressBegin()
+ decompression : ZSTDv05_decompressBegin()
+ variants _usingDict() are also allowed
+ copyCCtx() and copyDCtx() work too
- When a block is considered not compressible enough, ZSTDv05_compressBlock() result will be zero.
In which case, nothing is produced into `dst`.
+ User must test for such outcome and deal directly with uncompressed data
+ ZSTDv05_decompressBlock() doesn't accept uncompressed data as input !!
*/
size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDv05_STATIC_H */
/*
zstd_internal - common functions to include
Header File for include
Copyright (C) 2014-2016, Yann Collet.
BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
ext/zstd/lib/legacy/zstd_v05.c view on Meta::CPAN
# error "HUFv05_MAX_TABLELOG is too large !"
#endif
/* **************************************************************
* Error Management
****************************************************************/
unsigned HUFv05_isError(size_t code) { return ERR_isError(code); }
const char* HUFv05_getErrorName(size_t code) { return ERR_getErrorName(code); }
#define HUFv05_STATIC_ASSERT(c) { enum { HUFv05_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/* *******************************************************
* Huff0 : Huffman block decompression
*********************************************************/
typedef struct { BYTE byte; BYTE nbBits; } HUFv05_DEltX2; /* single-symbol decoding */
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUFv05_DEltX4; /* double-symbols decoding */
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
/*! HUFv05_readStats
Read compact Huffman tree, saved by HUFv05_writeCTable
@huffWeight : destination buffer
@return : size read from `src`
*/
static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize)
{
U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src;
size_t iSize;
size_t oSize;
U32 n;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
/* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
if (iSize >= (242)) { /* RLE */
static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
oSize = l[iSize-242];
memset(huffWeight, 1, hwSize);
iSize = 0;
}
else { /* Incompressible */
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
} } }
else { /* header compressed with FSEv05 (normal case) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSEv05_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
if (FSEv05_isError(oSize)) return oSize;
}
/* collect weight stats */
memset(rankStats, 0, (HUFv05_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32));
weightTotal = 0;
for (n=0; n<oSize; n++) {
if (huffWeight[n] >= HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
}
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BITv05_highbit32(weightTotal) + 1;
if (tableLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
{ /* determine last weight */
U32 total = 1 << tableLog;
U32 rest = total - weightTotal;
U32 verif = 1 << BITv05_highbit32(rest);
U32 lastWeight = BITv05_highbit32(rest) + 1;
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
huffWeight[oSize] = (BYTE)lastWeight;
rankStats[lastWeight]++;
}
/* check tree construction validity */
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
/* results */
*nbSymbolsPtr = (U32)(oSize+1);
*tableLogPtr = tableLog;
return iSize+1;
}
/*-***************************/
/* single-symbol decoding */
/*-***************************/
size_t HUFv05_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
{
BYTE huffWeight[HUFv05_MAX_SYMBOL_VALUE + 1];
U32 rankVal[HUFv05_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
size_t iSize;
U32 nbSymbols = 0;
U32 n;
U32 nextRankStart;
void* const dtPtr = DTable + 1;
HUFv05_DEltX2* const dt = (HUFv05_DEltX2*)dtPtr;
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUFv05_readStats(huffWeight, HUFv05_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
if (HUFv05_isError(iSize)) return iSize;
/* check result */
if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */
ext/zstd/lib/legacy/zstd_v05.c view on Meta::CPAN
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BITv05_initDStream(&bitD1, istart1, length1);
if (HUFv05_isError(errorCode)) return errorCode;
errorCode = BITv05_initDStream(&bitD2, istart2, length2);
if (HUFv05_isError(errorCode)) return errorCode;
errorCode = BITv05_initDStream(&bitD3, istart3, length3);
if (HUFv05_isError(errorCode)) return errorCode;
errorCode = BITv05_initDStream(&bitD4, istart4, length4);
if (HUFv05_isError(errorCode)) return errorCode;
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
for ( ; (endSignal==BITv05_DStream_unfinished) && (op4<(oend-7)) ; ) {
HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
HUFv05_DECODE_SYMBOLX2_1(op1, &bitD1);
HUFv05_DECODE_SYMBOLX2_1(op2, &bitD2);
HUFv05_DECODE_SYMBOLX2_1(op3, &bitD3);
HUFv05_DECODE_SYMBOLX2_1(op4, &bitD4);
HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
HUFv05_DECODE_SYMBOLX2_0(op1, &bitD1);
HUFv05_DECODE_SYMBOLX2_0(op2, &bitD2);
HUFv05_DECODE_SYMBOLX2_0(op3, &bitD3);
HUFv05_DECODE_SYMBOLX2_0(op4, &bitD4);
endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUFv05_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
HUFv05_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
HUFv05_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
HUFv05_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
/* check */
endSignal = BITv05_endOfDStream(&bitD1) & BITv05_endOfDStream(&bitD2) & BITv05_endOfDStream(&bitD3) & BITv05_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
}
size_t HUFv05_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUFv05_CREATE_STATIC_DTABLEX2(DTable, HUFv05_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t errorCode;
errorCode = HUFv05_readDTableX2 (DTable, cSrc, cSrcSize);
if (HUFv05_isError(errorCode)) return errorCode;
if (errorCode >= cSrcSize) return ERROR(srcSize_wrong);
ip += errorCode;
cSrcSize -= errorCode;
return HUFv05_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
/* *************************/
/* double-symbols decoding */
/* *************************/
static void HUFv05_fillDTableX4Level2(HUFv05_DEltX4* DTable, U32 sizeLog, const U32 consumed,
const U32* rankValOrigin, const int minWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
U32 nbBitsBaseline, U16 baseSeq)
{
HUFv05_DEltX4 DElt;
U32 rankVal[HUFv05_ABSOLUTEMAX_TABLELOG + 1];
U32 s;
/* get pre-calculated rankVal */
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill skipped values */
if (minWeight>1) {
U32 i, skipSize = rankVal[minWeight];
MEM_writeLE16(&(DElt.sequence), baseSeq);
DElt.nbBits = (BYTE)(consumed);
DElt.length = 1;
for (i = 0; i < skipSize; i++)
DTable[i] = DElt;
}
/* fill DTable */
for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */
const U32 symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight;
const U32 nbBits = nbBitsBaseline - weight;
const U32 length = 1 << (sizeLog-nbBits);
const U32 start = rankVal[weight];
U32 i = start;
const U32 end = start + length;
MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
DElt.nbBits = (BYTE)(nbBits + consumed);
DElt.length = 2;
ext/zstd/lib/legacy/zstd_v05.c view on Meta::CPAN
#define HUFv05_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUFv05_MAX_TABLELOG<=12)) \
ptr += HUFv05_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUFv05_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
ptr += HUFv05_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, BYTE* const pEnd, const HUFv05_DEltX4* const dt, const U32 dtLog)
{
BYTE* const pStart = p;
/* up to 8 symbols at a time */
while ((BITv05_reloadDStream(bitDPtr) == BITv05_DStream_unfinished) && (p < pEnd-7)) {
HUFv05_DECODE_SYMBOLX4_2(p, bitDPtr);
HUFv05_DECODE_SYMBOLX4_1(p, bitDPtr);
HUFv05_DECODE_SYMBOLX4_2(p, bitDPtr);
HUFv05_DECODE_SYMBOLX4_0(p, bitDPtr);
}
/* closer to the end */
while ((BITv05_reloadDStream(bitDPtr) == BITv05_DStream_unfinished) && (p <= pEnd-2))
HUFv05_DECODE_SYMBOLX4_0(p, bitDPtr);
while (p <= pEnd-2)
HUFv05_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
if (p < pEnd)
p += HUFv05_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
return p-pStart;
}
size_t HUFv05_decompress1X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const unsigned* DTable)
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const void* const dtPtr = DTable;
const HUFv05_DEltX4* const dt = ((const HUFv05_DEltX4*)dtPtr) +1;
size_t errorCode;
/* Init */
BITv05_DStream_t bitD;
errorCode = BITv05_initDStream(&bitD, istart, cSrcSize);
if (HUFv05_isError(errorCode)) return errorCode;
/* finish bitStreams one by one */
HUFv05_decodeStreamX4(ostart, &bitD, oend, dt, dtLog);
/* check */
if (!BITv05_endOfDStream(&bitD)) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUFv05_CREATE_STATIC_DTABLEX4(DTable, HUFv05_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUFv05_readDTableX4 (DTable, cSrc, cSrcSize);
if (HUFv05_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUFv05_decompress1X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
size_t HUFv05_decompress4X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const unsigned* DTable)
{
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable;
const HUFv05_DEltX4* const dt = ((const HUFv05_DEltX4*)dtPtr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
/* Init */
BITv05_DStream_t bitD1;
BITv05_DStream_t bitD2;
BITv05_DStream_t bitD3;
BITv05_DStream_t bitD4;
const size_t length1 = MEM_readLE16(istart);
const size_t length2 = MEM_readLE16(istart+2);
const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BITv05_initDStream(&bitD1, istart1, length1);
if (HUFv05_isError(errorCode)) return errorCode;
errorCode = BITv05_initDStream(&bitD2, istart2, length2);
if (HUFv05_isError(errorCode)) return errorCode;
errorCode = BITv05_initDStream(&bitD3, istart3, length3);
if (HUFv05_isError(errorCode)) return errorCode;
errorCode = BITv05_initDStream(&bitD4, istart4, length4);
if (HUFv05_isError(errorCode)) return errorCode;
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
for ( ; (endSignal==BITv05_DStream_unfinished) && (op4<(oend-7)) ; ) {
HUFv05_DECODE_SYMBOLX4_2(op1, &bitD1);
HUFv05_DECODE_SYMBOLX4_2(op2, &bitD2);
HUFv05_DECODE_SYMBOLX4_2(op3, &bitD3);
HUFv05_DECODE_SYMBOLX4_2(op4, &bitD4);
HUFv05_DECODE_SYMBOLX4_1(op1, &bitD1);
HUFv05_DECODE_SYMBOLX4_1(op2, &bitD2);
HUFv05_DECODE_SYMBOLX4_1(op3, &bitD3);
HUFv05_DECODE_SYMBOLX4_1(op4, &bitD4);
HUFv05_DECODE_SYMBOLX4_2(op1, &bitD1);
HUFv05_DECODE_SYMBOLX4_2(op2, &bitD2);
HUFv05_DECODE_SYMBOLX4_2(op3, &bitD3);
HUFv05_DECODE_SYMBOLX4_2(op4, &bitD4);
HUFv05_DECODE_SYMBOLX4_0(op1, &bitD1);
HUFv05_DECODE_SYMBOLX4_0(op2, &bitD2);
HUFv05_DECODE_SYMBOLX4_0(op3, &bitD3);
HUFv05_DECODE_SYMBOLX4_0(op4, &bitD4);
endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUFv05_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
HUFv05_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
HUFv05_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
HUFv05_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
/* check */
endSignal = BITv05_endOfDStream(&bitD1) & BITv05_endOfDStream(&bitD2) & BITv05_endOfDStream(&bitD3) & BITv05_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
}
size_t HUFv05_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUFv05_CREATE_STATIC_DTABLEX4(DTable, HUFv05_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUFv05_readDTableX4 (DTable, cSrc, cSrcSize);
if (HUFv05_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUFv05_decompress4X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
/* ********************************/
/* Generic decompression selector */
/* ********************************/
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
{
/* single, double, quad */
{{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */
{{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */
{{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */
{{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */
{{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */
{{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */
{{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */
{{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */
{{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */
{{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */
{{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */
{{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */
{{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */
{{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */
{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
};
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
static const decompressionAlgo decompress[3] = { HUFv05_decompress4X2, HUFv05_decompress4X4, NULL };
/* estimate decompression time */
U32 Q;
const U32 D256 = (U32)(dstSize >> 8);
U32 Dtime[3];
U32 algoNb = 0;
int n;
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
ext/zstd/lib/legacy/zstd_v05.c view on Meta::CPAN
}
static size_t ZSTDv05_decompressBlock_internal(ZSTDv05_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{ /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src;
size_t litCSize;
if (srcSize >= BLOCKSIZE) return ERROR(srcSize_wrong);
/* Decode literals sub-block */
litCSize = ZSTDv05_decodeLiteralsBlock(dctx, src, srcSize);
if (ZSTDv05_isError(litCSize)) return litCSize;
ip += litCSize;
srcSize -= litCSize;
return ZSTDv05_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
}
size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
ZSTDv05_checkContinuity(dctx, dst);
return ZSTDv05_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
}
/*! ZSTDv05_decompress_continueDCtx
* dctx must have been properly initialized */
static size_t ZSTDv05_decompress_continueDCtx(ZSTDv05_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* iend = ip + srcSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + maxDstSize;
size_t remainingSize = srcSize;
blockProperties_t blockProperties;
memset(&blockProperties, 0, sizeof(blockProperties));
/* Frame Header */
{ size_t frameHeaderSize;
if (srcSize < ZSTDv05_frameHeaderSize_min+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
frameHeaderSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;
if (srcSize < frameHeaderSize+ZSTDv05_blockHeaderSize) return ERROR(srcSize_wrong);
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
frameHeaderSize = ZSTDv05_decodeFrameHeader_Part2(dctx, src, frameHeaderSize);
if (ZSTDv05_isError(frameHeaderSize)) return frameHeaderSize;
}
/* Loop on each block */
while (1)
{
size_t decodedSize=0;
size_t cBlockSize = ZSTDv05_getcBlockSize(ip, iend-ip, &blockProperties);
if (ZSTDv05_isError(cBlockSize)) return cBlockSize;
ip += ZSTDv05_blockHeaderSize;
remainingSize -= ZSTDv05_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
switch(blockProperties.blockType)
{
case bt_compressed:
decodedSize = ZSTDv05_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize);
break;
case bt_raw :
decodedSize = ZSTDv05_copyRawBlock(op, oend-op, ip, cBlockSize);
break;
case bt_rle :
return ERROR(GENERIC); /* not yet supported */
break;
case bt_end :
/* end of frame */
if (remainingSize) return ERROR(srcSize_wrong);
break;
default:
return ERROR(GENERIC); /* impossible */
}
if (cBlockSize == 0) break; /* bt_end */
if (ZSTDv05_isError(decodedSize)) return decodedSize;
op += decodedSize;
ip += cBlockSize;
remainingSize -= cBlockSize;
}
return op-ostart;
}
size_t ZSTDv05_decompress_usingPreparedDCtx(ZSTDv05_DCtx* dctx, const ZSTDv05_DCtx* refDCtx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
ZSTDv05_copyDCtx(dctx, refDCtx);
ZSTDv05_checkContinuity(dctx, dst);
return ZSTDv05_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize);
}
size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
const void* dict, size_t dictSize)
{
ZSTDv05_decompressBegin_usingDict(dctx, dict, dictSize);
ZSTDv05_checkContinuity(dctx, dst);
return ZSTDv05_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize);
}
size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return ZSTDv05_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0);
}
size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
#if defined(ZSTDv05_HEAPMODE) && (ZSTDv05_HEAPMODE==1)
size_t regenSize;
ZSTDv05_DCtx* dctx = ZSTDv05_createDCtx();
if (dctx==NULL) return ERROR(memory_allocation);
regenSize = ZSTDv05_decompressDCtx(dctx, dst, maxDstSize, src, srcSize);
ZSTDv05_freeDCtx(dctx);
return regenSize;
#else
ZSTDv05_DCtx dctx;
return ZSTDv05_decompressDCtx(&dctx, dst, maxDstSize, src, srcSize);
#endif
}
/* ZSTD_errorFrameSizeInfoLegacy() :
assumes `cSize` and `dBound` are _not_ NULL */
static void ZSTD_errorFrameSizeInfoLegacy(size_t* cSize, unsigned long long* dBound, size_t ret)
{
*cSize = ret;
*dBound = ZSTD_CONTENTSIZE_ERROR;
}
void ZSTDv05_findFrameSizeInfoLegacy(const void *src, size_t srcSize, size_t* cSize, unsigned long long* dBound)
{
const BYTE* ip = (const BYTE*)src;
ext/zstd/lib/legacy/zstd_v05.c view on Meta::CPAN
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/* The objects defined into this file should be considered experimental.
* They are not labelled stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
*/
/* *************************************
* Constants
***************************************/
static size_t ZBUFFv05_blockHeaderSize = 3;
/* *** Compression *** */
static size_t ZBUFFv05_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
size_t length = MIN(maxDstSize, srcSize);
if (length > 0) {
memcpy(dst, src, length);
}
return length;
}
/** ************************************************
* Streaming decompression
*
* A ZBUFFv05_DCtx object is required to track streaming operation.
* Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources.
* Use ZBUFFv05_decompressInit() to start a new decompression operation.
* ZBUFFv05_DCtx objects can be reused multiple times.
*
* Use ZBUFFv05_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *maxDstSizePtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
* return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
* or 0 when a frame is completely decoded
* or an error code, which can be tested using ZBUFFv05_isError().
*
* Hint : recommended buffer sizes (not compulsory)
* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
* input : just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* **************************************************/
typedef enum { ZBUFFv05ds_init, ZBUFFv05ds_readHeader, ZBUFFv05ds_loadHeader, ZBUFFv05ds_decodeHeader,
ZBUFFv05ds_read, ZBUFFv05ds_load, ZBUFFv05ds_flush } ZBUFFv05_dStage;
/* *** Resource management *** */
#define ZSTDv05_frameHeaderSize_max 5 /* too magical, should come from reference */
struct ZBUFFv05_DCtx_s {
ZSTDv05_DCtx* zc;
ZSTDv05_parameters params;
char* inBuff;
size_t inBuffSize;
size_t inPos;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t hPos;
ZBUFFv05_dStage stage;
unsigned char headerBuffer[ZSTDv05_frameHeaderSize_max];
}; /* typedef'd to ZBUFFv05_DCtx within "zstd_buffered.h" */
ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void)
{
ZBUFFv05_DCtx* zbc = (ZBUFFv05_DCtx*)malloc(sizeof(ZBUFFv05_DCtx));
if (zbc==NULL) return NULL;
memset(zbc, 0, sizeof(*zbc));
zbc->zc = ZSTDv05_createDCtx();
zbc->stage = ZBUFFv05ds_init;
return zbc;
}
size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* zbc)
{
if (zbc==NULL) return 0; /* support free on null */
ZSTDv05_freeDCtx(zbc->zc);
free(zbc->inBuff);
free(zbc->outBuff);
free(zbc);
return 0;
}
/* *** Initialization *** */
size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* zbc, const void* dict, size_t dictSize)
{
zbc->stage = ZBUFFv05ds_readHeader;
zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0;
return ZSTDv05_decompressBegin_usingDict(zbc->zc, dict, dictSize);
}
size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* zbc)
{
return ZBUFFv05_decompressInitDictionary(zbc, NULL, 0);
}
ext/zstd/lib/legacy/zstd_v05.c view on Meta::CPAN
}
zbc->stage = ZBUFFv05ds_decodeHeader;
break;
}
/* fall-through */
case ZBUFFv05ds_loadHeader:
/* complete header from src */
{
size_t headerSize = ZBUFFv05_limitCopy(
zbc->headerBuffer + zbc->hPos, ZSTDv05_frameHeaderSize_max - zbc->hPos,
src, *srcSizePtr);
zbc->hPos += headerSize;
ip += headerSize;
headerSize = ZSTDv05_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos);
if (ZSTDv05_isError(headerSize)) return headerSize;
if (headerSize) {
/* not enough input to decode header : tell how many bytes would be necessary */
*maxDstSizePtr = 0;
return headerSize - zbc->hPos;
}
/* zbc->stage = ZBUFFv05ds_decodeHeader; break; */ /* useless : stage follows */
}
/* fall-through */
case ZBUFFv05ds_decodeHeader:
/* apply header to create / resize buffers */
{
size_t neededOutSize = (size_t)1 << zbc->params.windowLog;
size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
if (zbc->inBuffSize < neededInSize) {
free(zbc->inBuff);
zbc->inBuffSize = neededInSize;
zbc->inBuff = (char*)malloc(neededInSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
}
if (zbc->outBuffSize < neededOutSize) {
free(zbc->outBuff);
zbc->outBuffSize = neededOutSize;
zbc->outBuff = (char*)malloc(neededOutSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
} }
if (zbc->hPos) {
/* some data already loaded into headerBuffer : transfer into inBuff */
memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos);
zbc->inPos = zbc->hPos;
zbc->hPos = 0;
zbc->stage = ZBUFFv05ds_load;
break;
}
zbc->stage = ZBUFFv05ds_read;
/* fall-through */
case ZBUFFv05ds_read:
{
size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
if (neededInSize==0) { /* end of frame */
zbc->stage = ZBUFFv05ds_init;
notDone = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) {
/* directly decode from src */
size_t decodedSize = ZSTDv05_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
ip, neededInSize);
if (ZSTDv05_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize) break; /* this was just a header */
zbc->outEnd = zbc->outStart + decodedSize;
zbc->stage = ZBUFFv05ds_flush;
break;
}
if (ip==iend) { notDone = 0; break; } /* no more input */
zbc->stage = ZBUFFv05ds_load;
}
/* fall-through */
case ZBUFFv05ds_load:
{
size_t neededInSize = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
size_t toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFFv05_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip);
ip += loadedSize;
zbc->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
{
size_t decodedSize = ZSTDv05_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
zbc->inBuff, neededInSize);
if (ZSTDv05_isError(decodedSize)) return decodedSize;
zbc->inPos = 0; /* input is consumed */
if (!decodedSize) { zbc->stage = ZBUFFv05ds_read; break; } /* this was just a header */
zbc->outEnd = zbc->outStart + decodedSize;
zbc->stage = ZBUFFv05ds_flush;
/* break; */ /* ZBUFFv05ds_flush follows */
}
}
/* fall-through */
case ZBUFFv05ds_flush:
{
size_t toFlushSize = zbc->outEnd - zbc->outStart;
size_t flushedSize = ZBUFFv05_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
op += flushedSize;
zbc->outStart += flushedSize;
if (flushedSize == toFlushSize) {
zbc->stage = ZBUFFv05ds_read;
if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize)
zbc->outStart = zbc->outEnd = 0;
break;
}
/* cannot flush everything */
notDone = 0;
break;
}
default: return ERROR(GENERIC); /* impossible */
} }
*srcSizePtr = ip-istart;
*maxDstSizePtr = op-ostart;
{ size_t nextSrcSizeHint = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
if (nextSrcSizeHint > ZBUFFv05_blockHeaderSize) nextSrcSizeHint+= ZBUFFv05_blockHeaderSize; /* get next block header too */
nextSrcSizeHint -= zbc->inPos; /* already loaded*/
return nextSrcSizeHint;
}
}
/* *************************************
* Tool functions
***************************************/
unsigned ZBUFFv05_isError(size_t errorCode) { return ERR_isError(errorCode); }
const char* ZBUFFv05_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
size_t ZBUFFv05_recommendedDInSize(void) { return BLOCKSIZE + ZBUFFv05_blockHeaderSize /* block header size*/ ; }
size_t ZBUFFv05_recommendedDOutSize(void) { return BLOCKSIZE; }
( run in 0.790 second using v1.01-cache-2.11-cpan-39bf76dae61 )