Compress-Zstd
view release on metacpan or search on metacpan
ext/zstd/CONTRIBUTING.md view on Meta::CPAN
# Contributing to Zstandard
We want to make contributing to this project as easy and transparent as
possible.
## Our Development Process
New versions are being developed in the "dev" branch,
or in their own feature branch.
When they are deemed ready for a release, they are merged into "master".
As a consequences, all contributions must stage first through "dev"
or their own feature branch.
ext/zstd/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch view on Meta::CPAN
+ const char *const iend = (const char *)(input->src) + input->size;
+ const char *ip = istart;
+ char *const ostart = (char *)(output->dst) + output->pos;
+ char *const oend = (char *)(output->dst) + output->size;
+ char *op = ostart;
+ U32 someMoreWork = 1;
+
+ while (someMoreWork) {
+ switch (zds->stage) {
+ case zdss_init:
+ ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
+ /* fall-through */
+
+ case zdss_loadHeader: {
+ size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
+ if (ZSTD_isError(hSize))
+ return hSize;
+ if (hSize != 0) { /* need more input */
+ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
+ if (toLoad > (size_t)(iend - ip)) { /* not enough input to load full header */
+ memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip);
ext/zstd/contrib/linux-kernel/lib/zstd/decompress.c view on Meta::CPAN
const char *const iend = (const char *)(input->src) + input->size;
const char *ip = istart;
char *const ostart = (char *)(output->dst) + output->pos;
char *const oend = (char *)(output->dst) + output->size;
char *op = ostart;
U32 someMoreWork = 1;
while (someMoreWork) {
switch (zds->stage) {
case zdss_init:
ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
/* fall-through */
case zdss_loadHeader: {
size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
if (ZSTD_isError(hSize))
return hSize;
if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
if (toLoad > (size_t)(iend - ip)) { /* not enough input to load full header */
memcpy(zds->headerBuffer + zds->lhSize, ip, iend - ip);
ext/zstd/doc/zstd_compression_format.md view on Meta::CPAN
For broader compatibility, decoders are recommended to support
memory sizes of at least 8 MB.
This is only a recommendation,
each decoder is free to support higher or lower limits,
depending on local limitations.
__`Unused_bit`__
A decoder compliant with this specification version shall not interpret this bit.
It might be used in any future version,
to signal a property which is transparent to properly decode the frame.
An encoder compliant with this specification version must set this bit to zero.
__`Reserved_bit`__
This bit is reserved for some future feature.
Its value _must be zero_.
A decoder compliant with this specification version must ensure it is not set.
This bit may be used in a future revision,
to signal a feature that must be interpreted to decode the frame correctly.
ext/zstd/doc/zstd_manual.html view on Meta::CPAN
* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
* while compression work is performed in parallel, within worker threads.
* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
* More workers improve speed, but also increase memory usage.
* Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
ZSTD_c_jobSize=401, </b>/* Size of a compression job. This value is enforced only when nbWorkers >= 1.<b>
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
* The minimum size is automatically and transparently enforced */
ZSTD_c_overlapLog=402, </b>/* Control the overlap size, as a fraction of window size.<b>
* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
* It helps preserve compression ratio, while each job is compressed in parallel.
* This value is enforced only when nbWorkers >= 1.
* Larger values increase compression ratio, but decrease speed.
* Possible values range from 0 to 9 :
* - 0 means "default" : value will be determined by the library, depending on strategy
* - 1 means "no overlap"
* - 9 means "full overlap", using a full window size.
* Each intermediate rank increases/decreases load size by a factor 2 :
ext/zstd/lib/README.md view on Meta::CPAN
The API is exposed in `lib/dictBuilder/zdict.h`.
This module depends on both `lib/common` and `lib/compress` .
- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`.
This module depends on `lib/common` and `lib/decompress`.
To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation.
Specifying a number limits versions supported to that version onward.
For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`.
Decoding supported legacy format is a transparent capability triggered within decompression functions.
It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`.
Each version does also provide its own set of advanced API.
For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
- While invoking `make libzstd`, it's possible to define build macros
`ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the corresponding features.
This will also disable compilation of all dependencies
(eg. `ZSTD_LIB_COMPRESSION=0` will also disable dictBuilder).
ext/zstd/lib/compress/zstd_compress.c view on Meta::CPAN
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp)
{
DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp);
/* check conditions */
RETURN_ERROR_IF(output->pos > output->size, GENERIC);
RETURN_ERROR_IF(input->pos > input->size, GENERIC);
assert(cctx!=NULL);
/* transparent initialization stage */
if (cctx->streamStage == zcss_init) {
ZSTD_CCtx_params params = cctx->requestedParams;
ZSTD_prefixDict const prefixDict = cctx->prefixDict;
FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) ); /* Init the local dict if present. */
memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */
assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */
DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */
params.cParams = ZSTD_getCParamsFromCCtxParams(
&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
#ifdef ZSTD_MULTITHREAD
if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
}
if (params.nbWorkers > 0) {
ext/zstd/lib/compress/zstd_compress.c view on Meta::CPAN
cctx->appliedParams.nbWorkers = params.nbWorkers;
} else
#endif
{ FORWARD_IF_ERROR( ZSTD_resetCStream_internal(cctx,
prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
cctx->cdict,
params, cctx->pledgedSrcSizePlusOne-1) );
assert(cctx->streamStage == zcss_load);
assert(cctx->appliedParams.nbWorkers == 0);
} }
/* end of transparent initialization stage */
/* compression stage */
#ifdef ZSTD_MULTITHREAD
if (cctx->appliedParams.nbWorkers > 0) {
int const forceMaxProgress = (endOp == ZSTD_e_flush || endOp == ZSTD_e_end);
size_t flushMin;
assert(forceMaxProgress || endOp == ZSTD_e_continue /* Protection for a new flush type */);
if (cctx->cParamsChanged) {
ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams);
cctx->cParamsChanged = 0;
ext/zstd/lib/decompress/zstd_decompress.c view on Meta::CPAN
output->pos > output->size,
dstSize_tooSmall,
"forbidden. out: pos: %u vs size: %u",
(U32)output->pos, (U32)output->size);
DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
while (someMoreWork) {
switch(zds->streamStage)
{
case zdss_init :
DEBUGLOG(5, "stage zdss_init => transparent reset ");
zds->streamStage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
zds->legacyVersion = 0;
zds->hostageByte = 0;
/* fall-through */
case zdss_loadHeader :
DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
if (zds->legacyVersion) {
ext/zstd/lib/zstd.h view on Meta::CPAN
* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
* while compression work is performed in parallel, within worker threads.
* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
* More workers improve speed, but also increase memory usage.
* Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
* The minimum size is automatically and transparently enforced */
ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
* It helps preserve compression ratio, while each job is compressed in parallel.
* This value is enforced only when nbWorkers >= 1.
* Larger values increase compression ratio, but decrease speed.
* Possible values range from 0 to 9 :
* - 0 means "default" : value will be determined by the library, depending on strategy
* - 1 means "no overlap"
* - 9 means "full overlap", using a full window size.
* Each intermediate rank increases/decreases load size by a factor 2 :
ext/zstd/programs/zstd.1 view on Meta::CPAN
.IP
The default value is \fBwlog \- lhlog\fR\.
.
.SS "Example"
The following parameters sets advanced compression options to something similar to predefined level 19 for files bigger than 256 KB:
.
.P
\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
.
.SS "\-B#:"
Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custo...
.
.SH "BUGS"
Report bugs at: https://github\.com/facebook/zstd/issues
.
.SH "AUTHOR"
Yann Collet
ext/zstd/programs/zstd.1.md view on Meta::CPAN
The following parameters sets advanced compression options to something
similar to predefined level 19 for files bigger than 256 KB:
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
### -B#:
Select the size of each compression job.
This parameter is available only when multi-threading is enabled.
Default value is `4 * windowSize`, which means it varies depending on compression level.
`-B#` makes it possible to select a custom value.
Note that job size must respect a minimum value which is enforced transparently.
This minimum is either 1 MB, or `overlapSize`, whichever is largest.
BUGS
----
Report bugs at: https://github.com/facebook/zstd/issues
AUTHOR
------
Yann Collet
ext/zstd/tests/gzip/init.sh view on Meta::CPAN
export re_shell
exec "$re_shell_" $opts_ "$0" --no-reexec "$@"
echo "$ME_: exec failed" 1>&2
exit 127
fi
fi
# If this is bash, turn off all aliases.
test -n "$BASH_VERSION" && unalias -a
# Note that when supporting $EXEEXT (transparently mapping from PROG_NAME to
# PROG_NAME.exe), we want to support hyphen-containing names like test-acos.
# That is part of the shell-selection test above. Why use aliases rather
# than functions? Because support for hyphen-containing aliases is more
# widespread than that for hyphen-containing function names.
test -n "$EXEEXT" && shopt -s expand_aliases
# Enable glibc's malloc-perturbing option.
# This is useful for exposing code that depends on the fact that
# malloc-related functions often return memory that is mostly zeroed.
# If you have the time and cycles, use valgrind to do an even better job.
ext/zstd/zlibWrapper/gzguts.h view on Meta::CPAN
/* x.next: next output data to deliver or write */
/* x.pos: current position in uncompressed data */
/* used for both reading and writing */
int mode; /* see gzip modes above */
int fd; /* file descriptor */
char *path; /* path or fd for error messages */
unsigned size; /* buffer size, zero if not allocated yet */
unsigned want; /* requested buffer size, default is GZBUFSIZE */
unsigned char *in; /* input buffer (double-sized when writing) */
unsigned char *out; /* output buffer (double-sized when reading) */
int direct; /* 0 if processing gzip, 1 if transparent */
/* just for reading */
int how; /* 0: get header, 1: copy, 2: decompress */
z_off64_t start; /* where the gzip data started, for rewinding */
int eof; /* true if end of input file reached */
int past; /* true if read requested past end */
/* just for writing */
int level; /* compression level */
int strategy; /* compression strategy */
/* seek request */
z_off64_t skip; /* amount to skip (already rewound if backwards) */
ext/zstd/zlibWrapper/gzlib.c view on Meta::CPAN
}
mode++;
}
/* must provide an "r", "w", or "a" */
if (state.state->mode == GZ_NONE) {
free(state.state);
return NULL;
}
/* can't force transparent read */
if (state.state->mode == GZ_READ) {
if (state.state->direct) {
free(state.state);
return NULL;
}
state.state->direct = 1; /* for empty file */
}
/* save the path name for error messages */
#ifdef WIDECHAR
ext/zstd/zlibWrapper/gzread.c view on Meta::CPAN
/* get internal structure */
if (file == NULL)
return 0;
state = (gz_statep)file;
/* if the state is not known, but we can find out, then do so (this is
mainly for right after a gzopen() or gzdopen()) */
if (state.state->mode == GZ_READ && state.state->how == LOOK && state.state->x.have == 0)
(void)gz_look(state);
/* return 1 if transparent, 0 if processing a gzip stream */
return state.state->direct;
}
/* -- see zlib.h -- */
int ZEXPORT gzclose_r(file)
gzFile file;
{
int ret, err;
gz_statep state;
( run in 0.784 second using v1.01-cache-2.11-cpan-0a6323c29d9 )