Compress-Stream-Zstd
view release on metacpan or search on metacpan
ext/zstd/programs/fileio.c view on Meta::CPAN
}
void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value)
{
prefs->patchFromMode = value != 0;
}
void FIO_setContentSize(FIO_prefs_t* const prefs, int value)
{
prefs->contentSize = value != 0;
}
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
#ifdef ZSTD_MULTITHREAD
prefs->asyncIO = value;
#else
(void) prefs;
(void) value;
DISPLAYLEVEL(2, "Note : asyncio is disabled (lack of multithreading support) \n");
#endif
}
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
prefs->passThrough = (value != 0);
}
void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_paramSwitch_e value)
{
prefs->mmapDict = value;
}
/* FIO_ctx_t functions */
void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
fCtx->hasStdoutOutput = value;
}
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value)
{
fCtx->nbFilesTotal = value;
}
void FIO_determineHasStdinInput(FIO_ctx_t* const fCtx, const FileNamesTable* const filenames) {
size_t i = 0;
for ( ; i < filenames->tableSize; ++i) {
if (!strcmp(stdinmark, filenames->fileNames[i])) {
fCtx->hasStdinInput = 1;
return;
}
}
}
/*-*************************************
* Functions
***************************************/
/** FIO_removeFile() :
* @result : Unlink `fileName`, even if it's read-only */
static int FIO_removeFile(const char* path)
{
stat_t statbuf;
if (!UTIL_stat(path, &statbuf)) {
DISPLAYLEVEL(2, "zstd: Failed to stat %s while trying to remove it\n", path);
return 0;
}
if (!UTIL_isRegularFileStat(&statbuf)) {
DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path);
return 0;
}
#if defined(_WIN32) || defined(WIN32)
/* windows doesn't allow remove read-only files,
* so try to make it writable first */
if (!(statbuf.st_mode & _S_IWRITE)) {
UTIL_chmod(path, &statbuf, _S_IWRITE);
}
#endif
return remove(path);
}
/** FIO_openSrcFile() :
* condition : `srcFileName` must be non-NULL. `prefs` may be NULL.
* @result : FILE* to `srcFileName`, or NULL if it fails */
static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName, stat_t* statbuf)
{
int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0;
assert(srcFileName != NULL);
assert(statbuf != NULL);
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input \n");
SET_BINARY_MODE(stdin);
return stdin;
}
if (!UTIL_stat(srcFileName, statbuf)) {
DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n",
srcFileName, strerror(errno));
return NULL;
}
if (!UTIL_isRegularFileStat(statbuf)
&& !UTIL_isFIFOStat(statbuf)
&& !(allowBlockDevices && UTIL_isBlockDevStat(statbuf))
) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
return NULL;
}
{ FILE* const f = fopen(srcFileName, "rb");
if (f == NULL)
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return f;
}
}
/** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE*
FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
const char* srcFileName, const char* dstFileName,
const int mode)
{
int isDstRegFile;
if (prefs->testMode) return NULL; /* do not open file in test mode */
assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output \n");
SET_BINARY_MODE(stdout);
if (prefs->sparseFileSupport == 1) {
prefs->sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
}
return stdout;
}
/* ensure dst is not the same as src */
if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) {
DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n");
return NULL;
}
isDstRegFile = UTIL_isRegularFile(dstFileName); /* invoke once */
if (prefs->sparseFileSupport == 1) {
prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
if (!isDstRegFile) {
prefs->sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is disabled when output is not a file \n");
}
}
if (isDstRegFile) {
ext/zstd/programs/fileio.c view on Meta::CPAN
dstFileName);
return NULL;
}
DISPLAY("zstd: %s already exists; ", dstFileName);
if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput))
return NULL;
}
/* need to unlink */
FIO_removeFile(dstFileName);
}
{
#if defined(_WIN32)
/* Windows requires opening the file as a "binary" file to avoid
* mangling. This macro doesn't exist on unix. */
const int openflags = O_WRONLY|O_CREAT|O_TRUNC|O_BINARY;
const int fd = _open(dstFileName, openflags, mode);
FILE* f = NULL;
if (fd != -1) {
f = _fdopen(fd, "wb");
}
#else
const int openflags = O_WRONLY|O_CREAT|O_TRUNC;
const int fd = open(dstFileName, openflags, mode);
FILE* f = NULL;
if (fd != -1) {
f = fdopen(fd, "wb");
}
#endif
if (f == NULL) {
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
} else {
/* An increased buffer size can provide a significant performance
* boost on some platforms. Note that providing a NULL buf with a
* size that's not 0 is not defined in ANSI C, but is defined in an
* extension. There are three possibilities here:
* 1. Libc supports the extended version and everything is good.
* 2. Libc ignores the size when buf is NULL, in which case
* everything will continue as if we didn't call `setvbuf()`.
* 3. We fail the call and execution continues but a warning
* message might be shown.
* In all cases due execution continues. For now, I believe that
* this is a more cost-effective solution than managing the buffers
* allocations ourselves (will require an API change).
*/
if (setvbuf(f, NULL, _IOFBF, 1 MB)) {
DISPLAYLEVEL(2, "Warning: setvbuf failed for %s\n", dstFileName);
}
}
return f;
}
}
/* FIO_getDictFileStat() :
*/
static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) {
assert(dictFileStat != NULL);
if (fileName == NULL) return;
if (!UTIL_stat(fileName, dictFileStat)) {
EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno));
}
if (!UTIL_isRegularFileStat(dictFileStat)) {
EXM_THROW(32, "Dictionary %s must be a regular file.", fileName);
}
}
/* FIO_setDictBufferMalloc() :
* allocates a buffer, pointed by `dict->dictBuffer`,
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
FILE* fileHandle;
U64 fileSize;
void** bufferPtr = &dict->dictBuffer;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb");
if (fileHandle == NULL) {
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
}
fileSize = UTIL_getFileSizeStat(dictFileStat);
{
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
}
*bufferPtr = malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
if (readSize != fileSize) {
EXM_THROW(35, "Error reading dictionary file %s : %s",
fileName, strerror(errno));
}
}
fclose(fileHandle);
return (size_t)fileSize;
}
#if (PLATFORM_POSIX_VERSION > 0)
#include <sys/mman.h>
static void FIO_munmap(FIO_Dict_t* dict)
{
munmap(dict->dictBuffer, dict->dictBufferSize);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
ext/zstd/programs/fileio.c view on Meta::CPAN
if (transferStat) {
UTIL_setFDStat(dstFd, dstFileName, srcFileStat);
}
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName);
if (AIO_WritePool_closeFile(ress.writeCtx)) { /* error closing file */
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result=1;
}
if (transferStat) {
UTIL_utime(dstFileName, srcFileStat);
}
if ( (result != 0) /* operation failure */
&& strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */
) {
FIO_removeFile(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */
}
}
return result;
}
/* List used to compare file extensions (used with --exclude-compressed flag)
* Different from the suffixList and should only apply to ZSTD compress operationResult
*/
static const char *compressedFileExtensions[] = {
ZSTD_EXTENSION,
TZSTD_EXTENSION,
GZ_EXTENSION,
TGZ_EXTENSION,
LZMA_EXTENSION,
XZ_EXTENSION,
TXZ_EXTENSION,
LZ4_EXTENSION,
TLZ4_EXTENSION,
NULL
};
/*! FIO_compressFilename_srcFile() :
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
static int
FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t ress,
const char* dstFileName,
const char* srcFileName,
int compressionLevel)
{
int result;
FILE* srcFile;
stat_t srcFileStat;
U64 fileSize = UTIL_FILESIZE_UNKNOWN;
DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
if (strcmp(srcFileName, stdinmark)) {
if (UTIL_stat(srcFileName, &srcFileStat)) {
/* failure to stat at all is handled during opening */
/* ensure src is not a directory */
if (UTIL_isDirectoryStat(&srcFileStat)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return 1;
}
/* ensure src is not the same as dict (if present) */
if (ress.dictFileName != NULL && UTIL_isSameFileStat(srcFileName, ress.dictFileName, &srcFileStat, &ress.dictFileStat)) {
DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
return 1;
}
}
}
/* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
* YES => ZSTD will skip compression of the file and will return 0.
* NO => ZSTD will resume with compress operation.
*/
if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) {
DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
return 0;
}
srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
if (srcFile == NULL) return 1; /* srcFile could not be opened */
/* Don't use AsyncIO for small files */
if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
fileSize = UTIL_getFileSizeStat(&srcFileStat);
if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
AIO_ReadPool_setAsync(ress.readCtx, 0);
AIO_WritePool_setAsync(ress.writeCtx, 0);
} else {
AIO_ReadPool_setAsync(ress.readCtx, 1);
AIO_WritePool_setAsync(ress.writeCtx, 1);
}
AIO_ReadPool_setFile(ress.readCtx, srcFile);
result = FIO_compressFilename_dstFile(
fCtx, prefs, ress,
dstFileName, srcFileName,
&srcFileStat, compressionLevel);
AIO_ReadPool_closeFile(ress.readCtx);
if ( prefs->removeSrcFile /* --rm */
&& result == 0 /* success */
&& strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */
) {
/* We must clear the handler, since after this point calling it would
* delete both the source and destination files.
*/
clearHandler();
if (FIO_removeFile(srcFileName))
EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno));
}
return result;
}
( run in 0.582 second using v1.01-cache-2.11-cpan-39bf76dae61 )