view release on metacpan or  search on metacpan

ext/zstd/tests/fuzzer.c  view on Meta::CPAN

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
 * in the COPYING file in the root directory of this source tree).
 * You may select, at your option, one of the above-listed licenses.
 */


/*-************************************
*  Compiler specific
**************************************/
#ifdef _MSC_VER    /* Visual Studio */
#  define _CRT_SECURE_NO_WARNINGS   /* fgets */
#  pragma warning(disable : 4127)   /* disable: C4127: conditional expression is constant */
#  pragma warning(disable : 4204)   /* disable: C4204: non-constant aggregate initializer */
#endif


/*-************************************
*  Includes
**************************************/
#include <stdlib.h>       /* free */
#include <stdio.h>        /* fgets, sscanf */
#include <string.h>       /* strcmp */
#include <time.h>         /* time(), time_t */
#undef NDEBUG             /* always enable assert() */
#include <assert.h>
#define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "debug.h"        /* DEBUG_STATIC_ASSERT */
#include "fse.h"
#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
#include "zstd.h"         /* ZSTD_VERSION_STRING */
#include "zstd_errors.h"  /* ZSTD_getErrorCode */
#define ZDICT_STATIC_LINKING_ONLY
#include "zdict.h"        /* ZDICT_trainFromBuffer */
#include "mem.h"
#include "datagen.h"      /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
#include "xxhash.h"       /* XXH64 */
#include "util.h"
#include "timefn.h"       /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
/* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
#include "threading.h"    /* ZSTD_pthread_create, ZSTD_pthread_join */


/*-************************************
*  Constants
**************************************/
#define GB *(1U<<30)

static const int FUZ_compressibility_default = 50;
static const int nbTestsDefault = 30000;


/*-************************************
*  Display Macros
**************************************/
#define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2;

static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;

#define DISPLAYUPDATE(l, ...) \
    if (g_displayLevel>=l) { \
        if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
        { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
        if (g_displayLevel>=4) fflush(stderr); } \
    }


/*-*******************************************************
*  Compile time test
*********************************************************/
#undef MIN
#undef MAX
/* Declaring the function, to avoid -Wmissing-prototype */
void FUZ_bug976(void);
void FUZ_bug976(void)
{   /* these constants shall not depend on MIN() macro */
    DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
    DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
}


/*-*******************************************************
*  Internal functions
*********************************************************/
#define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))

#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static U32 FUZ_rand(U32* src)
{
    static const U32 prime1 = 2654435761U;
    static const U32 prime2 = 2246822519U;
    U32 rand32 = *src;
    rand32 *= prime1;
    rand32 += prime2;
    rand32  = FUZ_rotl32(rand32, 13);
    *src = rand32;

ext/zstd/tests/fuzzer.c  view on Meta::CPAN

        ZSTD_freeCCtx(largeCCtx);
    }
    DISPLAYLEVEL(3, "OK \n");

    /* Static CCtx tests */
#define STATIC_CCTX_LEVEL 4
    DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
    {   size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
        void* const staticCCtxBuffer = malloc(staticCStreamSize);
        size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
        void* const staticDCtxBuffer = malloc(staticDCtxSize);
        DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize);
        if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
            free(staticCCtxBuffer);
            free(staticDCtxBuffer);
            DISPLAY("Not enough memory, aborting\n");
            testResult = 1;
            goto _end;
        }
        {   size_t const smallInSize = 32 KB;
            ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0);
            size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small);
            size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL);
            ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize);
            ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
            DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize);
            DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize);
            if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++);
            CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
                                  compressedBuffer, compressedBufferSize,
                                  CNBuffer, smallInSize, STATIC_CCTX_LEVEL) );
            DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
                            (unsigned)cSize, (double)cSize/smallInSize*100);

            DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++);
            {   size_t const r = ZSTD_compressCCtx(staticCCtx,
                                  compressedBuffer, compressedBufferSize,
                                  CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL);
                if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error;
            }
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
            staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
            DISPLAYLEVEL(4, "staticCCtxBuffer = %p,  staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx);
            if (staticCCtx == NULL) goto _output_error;
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++);
            CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
                                  compressedBuffer, compressedBufferSize,
                                  CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
            DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
                            (unsigned)cSize, (double)cSize/CNBuffSize*100);

            DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++);
            {   int nbc;
                assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR);  /* ensure size down scenario */
                assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3);
                for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) {
                    CHECK_Z(ZSTD_compressCCtx(staticCCtx,
                                  compressedBuffer, compressedBufferSize,
                                  (char*)CNBuffer + nbc, smallInSize,
                                  STATIC_CCTX_LEVEL) );
            }   }
            DISPLAYLEVEL(3, "OK \n")

            DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
            CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) );
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++);
            CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
                                  compressedBuffer, compressedBufferSize,
                                  CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
            DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
                            (unsigned)cSize, (double)cSize/CNBuffSize*100);

            DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
            { size_t const r = ZSTD_decompressDCtx(staticDCtx,
                                                decodedBuffer, CNBuffSize,
                                                compressedBuffer, cSize);
              if (r != CNBuffSize) goto _output_error; }
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
            if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
            { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
              if (!ZSTD_isError(r)) goto _output_error; }
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
            CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
            CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
            {   ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
                ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
                size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */
                if (!ZSTD_isError(r)) goto _output_error;
            }
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++);
            staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize);
            assert(staticCCtx != NULL);
            CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
            {   ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
                ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
                CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
            }
            DISPLAYLEVEL(3, "OK \n");

            DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1);
            CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */
            {   ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };