view release on metacpan or  search on metacpan

bpc_refCount.c  view on Meta::CPAN

/*
 * Routines to provide reference counting
 *
 * Copyright (C) 2013 Craig Barratt.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, visit the http://fsf.org website.
 */

#include "backuppc.h"

/*
 * magic number that appears at the start of the reference count (or delta count file)
 */
#define BPC_POOL_REF_MAGIC    (0x178e553c)

#define CONV_BUF_TO_UINT32(buf)    ((buf)[0] << 24 | (buf)[1] << 16 | (buf)[2] << 8 | (buf)[3])

#define CONV_UINT32_TO_BUF(buf, val)   { *(buf)++ = ((val) >> 24) & 0xff;               \
                                         *(buf)++ = ((val) >> 16) & 0xff;               \
                                         *(buf)++ = ((val) >> 8)  & 0xff;               \
                                         *(buf)++ = ((val) >> 0)  & 0xff; }

typedef struct {
    bpc_hashtable_key key;
    int32 count;
    bpc_digest digest;
} DigestInfo;

typedef struct {
    int fd;
    uchar *bufP;
    int errorCnt;
    uchar buf[4 * 65536];
} write_info;

void bpc_poolRefInit(bpc_refCount_info *info, int entryCnt)
{
    bpc_hashtable_create(&info->ht, entryCnt, sizeof(DigestInfo));
}

void bpc_poolRefDestroy(bpc_refCount_info *info)
{
    bpc_hashtable_destroy(&info->ht);
}

void bpc_poolRefSet(bpc_refCount_info *info, bpc_digest *digest, int32 count)
{
    DigestInfo *d = bpc_hashtable_find(&info->ht, digest->digest, digest->len, 1);
    if ( d->key.key == digest ) {
        /*
         * new entry - copy in digest
         */
        d->digest  = *digest;
        d->key.key = d->digest.digest;
    }
    d->count = count;
    return;
}

int bpc_poolRefGet(bpc_refCount_info *info, bpc_digest *digest, int32 *count)
{

    DigestInfo *d = bpc_hashtable_find(&info->ht, digest->digest, digest->len, 0);
    if ( !d ) return -1;
    *count = d->count;
    return 0;
}

int bpc_poolRefDelete(bpc_refCount_info *info, bpc_digest *digest)

bpc_refCount.c  view on Meta::CPAN

    uchar *bufP = buf;

    if ( fd < 0 ) {
        bpc_logErrf("bpc_poolRefFileRead: can't open %s (errno %d)\n", fileName, errno);
        return -1;
    }
    if ( bpc_poolRef_read_more_data(fd, buf, sizeof(buf), &nRead, &bufP, fileName) < 0 ) {
        bpc_logErrf("bpc_poolRefFileRead: can't read data from %s (errno %d)\n", fileName, errno);
        return -1;
    }
    magic = CONV_BUF_TO_UINT32(bufP);
    bufP += 4;

    if ( magic != BPC_POOL_REF_MAGIC ) {
        bpc_logErrf("bpc_poolRefFileRead: bad magic number 0x%x (expected 0x%x)\n", magic, BPC_POOL_REF_MAGIC);
        return -1;
    }

    entryCnt = getVarInt(&bufP, buf + nRead);
    if ( BPC_LogLevel >= 4 ) bpc_logMsgf("bpc_poolRefFileRead: got %d entries (nRead = %d)\n", entryCnt, nRead);
    /*
     * make sure the hash table is big enough in one go to avoid multiple doublings
     */
    bpc_hashtable_growSize(&info->ht, entryCnt * 4 / 3);

    for ( i = 0 ; i < entryCnt ; i++ ) {
        DigestInfo *digestInfo;

        if ( nRead == sizeof(buf) && bufP > buf + nRead - 64
                && bpc_poolRef_read_more_data(fd, buf, sizeof(buf), &nRead, &bufP, fileName) < 0 ) {
            bpc_logErrf("bpc_poolRefFileRead: can't read more data from %s (errno %d)\n", fileName, errno);
            return -1;
        }
        digest.len = *bufP++;
        if ( digest.len > (int)sizeof(digest.digest) ) digest.len = sizeof(digest.digest);
        memcpy(digest.digest, bufP, digest.len);
        bufP += digest.len;
        count = getVarInt(&bufP, buf + nRead);

        if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_poolRefFileRead: entry %d: digest len = %d, digest = 0x%02x%02x%02x...., count = %d\n",
                                              i, digest.len, digest.digest[0], digest.digest[1], digest.digest[2], count);

        digestInfo = bpc_hashtable_find(&info->ht, digest.digest, digest.len, 1);

        if ( digestInfo->key.key == digest.digest ) {
            /*
             * new entry since the key points to our key - copy info into new node and set key locally
             */
            digestInfo->digest  = digest;
            digestInfo->key.key = digestInfo->digest.digest;
        }
        digestInfo->count = count;
    }

    close(fd);

    return 0;
}

/*
 * Mark this host backup as needing an fsck.  Multiple requests can be supported with
 * unique numbers.  ext == 0 is used for the overall backup process, and it is removed when
 * the backup finished.  Various errors can use other extensions.  If any files are
 * present, an fsck is done either by the next backup, BackupPC_refCountUpdate or
 * BackupPC_fsck.
 */
void bpc_poolRefRequestFsck(char *backupDir, int ext)
{
    char fileName[BPC_MAXPATHLEN];
    int fd;

    snprintf(fileName, sizeof(fileName), "%s/refCnt/needFsck%d", backupDir, ext);
    if ( (fd = open(fileName, O_CREAT | O_WRONLY, 0660)) < 0 ) {
        bpc_logErrf("bpc_poolRefRequestFsck: can't open/create fsck request file %s (errno %d)\n", fileName, errno);
    }
}

/***********************************************************************
 * Reference count deltas - we maintain two hash tables for uncompressed
 * and compressed deltas.
 ***********************************************************************/

/*
 * Legacy support for <= 4.0.0beta3.
 */
static bpc_deltaCount_info DeltaInfoOld;

static int OutputFileCnt = 0;

void bpc_poolRefDeltaFileInit(bpc_deltaCount_info *info, char *hostDir)
{
    if ( snprintf(info->targetDir, sizeof(info->targetDir), "%s", hostDir)
		>= (int)sizeof(info->targetDir) - 1 ) {
	bpc_logErrf("bpc_poolRefDeltaFileInit: targetDir %s truncated\n", hostDir);
    }
    bpc_poolRefInit(&info->refCnt[0], 256);
    bpc_poolRefInit(&info->refCnt[1], 1 << 20);
    info->refCnt[0].initDone = info->refCnt[1].initDone = 1;
}

void bpc_poolRefDeltaFileDestroy(bpc_deltaCount_info *info)
{
    bpc_poolRefDestroy(&info->refCnt[0]);
    bpc_poolRefDestroy(&info->refCnt[1]);
}

uint32 bpc_poolRefDeltaFileFlush(bpc_deltaCount_info *info)
{
    char tempFileName[BPC_MAXPATHLEN], finalFileName[BPC_MAXPATHLEN];
    int compress;
    int errorCnt = 0;
    int fd;

    if ( !info ) info = &DeltaInfoOld;         /* backward compatibility */
    if ( !info->refCnt[0].initDone ) return 1;
    for ( compress = 0 ; compress < 2 ; compress++ ) {
        uint entryCnt = bpc_hashtable_entryCount(&info->refCnt[compress].ht);

        if ( entryCnt == 0 ) continue;

        do {
            if ( snprintf(tempFileName, sizeof(tempFileName), "%s/refCnt/tpoolCntDelta_%d_%d_%d_%d",
                          info->targetDir, compress, BPC_TmpFileUnique, OutputFileCnt, getpid()) >= (int)sizeof(tempFileName) - 1 ) {
                bpc_logErrf("bpc_poolRefDeltaFileFlush: pool delta file name %s truncated\n", tempFileName);
                errorCnt++;
            }
            if ( (fd = open(tempFileName, O_RDONLY, 0666)) >= 0 ) {
                close(fd);
                OutputFileCnt++;
            }
        } while ( fd >= 0 );

        errorCnt += bpc_poolRefFileWrite(&info->refCnt[compress], tempFileName);

        if ( snprintf(finalFileName, sizeof(finalFileName), "%s/refCnt/poolCntDelta_%d_%d_%d_%d",
                      info->targetDir, compress, BPC_TmpFileUnique >= 0 ? BPC_TmpFileUnique : 0,
                      OutputFileCnt, getpid()) >= (int)sizeof(finalFileName) - 1 ) {
            bpc_logErrf("bpc_poolRefDeltaFileFlush: pool delta file name %s truncated\n", finalFileName);
            errorCnt++;
        }
        if ( errorCnt ) {
            unlink(tempFileName);
            continue;
        }
        if ( rename(tempFileName, finalFileName) != 0 ) {
            bpc_logErrf("bpc_poolRefDeltaFileFlush: can't rename %s to %s (errno %d)\n", tempFileName, finalFileName, errno);
            unlink(tempFileName);
            errorCnt++;
        }
        if ( !errorCnt ) {
            bpc_hashtable_erase(&info->refCnt[compress].ht);
        }
    }
    OutputFileCnt++;
    if ( errorCnt ) {
        /*
         * Need to fsck this particular backup on this host
         */
        bpc_poolRefRequestFsck(info->targetDir, getpid());
    }
    return errorCnt;
}

void bpc_poolRefDeltaUpdate(bpc_deltaCount_info *info, int compress, bpc_digest *digest, int32 count)
{
    DigestInfo *digestInfo;

    if ( !info ) info = &DeltaInfoOld;         /* backward compatibility */
    if ( !digest || digest->len == 0 ) return;
    if ( !info->refCnt[0].initDone ) return;

    digestInfo = bpc_hashtable_find(&info->refCnt[compress ? 1 : 0].ht, digest->digest, digest->len, 1);
    if ( digestInfo->key.key == digest->digest ) {
        /*
         * new entry since the key points to our key - copy info into new node and set key locally
         */
        digestInfo->digest  = *digest;
        digestInfo->key.key = digestInfo->digest.digest;
    }
    digestInfo->count += count;
    if ( BPC_LogLevel >= 8 ) {
        char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1];

        bpc_digest_digest2str(&digestInfo->digest, hexStr);
        bpc_logMsgf("bpc_poolRefDeltaUpdate(%s, %d), count now %d\n", hexStr, count, digestInfo->count);
    }
    if ( bpc_hashtable_entryCount(&info->refCnt[compress ? 1 : 0].ht) > (1 << 20) ) {
        bpc_poolRefDeltaFileFlush(info);
    }
}

void bpc_poolRefDeltaPrint(bpc_deltaCount_info *info)
{
    if ( !info ) info = &DeltaInfoOld;         /* backward compatibility */
    if ( !info->refCnt[0].initDone ) return;
    fprintf(stderr, "Uncompressed HT:\n");
    bpc_hashtable_iterate(&info->refCnt[0].ht, (void*)bpc_poolRefPrintEntry, NULL);
    fprintf(stderr, "Compressed HT:\n");
    bpc_hashtable_iterate(&info->refCnt[1].ht, (void*)bpc_poolRefPrintEntry, NULL);
}

/*
 * Legacy support for <= 4.0.0beta3.
 */
void bpc_poolRefDeltaFileInitOld(char *hostDir)
{
    bpc_poolRefDeltaFileInit(&DeltaInfoOld, hostDir);
}

uint32 bpc_poolRefDeltaFileFlushOld(void)
{
    return bpc_poolRefDeltaFileFlush(&DeltaInfoOld);
}

/*
 * Increment/decrement the reference count for the given digest
 */