BackupPC-XS
view release on metacpan or search on metacpan
bpc_poolWrite.c view on Meta::CPAN
bpc_logErrf("bpc_poolWrite_write: can't open %s for reading", info->tmpFileName);
return -1;
}
info->fdOpen = 1;
md5_result(&info->md5, info->digest.digest);
info->digest.len = MD5_DIGEST_LEN;
if ( BPC_PoolV3Enabled ) {
bpc_digest_buffer2MD5_v3(&info->digest_v3, info->buffer, info->fileSize);
if ( BPC_LogLevel >= 8 ) {
char hexStr_v3[BPC_DIGEST_LEN_MAX * 2 + 1], hexStr[BPC_DIGEST_LEN_MAX * 2 + 1];
bpc_digest_digest2str(&info->digest, hexStr);
bpc_digest_digest2str(&info->digest_v3, hexStr_v3);
bpc_logMsgf("bpc_poolWrite_write: digest is %s, v3 is %s\n", hexStr, hexStr_v3);
}
} else if ( BPC_LogLevel >= 8 ) {
char hexStr[BPC_DIGEST_LEN_MAX * 2 + 1];
bpc_digest_digest2str(&info->digest, hexStr);
bpc_logMsgf("bpc_poolWrite_write: digest is %s\n", hexStr);
}
info->state = 2;
}
}
if ( info->state == 2 ) {
uint32 ext = 0;
char poolPath[BPC_MAXPATHLEN];
STRUCT_STAT st;
/*
* In this state we have either the full file in info->buffer, or the full file
* is opened for reading with info->fd. We also have digests computed.
*
* We figure out the list of candidate files to match. If there are any
* new digest files then we just try to match them. Otherwise we also
* try to match any old V3 files.
*
* Since the empty file is never stored in the pool, we have to make sure
* that any digest that collides (ie: 0xd41d8cd98f00b204e9800998ecf8427e)
* doesn't use the first slot (ie: make sure it has an extension > 0)
*/
static uchar zeroLenMD5[] = {
0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e
};
if ( info->fileSize > 0 && !memcmp(info->digest.digest, zeroLenMD5, sizeof(zeroLenMD5)) ) {
ext++;
}
info->digestExtZeroLen = -1;
while ( 1 ) {
bpc_digest_append_ext(&info->digest, ext);
bpc_digest_md52path(poolPath, info->compress, &info->digest);
/*
* For >= V4.x pool, don't attempt to match pool files that
* are empty, since in >= V4.x we don't rename pool
* files in a repeated chain and instead replace them
* with an empty file.
* If the candidate has the other execute bit set, we do a safe
* reset of the bit and allow matches to occur. This is used to flag
* pool files that will be deleted next time BackupPC_refCountUpdate
* runs, so resetting that bit prevents the deletion.
*/
if ( stat(poolPath, &st) ) break;
if ( S_ISREG(st.st_mode) ) {
if ( st.st_size > 0 ) {
bpc_candidate_file *candidateFile;
if ( (st.st_mode & S_IXOTH) && bpc_poolWrite_unmarkPendingDelete(poolPath) ) {
bpc_logErrf("Couldn't unmark candidate matching file %s (skipped; errno = %d)\n", poolPath, errno);
info->errorCnt++;
break;
}
candidateFile = malloc(sizeof(bpc_candidate_file));
if ( !candidateFile ) {
info->errorCnt++;
bpc_logErrf("bpc_poolWrite_write: can't allocate bpc_candidate_file\n");
return -1;
}
candidateFile->digest = info->digest;
candidateFile->fileSize = st.st_size;
candidateFile->v3File = 0;
strcpy(candidateFile->fileName, poolPath);
candidateFile->next = info->candidateList;
info->candidateList = candidateFile;
if ( BPC_LogLevel >= 7 ) bpc_logMsgf("Candidate matching file %s\n", candidateFile->fileName);
} else if ( info->digestExtZeroLen < 0 ) {
/*
* Remember the first empty file in case we have to insert a
* new pool file here.
*/
info->digestExtZeroLen = ext;
}
}
ext++;
}
/*
* Remember the next open slot in case we have to add a new pool
* file here.
*/
info->digestExtOpen = ext;
bpc_digest_append_ext(&info->digest, 0);
if ( BPC_PoolV3Enabled && !info->candidateList ) {
/*
* No matching candidate files so far, so now look in V3 pool
*/
ext = 0;
while ( 1 ) {
bpc_digest_append_ext(&info->digest_v3, ext);
bpc_digest_md52path_v3(poolPath, info->compress, &info->digest_v3);
ext++;
/*
* For V3.x pool, don't attempt to match pool files:
* - that already have too many hardlinks.
* - with only one link since starting in BackupPC v3.0,
* BackupPC_nightly could be running in parallel (and
* removing those files). This doesn't eliminate all
* possible race conditions, but just reduces the
* odds. Other design steps eliminate the remaining
* race conditions of linking vs removing.
*/
if ( stat(poolPath, &st) ) break;
if ( S_ISREG(st.st_mode)
&& 1 < st.st_nlink && st.st_nlink < (unsigned)BPC_HardLinkMax ) {
bpc_candidate_file *candidateFile = malloc(sizeof(bpc_candidate_file));
if ( !candidateFile ) {
info->errorCnt++;
bpc_logErrf("bpc_poolWrite_write: can't allocate bpc_candidate_file\n");
return -1;
}
candidateFile->digest = info->digest_v3;
candidateFile->fileSize = st.st_size;
candidateFile->v3File = 1;
strcpy(candidateFile->fileName, poolPath);
candidateFile->next = info->candidateList;
info->candidateList = candidateFile;
if ( BPC_LogLevel >= 7 ) bpc_logMsgf("Candidate v3 matching file %s\n", candidateFile->fileName);
}
}
bpc_digest_append_ext(&info->digest_v3, 0);
}
/*
* Open the first set of candidate files.
*/
bpc_poolWrite_updateMatches(info);
info->state = 3;
}
if ( info->state == 3 ) {
/*
* In this state we are continuing to match against candidate files
*/
while ( 1 ) {
int i, replaceCnt = 0, nMatch = 0;
uchar *buf0 = TempBuf;
uchar *buf1 = TempBuf + COMPARE_BUF_SZ;
uchar *buf;
OFF_T nread0;
if ( info->fdOpen ) {
nread0 = bpc_fileZIO_read(&info->fd, buf0, COMPARE_BUF_SZ);
buf = buf0;
} else {
nread0 = COMPARE_BUF_SZ;
if ( nread0 > info->bufferIdx - info->matchPosn ) nread0 = info->bufferIdx - info->matchPosn;
buf = info->buffer + info->matchPosn;
}
for ( i = 0 ; i < BPC_POOL_WRITE_CONCURRENT_MATCH ; i++ ) {
OFF_T nread1;
if ( !info->match[i].used ) continue;
nMatch++;
/*
* Try to read an extra byte when we expect EOF, to make sure the candidate file is also at EOF
*/
nread1 = bpc_fileZIO_read(&info->match[i].fd, buf1, nread0 > 0 ? nread0 : 1);
if ( BPC_LogLevel >= 9 ) bpc_logMsgf("Read %d bytes of %d from match[%d] (%s)\n", (int)nread1, (int)nread0, i, info->match[i].fileName);
if ( nread0 != nread1 || (nread0 > 0 && memcmp(buf, buf1, nread0)) ) {
bpc_fileZIO_close(&info->match[i].fd);
if ( BPC_LogLevel >= 8 ) bpc_logMsgf("match[%d] no longer matches\n", i);
info->match[i].used = 0;
replaceCnt++;
}
bpc_poolWrite.c view on Meta::CPAN
int bpc_poolWrite_copyToPool(bpc_poolWrite_info *info, char *poolPath, char *fileName)
{
int fdRead, fdWrite;
int nRead, nWrite;
if ( (fdWrite = open(poolPath, O_WRONLY | O_CREAT | O_EXCL, 0666)) < 0 ) {
info->errorCnt++;
bpc_logErrf("bpc_poolWrite_copyToPool: can't open/create %s for writing", poolPath);
return -1;
}
if ( (fdRead = open(fileName, O_RDONLY)) < 0 ) {
info->errorCnt++;
bpc_logErrf("bpc_poolWrite_copyToPool: can't open %s for reading", fileName);
return -1;
}
while ( (nRead = read(fdRead, info->buffer, sizeof(info->buffer))) > 0 ) {
uchar *p = info->buffer;
int thisWrite;
nWrite = 0;
while ( nWrite < nRead ) {
do {
thisWrite = write(fdWrite, p, nRead - nWrite);
} while ( thisWrite < 0 && errno == EINTR );
if ( thisWrite < 0 ) {
info->errorCnt++;
bpc_logErrf("bpc_poolWrite_copyToPool: write to %s failed (errno = %d)", poolPath, errno);
close(fdWrite);
close(fdRead);
unlink(poolPath);
return -1;
}
p += thisWrite;
nWrite += thisWrite;
}
}
close(fdWrite);
close(fdRead);
return 0;
}
void bpc_poolWrite_addToPool(bpc_poolWrite_info *info, char *fileName, int v3PoolFile)
{
STRUCT_STAT st;
char poolPath[BPC_MAXPATHLEN];
int redo = 0;
if ( bpc_poolWrite_createPoolDir(info, &info->digest) ) return;
/*
* If originally present, make sure the zero-length file is still there (and still
* zero-length), and the open slot is still open. If not, it probably means someone
* beat us to it, and we should re-do the whole pool matching to see if the newly
* added pool file now matches.
*/
if ( info->digestExtZeroLen >= 0 ) {
bpc_digest_append_ext(&info->digest, info->digestExtZeroLen);
bpc_digest_md52path(poolPath, info->compress, &info->digest);
if ( stat(poolPath, &st) || st.st_size != 0 ) {
redo = 1;
}
}
if ( !redo ) {
bpc_digest_append_ext(&info->digest, info->digestExtOpen);
bpc_digest_md52path(poolPath, info->compress, &info->digest);
if ( !stat(poolPath, &st) ) {
redo = 1;
}
}
/*
* Try to insert the new file at the zero-length file slot (if present).
*/
if ( !redo && info->digestExtZeroLen >= 0 ) {
char lockFile[BPC_MAXPATHLEN];
int lockFd;
/*
* We can replace a zero-length file, but only via locking to
* avoid race conditions. Since the hardlinking code below doesn't
* use a lock, we can't remove the file and use a hardlink
* because of race conditions - another process might be
* inserting with the same digest and grab the slot.
*
* So we make sure we have exclusive access via a lock file,
* check that the file is still zero-length, and then rename
* the file. If that fails then we redo everything.
*/
bpc_digest_append_ext(&info->digest, info->digestExtZeroLen);
bpc_digest_md52path(poolPath, info->compress, &info->digest);
if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_poolWrite_addToPool: replacing empty pool file %s with %s\n", poolPath, fileName);
snprintf(lockFile, BPC_MAXPATHLEN, "%s.lock", poolPath);
lockFd = bpc_lockRangeFile(lockFile, 0, 1, 1);
/*
* If we don't have the lock, or the file is no longer zero length, or the rename fails,
* then try again.
*/
if ( lockFd < 0 || stat(poolPath, &st) || st.st_size != 0 || rename(fileName, poolPath) ) {
if ( BPC_LogLevel >= 5 ) {
bpc_logMsgf("bpc_poolWrite_addToPool: lock/rename failed: need to repeat write (lockFd = %d, size = %lu, errno = %d)\n",
lockFd, (unsigned long)st.st_size, errno);
}
if ( lockFd >= 0 ) {
bpc_unlockRangeFile(lockFd);
}
unlink(lockFile);
redo = 1;
} else {
chmod(poolPath, 0444);
stat(poolPath, &st);
info->retValue = v3PoolFile ? 2 : 0;
info->poolFileSize = st.st_size;
bpc_unlockRangeFile(lockFd);
unlink(lockFile);
return;
}
}
/*
* Now try to link the file to the new empty slot at the end
*/
if ( !redo ) {
int linkOk, statOk;
ino_t fileIno, poolIno;
/*
* Since this is a new slot, there is no need to do locking since
* the link or open operations below are atomic/exclusive.
*
* First try to hardlink to the empty pool file slot
*/
bpc_digest_append_ext(&info->digest, info->digestExtOpen);
bpc_digest_md52path(poolPath, info->compress, &info->digest);
if ( stat(fileName, &st) ) {
info->errorCnt++;
bpc_logErrf("bpc_poolWrite_addToPool: can't stat %s\n", fileName);
return;
}
fileIno = st.st_ino;
linkOk = !link(fileName, poolPath);
if ( !(statOk = !stat(poolPath, &st)) ) linkOk = 0;
poolIno = st.st_ino;
if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_poolWrite_addToPool: link %s -> %s (linkOk = %d, statOk = %d, ino = %lu/%lu)\n",
poolPath, fileName, linkOk, statOk, (unsigned long)fileIno, (unsigned long)poolIno);
/*
* make sure the link really worked by checking inode numbers
* TODO: test these different cases.
*/
if ( statOk && fileIno == poolIno ) {
/*
* remove the original file and return
*/
unlink(fileName);
chmod(poolPath, 0444);
info->retValue = v3PoolFile ? 2 : 0;
info->poolFileSize = st.st_size;
return;
}
/*
* Something failed. If the stat failed, the hardlink failure wasn't due
* to another file being added by someone else. Perhaps the cpool is
* split across multiple file systems?
*/
if ( !statOk ) {
/*
* The hardlink failed. This could be due to hitting the hardlink
* limit, or the fact that fileName and poolPath are on different
* file systems, or the fileName didn't get written.
* Just copy the file instead (assuming fileName got written).
*/
bpc_poolWrite_copyToPool(info, poolPath, fileName);
return;
}
}
/*
* We need to redo the pool write, since it appears someone else has added
* a pool file with the same digest.
*/
bpc_poolWrite_repeatPoolWrite(info, fileName);
}
/*
* Safely remove the o+x permission that marks a file for future deletion.
* Similar locking is done by BackupPC_refCountUpdate so we can avoid any
* race conditions with the file actually being deleted.
*
* Returns 0 on success.
*/
int bpc_poolWrite_unmarkPendingDelete(char *poolPath)
{
char lockFile[BPC_MAXPATHLEN], *p;
STRUCT_STAT st;
int lockFd;
/*
* The lock file is in the first level of pool sub directories - one level
* up from the full path. So we need to find the 2nd last '/'.
*/
snprintf(lockFile, BPC_MAXPATHLEN, "%s", poolPath);
if ( !(p = strrchr(lockFile, '/')) ) return -1;
*p = '\0';
if ( !(p = strrchr(lockFile, '/')) ) return -1;
snprintf(p + 1, BPC_MAXPATHLEN - (p + 1 - lockFile), "%s", "LOCK");
if ( (lockFd = bpc_lockRangeFile(lockFile, 0, 1, 1)) < 0 ) return -1;
if ( !stat(poolPath, &st) && !chmod(poolPath, st.st_mode & ~S_IXOTH & ~S_IFMT) ) {
if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_poolWrite_unmarkPendingDelete(%s) succeeded\n", poolPath);
bpc_unlockRangeFile(lockFd);
return 0;
} else {
bpc_logErrf("bpc_poolWrite_unmarkPendingDelete(%s) failed; errno = %d\n", poolPath, errno);
bpc_unlockRangeFile(lockFd);
return -1;
}
}
( run in 1.240 second using v1.01-cache-2.11-cpan-39bf76dae61 )