BackupPC-XS
view release on metacpan or search on metacpan
bpc_attrib.c view on Meta::CPAN
/*
* Routines for read/writing/managing file attributes
*
* Copyright (C) 2013 Craig Barratt.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, visit the http://fsf.org website.
*/
#include "backuppc.h"
/*
* Type of attribute file. This is saved as a magic number at the
* start of the file. This type is for V3 and earlier.
*/
#define BPC_ATTRIB_TYPE_UNIX (0x17555555)
/*
* super set of UNIX, including extended attribs and digest, for 4.x+
*/
#define BPC_ATTRIB_TYPE_XATTR (0x17565353)
/*
* starting in 4.x, attrib files in the pc backup tree are
* just digests that give the location of the real attrib file
* in the pool. attrib files written in the pc backup tree
* start with this magic number, followed by the digest.
*/
#define BPC_ATTRIB_TYPE_DIGEST (0x17585451)
static char *FileType2Text[] = {
"file",
"hardlink",
"symlink",
"chardev",
"blockdev",
"dir",
"fifo",
"?",
"socket",
"?",
"deleted",
};
/*
* Ensure by default we use old-style "attrib" files for <= 4.0.0alpha3 compatibility.
* Newer versions of BackupPC turn this off to use new attribute file naming.
*/
static int WriteOldStyleAttribFile = 1;
static int KeepOldAttribFiles = 1;
void bpc_attrib_backwardCompat(int writeOldStyleAttribFile, int keepOldAttribFiles)
{
if ( writeOldStyleAttribFile >= 0 ) WriteOldStyleAttribFile = writeOldStyleAttribFile;
if ( keepOldAttribFiles >= 0 ) KeepOldAttribFiles = keepOldAttribFiles;
if ( BPC_LogLevel >= 5 ) {
bpc_logMsgf("bpc_attrib_backwardCompat: WriteOldStyleAttribFile = %d, KeepOldAttribFiles = %d\n",
WriteOldStyleAttribFile, KeepOldAttribFiles);
}
}
#define CONV_BUF_TO_UINT32(buf) ((buf)[0] << 24 | (buf)[1] << 16 | (buf)[2] << 8 | (buf)[3])
#define CONV_UINT32_TO_BUF(buf, val) { *(buf)++ = ((val) >> 24) & 0xff; \
*(buf)++ = ((val) >> 16) & 0xff; \
*(buf)++ = ((val) >> 8) & 0xff; \
*(buf)++ = ((val) >> 0) & 0xff; }
/*
* Note on xattr keys: they are treated as opaque strings of bytes, and the convention
* is to include the '\0' byte termination in the keyLen (ie: it is strlen(key) + 1).
*/
bpc_attrib_xattr *bpc_attrib_xattrGet(bpc_attrib_file *file, void *key, int keyLen, int allocate_if_missing)
{
return (bpc_attrib_xattr*)bpc_hashtable_find(&file->xattrHT, key, keyLen, allocate_if_missing);
}
void bpc_attrib_xattrDestroy(bpc_attrib_xattr *xattr)
{
if ( xattr->key.key ) free(xattr->key.key);
if ( xattr->value ) free(xattr->value);
}
int bpc_attrib_xattrDelete(bpc_attrib_file *file, void *key, int keyLen)
{
bpc_attrib.c view on Meta::CPAN
}
void bpc_attrib_fileDestroy(bpc_attrib_file *file)
{
if ( file->name) free(file->name);
bpc_hashtable_iterate(&file->xattrHT, (void*)bpc_attrib_xattrDestroy, NULL);
bpc_hashtable_destroy(&file->xattrHT);
}
/*
* Return the attributes for the given file.
* If allocate_if_missing == 0 and not present, then NULL is returned.
* If allocate_if_missing != 0 and not present, then an empty struct is returned with the key filled in,
* and file->name is NULL.
*/
bpc_attrib_file *bpc_attrib_fileGet(bpc_attrib_dir *dir, char *fileName, int allocate_if_missing)
{
return bpc_hashtable_find(&dir->filesHT, (uchar*)fileName, strlen(fileName), allocate_if_missing);
}
/*
* Initialize an empty file structure (ie: one returned by bpc_attrib_fileGet() that is empty)
*/
void bpc_attrib_fileInit(bpc_attrib_file *file, char *fileName, int xattrNumEntries)
{
int fileNameLen = strlen(fileName);
if ( file->name ) bpc_attrib_fileDestroy(file);
file->name = (char*)malloc(fileNameLen + 1);
if ( !file->name ) {
bpc_logErrf("bpc_attrib_fileInit: can't allocate %d bytes for file name\n", fileNameLen + 1);
return;
}
memcpy(file->name, fileName, fileNameLen + 1);
file->isTemp = 0;
file->key.key = file->name;
bpc_hashtable_create(&file->xattrHT, 16 + xattrNumEntries, sizeof(bpc_attrib_xattr));
}
/*
* Copy all the attributes from fileSrc to fileDest. fileDest should already have a
* valid allocated fileName and allocated xattr hash. The fileDest xattr hash is
* emptied before the copy, meaning it is over written.
*
* If overwriteEmptyDigest == 0, an empty digest in fileSrc will not overwrite fileDest.
*/
void bpc_attrib_fileCopyOpt(bpc_attrib_file *fileDest, bpc_attrib_file *fileSrc, int overwriteEmptyDigest)
{
if ( fileDest == fileSrc ) return;
fileDest->type = fileSrc->type;
fileDest->compress = fileSrc->compress;
fileDest->mode = fileSrc->mode;
fileDest->isTemp = fileSrc->isTemp;
fileDest->uid = fileSrc->uid;
fileDest->gid = fileSrc->gid;
fileDest->nlinks = fileSrc->nlinks;
fileDest->mtime = fileSrc->mtime;
fileDest->size = fileSrc->size;
fileDest->inode = fileSrc->inode;
fileDest->backupNum = fileSrc->backupNum;
if ( fileSrc->digest.len > 0 || overwriteEmptyDigest ) {
fileDest->digest = fileSrc->digest;
}
bpc_hashtable_iterate(&fileDest->xattrHT, (void*)bpc_attrib_xattrDestroy, NULL);
bpc_hashtable_erase(&fileDest->xattrHT);
bpc_hashtable_iterate(&fileSrc->xattrHT, (void*)bpc_attrib_xattrCopy, fileDest);
}
/*
* Copy all the attributes from fileSrc to fileDest. fileDest should already have a
* valid allocated fileName and allocated xattr hash. The fileDest xattr hash is
* emptied before the copy, meaning it is over written.
*/
void bpc_attrib_fileCopy(bpc_attrib_file *fileDest, bpc_attrib_file *fileSrc)
{
if ( fileDest == fileSrc ) return;
bpc_attrib_fileCopyOpt(fileDest, fileSrc, 1);
}
/*
* Check if two file attribute structures are the same. Returns 0 if they are the same.
*/
int bpc_attrib_fileCompare(bpc_attrib_file *file0, bpc_attrib_file *file1)
{
uint idx = 0;
if ( file0->type != file1->type
|| file0->compress != file1->compress
|| file0->mode != file1->mode
|| file0->uid != file1->uid
|| file0->gid != file1->gid
|| file0->nlinks != file1->nlinks
|| file0->mtime != file1->mtime
|| file0->size != file1->size
|| file0->inode != file1->inode
|| file0->digest.len != file1->digest.len
|| memcmp(file0->digest.digest, file1->digest.digest, file0->digest.len)
|| bpc_attrib_xattrCount(file0) != bpc_attrib_xattrCount(file1) ) {
if ( BPC_LogLevel >= 9 ) bpc_logMsgf("bpc_attrib_fileCompare: %s %s differ\n", file0->name, file1->name);
return 1;
}
while ( 1 ) {
bpc_attrib_xattr *xattr0 = bpc_hashtable_nextEntry(&file0->xattrHT, &idx), *xattr1;
if ( !xattr0 ) return 0;
if ( !(xattr1 = bpc_attrib_xattrGet(file1, xattr0->key.key, xattr0->key.keyLen, 0)) ) return 1;
if ( xattr0->valueLen != xattr1->valueLen || memcmp(xattr0->value, xattr1->value, xattr0->valueLen) ) return 1;
}
}
void bpc_attrib_fileDeleteName(bpc_attrib_dir *dir, char *fileName)
{
bpc_attrib_file *file = bpc_hashtable_find(&dir->filesHT, (uchar*)fileName, strlen(fileName), 0);
if ( !file ) return;
bpc_attrib_fileDestroy(file);
bpc_hashtable_nodeDelete(&dir->filesHT, file);
}
int bpc_attrib_fileIterate(bpc_attrib_dir *dir, bpc_attrib_file **file, uint *idx)
bpc_attrib.c view on Meta::CPAN
file->mtime = getVarInt(&bufP, bufEnd);
file->mode = getVarInt(&bufP, bufEnd);
file->uid = getVarInt(&bufP, bufEnd);
file->gid = getVarInt(&bufP, bufEnd);
file->size = getVarInt(&bufP, bufEnd);
file->inode = getVarInt(&bufP, bufEnd);
file->compress = getVarInt(&bufP, bufEnd);
file->nlinks = getVarInt(&bufP, bufEnd);
file->digest.len = getVarInt(&bufP, bufEnd);
file->isTemp = 0;
if ( file->digest.len > 0 && bufP + file->digest.len <= bufEnd ) {
memcpy(file->digest.digest, bufP, file->digest.len);
}
bufP += file->digest.len;
for ( i = 0 ; i < xattrNumEntries ; i++ ) {
uint keyLen = getVarInt(&bufP, bufEnd);
uint valueLen = getVarInt(&bufP, bufEnd);
if ( bufP + keyLen + valueLen <= bufEnd ) {
if ( xattrFixup && bufP[keyLen - 1] != 0x0 ) {
*xattrFixup = 1;
}
bpc_attrib_xattrSetValue(file, bufP, keyLen, bufP + keyLen, valueLen);
}
bufP += keyLen + valueLen;
}
return bufP;
}
/*
* Extract an entire packed file structure, starting with the fileName length varint.
* Returns next unused buffer location. It is assumed the file structure is already
* initialized and has a valid fileName allocated, so we don't allocate it here.
*
* If there isn't enough data to extract a complete file structure, the return value
* will be greater than bufEnd. You should gather more data and re-call the function.
* On certain errors, returns NULL;
*/
uchar *bpc_attrib_buf2fileFull(bpc_attrib_file *file, uchar *bufP, uchar *bufEnd)
{
uint fileNameLen, xattrNumEntries;
fileNameLen = getVarInt(&bufP, bufEnd);
if ( fileNameLen > BPC_MAXPATHLEN - 1 ) {
bpc_logErrf("bpc_attrib_buf2fileFull: got unreasonable file name length %d\n", fileNameLen);
return NULL;
}
bufP += fileNameLen;
bpc_attrib_xattrDeleteAll(file);
xattrNumEntries = getVarInt(&bufP, bufEnd);
if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_buf2fileFull: xattrNumEntries = %d\n", xattrNumEntries);
bufP = bpc_attrib_buf2file(file, bufP, bufEnd, xattrNumEntries, NULL);
return bufP;
}
/*
* Read the attribute file at dirPath/attribFilePath and populate dir
*/
int bpc_attrib_dirRead(bpc_attrib_dir *dir, char *dirPath, char *attribFilePath, int backupNum)
{
char attribPath[BPC_MAXPATHLEN];
bpc_fileZIO_fd fd;
size_t nRead;
uint32 magic = 0;
uchar buf[8 * 65536], *bufP;
STRUCT_STAT st;
char *p, *attribFileName;
bpc_attrib_attribFilePath(attribPath, dirPath, attribFilePath);
dir->digest.len = 0;
/*
* attribFileName points to the last portion of attribFilePath, or the whole
* string if it doesn't contain '/'
*/
if ( (attribFileName = strrchr(attribFilePath, '/')) ) {
attribFileName++;
} else {
attribFileName = attribFilePath;
}
if ( BPC_LogLevel >= 6 ) bpc_logMsgf("bpc_attrib_dirRead(%s); dirPath = %s, attribFilePath = %s, attribFileName = %s\n",
attribPath, dirPath, attribFilePath, attribFileName);
if ( (p = strchr(attribFileName, '_')) && !stat(attribPath, &st) && S_ISREG(st.st_mode) ) {
/*
* Explicit path name to new-style attrib file, and it exists; extract digest
*/
if ( !strcmp(p + 1, "0") ) return 0;
bpc_digest_str2digest(&dir->digest, p + 1);
if ( BPC_LogLevel >= 6 ) {
char str[256];
bpc_digest_digest2str(&dir->digest, str);
bpc_logMsgf("bpc_attrib_dirRead: called with attrib file %s: digest = %s, len = %d\n",
attribPath, str, dir->digest.len);
}
/*
* Write new type attrib files (since we found a new-style one)
*/
WriteOldStyleAttribFile = 0;
magic = BPC_ATTRIB_TYPE_XATTR;
} else if ( stat(attribPath, &st) || !S_ISREG(st.st_mode) || strchr(attribFileName, '_') ) {
DIR *dirOs;
struct dirent *dp;
int attribFileNameLen = strlen(attribFileName);
char attribDirPath[BPC_MAXPATHLEN];
/*
* Starting in 0.50, the attrib files are zero length with the digest encoded in
* the file name, so there is no file just called "attrib". Look in the directory
* to find it.
*/
strcpy(attribDirPath, attribPath);
if ( (p = strrchr(attribDirPath, '/')) ) {
*p = '\0';
} else {
strcpy(attribDirPath, ".");
}
if ( !(dirOs = opendir(attribDirPath)) ) {
/*
bpc_attrib.c view on Meta::CPAN
strcpy(attribPathTemp, attribPath);
strcat(attribPathTemp, "_0");
if ( rename(attribPath, attribPathTemp) ) {
bpc_logErrf("bpc_attrib_dirRead: rename of empty attrib file from %s to %s failed\n", attribPath, attribPathTemp);
return -1;
} else if ( BPC_LogLevel >= 6 ) {
bpc_logMsgf("bpc_attrib_dirRead: renamed empty attrib file %s -> %s\n", attribPath, attribPathTemp);
}
}
return 0;
}
if ( nRead < 4 ) {
bpc_logErrf("bpc_attrib_dirRead: can't read at least 4 bytes from %s\n", attribPath);
bpc_fileZIO_close(&fd);
return -1;
}
magic = CONV_BUF_TO_UINT32(buf);
}
if ( magic == BPC_ATTRIB_TYPE_DIGEST ) {
char attribPathNew[BPC_MAXPATHLEN];
int fdNum;
size_t digestLen = nRead - 4, attribPathLen = strlen(attribPath);
if ( nRead < 20 ) {
bpc_logErrf("bpc_attrib_dirRead: can't read at least 20 bytes from %s\n", attribPath);
return -1;
}
bpc_fileZIO_close(&fd);
if ( digestLen > sizeof(dir->digest.digest) ) digestLen = sizeof(dir->digest.digest);
memcpy(dir->digest.digest, buf + 4, digestLen);
dir->digest.len = digestLen;
if ( !KeepOldAttribFiles ) {
/*
* replace the attrib file with a new-style attrib file where the digest is encoded
* in a zero length file name
*/
if ( attribPathLen + dir->digest.len * 2 + 2 >= sizeof(attribPathNew) ) {
bpc_logErrf("bpc_attrib_dirRead: new digest path too long (%d, %d, %d)\n",
attribPathLen, dir->digest.len, sizeof(attribPathNew));
return -1;
}
strcpy(attribPathNew, attribPath);
attribPathNew[attribPathLen++] = '_';
bpc_digest_digest2str(&dir->digest, attribPathNew + attribPathLen);
if ( (fdNum = open(attribPathNew, O_WRONLY | O_CREAT | O_TRUNC, 0660)) < 0 ) {
bpc_logErrf("bpc_attrib_dirRead: can't open/create empty attrib file %s\n", attribPathNew);
return -1;
}
close(fdNum);
unlink(attribPath);
if ( BPC_LogLevel >= 4 ) bpc_logMsgf("bpc_attrib_dirRead: replaced %s with %s\n",
attribPath, attribPathNew);
}
}
if ( dir->digest.len > 0 ) {
/*
* Handle V4+ case - open the pool file directly
* For V3, digest.len == 0 since we opened the attrib file above (it is stored hardlinked in the backup
* directory; there is no digest)
*/
bpc_digest_md52path(attribPath, dir->compress, &dir->digest);
if ( bpc_fileZIO_open(&fd, attribPath, 0, dir->compress) ) {
bpc_logErrf("bpc_attrib_dirRead: can't open %s\n", attribPath);
return -1;
}
nRead = bpc_fileZIO_read(&fd, buf, sizeof(buf));
if ( nRead < 4 ) {
bpc_logErrf("bpc_attrib_dirRead: can't read at least 4 bytes from %s\n", attribPath);
bpc_fileZIO_close(&fd);
return -1;
}
magic = CONV_BUF_TO_UINT32(buf);
}
bufP = buf + 4;
if ( magic == BPC_ATTRIB_TYPE_XATTR ) {
int retry = 0;
while ( bufP < buf + nRead ) {
uint fileNameLen, xattrNumEntries;
char *fileName;
bpc_attrib_file *file;
uchar *bufPsave = bufP;
int xattrFixup = 0;
if ( nRead == sizeof(buf) && bufP > buf + nRead - 2 * BPC_MAXPATHLEN
&& read_more_data(&fd, buf, sizeof(buf), &nRead, &bufP, attribPath) ) {
bpc_fileZIO_close(&fd);
return -1;
}
fileNameLen = getVarInt(&bufP, buf + nRead);
if ( fileNameLen > BPC_MAXPATHLEN - 1 ) {
bpc_logErrf("bpc_attrib_dirRead: got unreasonable file name length %d\n", fileNameLen);
bpc_fileZIO_close(&fd);
return -1;
}
/*
* Save the fileName, but it's not NULL terminated yet.
* After we consume the next varint, we can safely NULL-terminate
* the fileName, which allows us to look up or create the file entry.
*/
fileName = (char*)bufP;
bufP += fileNameLen;
xattrNumEntries = getVarInt(&bufP, buf + nRead);
fileName[fileNameLen] = '\0';
file = bpc_attrib_fileGet(dir, fileName, 1);
bpc_attrib_fileInit(file, fileName, xattrNumEntries);
file->backupNum = backupNum;
bufP = bpc_attrib_buf2file(file, bufP, buf + nRead, xattrNumEntries, &xattrFixup);
dir->needRewrite |= xattrFixup;
if ( bufP > buf + nRead ) {
/*
* Need to get more data and try again. We have allocated file->name,
* and perhaps partially filled the xattr structure, which will be ok
* on the retry since the same structure will be used.
*/
if ( retry ) {
bpc_logErrf("bpc_attrib_dirRead: BOTCH: couldn't complete file conversion on retry (%ld,%ld,%ld)\n",
bufP - buf, bufPsave - buf, nRead);
bpc_fileZIO_close(&fd);
return -1;
}
if ( BPC_LogLevel >= 7 ) bpc_logMsgf("bpc_attrib_dirRead: retrying file conversion\n");
bufP = bufPsave;
if ( read_more_data(&fd, buf, sizeof(buf), &nRead, &bufP, attribPath) ) {
bpc_fileZIO_close(&fd);
return -1;
}
retry = 1;
} else {
retry = 0;
}
if ( !retry && BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirRead(%s): Got file %s: type = %d, mode = 0%o, uid/gid = %d/%d, size = %d\n",
attribPath, file->name, file->type, file->mode, file->uid, file->gid, file->size);
}
} else if ( magic == BPC_ATTRIB_TYPE_UNIX ) {
while ( bufP < buf + nRead ) {
uint fileNameLen;
char *fileName;
bpc_attrib_file *file;
int64 sizeDiv4GB;
uint type;
if ( nRead == sizeof(buf) && bufP > buf + nRead - 2 * BPC_MAXPATHLEN
&& read_more_data(&fd, buf, sizeof(buf), &nRead, &bufP, attribPath) ) {
bpc_fileZIO_close(&fd);
return -1;
}
fileNameLen = getVarInt_v3(&bufP, buf + nRead);
if ( fileNameLen > 2 * BPC_MAXPATHLEN - 16 ) {
bpc_logErrf("bpc_attrib_dirRead: got unreasonable file name length %d\n", fileNameLen);
bpc_fileZIO_close(&fd);
return -1;
}
/*
* Save the fileName, but it's not NULL terminated yet.
* After we get the next data, we can safely NULL-terminate the fileName.
*/
fileName = (char*)bufP;
bufP += fileNameLen;
type = getVarInt_v3(&bufP, buf + nRead);
fileName[fileNameLen] = '\0';
file = bpc_attrib_fileGet(dir, fileName, 1);
bpc_attrib_fileInit(file, fileName, 0);
file->type = type;
file->mode = getVarInt_v3(&bufP, buf + nRead);
file->uid = getVarInt_v3(&bufP, buf + nRead);
file->gid = getVarInt_v3(&bufP, buf + nRead);
sizeDiv4GB = getVarInt_v3(&bufP, buf + nRead);
file->size = (sizeDiv4GB << 32) + getVarInt_v3(&bufP, buf + nRead);
file->mtime = CONV_BUF_TO_UINT32(bufP); bufP += 4;
file->compress = dir->compress;
file->backupNum = backupNum;
if ( BPC_LogLevel >= 8 ) bpc_logMsgf("bpc_attrib_dirRead(%s): Got v3 file %s: type = %d, mode = 0%o, uid/gid = %d/%d, size = %d\n",
attribPath, file->name, file->type, file->mode, file->uid, file->gid, file->size);
}
} else {
bpc_logErrf("Unexpected magic number 0x%x read from %s\n", magic, attribPath);
return -1;
}
/* TODO: make sure we are at EOF? */
bpc_fileZIO_close(&fd);
return 0;
}
typedef struct {
uchar *bufP;
uchar *bufEnd;
uint numEntries;
} buf_info;
typedef struct {
bpc_poolWrite_info fd;
uchar buf[4 * 65536];
uchar *bufP;
} write_info;
static void write_file_flush(write_info *info)
{
if ( info->bufP > info->buf ) {
if ( BPC_LogLevel >= 7 ) bpc_logMsgf("write_file_flush: writing %lu bytes to pool\n", (unsigned long)(info->bufP - info->buf));
bpc_poolWrite_write(&info->fd, info->buf, info->bufP - info->buf);
}
info->bufP = info->buf;
}
static void bpc_attrib_xattrWrite(bpc_attrib_xattr *xattr, buf_info *info)
{
setVarInt(&info->bufP, info->bufEnd, xattr->key.keyLen);
setVarInt(&info->bufP, info->bufEnd, xattr->valueLen);
if ( xattr->key.keyLen >= 1 && info->bufP + xattr->key.keyLen <= info->bufEnd ) {
memcpy(info->bufP, xattr->key.key, xattr->key.keyLen);
if ( info->bufP[xattr->key.keyLen - 1] != 0x0 ) {
info->bufP[xattr->key.keyLen - 1] = 0x0;
bpc_logMsgf("bpc_attrib_xattrWrite: BOTCH: truncated xattr name '%s' to match keyLen %u\n", info->bufP, xattr->key.keyLen);
}
}
info->bufP += xattr->key.keyLen;
if ( info->bufP + xattr->valueLen <= info->bufEnd ) {
memcpy(info->bufP, xattr->value, xattr->valueLen);
}
info->bufP += xattr->valueLen;
info->numEntries++;
}
/*
* Write a file structure to the memory buffer. Returns the next unused buffer
* pointer. If the buffer is exhausted, no data is written past the buffer end,
* Therefore, if the return value is greater than bufEnd, then the conversion
* failed to fit. The routine can be called again with at least (bufP - buf)
( run in 1.052 second using v1.01-cache-2.11-cpan-39bf76dae61 )