Alien-SVN
view release on metacpan or search on metacpan
src/subversion/tools/dev/fsfs-reorg.c view on Meta::CPAN
/* fsfs-reorg.c -- prototypic tool to reorganize packed FSFS repositories
* to reduce seeks
*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*/
#include <assert.h>
#include <apr.h>
#include <apr_general.h>
#include <apr_file_io.h>
#include <apr_poll.h>
#include "svn_pools.h"
#include "svn_diff.h"
#include "svn_io.h"
#include "svn_utf.h"
#include "svn_dirent_uri.h"
#include "svn_sorts.h"
#include "svn_delta.h"
#include "svn_hash.h"
#include "private/svn_string_private.h"
#include "private/svn_subr_private.h"
#include "private/svn_dep_compat.h"
#ifndef _
#define _(x) x
#endif
#define ERROR_TAG "fsfs-reporg: "
/* forward declarations */
typedef struct noderev_t noderev_t;
typedef struct revision_info_t revision_info_t;
/* A FSFS rev file is sequence of fragments and unused space (the latter
* only being inserted by this tool and not during ordinary SVN operation).
*
* This type defines the type of any fragment.
*
* Please note that the classification as "property", "dir" or "file"
* fragments is only to be used while determining the future placement
* of a representation. If the rep is shared, the same rep may be used
* as *any* of the 3 kinds.
*/
enum fragment_kind_t
{
/* the 2 number line containing changes and root node offsets */
header_fragment,
/* list of all changes in a revision */
changes_fragment,
/* (the textual representation of) a noderev */
noderev_fragment,
/* a property rep (including PLAIN / DELTA header) */
property_fragment,
/* a directory rep (including PLAIN / DELTA header) */
dir_fragment,
/* a file rep (including PLAIN / DELTA header) */
file_fragment
};
/* A fragment. This is used to represent the final ordering, i.e. there
* will be an array containing elements of this type that basically put
* a fragment at some location in the target file.
*/
typedef struct fragment_t
{
/* position in the target file */
apr_size_t position;
/* kind of fragment */
enum fragment_kind_t kind;
/* pointer to the fragment struct; type depends on KIND */
void *data;
} fragment_t;
/* Location info for a single revision.
*/
typedef struct revision_location_t
{
/* pack file offset (manifest value), 0 for non-packed files */
apr_size_t offset;
/* offset of the changes list relative to OFFSET */
apr_size_t changes;
/* length of the changes list on bytes */
apr_size_t changes_len;
/* first offset behind the revision data in the pack file (file length
* for non-packed revs) */
apr_size_t end;
} revision_location_t;
/* Absolute position and size of some item.
*/
typedef struct location_t
{
/* absolute offset in the file */
apr_size_t offset;
/* item length in bytes */
apr_size_t size;
} location_t;
/* A parsed directory entry. Note that instances of this struct may be
* shared between different DIRECTORY_T containers.
*/
typedef struct direntry_t
{
/* (local) entry / path name */
const char *name;
/* strlen (name) */
apr_size_t name_len;
/* node rev providing ID and representation(s) */
noderev_t *node;
} direntry_t;
/* Representation of a parsed directory content.
*/
typedef struct directory_t
{
/* array of pointers to DIRENTRY_T */
apr_array_header_t *entries;
/* MD5 of the textual representation. Will be set lazily as a side-effect
* of determining the length of this dir's textual representation. */
unsigned char target_md5[16];
/* (expanded) length of the textual representation.
* Determined lazily during the write process. */
apr_size_t size;
} directory_t;
/* A representation fragment.
*/
typedef struct representation_t
{
/* location in the source file */
location_t original;
/* location in the reordered target file */
location_t target;
/* length of the PLAIN / DELTA line in the source file in bytes */
apr_size_t header_size;
/* deltification base, or NULL if there is none */
struct representation_t *delta_base;
/* revision that contains this representation
* (may be referenced by other revisions, though) */
revision_info_t *revision;
/* representation content parsed as a directory. This will be NULL, if
* *no* directory noderev uses this representation. */
directory_t *dir;
/* the source content has a PLAIN header, so we may simply copy the
* source content into the target */
svn_boolean_t is_plain;
/* coloring flag used in the reordering algorithm to keep track of
* representations that still need to be placed. */
svn_boolean_t covered;
} representation_t;
/* A node rev.
*/
struct noderev_t
{
/* location within the source file */
location_t original;
/* location within the reorganized target file. */
location_t target;
/* predecessor node, or NULL if there is none */
noderev_t *predecessor;
/* content representation; may be NULL if there is none */
representation_t *text;
/* properties representation; may be NULL if there is none */
representation_t *props;
/* revision that this noderev belongs to */
revision_info_t *revision;
/* coloring flag used in the reordering algorithm to keep track of
* representations that still need to be placed. */
svn_boolean_t covered;
};
/* Represents a single revision.
* There will be only one instance per revision. */
struct revision_info_t
{
/* number of this revision */
svn_revnum_t revision;
/* position in the source file */
revision_location_t original;
/* position in the reorganized target file */
revision_location_t target;
/* noderev of the root directory */
noderev_t *root_noderev;
/* all noderevs_t of this revision (ordered by source file offset),
* i.e. those that point back to this struct */
apr_array_header_t *node_revs;
/* all representation_t of this revision (ordered by source file offset),
* i.e. those that point back to this struct */
apr_array_header_t *representations;
};
/* Represents a packed revision file.
*/
typedef struct revision_pack_t
{
/* first revision in the pack file */
svn_revnum_t base;
/* revision_info_t* of all revisions in the pack file; in revision order. */
apr_array_header_t *info;
/* list of fragments to place in the target pack file; in target order. */
apr_array_header_t *fragments;
/* source pack file length */
apr_size_t filesize;
/* temporary value. Equal to the number of bytes in the target pack file
* already allocated to fragments. */
apr_size_t target_offset;
} revision_pack_t;
/* Cache for revision source content. All content is stored in DATA and
* the HASH maps revision number to an svn_string_t instance whose data
* member points into DATA.
*
* Once TOTAL_SIZE exceeds LIMIT, all content will be discarded. Similarly,
* the hash gets cleared every 10000 insertions to keep the HASH_POOL
* memory usage in check.
*/
typedef struct content_cache_t
{
/* pool used for HASH */
apr_pool_t *hash_pool;
/* svn_revnum_t -> svn_string_t.
* The strings become (potentially) invalid when adding new cache entries. */
apr_hash_t *hash;
/* data buffer. the first TOTAL_SIZE bytes are actually being used. */
char *data;
/* DATA capacity */
apr_size_t limit;
/* number of bytes used in DATA */
apr_size_t total_size;
/* number of insertions since the last hash cleanup */
apr_size_t insert_count;
} content_cache_t;
/* A cached directory. In contrast to directory_t, this stored the data as
* the plain hash that the normal FSFS will use to serialize & diff dirs.
*/
typedef struct dir_cache_entry_t
{
/* revision containing the representation */
svn_revnum_t revision;
/* offset of the representation within that revision */
apr_size_t offset;
/* key-value representation of the directory entries */
apr_hash_t *hash;
} dir_cache_entry_t;
/* Directory cache. (revision, offset) will be mapped directly into the
* ENTRIES array of ENTRY_COUNT buckets (many entries will be NULL).
* Two alternating pools will be used to allocate dir content.
*
* If the INSERT_COUNT exceeds a given limit, the pools get exchanged and
* the older of the two will be cleared. This is to keep dir objects valid
* for at least one insertion.
*/
typedef struct dir_cache_t
{
/* fixed-size array of ENTRY_COUNT elements */
dir_cache_entry_t *entries;
src/subversion/tools/dev/fsfs-reorg.c view on Meta::CPAN
pool, scratch_pool));
else if (key_matches(&key, "props"))
SVN_ERR(parse_representation(&result->props, fs, file_content,
&value, revision_info,
pool, scratch_pool));
}
/* link noderev to revision info */
result->revision = revision_info;
result->original.size = offset - result->original.offset;
svn_sort__array_insert(&result,
revision_info->node_revs,
svn_sort__bsearch_lower_bound(&offset,
revision_info->node_revs,
compare_noderev_offsets));
/* if this is a directory, read and process that recursively */
if (is_dir)
SVN_ERR(parse_dir(fs, file_content, result->text,
pool, scratch_pool));
/* done */
svn_pool_destroy(scratch_pool);
*noderev = result;
return SVN_NO_ERROR;
}
/* Simple utility to print a REVISION number and make it appear immediately.
*/
static void
print_progress(svn_revnum_t revision)
{
printf("%8ld", revision);
fflush(stdout);
}
/* Read the content of the pack file staring at revision BASE and store it
* in FS. Use POOL for allocations.
*/
static svn_error_t *
read_pack_file(fs_fs_t *fs,
svn_revnum_t base,
apr_pool_t *pool)
{
apr_array_header_t *manifest = NULL;
apr_pool_t *local_pool = svn_pool_create(pool);
apr_pool_t *iter_pool = svn_pool_create(local_pool);
int i;
svn_stringbuf_t *file_content;
revision_pack_t *revisions;
const char *pack_folder = get_pack_folder(fs, base, local_pool);
/* read the whole pack file into memory */
SVN_ERR(read_rev_or_pack_file(&file_content, fs, base, local_pool));
/* create the revision container */
revisions = apr_pcalloc(pool, sizeof(*revisions));
revisions->base = base;
revisions->fragments = NULL;
revisions->info = apr_array_make(pool,
fs->max_files_per_dir,
sizeof(revision_info_t*));
revisions->filesize = file_content->len;
APR_ARRAY_PUSH(fs->packs, revision_pack_t*) = revisions;
/* parse the manifest file */
SVN_ERR(read_manifest(&manifest, fs, pack_folder, local_pool));
if (manifest->nelts != fs->max_files_per_dir)
return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, NULL);
/* process each revision in the pack file */
for (i = 0; i < manifest->nelts; ++i)
{
apr_size_t root_node_offset;
svn_string_t rev_content;
/* create the revision info for the current rev */
revision_info_t *info = apr_pcalloc(pool, sizeof(*info));
info->node_revs = apr_array_make(iter_pool, 4, sizeof(noderev_t*));
info->representations = apr_array_make(iter_pool, 4, sizeof(representation_t*));
info->revision = base + i;
info->original.offset = APR_ARRAY_IDX(manifest, i, apr_size_t);
info->original.end = i+1 < manifest->nelts
? APR_ARRAY_IDX(manifest, i+1 , apr_size_t)
: file_content->len;
SVN_ERR(read_revision_header(&info->original.changes,
&info->original.changes_len,
&root_node_offset,
file_content,
APR_ARRAY_IDX(manifest, i , apr_size_t),
info->original.end,
iter_pool));
/* put it into our containers */
APR_ARRAY_PUSH(revisions->info, revision_info_t*) = info;
APR_ARRAY_PUSH(fs->revisions, revision_info_t*) = info;
/* cache the revision content */
rev_content.data = file_content->data + info->original.offset;
rev_content.len = info->original.end - info->original.offset;
set_cached_content(fs->cache, info->revision, &rev_content);
/* parse the revision content recursively. */
SVN_ERR(read_noderev(&info->root_noderev, fs, file_content,
root_node_offset, info, pool, iter_pool));
/* copy dynamically grown containers from temp into result pool */
info->node_revs = apr_array_copy(pool, info->node_revs);
info->representations = apr_array_copy(pool, info->representations);
/* destroy temps */
svn_pool_clear(iter_pool);
}
/* one more pack file processed */
print_progress(base);
svn_pool_destroy(local_pool);
return SVN_NO_ERROR;
}
/* Read the content of REVSION file and store it in FS.
* Use POOL for allocations.
*/
static svn_error_t *
read_revision_file(fs_fs_t *fs,
svn_revnum_t revision,
apr_pool_t *pool)
{
apr_size_t root_node_offset;
apr_pool_t *local_pool = svn_pool_create(pool);
svn_stringbuf_t *file_content;
svn_string_t rev_content;
revision_pack_t *revisions = apr_pcalloc(pool, sizeof(*revisions));
revision_info_t *info = apr_pcalloc(pool, sizeof(*info));
/* read the whole pack file into memory */
SVN_ERR(read_rev_or_pack_file(&file_content, fs, revision, local_pool));
/* create the revision info for the current rev */
info->node_revs = apr_array_make(pool, 4, sizeof(noderev_t*));
info->representations = apr_array_make(pool, 4, sizeof(representation_t*));
info->revision = revision;
info->original.offset = 0;
info->original.end = file_content->len;
SVN_ERR(read_revision_header(&info->original.changes,
&info->original.changes_len,
&root_node_offset,
file_content,
0,
info->original.end,
local_pool));
/* put it into our containers */
APR_ARRAY_PUSH(fs->revisions, revision_info_t*) = info;
/* create a pseudo-pack file container for just this rev to keep our
* data structures as uniform as possible.
*/
revisions->base = revision;
revisions->fragments = NULL;
revisions->info = apr_array_make(pool, 1, sizeof(revision_info_t*));
revisions->filesize = file_content->len;
APR_ARRAY_PUSH(revisions->info, revision_info_t*) = info;
APR_ARRAY_PUSH(fs->packs, revision_pack_t*) = revisions;
/* cache the revision content */
rev_content.data = file_content->data + info->original.offset;
rev_content.len = info->original.end - info->original.offset;
set_cached_content(fs->cache, info->revision, &rev_content);
/* parse the revision content recursively. */
SVN_ERR(read_noderev(&info->root_noderev, fs, file_content,
root_node_offset, info,
pool, local_pool));
APR_ARRAY_PUSH(info->node_revs, noderev_t*) = info->root_noderev;
/* show progress every 1000 revs or so */
if (revision % fs->max_files_per_dir == 0)
print_progress(revision);
svn_pool_destroy(local_pool);
return SVN_NO_ERROR;
}
/* Read the repository at PATH beginning with revision START_REVISION and
* return the result in *FS. Allocate caches with MEMSIZE bytes total
* capacity. Use POOL for non-cache allocations.
*/
static svn_error_t *
read_revisions(fs_fs_t **fs,
const char *path,
svn_revnum_t start_revision,
apr_size_t memsize,
apr_pool_t *pool)
{
svn_revnum_t revision;
apr_size_t content_cache_size;
apr_size_t window_cache_size;
apr_size_t dir_cache_size;
/* determine cache sizes */
if (memsize < 100)
memsize = 100;
content_cache_size = memsize * 7 / 10 > 4000 ? 4000 : memsize * 7 / 10;
window_cache_size = memsize * 2 / 10 * 1024 * 1024;
dir_cache_size = (memsize / 10) * 16000;
/* read repo format and such */
SVN_ERR(fs_open(fs, path, pool));
/* create data containers and caches */
(*fs)->start_revision = start_revision
- (start_revision % (*fs)->max_files_per_dir);
(*fs)->revisions = apr_array_make(pool,
(*fs)->max_revision + 1 - (*fs)->start_revision,
sizeof(revision_info_t *));
(*fs)->packs = apr_array_make(pool,
((*fs)->min_unpacked_rev - (*fs)->start_revision)
/ (*fs)->max_files_per_dir,
sizeof(revision_pack_t *));
(*fs)->null_base = apr_pcalloc(pool, sizeof(*(*fs)->null_base));
(*fs)->cache = create_content_cache
(apr_allocator_owner_get
(svn_pool_create_allocator(FALSE)),
content_cache_size * 1024 * 1024);
(*fs)->dir_cache = create_dir_cache
(apr_allocator_owner_get
(svn_pool_create_allocator(FALSE)),
dir_cache_size);
(*fs)->window_cache = create_window_cache
(apr_allocator_owner_get
(svn_pool_create_allocator(FALSE)),
10000, window_cache_size);
/* read all packed revs */
for ( revision = start_revision
; revision < (*fs)->min_unpacked_rev
; revision += (*fs)->max_files_per_dir)
SVN_ERR(read_pack_file(*fs, revision, pool));
/* read non-packed revs */
for ( ; revision <= (*fs)->max_revision; ++revision)
SVN_ERR(read_revision_file(*fs, revision, pool));
return SVN_NO_ERROR;
}
/* Return the maximum number of decimal digits required to represent offsets
* in the given PACK file.
*/
static apr_size_t
get_max_offset_len(const revision_pack_t *pack)
{
/* the pack files may grow a few percent.
* Fudge it up to be on safe side.
*/
apr_size_t max_future_size = pack->filesize * 2 + 10000;
apr_size_t result = 0;
while (max_future_size > 0)
{
++result;
max_future_size /= 10;
}
return result;
}
/* Create the fragments container in PACK and add revision header fragments
* to it. Use POOL for allocations.
*/
static svn_error_t *
add_revisions_pack_heads(revision_pack_t *pack,
apr_pool_t *pool)
{
int i;
revision_info_t *info;
apr_size_t offset_len = get_max_offset_len(pack);
fragment_t fragment;
/* allocate fragment arrays */
int fragment_count = 1;
for (i = 0; i < pack->info->nelts; ++i)
{
info = APR_ARRAY_IDX(pack->info, i, revision_info_t*);
fragment_count += info->node_revs->nelts
+ info->representations->nelts
+ 2;
}
pack->target_offset = pack->info->nelts > 1 ? 64 : 0;
pack->fragments = apr_array_make(pool,
fragment_count,
sizeof(fragment_t));
/* put revision headers first */
for (i = 0; i < pack->info->nelts - 1; ++i)
{
info = APR_ARRAY_IDX(pack->info, i, revision_info_t*);
info->target.offset = pack->target_offset;
fragment.data = info;
fragment.kind = header_fragment;
fragment.position = pack->target_offset;
APR_ARRAY_PUSH(pack->fragments, fragment_t) = fragment;
pack->target_offset += 2 * offset_len + 3;
}
info = APR_ARRAY_IDX(pack->info, pack->info->nelts - 1, revision_info_t*);
info->target.offset = pack->target_offset;
/* followed by the changes list */
for (i = 0; i < pack->info->nelts; ++i)
{
info = APR_ARRAY_IDX(pack->info, i, revision_info_t*);
info->target.changes = pack->target_offset - info->target.offset;
info->target.changes_len = info->original.changes_len;
fragment.data = info;
fragment.kind = changes_fragment;
fragment.position = pack->target_offset;
APR_ARRAY_PUSH(pack->fragments, fragment_t) = fragment;
pack->target_offset += info->original.changes_len;
}
return SVN_NO_ERROR;
}
/* For the revision given by INFO in FS, return the fragment container in
* *FRAGMENTS and the current placement offset in *CURRENT_POS.
*/
static svn_error_t *
get_target_offset(apr_size_t **current_pos,
apr_array_header_t **fragments,
fs_fs_t *fs,
revision_info_t *info)
{
int i;
revision_pack_t *pack;
svn_revnum_t revision = info->revision;
/* identify the pack object */
if (fs->min_unpacked_rev > revision)
{
i = (revision - fs->start_revision) / fs->max_files_per_dir;
}
else
{
i = (fs->min_unpacked_rev - fs->start_revision) / fs->max_files_per_dir;
i += revision - fs->min_unpacked_rev;
}
/* extract the desired info from it */
pack = APR_ARRAY_IDX(fs->packs, i, revision_pack_t*);
*current_pos = &pack->target_offset;
*fragments = pack->fragments;
return SVN_NO_ERROR;
}
/* forward declaration */
static svn_error_t *
add_noderev_recursively(fs_fs_t *fs,
noderev_t *node,
apr_pool_t *pool);
/* Place fragments for the given REPRESENTATION of the given KIND, iff it
* has not been covered, yet. Place the base reps along the deltification
* chain as far as those reps have not been covered, yet. If REPRESENTATION
* is a directory, recursively place its elements.
*
* Use POOL for allocations.
*/
static svn_error_t *
add_representation_recursively(fs_fs_t *fs,
representation_t *representation,
enum fragment_kind_t kind,
apr_pool_t *pool)
{
apr_size_t *current_pos;
apr_array_header_t *fragments;
fragment_t fragment;
/* place REPRESENTATION only once and only if it exists and will not
* be covered later as a directory. */
if ( representation == NULL
|| representation->covered
|| (representation->dir && kind != dir_fragment)
|| representation == fs->null_base)
return SVN_NO_ERROR;
/* add and place a fragment for REPRESENTATION */
SVN_ERR(get_target_offset(¤t_pos, &fragments,
fs, representation->revision));
representation->target.offset = *current_pos;
representation->covered = TRUE;
fragment.data = representation;
fragment.kind = kind;
fragment.position = *current_pos;
APR_ARRAY_PUSH(fragments, fragment_t) = fragment;
/* determine the size of data to be added to the target file */
if ( kind != dir_fragment
&& representation->delta_base && representation->delta_base->dir)
{
/* base rep is a dir -> would change -> need to store it as fulltext
* in our target file */
apr_pool_t *text_pool = svn_pool_create(pool);
svn_stringbuf_t *content;
SVN_ERR(get_combined_window(&content, fs, representation, text_pool));
representation->target.size = content->len;
*current_pos += representation->target.size + 13;
svn_pool_destroy(text_pool);
}
else
if ( kind == dir_fragment
|| (representation->delta_base && representation->delta_base->dir))
{
/* deltified directories may grow considerably */
if (representation->original.size < 50)
*current_pos += 300;
else
*current_pos += representation->original.size * 3 + 150;
}
else
{
/* plain / deltified content will not change but the header may
* grow slightly due to larger offsets. */
representation->target.size = representation->original.size;
if (representation->delta_base &&
(representation->delta_base != fs->null_base))
*current_pos += representation->original.size + 50;
else
*current_pos += representation->original.size + 13;
}
/* follow the delta chain and place base revs immediately after this */
if (representation->delta_base)
SVN_ERR(add_representation_recursively(fs,
representation->delta_base,
kind,
pool));
/* finally, recurse into directories */
if (representation->dir)
{
int i;
apr_array_header_t *entries = representation->dir->entries;
for (i = 0; i < entries->nelts; ++i)
{
direntry_t *entry = APR_ARRAY_IDX(entries, i, direntry_t *);
if (entry->node)
SVN_ERR(add_noderev_recursively(fs, entry->node, pool));
}
}
return SVN_NO_ERROR;
}
/* Place fragments for the given NODE in FS, iff it has not been covered,
* yet. Place the reps (text, props) immediately after the node.
*
* Use POOL for allocations.
*/
static svn_error_t *
add_noderev_recursively(fs_fs_t *fs,
noderev_t *node,
apr_pool_t *pool)
{
apr_size_t *current_pos;
apr_array_header_t *fragments;
fragment_t fragment;
/* don't add it twice */
if (node->covered)
return SVN_NO_ERROR;
/* add and place a fragment for NODE */
SVN_ERR(get_target_offset(¤t_pos, &fragments, fs, node->revision));
node->covered = TRUE;
node->target.offset = *current_pos;
fragment.data = node;
fragment.kind = noderev_fragment;
fragment.position = *current_pos;
APR_ARRAY_PUSH(fragments, fragment_t) = fragment;
/* size may slightly increase */
*current_pos += node->original.size + 40;
/* recurse into representations */
if (node->text && node->text->dir)
SVN_ERR(add_representation_recursively(fs, node->text, dir_fragment, pool));
else
SVN_ERR(add_representation_recursively(fs, node->text, file_fragment, pool));
SVN_ERR(add_representation_recursively(fs, node->props, property_fragment, pool));
return SVN_NO_ERROR;
}
/* Place a fragment for the last revision in PACK. Use POOL for allocations.
*/
static svn_error_t *
add_revisions_pack_tail(revision_pack_t *pack,
apr_pool_t *pool)
{
int i;
revision_info_t *info;
apr_size_t offset_len = get_max_offset_len(pack);
fragment_t fragment;
/* put final revision header last and fix up revision lengths */
info = APR_ARRAY_IDX(pack->info, pack->info->nelts-1, revision_info_t*);
fragment.data = info;
fragment.kind = header_fragment;
fragment.position = pack->target_offset;
APR_ARRAY_PUSH(pack->fragments, fragment_t) = fragment;
pack->target_offset += 2 * offset_len + 3;
/* end of target file reached. Store that info in all revs. */
for (i = 0; i < pack->info->nelts; ++i)
{
info = APR_ARRAY_IDX(pack->info, i, revision_info_t*);
info->target.end = pack->target_offset;
}
return SVN_NO_ERROR;
}
/* Place all fragments for all revisions / packs in FS.
* Use POOL for allocations.
*/
static svn_error_t *
reorder_revisions(fs_fs_t *fs,
apr_pool_t *pool)
{
int i, k;
/* headers and changes */
for (i = 0; i < fs->packs->nelts; ++i)
{
revision_pack_t *pack = APR_ARRAY_IDX(fs->packs, i, revision_pack_t*);
SVN_ERR(add_revisions_pack_heads(pack, pool));
}
/* representations & nodes */
for (i = fs->revisions->nelts-1; i >= 0; --i)
{
revision_info_t *info = APR_ARRAY_IDX(fs->revisions, i, revision_info_t*);
for (k = info->node_revs->nelts - 1; k >= 0; --k)
{
noderev_t *node = APR_ARRAY_IDX(info->node_revs, k, noderev_t*);
SVN_ERR(add_noderev_recursively(fs, node, pool));
}
if (info->revision % fs->max_files_per_dir == 0)
print_progress(info->revision);
}
/* pack file tails */
for (i = 0; i < fs->packs->nelts; ++i)
{
revision_pack_t *pack = APR_ARRAY_IDX(fs->packs, i, revision_pack_t*);
SVN_ERR(add_revisions_pack_tail(pack, pool));
}
return SVN_NO_ERROR;
}
/* forward declaration */
static svn_error_t *
get_fragment_content(svn_string_t **content,
fs_fs_t *fs,
fragment_t *fragment,
apr_pool_t *pool);
/* Directory content may change and with it, the deltified representations
* may significantly. This function causes all directory target reps in
* PACK of FS to be built and their new MD5 as well as rep sizes be updated.
* We must do that before attempting to write noderevs.
*
* Use POOL for allocations.
*/
static svn_error_t *
update_noderevs(fs_fs_t *fs,
revision_pack_t *pack,
apr_pool_t *pool)
{
int i;
apr_pool_t *itempool = svn_pool_create(pool);
for (i = 0; i < pack->fragments->nelts; ++i)
{
fragment_t *fragment = &APR_ARRAY_IDX(pack->fragments, i, fragment_t);
if (fragment->kind == dir_fragment)
{
svn_string_t *content;
/* request updated rep content but ignore the result.
* We are only interested in the MD5, content and rep size updates. */
SVN_ERR(get_fragment_content(&content, fs, fragment, itempool));
svn_pool_clear(itempool);
}
}
svn_pool_destroy(itempool);
return SVN_NO_ERROR;
}
/* Determine the target size of the FRAGMENT in FS and return the value
* in *LENGTH. If ADD_PADDING has been set, slightly fudge the numbers
* to account for changes in offset lengths etc. Use POOL for temporary
* allocations.
*/
static svn_error_t *
get_content_length(apr_size_t *length,
fs_fs_t *fs,
fragment_t *fragment,
svn_boolean_t add_padding,
apr_pool_t *pool)
{
svn_string_t *content;
SVN_ERR(get_fragment_content(&content, fs, fragment, pool));
if (add_padding)
switch (fragment->kind)
{
case dir_fragment:
*length = content->len + 16;
break;
case noderev_fragment:
*length = content->len + 3;
break;
default:
*length = content->len;
break;
}
else
*length = content->len;
return SVN_NO_ERROR;
}
/* Move the FRAGMENT to global file offset NEW_POSITION. Update the target
* location info of the underlying object as well.
*/
static void
move_fragment(fragment_t *fragment,
apr_size_t new_position)
{
revision_info_t *info;
representation_t *representation;
noderev_t *node;
/* move the fragment */
fragment->position = new_position;
/* move the underlying object */
switch (fragment->kind)
{
case header_fragment:
info = fragment->data;
info->target.offset = new_position;
break;
case changes_fragment:
info = fragment->data;
info->target.changes = new_position - info->target.offset;
break;
case property_fragment:
case file_fragment:
case dir_fragment:
representation = fragment->data;
representation->target.offset = new_position;
break;
case noderev_fragment:
node = fragment->data;
node->target.offset = new_position;
break;
}
}
/* Move the fragments in PACK's target fragment list to their final offsets.
* This may require several iterations if the fudge factors turned out to
* be insufficient. Use POOL for allocations.
*/
static svn_error_t *
pack_revisions(fs_fs_t *fs,
revision_pack_t *pack,
apr_pool_t *pool)
{
int i;
fragment_t *fragment, *next;
svn_boolean_t needed_to_expand;
revision_info_t *info;
apr_size_t current_pos, len, old_len;
apr_pool_t *itempool = svn_pool_create(pool);
/* update all directory reps. Chances are that most of the target rep
* sizes are now close to accurate. */
SVN_ERR(update_noderevs(fs, pack, pool));
/* compression phase: pack all fragments tightly with only a very small
* fudge factor. This should cause offsets to shrink, thus all the
* actual fragment rate should tend to be even smaller afterwards. */
current_pos = pack->info->nelts > 1 ? 64 : 0;
for (i = 0; i + 1 < pack->fragments->nelts; ++i)
{
fragment = &APR_ARRAY_IDX(pack->fragments, i, fragment_t);
SVN_ERR(get_content_length(&len, fs, fragment, TRUE, itempool));
move_fragment(fragment, current_pos);
current_pos += len;
svn_pool_clear(itempool);
}
/* don't forget the final fragment (last revision's revision header) */
fragment = &APR_ARRAY_IDX(pack->fragments, pack->fragments->nelts-1, fragment_t);
fragment->position = current_pos;
/* expansion phase: check whether all fragments fit into their allotted
* slots. Grow them geometrically if they don't fit. Retry until they
* all do fit.
* Note: there is an upper limit to which fragments can grow. So, this
* loop will terminate. Often, no expansion will be necessary at all. */
do
{
needed_to_expand = FALSE;
current_pos = pack->info->nelts > 1 ? 64 : 0;
for (i = 0; i + 1 < pack->fragments->nelts; ++i)
{
fragment = &APR_ARRAY_IDX(pack->fragments, i, fragment_t);
next = &APR_ARRAY_IDX(pack->fragments, i + 1, fragment_t);
old_len = next->position - fragment->position;
SVN_ERR(get_content_length(&len, fs, fragment, FALSE, itempool));
if (len > old_len)
{
len = (apr_size_t)(len * 1.1) + 10;
needed_to_expand = TRUE;
}
else
len = old_len;
if (i == pack->info->nelts - 1)
{
info = APR_ARRAY_IDX(pack->info, pack->info->nelts - 1, revision_info_t*);
info->target.offset = current_pos;
}
move_fragment(fragment, current_pos);
current_pos += len;
svn_pool_clear(itempool);
}
fragment = &APR_ARRAY_IDX(pack->fragments, pack->fragments->nelts-1, fragment_t);
fragment->position = current_pos;
/* update the revision
* sizes (they all end at the end of the pack file now) */
SVN_ERR(get_content_length(&len, fs, fragment, FALSE, itempool));
current_pos += len;
for (i = 0; i < pack->info->nelts; ++i)
{
info = APR_ARRAY_IDX(pack->info, i, revision_info_t*);
info->target.end = current_pos;
}
}
while (needed_to_expand);
svn_pool_destroy(itempool);
return SVN_NO_ERROR;
}
/* Write reorg'ed target content for PACK in FS. Use POOL for allocations.
*/
static svn_error_t *
write_revisions(fs_fs_t *fs,
revision_pack_t *pack,
apr_pool_t *pool)
{
int i;
fragment_t *fragment = NULL;
svn_string_t *content;
apr_pool_t *itempool = svn_pool_create(pool);
apr_pool_t *iterpool = svn_pool_create(pool);
apr_file_t *file;
apr_size_t current_pos = 0;
svn_stringbuf_t *null_buffer = svn_stringbuf_create_empty(iterpool);
/* create the target file */
const char *dir = apr_psprintf(iterpool, "%s/new/%ld%s",
fs->path, pack->base / fs->max_files_per_dir,
pack->info->nelts > 1 ? ".pack" : "");
SVN_ERR(svn_io_make_dir_recursively(dir, pool));
SVN_ERR(svn_io_file_open(&file,
pack->info->nelts > 1
? apr_psprintf(iterpool, "%s/pack", dir)
: apr_psprintf(iterpool, "%s/%ld", dir, pack->base),
APR_WRITE | APR_CREATE | APR_BUFFERED,
APR_OS_DEFAULT,
iterpool));
/* write all fragments */
for (i = 0; i < pack->fragments->nelts; ++i)
{
apr_size_t padding;
/* get fragment content to write */
fragment = &APR_ARRAY_IDX(pack->fragments, i, fragment_t);
SVN_ERR(get_fragment_content(&content, fs, fragment, itempool));
SVN_ERR_ASSERT(fragment->position >= current_pos);
/* number of bytes between this and the previous fragment */
if ( fragment->kind == header_fragment
&& i+1 < pack->fragments->nelts)
/* special case: header fragments are aligned to the slot end */
padding = APR_ARRAY_IDX(pack->fragments, i+1, fragment_t).position -
content->len - current_pos;
else
/* standard case: fragments are aligned to the slot start */
padding = fragment->position - current_pos;
/* write padding between fragments */
if (padding)
{
while (null_buffer->len < padding)
svn_stringbuf_appendbyte(null_buffer, 0);
SVN_ERR(svn_io_file_write_full(file,
null_buffer->data,
padding,
NULL,
itempool));
current_pos += padding;
}
/* write fragment content */
SVN_ERR(svn_io_file_write_full(file,
content->data,
content->len,
NULL,
itempool));
current_pos += content->len;
svn_pool_clear(itempool);
}
apr_file_close(file);
/* write new manifest file */
if (pack->info->nelts > 1)
{
svn_stream_t *stream;
SVN_ERR(svn_io_file_open(&file,
apr_psprintf(iterpool, "%s/manifest", dir),
APR_WRITE | APR_CREATE | APR_BUFFERED,
APR_OS_DEFAULT,
iterpool));
stream = svn_stream_from_aprfile2(file, FALSE, iterpool);
for (i = 0; i < pack->info->nelts; ++i)
{
revision_info_t *info = APR_ARRAY_IDX(pack->info, i,
revision_info_t *);
SVN_ERR(svn_stream_printf(stream, itempool,
"%" APR_SIZE_T_FMT "\n",
info->target.offset));
svn_pool_clear(itempool);
}
}
/* cleanup */
svn_pool_destroy(itempool);
svn_pool_destroy(iterpool);
return SVN_NO_ERROR;
}
/* Write reorg'ed target content for all revisions in FS. To maximize
* data locality, pack and write in one go per pack file.
* Use POOL for allocations.
*/
static svn_error_t *
pack_and_write_revisions(fs_fs_t *fs,
apr_pool_t *pool)
{
int i;
SVN_ERR(svn_io_make_dir_recursively(apr_psprintf(pool, "%s/new",
fs->path),
pool));
for (i = 0; i < fs->packs->nelts; ++i)
{
revision_pack_t *pack = APR_ARRAY_IDX(fs->packs, i, revision_pack_t*);
if (pack->base % fs->max_files_per_dir == 0)
print_progress(pack->base);
src/subversion/tools/dev/fsfs-reorg.c view on Meta::CPAN
update_text(svn_stringbuf_t *node_rev,
const char *key,
representation_t *representation,
apr_pool_t *scratch_pool)
{
apr_size_t key_len = strlen(key);
char *pos = strstr(node_rev->data, key);
char *val_pos;
if (!pos)
return;
val_pos = pos + key_len;
if (representation->dir)
{
/* for directories, we need to write all rep info anew */
char *newline_pos = strchr(val_pos, '\n');
svn_checksum_t checksum;
const char* temp = apr_psprintf(scratch_pool, "%ld %" APR_SIZE_T_FMT " %"
APR_SIZE_T_FMT" %" APR_SIZE_T_FMT " %s",
representation->revision->revision,
representation->target.offset - representation->revision->target.offset,
representation->target.size,
representation->dir->size,
svn_checksum_to_cstring(&checksum,
scratch_pool));
checksum.digest = representation->dir->target_md5;
checksum.kind = svn_checksum_md5;
svn_stringbuf_replace(node_rev,
val_pos - node_rev->data, newline_pos - val_pos,
temp, strlen(temp));
}
else
{
/* ordinary representation: replace offset and rep size only.
* Content size and checksums are unchanged. */
const char* temp;
char *end_pos = strchr(val_pos, ' ');
val_pos = end_pos + 1;
end_pos = strchr(strchr(val_pos, ' ') + 1, ' ');
temp = apr_psprintf(scratch_pool, "%" APR_SIZE_T_FMT " %" APR_SIZE_T_FMT,
representation->target.offset - representation->revision->target.offset,
representation->target.size);
svn_stringbuf_replace(node_rev,
val_pos - node_rev->data, end_pos - val_pos,
temp, strlen(temp));
}
}
/* Get the target content (data block as to be written to the file) for
* the given FRAGMENT in FS. Return the content in *CONTENT. Use POOL
* for allocations.
*
* Note that, as a side-effect, this will update the target rep. info for
* directories.
*/
static svn_error_t *
get_fragment_content(svn_string_t **content,
fs_fs_t *fs,
fragment_t *fragment,
apr_pool_t *pool)
{
revision_info_t *info;
representation_t *representation;
noderev_t *node;
svn_string_t *revision_content, *base_content;
svn_stringbuf_t *header, *node_rev, *text;
apr_size_t header_size;
svn_checksum_t *checksum = NULL;
switch (fragment->kind)
{
/* revision headers can be constructed from target position info */
case header_fragment:
info = fragment->data;
*content = svn_string_createf(pool,
"\n%" APR_SIZE_T_FMT " %" APR_SIZE_T_FMT "\n",
info->root_noderev->target.offset - info->target.offset,
info->target.changes);
return SVN_NO_ERROR;
/* The changes list remains untouched */
case changes_fragment:
info = fragment->data;
SVN_ERR(get_content(&revision_content, fs, info->revision, pool));
*content = svn_string_create_empty(pool);
(*content)->data = revision_content->data + info->original.changes;
(*content)->len = info->target.changes_len;
return SVN_NO_ERROR;
/* property and file reps get new headers any need to be rewritten,
* iff the base rep is a directory. The actual (deltified) content
* remains unchanged, though. MD5 etc. do not change. */
case property_fragment:
case file_fragment:
representation = fragment->data;
SVN_ERR(get_content(&revision_content, fs,
representation->revision->revision, pool));
if (representation->delta_base)
if (representation->delta_base->dir)
{
/* if the base happens to be a directory, reconstruct the
* full text and represent it as PLAIN rep. */
SVN_ERR(get_combined_window(&text, fs, representation, pool));
representation->target.size = text->len;
svn_stringbuf_insert(text, 0, "PLAIN\n", 6);
svn_stringbuf_appendcstr(text, "ENDREP\n");
*content = svn_stringbuf__morph_into_string(text);
return SVN_NO_ERROR;
}
else
/* construct a new rep header */
if (representation->delta_base == fs->null_base)
header = svn_stringbuf_create("DELTA\n", pool);
else
header = svn_stringbuf_createf(pool,
"DELTA %ld %" APR_SIZE_T_FMT " %" APR_SIZE_T_FMT "\n",
representation->delta_base->revision->revision,
representation->delta_base->target.offset
- representation->delta_base->revision->target.offset,
representation->delta_base->target.size);
else
header = svn_stringbuf_create("PLAIN\n", pool);
/* if it exists, the actual delta base is unchanged. Hence, this
* rep is unchanged even if it has been deltified. */
header_size = strchr(revision_content->data +
representation->original.offset, '\n') -
revision_content->data -
representation->original.offset + 1;
svn_stringbuf_appendbytes(header,
revision_content->data +
representation->original.offset +
header_size,
representation->original.size);
svn_stringbuf_appendcstr(header, "ENDREP\n");
*content = svn_stringbuf__morph_into_string(header);
return SVN_NO_ERROR;
/* directory reps need to be rewritten (and deltified) completely.
* As a side-effect, update the MD5 and target content size. */
case dir_fragment:
/* construct new content and update MD5 */
representation = fragment->data;
SVN_ERR(get_updated_dir(&revision_content, fs, representation,
pool, pool));
SVN_ERR(svn_checksum(&checksum, svn_checksum_md5,
revision_content->data, revision_content->len,
pool));
memcpy(representation->dir->target_md5,
checksum->digest,
sizeof(representation->dir->target_md5));
/* deltify against the base rep if necessary */
if (representation->delta_base)
{
if (representation->delta_base->dir == NULL)
{
/* dummy or non-dir base rep -> self-compress only */
header = svn_stringbuf_create("DELTA\n", pool);
base_content = svn_string_create_empty(pool);
}
else
{
/* deltify against base rep (which is a directory, too)*/
representation_t *base_rep = representation->delta_base;
header = svn_stringbuf_createf(pool,
"DELTA %ld %" APR_SIZE_T_FMT " %" APR_SIZE_T_FMT "\n",
base_rep->revision->revision,
base_rep->target.offset - base_rep->revision->target.offset,
base_rep->target.size);
SVN_ERR(get_updated_dir(&base_content, fs, base_rep,
pool, pool));
}
/* run deltification and update target content size */
header_size = header->len;
SVN_ERR(diff_stringbufs(header, base_content,
revision_content, pool));
representation->dir->size = revision_content->len;
representation->target.size = header->len - header_size;
svn_stringbuf_appendcstr(header, "ENDREP\n");
*content = svn_stringbuf__morph_into_string(header);
}
else
{
/* no delta base (not even a dummy) -> PLAIN rep */
representation->target.size = revision_content->len;
representation->dir->size = revision_content->len;
*content = svn_string_createf(pool, "PLAIN\n%sENDREP\n",
revision_content->data);
}
return SVN_NO_ERROR;
/* construct the new noderev content. No side-effects.*/
case noderev_fragment:
/* get the original noderev as string */
node = fragment->data;
SVN_ERR(get_content(&revision_content, fs,
node->revision->revision, pool));
node_rev = svn_stringbuf_ncreate(revision_content->data +
node->original.offset,
node->original.size,
pool);
/* update the values that may have hanged for target */
update_id(node_rev, "id: ", node);
update_id(node_rev, "pred: ", node->predecessor);
update_text(node_rev, "text: ", node->text, pool);
update_text(node_rev, "props: ", node->props, pool);
*content = svn_stringbuf__morph_into_string(node_rev);
return SVN_NO_ERROR;
}
SVN_ERR_ASSERT(0);
return SVN_NO_ERROR;
}
/* In the repository at PATH, restore the original content in case we ran
* this reorg tool before. Use POOL for allocations.
*/
static svn_error_t *
prepare_repo(const char *path, apr_pool_t *pool)
{
svn_node_kind_t kind;
const char *old_path = svn_dirent_join(path, "db/old", pool);
const char *new_path = svn_dirent_join(path, "new", pool);
const char *revs_path = svn_dirent_join(path, "db/revs", pool);
const char *old_rep_cache_path = svn_dirent_join(path, "db/rep-cache.db.old", pool);
const char *rep_cache_path = svn_dirent_join(path, "db/rep-cache.db", pool);
/* is there a backup? */
SVN_ERR(svn_io_check_path(old_path, &kind, pool));
if (kind == svn_node_dir)
{
/* yes, restore the org content from it */
SVN_ERR(svn_io_remove_dir2(new_path, TRUE, NULL, NULL, pool));
SVN_ERR(svn_io_file_move(revs_path, new_path, pool));
SVN_ERR(svn_io_file_move(old_path, revs_path, pool));
SVN_ERR(svn_io_remove_dir2(new_path, TRUE, NULL, NULL, pool));
}
/* same for the rep cache db */
SVN_ERR(svn_io_check_path(old_rep_cache_path, &kind, pool));
if (kind == svn_node_file)
SVN_ERR(svn_io_file_move(old_rep_cache_path, rep_cache_path, pool));
return SVN_NO_ERROR;
}
/* In the repository at PATH, create a backup of the orig content and
* replace it with the reorg'ed. Use POOL for allocations.
*/
static svn_error_t *
activate_new_revs(const char *path, apr_pool_t *pool)
( run in 1.035 second using v1.01-cache-2.11-cpan-b16cb0d3907 )