Alien-Libjio
view release on metacpan or search on metacpan
o Remove requirement for Config; we use notes to determine $(MAKE)
Changed my mail address to jawnsy@cpan.org
1.002 2009-08-18
Updated test sequence
Remove modules available in core since Perl 5.6 from requires
Added ignore expressions for libjio's build-flags
and libjio.so.0.90 (the symlink)
Override clean action to clean libjio files
1.001 2009-08-17
Integrate version 0.90 of libjio
Remove generated files from the repository
MANIFEST.SKIP view on Meta::CPAN
\.bak$
# Devel::Cover files.
^cover_db/
# Test::Kwalitee cache files.
^Debian_CPANTS\.txt
# Compiled files: libjio
\.(o|a|so|lib|dll|mak)$
^libjio/libjio/build/build-flags$
^libjio/libjio/build/libjio.pc$
^libjio/libjio/build/jiofsck$
^libjio/libjio/build/libjio.so # .so symlink
# Debian package control files
^debian/
examples/libjio-config view on Meta::CPAN
use strict;
use warnings;
use Alien::Libjio;
my $jio = Alien::Libjio->new();
if ($jio->installed) {
print "libjio is installed on your system.\n";
printf " Detection mechanism: %s\n", $jio->how;
printf " Compiler flags: %s\n",
ref scalar $jio->cflags ? join(' ', $jio->cflags) : '(null)';
printf " Linker flags: %s\n",
ref scalar $jio->ldflags ? join(' ', $jio->ldflags) : '(null)';
printf " Module version: %s\n", $jio->version
if $jio->method eq 'pkg-config';
}
else {
print {*STDERR} "libjio is not installed on your system\n";
}
lib/Alien/Libjio.pm view on Meta::CPAN
sub version {
my ($self) = @_;
Carp::croak('You must call this method as an object') unless ref($self);
return $self->{version};
}
sub ldflags {
my ($self) = @_;
Carp::croak('You must call this method as an object') unless ref($self);
# Return early if called in void context
return unless defined wantarray;
# If calling in array context, dereference and return
return @{ $self->{ldflags} } if wantarray;
return $self->{ldflags};
}
# Glob to create an alias to ldflags
*linker_flags = *ldflags;
sub cflags {
my ($self) = @_;
Carp::croak('You must call this method as an object') unless ref($self);
# Return early if called in void context
return unless defined wantarray;
# If calling in array context, dereference and return
return @{ $self->{cflags} } if wantarray;
return $self->{cflags};
}
*compiler_flags = *cflags;
sub method {
my ($self) = @_;
Carp::croak('You must call this method as an object') unless ref($self);
return $self->{method};
}
*how = *method;
lib/Alien/Libjio.pm view on Meta::CPAN
# The value we got back
return <$read>;
}
return (undef, <$read>) if wantarray;
return;
}
sub _try_pkg_config {
my ($self) = @_;
my ($value, $err) = _get_pc('cflags');
return unless (defined $value && length $value);
#if (defined $err && length $err) {
# #warn "Problem with pkg-config; using ExtUtils::Liblist instead\n";
# return;
#}
$self->{installed} = 1;
$self->{method} = 'pkg-config';
# pkg-config returns things with a newline, so remember to remove it
$self->{cflags} = [ split(' ', $value) ];
$self->{ldflags} = [ split(' ', _get_pc('libs')) ];
$self->{version} = _get_pc('modversion');
return 1;
}
sub _try_liblist {
my ($self) = @_;
use ExtUtils::Liblist ();
local $SIG{__WARN__} = sub { }; # mask warnings
my (undef, undef, $ldflags, $ldpath) = ExtUtils::Liblist->ext('-ljio');
return unless (defined($ldflags) && length($ldflags));
$self->{installed} = 1;
$self->{method} = 'ExtUtils::Liblist';
# Empty out cflags; initialize it
$self->{cflags} = [];
my $read;
my $pid = open3(undef, $read, undef, 'getconf', 'LFS_CFLAGS');
# We're using blocking wait, so the return value doesn't matter
## no critic(RequireCheckedSyscalls)
waitpid($pid, 0);
# Check the status code
if (($? >> 8) == 0) {
# This only takes the first line
push(@{ $self->{cflags} }, split(' ', <$read>));
}
else {
warn 'Problem using getconf: ', <$read>, "\n";
push(@{ $self->{cflags} },
'-D_LARGEFILE_SOURCE',
'-D_FILE_OFFSET_BITS=64',
);
}
# Used for resolving the include path, relative to lib
use Cwd ();
use File::Spec ();
push(@{ $self->{cflags} },
# The include path is taken as: $libpath/../include
'-I' . Cwd::realpath(File::Spec->catfile(
$ldpath,
File::Spec->updir(),
'include'
))
);
push(@{ $self->{ldflags} },
'-L' . $ldpath,
$ldflags,
);
return 1;
}
__END__
=pod
=head1 NAME
lib/Alien/Libjio.pm view on Meta::CPAN
=head1 VERSION
version 1.004
=head1 SYNOPSIS
use Alien::Libjio;
my $jio = Alien::Libjio->new;
my $ldflags = $jio->ldflags;
my $cflags = $jio->cflags;
=head1 DESCRIPTION
To ensure reliability, some file systems and databases provide support for
something known as journalling. The idea is to ensure data consistency by
creating a log of actions to be taken (called a Write Ahead Log) before
committing them to disk. That way, if a transaction were to fail due to a
system crash or other unexpected event, the write ahead log could be used to
finish writing the data.
lib/Alien/Libjio.pm view on Meta::CPAN
writes are important. In other cases, the filesystem does not provide native
journalling support, so other tricks may be used to ensure data integrity,
such as writing to a separate temporary file and then overwriting the file
instead of modifying it in-place. Unfortunately, this method cannot handle
threaded operations appropriately.
Thankfully, Alberto Bertogli published a userspace C library called libjio
that can provide these features in a small (less than 1500 lines of code)
library with no external dependencies.
This package is designed to install it, and provide a way to get the flags
necessary to compile programs using it. It is particularly useful for Perl XS
programs that use it, such as B<IO::Journal>.
=head1 METHODS
=head2 Alien::Libjio->new
Creates a new C<Alien::Libjio> object, which essentially just has a few
convenience methods providing useful information like compiler and linker
flags.
Example code:
my $jio = Alien::Libjio->new();
This method will return an appropriate B<Alien::Libjio> object or throw an
exception on error.
=head2 $jio->installed
lib/Alien/Libjio.pm view on Meta::CPAN
Determine the installed version of libjio, as a string.
Currently versions are simply floating-point numbers, so you can treat the
version number as such, but this behaviour is subject to change.
Example code:
my $version = $jio->version;
=head2 $jio->ldflags
=head2 $jio->linker_flags
This returns the flags required to link C code with the local installation of
libjio (typically in the LDFLAGS variable). It is particularly useful for
building and installing Perl XS modules such as L<IO::Journal>.
In scalar context, it returns an array reference suitable for passing to
other build systems, particularly L<Module::Build>. In list context, it gives
a normal array so that C<join> and friends will work as expected.
Example code:
my $ldflags = $jio->ldflags;
my @ldflags = @{ $jio->ldflags };
my $ldstring = join(' ', $jio->ldflags);
# or:
# my $ldflags = $jio->linker_flags;
=head2 $jio->cflags
=head2 $jio->compiler_flags
This method returns the compiler option flags to compile C code which uses
the libjio library (typically in the CFLAGS variable). It is particularly
useful for building and installing Perl XS modules such as L<IO::Journal>.
Example code:
my $cflags = $jio->cflags;
my @cflags = @{ $jio->cflags };
my $ccstring = join(' ', $jio->cflags);
# or:
# my $cflags = $jio->compiler_flags;
=head2 $jio->method
=head2 $jio->how
This method returns the method the module used to find information about
libjio. The following methods are currently used (in priority order):
=over
libjio/UPGRADING view on Meta::CPAN
Normally, nothing should be required when upgrading between two stable
releases from the same branch.
------- 1.00: Stable release
-> 0.90 (On-disk format change, pre 1.0 freeze)
- The way transactions are stored on disk has changed. It is mandatory that
you jfsck all your files before upgrading. Hopefully this will be the last
backwards-incompatible format change before 1.0.
- jtrans_new() now takes an additional flags parameter.
- jopen() jflags parameter is now unsigned.
- J_NOCLEANUP was removed in favour of J_CLEANUP, and the default behaviour
of jfsck() is now not to clean up unless J_CLEANUP is passed in the flags.
- jtrans_add() renamed to jtrans_add_w().
- jtrans_commit() returns 0 on success, instead of the amount of bytes
written.
-> 0.50 (Big API change)
- Structures are now opaque types:
struct jfs -> jfs_t; jopen() returns a pointer to one, jclose() frees it.
struct jtrans -> jtrans_t; jtrans_new() returns a pointer to one,
jtrans_free() frees it.
- Renamed jtrans_init() to jtrans_new().
- jtrans_commit() returns -1 on recovered errors, -2 on unrecovered errors
(which are an indication of a severe underlying condition).
- jtrans_add() returns 0 on success and -1 on errors (it used to return 1 on
success and 0 on errors).
- jfsck() now has an additional "flags" parameter, which should be set to 0
to select the default behaviour.
- jfsck_cleanup() was removed, and now jfsck() cleans up by default.
-> 0.25
- It is no longer necessary to pass O_SYNC to jopen() if lingering
transactions are not in use.
- libjio.h can no longer be included from C++ source without surrounding it
by an 'extern "C"'. This obviously should only affect C++ applications.
-> 0.24
libjio/UPGRADING view on Meta::CPAN
-> 0.22
- Applications need to be recompiled due to a change in the jfs structure.
-> 0.20
- Applications need to be recompiled due to a change in the jfs structure.
- When you link your applications with libjio, you need to make sure you
compile using large file support.
-> 0.19
- Applications need to be recompiled due to a change in the flag numbering.
- A flag number has changed, from J_COMMITED to J_COMMITTED. If you used
that flag, you need to rename it.
libjio/bindings/preload/libjio_preload.c view on Meta::CPAN
/* maximum number of simultaneous open file descriptors we support */
#define MAXFD (4096 * 2)
/* recursion counter, per-thread */
static int __thread called = 0;
/* C library functions, filled via the dynamic loader */
static void *libc;
static int (*c_open)(const char *pathname, int flags, mode_t mode);
static int (*c_open64)(const char *pathname, int flags, mode_t mode);
static int (*c_close)(int fd);
static int (*c_unlink)(const char *pathname);
static ssize_t (*c_read)(int fd, void *buf, size_t count);
static ssize_t (*c_pread)(int fd, void *buf, size_t count, off_t offset);
static ssize_t (*c_pread64)(int fd, void *buf, size_t count, off64_t offset);
static ssize_t (*c_readv)(int fd, const struct iovec *vector, int count);
static ssize_t (*c_write)(int fd, const void *buf, size_t count);
static ssize_t (*c_pwrite)(int fd, const void *buf, size_t count, off_t offset);
static ssize_t (*c_pwrite64)(int fd, const void *buf, size_t count, off64_t offset);
static ssize_t (*c_writev)(int fd, const struct iovec *vector, int count);
libjio/bindings/preload/libjio_preload.c view on Meta::CPAN
libc_load(dup2);
printd("done\n");
return 1;
}
/*
* wrappers
*/
int open(const char *pathname, int flags, ...)
{
int r, fd;
jfs_t *fs;
mode_t mode;
struct stat st;
va_list l;
if (flags & O_CREAT) {
va_start(l, flags);
mode = va_arg(l, mode_t);
va_end(l);
} else {
/* set it to 0, it's ignored anyway */
mode = 0;
}
if (called) {
printd("orig (r)\n");
return (*c_open)(pathname, flags, mode);
}
printd("libjio\n");
/* skip special files */
r = stat(pathname, &st);
if (r == 0 && ( S_ISDIR(st.st_mode) \
|| S_ISCHR(st.st_mode) \
|| S_ISFIFO(st.st_mode) ) ) {
printd("orig (s)\n");
return (*c_open)(pathname, flags, mode);
}
/* skip /proc and /sys (not /dev, the problematic files are taken care
* of with the stat test above */
/* FIXME: this breaks with relative paths */
if ( (strncmp("/proc", pathname, 5) == 0) ||
(strncmp("/sys", pathname, 4) == 0) ) {
printd("orig (f)\n");
return (*c_open)(pathname, flags, mode);
}
rec_inc();
fs = jopen(pathname, flags, mode, 0);
if (fs == NULL) {
rec_dec();
return -1;
}
rec_dec();
fd = jfileno(fs);
fd_lock(fd);
fd_table[fd].fd = fd;
fd_table[fd].refcount = malloc(sizeof(unsigned int));
*fd_table[fd].refcount = 1;
fd_table[fd].fs = fs;
fd_unlock(fd);
printd("return %d\n", fd);
return fd;
}
/* exact copy of open(), but call c_open64 instead of c_open */
int open64(const char *pathname, int flags, ...)
{
int r, fd;
jfs_t *fs;
mode_t mode;
struct stat st;
va_list l;
if (flags & O_CREAT) {
va_start(l, flags);
mode = va_arg(l, mode_t);
va_end(l);
} else {
/* set it to 0, it's ignored anyway */
mode = 0;
}
if (called) {
printd("orig (r)\n");
return (*c_open64)(pathname, flags, mode);
}
printd("libjio\n");
/* skip special files */
r = stat(pathname, &st);
if (r == 0 && ( S_ISDIR(st.st_mode) \
|| S_ISCHR(st.st_mode) \
|| S_ISFIFO(st.st_mode) ) ) {
printd("orig (s)\n");
return (*c_open64)(pathname, flags, mode);
}
/* skip /proc and /sys (not /dev, the problematic files are taken care
* of with the stat test above */
/* FIXME: this breaks with relative paths */
if ( (strncmp("/proc", pathname, 5) == 0) ||
(strncmp("/sys", pathname, 4) == 0) ) {
printd("orig (f)\n");
return (*c_open64)(pathname, flags, mode);
}
rec_inc();
fs = jopen(pathname, flags, mode, 0);
if (fs == NULL) {
rec_dec();
return -1;
}
rec_dec();
fd = jfileno(fs);
fd_lock(fd);
fd_table[fd].fd = fd;
libjio/bindings/python/libjio.c view on Meta::CPAN
/* new_trans */
PyDoc_STRVAR(jf_new_trans__doc,
"new_trans()\n\
\n\
Returns an object representing a new empty transaction.\n\
It's a wrapper to jtrans_new().\n");
static PyObject *jf_new_trans(jfile_object *fp, PyObject *args)
{
jtrans_object *tp;
unsigned int flags = 0;
if (!PyArg_ParseTuple(args, "|I:new_trans", &flags))
return NULL;
#ifdef PYTHON3
tp = (jtrans_object *) jtrans_type.tp_alloc(&jtrans_type, 0);
#elif PYTHON2
tp = PyObject_New(jtrans_object, &jtrans_type);
#endif
if (tp == NULL)
return NULL;
tp->ts = jtrans_new(fp->fs, flags);
if(tp->ts == NULL) {
return PyErr_NoMemory();
}
/* increment the reference count, it's decremented on deletion */
tp->jfile = fp;
Py_INCREF(fp);
tp->views = NULL;
tp->nviews = 0;
libjio/bindings/python/libjio.c view on Meta::CPAN
#endif
/*
* The module
*/
/* open */
PyDoc_STRVAR(jf_open__doc,
"open(name[, flags[, mode[, jflags]]])\n\
\n\
Opens a file, returns a file object.\n\
The arguments flags, mode and jflags are the same as jopen(); the constants\n\
needed are defined in the module.\n\
It's a wrapper to jopen().\n");
static PyObject *jf_open(PyObject *self, PyObject *args)
{
char *file;
int flags = O_RDONLY;
int mode = 0600;
unsigned int jflags = 0;
jfile_object *fp;
flags = O_RDWR;
mode = 0600;
jflags = 0;
if (!PyArg_ParseTuple(args, "s|iiI:open", &file, &flags, &mode,
&jflags))
return NULL;
#ifdef PYTHON3
fp = (jfile_object *) jfile_type.tp_alloc(&jfile_type, 0);
#elif PYTHON2
fp = PyObject_New(jfile_object, &jfile_type);
#endif
if (fp == NULL)
return NULL;
fp->fs = jopen(file, flags, mode, jflags);
if (fp->fs == NULL) {
return PyErr_SetFromErrno(PyExc_IOError);
}
if (PyErr_Occurred()) {
jclose(fp->fs);
return NULL;
}
return (PyObject *) fp;
}
/* jfsck */
PyDoc_STRVAR(jf_jfsck__doc,
"jfsck(name[, jdir] [, flags])\n\
\n\
Checks the integrity of the file with the given name, using (optionally) jdir\n\
as the journal directory and the given flags; returns a dictionary with all\n\
the different values of the check (equivalent to the 'struct jfsck_result').\n\
If the path is incorrect, or there is no journal associated with it, an\n\
IOError will be raised.\n\
It's a wrapper to jfsck().\n");
static PyObject *jf_jfsck(PyObject *self, PyObject *args, PyObject *kw)
{
int rv;
unsigned int flags;
char *name, *jdir = NULL;
struct jfsck_result res;
PyObject *dict;
char *keywords[] = { "name", "jdir", "flags", NULL };
if (!PyArg_ParseTupleAndKeywords(args, kw, "s|sI:jfsck",
keywords, &name, &jdir, &flags))
return NULL;
dict = PyDict_New();
if (dict == NULL)
return PyErr_NoMemory();
Py_BEGIN_ALLOW_THREADS
rv = jfsck(name, jdir, &res, flags);
Py_END_ALLOW_THREADS
if (rv == J_ENOMEM) {
Py_XDECREF(dict);
return PyErr_NoMemory();
} else if (rv != 0) {
Py_XDECREF(dict);
PyErr_SetObject(PyExc_IOError, PyLong_FromLong(rv));
return NULL;
}
libjio/bindings/python/libjio.c view on Meta::CPAN
PyModule_AddIntConstant(m, "J_RDONLY", J_RDONLY);
/* enum jfsck_return */
PyModule_AddIntConstant(m, "J_ESUCCESS", J_ESUCCESS);
PyModule_AddIntConstant(m, "J_ENOENT", J_ENOENT);
PyModule_AddIntConstant(m, "J_ENOJOURNAL", J_ENOJOURNAL);
PyModule_AddIntConstant(m, "J_ENOMEM", J_ENOMEM);
PyModule_AddIntConstant(m, "J_ECLEANUP", J_ECLEANUP);
PyModule_AddIntConstant(m, "J_EIO", J_EIO);
/* jfsck() flags */
PyModule_AddIntConstant(m, "J_CLEANUP", J_CLEANUP);
/* open constants (at least the POSIX ones) */
PyModule_AddIntConstant(m, "O_RDONLY", O_RDONLY);
PyModule_AddIntConstant(m, "O_WRONLY", O_WRONLY);
PyModule_AddIntConstant(m, "O_RDWR", O_RDWR);
PyModule_AddIntConstant(m, "O_CREAT", O_CREAT);
PyModule_AddIntConstant(m, "O_EXCL", O_EXCL);
PyModule_AddIntConstant(m, "O_TRUNC", O_TRUNC);
PyModule_AddIntConstant(m, "O_APPEND", O_APPEND);
libjio/doc/guide.rst view on Meta::CPAN
Then second is *jtrans_t*, usually called the transaction structure, which
represents a single transaction.
Basic operation
---------------
First of all, as with regular I/O, you need to open your files. This is done
with *jopen()*, which looks a lot like *open()* but returns a file structure
instead of a file descriptor (this will be very common among all the
functions), and adds a new parameter *jflags* that can be used to modify some
library behaviour we'll see later, and is normally not used.
Now that you have opened a file, the next thing to do would be to create a
transaction. This is what *jtrans_new()* is for: it takes a file structure and
returns a new transaction structure.
To add a write operation to the transaction, use *jtrans_add_w()*. You can add
as many operations as you want. Operations within a transaction may overlap,
and will be applied in order.
libjio/doc/guide.rst view on Meta::CPAN
file = jopen("filename", O_RDWR | O_CREAT, 0600, 0);
trans = jtrans_new(file, 0);
jtrans_add_w(trans, buf, strlen(buf), 0);
jtrans_commit(trans);
jtrans_free(trans);
jclose(file);
As we've seen, you open the file and initialize the structure with *jopen()*
(with the parameter *jflags* being the last 0), create a new transaction with
*jtrans_new()*, then add an operation with *jtrans_add_w()* (the last 0 is the
offset, in this case the beginning of the file), commit the transaction with
*jtrans_commit()*, free it with *jtrans_free()*, and finally close the file
with *jclose()*.
Reading is much easier: the library provides three functions, *jread()*,
*jpread()* and *jreadv()*, that behave exactly like *read()*, *pread()* and
*readv()*, except that they play safe with libjio's writing code. You should
use these to read from files when using libjio.
libjio/doc/guide.rst view on Meta::CPAN
operations, etc.) and all the wrappers are safe and don't require any special
considerations.
Lingering transactions
----------------------
If you need to increase performance, you can use lingering transactions. In
this mode, transactions take up more disk space but allows you to do the
synchronous write only once, making commits much faster. To use them, just add
*J_LINGER* to the *jflags* parameter in *jopen()*. You should call *jsync()*
frequently to avoid using up too much space, or start an asynchronous thread
that calls *jsync()* automatically using *jfs_autosync_start()*. Note that
files opened with this mode must not be opened by more than one process at the
same time.
Disk layout
-----------
The library creates a single directory for each file opened, named after it.
libjio/doc/guide.rst view on Meta::CPAN
Besides the UNIX-alike API you can find an ANSI C alike API, which emulates
the traditional *fread()*, *fwrite()*, etc. It's still in development and has
not been tested carefully, so I won't spend time documenting them. Let me know
if you need them.
Compiling and linking
---------------------
If you have *pkg-config* in your build environment, then you can get the build
flags you need to use when building and linking against the library by
running::
pkg-config --cflags --libs libjio
If *pkg-config* is not available, you have to make sure your application uses
the Large File Support (*"LFS"* from now on), to be able to handle large files
properly. This means that you will have to pass some special standard flags to
the compiler, so your C library uses the same data types as the library. For
instance, on 32-bit platforms (like x86), when using LFS, offsets are usually
64 bits, as opposed to the usual 32.
The library is always built with LFS; however, linking it against an
application without LFS support could lead to serious problems because this
kind of size differences and ABI compatibility.
The Single Unix Specification standard proposes a simple and practical way to
get the flags you need to pass your C compiler to tell you want to compile
your application with LFS: use a program called "getconf" which should be
called like "getconf LFS_CFLAGS", and it outputs the appropiate parameters.
In the end, the command line would be something like::
gcc `getconf LFS_CFLAGS` app.c -ljio -o app
If you want more detailed information or examples, you can check out how the
library and sample applications get built.
libjio/doc/libjio.rst view on Meta::CPAN
warranties a lot and eases many things from its user, you should still be
careful when doing strange things with files while working on them.
The transaction file
~~~~~~~~~~~~~~~~~~~~
The transaction file is composed of three main parts: the header, the
operations, and the trailer.
The header holds basic information about the transaction itself, including the
version, the transaction ID, and its flags.
Then the operation part has all the operations one after the other, prepending
the operation data with a per-operation header that includes the length of the
data and the offset of the file where it should be applied, and then the data
itself.
Finally, the trailer contains the number of operations included in it and a
checksum of the whole file. Both fields are used to detect broken or corrupted
transactions.
libjio/doc/libjio.rst view on Meta::CPAN
Well, so much for talking, now let's get real; libjio applies commits in a
very simple and straightforward way, inside jtrans_commit():
- Lock the file offsets where the commit takes place
- Open the transaction file
- Write the header
- Read all the previous data from the file
- Write the previous data in the transaction
- Write the data to the file
- Mark the transaction as committed by setting a flag in the header
- Unlink the transaction file
- Unlock the offsets where the commit takes place
This may seem like a lot of steps, but they're not as much as it looks like
inside the code, and allows a recovery from interruptions in every step of the
way, and even in the middle of a step.
The rollback procedure
----------------------
libjio/libjio/Makefile view on Meta::CPAN
OBJS = $(addprefix $O/,autosync.o checksum.o common.o compat.o trans.o \
check.o journal.o unix.o ansi.o)
# targets
default: all
all: $O/libjio.so $O/libjio.a $O/libjio.pc $O/jiofsck
# used to rebuild everything when the build flags have changed
BF = $(CC) ~ $(ALL_CFLAGS) ~ $(PREFIX)
$O/build-flags: .force-build-flags
@mkdir -p $O
@if [ x"$(BF)" != x"`cat $O/build-flags 2>/dev/null`" ]; then \
if [ -f $O/build-flags ]; then \
echo "build flags changed, rebuilding"; \
fi; \
echo "$(BF)" > $O/build-flags; \
fi
$(OBJS): $O/build-flags
$O/%.o: %.c
@mkdir -p $O
$(N_CC) $(ALL_CFLAGS) -MMD -MF $@.mak -MP -MT $@ -c $< -o $@
sinclude $(OBJS:.o=.o.mak)
$O/libjio.so: $O/build-flags $(OBJS)
$(N_CC) -shared $(ALL_LDFLAGS) \
-Wl,-soname,libjio.so.$(LIB_SO_VER) \
$(LIBS) $(OBJS) -o $O/libjio.so.$(LIB_VER)
@echo " LN libjio.so.$(LIB_VER)"
@ln -fs libjio.so.$(LIB_VER) $O/libjio.so
$O/libjio.a: $O/build-flags $(OBJS)
$(N_AR) cr $@ $(OBJS)
$O/libjio.pc: $O/build-flags libjio.pc.in
@echo " GEN libjio.pc"
@cat libjio.pc.in | \
sed 's@++PREFIX++@$(DESTDIR)@g' | \
sed 's@++VERSION++@$(LIB_VER)@g' | \
sed 's@++CFLAGS++@$(MANDATORY_CFLAGS)@g' \
> $O/libjio.pc
$O/jiofsck: $O/build-flags $O/jiofsck.o $O/libjio.a
$(N_CC) $(ALL_LDFLAGS) $O/jiofsck.o $O/libjio.a $(LIBS) -o $@
install: all
$(INSTALL) -d $(PREFIX)/lib
$(INSTALL) -m 0755 $O/libjio.so.$(LIB_VER) $(PREFIX)/lib
ln -fs libjio.so.$(LIB_VER) $(PREFIX)/lib/libjio.so
ln -fs libjio.so.$(LIB_VER) $(PREFIX)/lib/libjio.so.$(LIB_SO_VER)
$(INSTALL) -m 0644 $O/libjio.a $(PREFIX)/lib
$(INSTALL) -d $(PREFIX)/include
$(INSTALL) -m 0644 libjio.h $(PREFIX)/include
libjio/libjio/Makefile view on Meta::CPAN
@echo "Please run ldconfig to update your library cache"
@echo
doxygen:
$(MAKE) LIB_VER=$(LIB_VER) -C doxygen
clean:
rm -f $O/libjio.a $O/libjio.so $O/libjio.so.$(LIB_VER) $O/libjio.pc
rm -f $(OBJS) $O/jiofsck.o $O/jiofsck
rm -f $O/*.bb $O/*.bbg $O/*.da $O/*.gcov $O/*.gcno $O/*.gcda $O/gmon.out
rm -f $O/build-flags $O/*.o.mak
$(MAKE) -C doxygen $@
.PHONY: default all install clean doxygen .force-build-flags
libjio/libjio/ansi.c view on Meta::CPAN
* The API is not nice, and I wouldn't recommend it for any serious I/O.
*
* Note that it is still experimental and it hasn't received too much testing
* (that's why it's not even documented), so use it at your own risk.
*/
/* fopen() wrapper */
struct jfs *jfopen(const char *path, const char *mode)
{
int flags;
int pos_at_the_beginning;
struct jfs *fs;
if (strlen(mode) < 1)
return NULL;
if (mode[0] == 'r') {
pos_at_the_beginning = 1;
if (strlen(mode) > 1 && strchr(mode, '+'))
flags = O_RDWR;
else
flags = O_RDONLY;
} else if (mode[0] == 'a') {
if (strlen(mode) > 1 && strchr(mode, '+'))
pos_at_the_beginning = 1;
else
pos_at_the_beginning = 0;
flags = O_RDWR | O_CREAT | O_APPEND;
} else if (mode[0] == 'w') {
pos_at_the_beginning = 1;
flags = O_RDWR | O_CREAT | O_TRUNC;
} else {
return NULL;
}
fs = jopen(path, flags, 0666, 0);
if (fs == NULL)
return NULL;
if (pos_at_the_beginning)
lseek(fs->fd, 0, SEEK_SET);
else
lseek(fs->fd, 0, SEEK_END);
return fs;
}
libjio/libjio/ansi.c view on Meta::CPAN
/* fileno() wrapper */
int jfileno(struct jfs *stream)
{
return stream->fd;
}
/* feof() wrapper */
int jfeof(struct jfs *stream)
{
/* ANSI expects that when an EOF is reached in any operation (like
* fread() or fwrite()) some internal flag is set, and this function
* can be used to check if it is set or unset.
* As we don't do that (it's pointless for this kind of I/O), this
* just checks if the file pointer is at the end of the file */
off_t curpos, endpos;
pthread_mutex_lock(&(stream->lock));
curpos = lseek(jfileno(stream), 0, SEEK_CUR);
endpos = lseek(jfileno(stream), 0, SEEK_END);
libjio/libjio/check.c view on Meta::CPAN
return -1;
if (rmdir(jdir) != 0)
return -1;
return 0;
}
/* Check the journal and fix the incomplete transactions */
enum jfsck_return jfsck(const char *name, const char *jdir,
struct jfsck_result *res, unsigned int flags)
{
int tfd, rv, i, ret;
unsigned int maxtid;
char jlockfile[PATH_MAX], tname[PATH_MAX], brokenname[PATH_MAX];
struct stat sinfo;
struct jfs fs;
struct jtrans *curts;
struct operation *tmpop;
DIR *dir;
struct dirent *dent;
libjio/libjio/check.c view on Meta::CPAN
rv = fill_trans(map, filelen, curts);
if (rv == -1) {
res->broken++;
goto loop;
} else if (rv == -2) {
res->corrupt++;
goto loop;
}
/* remove flags from the transaction, so we don't have issues
* re-committing */
curts->flags = 0;
rv = jtrans_commit(curts);
if (rv < 0) {
ret = J_EIO;
goto exit;
}
res->reapplied++;
loop:
libjio/libjio/check.c view on Meta::CPAN
free(curts->op->pdata);
free(curts->op);
curts->op = tmpop;
}
pthread_mutex_destroy(&(curts->lock));
free(curts);
res->total++;
}
if (flags & J_CLEANUP) {
if (jfsck_cleanup(name, fs.jdir) < 0) {
ret = J_ECLEANUP;
}
}
exit:
if (fs.fd >= 0)
close(fs.fd);
if (fs.jfd >= 0)
close(fs.jfd);
libjio/libjio/common.h view on Meta::CPAN
/** Journal directory file descriptor */
int jdirfd;
/** Journal's lock file descriptor */
int jfd;
/** Journal's lock file mmap */
unsigned int *jmap;
/** Journal flags */
uint32_t flags;
/** Flags passed to the real open() */
uint32_t open_flags;
/** Lingering transactions (linked list) */
struct jlinger *ltrans;
/** Length of all the lingered transactions */
size_t ltrans_len;
/** Lingering transactions' lock */
pthread_mutex_t ltlock;
libjio/libjio/fiu-local.h view on Meta::CPAN
* http://blitiri.com.ar/p/libfiu.
*/
#ifndef _FIU_LOCAL_H
#define _FIU_LOCAL_H
/* Only define the stubs when fiu is disabled, otherwise use the real fiu.h
* header */
#ifndef FIU_ENABLE
#define fiu_init(flags) 0
#define fiu_fail(name) 0
#define fiu_failinfo() NULL
#define fiu_do_on(name, action)
#define fiu_exit_on(name)
#define fiu_return_on(name, retval)
#else
#include <fiu.h>
libjio/libjio/jiofsck.c view on Meta::CPAN
# jiofsck file\n\
# jiofsck clean=1 file\n\
# jiofsck dir=/tmp/journal file\n\
# jiofsck clean=1 dir=/tmp/journal file\n\
\n");
}
int main(int argc, char **argv)
{
int i, do_cleanup;
unsigned int flags;
char *file, *jdir;
struct jfsck_result res;
enum jfsck_return rv;
file = jdir = NULL;
do_cleanup = 0;
if (argc < 2) {
libjio/libjio/jiofsck.c view on Meta::CPAN
do_cleanup = 1;
} else if (strncmp("dir=", argv[i], 4) == 0) {
jdir = argv[i] + 4;
} else {
file = argv[i];
}
}
memset(&res, 0, sizeof(res));
flags = 0;
if (do_cleanup)
flags |= J_CLEANUP;
printf("Checking journal: ");
fflush(stdout);
rv = jfsck(file, jdir, &res, flags);
switch (rv) {
case J_ESUCCESS:
printf("done\n");
break;
case J_ENOENT:
printf("No such file or directory\n");
return 1;
case J_ENOJOURNAL:
printf("No journal associated to the file, "
libjio/libjio/journal.c view on Meta::CPAN
* \ /
* +--------------- operations ----------------+
*
* The details of each part can be seen on the following structures. All
* integers are stored in network byte order.
*/
/** Transaction file header */
struct on_disk_hdr {
uint16_t ver;
uint16_t flags;
uint32_t trans_id;
} __attribute__((packed));
/** Transaction file operation header */
struct on_disk_ophdr {
uint32_t len;
uint64_t offset;
} __attribute__((packed));
/** Transaction file trailer */
libjio/libjio/journal.c view on Meta::CPAN
uint32_t numops;
uint32_t checksum;
} __attribute__((packed));
/* Convert structs to/from host to network (disk) endian */
static void hdr_hton(struct on_disk_hdr *hdr)
{
hdr->ver = htons(hdr->ver);
hdr->flags = htons(hdr->flags);
hdr->trans_id = htonl(hdr->trans_id);
}
static void hdr_ntoh(struct on_disk_hdr *hdr)
{
hdr->ver = ntohs(hdr->ver);
hdr->flags = ntohs(hdr->flags);
hdr->trans_id = ntohl(hdr->trans_id);
}
static void ophdr_hton(struct on_disk_ophdr *ophdr)
{
ophdr->len = htonl(ophdr->len);
ophdr->offset = htonll(ophdr->offset);
}
static void ophdr_ntoh(struct on_disk_ophdr *ophdr)
libjio/libjio/journal.c view on Meta::CPAN
return access(broken_path, F_OK) == 0;
}
/*
* Journal functions
*/
/** Create a new transaction in the journal. Returns a pointer to an opaque
* jop_t (that is freed using journal_free), or NULL if there was an error. */
struct journal_op *journal_new(struct jfs *fs, unsigned int flags)
{
int fd, id;
ssize_t rv;
char *name = NULL;
struct journal_op *jop = NULL;
struct on_disk_hdr hdr;
struct iovec iov[1];
if (is_broken(fs))
goto error;
libjio/libjio/journal.c view on Meta::CPAN
jop->numops = 0;
jop->name = name;
jop->csum = 0;
jop->fs = fs;
fiu_exit_on("jio/commit/created_tf");
/* save the header */
hdr.ver = 1;
hdr.trans_id = id;
hdr.flags = flags;
hdr_hton(&hdr);
iov[0].iov_base = (void *) &hdr;
iov[0].iov_len = sizeof(hdr);
rv = swritev(fd, iov, 1);
if (rv != sizeof(hdr))
goto unlink_error;
jop->csum = checksum_buf(jop->csum, (unsigned char *) &hdr,
sizeof(hdr));
libjio/libjio/journal.c view on Meta::CPAN
p = map;
memcpy(&hdr, p, sizeof(hdr));
p += sizeof(hdr);
hdr_ntoh(&hdr);
if (hdr.ver != 1)
return -1;
ts->id = hdr.trans_id;
ts->flags = hdr.flags;
ts->numops_r = 0;
ts->numops_w = 0;
ts->len_w = 0;
for (;;) {
if (p + sizeof(ophdr) > map + len)
goto error;
memcpy(&ophdr, p, sizeof(ophdr));
p += sizeof(ophdr);
libjio/libjio/journal.h view on Meta::CPAN
int id;
int fd;
int numops;
char *name;
uint32_t csum;
struct jfs *fs;
};
typedef struct journal_op jop_t;
struct journal_op *journal_new(struct jfs *fs, unsigned int flags);
int journal_add_op(struct journal_op *jop, unsigned char *buf, size_t len,
off_t offset);
void journal_pre_commit(struct journal_op *jop);
int journal_commit(struct journal_op *jop);
int journal_free(struct journal_op *jop, int do_unlink);
int fill_trans(unsigned char *map, off_t len, struct jtrans *ts);
#endif
libjio/libjio/libjio.3 view on Meta::CPAN
.TH libjio 3 "21/Feb/2004"
.SH NAME
libjio - A library for Journaled I/O
.SH SYNOPSIS
.nf
.B #include <libjio.h>
.BI "jfs_t *jopen(const char *" name ", int " flags ", int " mode ",
.BI " unsigned int " jflags ");"
.BI "ssize_t jread(jfs_t *" fs ", void *" buf ", size_t " count ");"
.BI "ssize_t jpread(jfs_t *" fs ", void *" buf ", size_t " count ","
.BI " off_t " offset ");"
.BI "ssize_t jreadv(jfs_t *" fs ", struct iovec *" vector ","
.BI " int " count ");"
.BI "ssize_t jwrite(jfs_t *" fs ", const void *" buf ", size_t " count ");"
.BI "ssize_t jpwrite(jfs_t *" fs ", const void *" buf ", size_t " count ","
.BI " off_t " offset ");"
.BI "ssize_t jwritev(jfs_t *" fs ", const struct iovec *" vector ","
.BI " int " count ");"
.BI "int jtruncate(jfs_t *" fs ", off_t " lenght ");"
.BI "off_t jlseek(jfs_t *" fs ", off_t " offset ", int " whence ");"
.BI "int jclose(jfs_t *" fs ");"
.BI "jtrans_t *jtrans_new(jfs_t *" fs ", unsigned int " flags ");"
.BI "int jtrans_commit(jtrans_t *" ts ");"
.BI "int jtrans_add_r(jtrans_t *" ts ", void *" buf ","
.BI " size_t " count ", off_t " offset ");"
.BI "int jtrans_add_w(jtrans_t *" ts ", const void *" buf ","
.BI " size_t " count ", off_t " offset ");"
.BI "int jtrans_rollback(jtrans_t *" ts ");"
.BI "void jtrans_free(jtrans_t *" ts ");"
.BI "int jsync(jfs_t *" fs ");"
.BI "int jfs_autosync_start(jfs_t *" fs ", time_t " max_sec ","
.BI " size_t " max_bytes ");"
.BI "int jfs_autosync_stop(jfs_t *" fs ");"
.BI "int jmove_journal(jfs_t *" fs ", const char *" newpath ");"
.BI "enum jfsck_return jfsck(const char *" name ", const char *" jdir ","
.BI " jfsck_result *" res ", unsigned int " flags ");"
.BR "struct jfsck_result" " {"
int total; /* total transactions files we looked at */
int invalid; /* invalid files in the journal directory */
int in_progress; /* transactions in progress */
int broken; /* transactions broken */
int rollbacked; /* transactions that were rollbacked */
...
};
libjio/libjio/libjio.3 view on Meta::CPAN
.BR jfs_autosync_start() .
The thread is also stopped automatically when
.B jclose()
is called.
.B jfsck()
takes as the first two parameters the path to the file to check and the path
to the journal directory (usually NULL for the default, unless you've changed
it manually using
.BR jmove_journal() ),
and optionally a flags parameter, which can be 0 for the default behaviour, or
J_CLEANUP to indicate that the journal should be cleaned up after successful
recovery.
It is used to perform journal checking and recovery in case of a crash. It
must be performed when nobody else is using the file (like in the case of a
filesystem which can't be mounted), and it returns 0 if success or an error
code != 0 in case of a failure. If it succeeded, it will fill jfsck_result
summarizing the outcome of the operation. The error codes can be either
.I J_ENOENT
(no such file),
libjio/libjio/libjio.h view on Meta::CPAN
/*
* Core functions
*/
/** Open a file.
*
* Takes the same parameters as the UNIX open(2), with an additional one for
* internal flags.
*
* The only supported internal flag is J_LINGER, which enables lingering
* transactions.
*
* @param name path to the file to open
* @param flags flags to pass to open(2)
* @param mode mode to pass to open(2)
* @param jflags journal flags
* @returns a new jfs_t that identifies the open file on success, or NULL on
* error
* @see jclose(), open()
* @ingroup basic
*/
jfs_t *jopen(const char *name, int flags, int mode, unsigned int jflags);
/** Close a file opened with jopen().
*
* After a call to this function, the memory allocated for the open file will
* be freed.
*
* If there was an autosync thread started for this file, it will be stopped.
*
* @param fs open file
* @returns 0 on success, -1 on error
libjio/libjio/libjio.h view on Meta::CPAN
/** Sync a file. Makes sense only when using lingering transactions.
*
* @param fs open file
* @returns 0 on success, -1 on error
* @ingroup basic
*/
int jsync(jfs_t *fs);
/** Create a new transaction.
*
* Note that the final flags to use in the transaction will be the result of
* ORing the flags parameter with fs' flags.
*
* @param fs open file the transaction will apply to
* @param flags transaction flags
* @returns a new transaction (must be freed using jtrans_free())
* @see jtrans_free()
* @ingroup basic
*/
jtrans_t *jtrans_new(jfs_t *fs, unsigned int flags);
/** Add a write operation to a transaction.
*
* A write operation consists of a buffer, its length, and the offset to write
* it to.
*
* The file will not be touched (not even locked) until commit time, where the
* first count bytes of buf will be written at offset.
*
* Operations will be applied in order, and overlapping operations are
libjio/libjio/libjio.h view on Meta::CPAN
*/
/** Check and repair the given path.
*
* The file MUST NOT be in use by any other thread or process. This
* requirement will be lifted in future releases.
*
* @param name path to the file to check
* @param jdir journal directory of the given file, use NULL for the default
* @param res structure where to store the result
* @param flags flags that change the checking behaviour, currently only
* J_CLEANUP is supported, which removes the journal directory after a
* successful recovery
* @see struct jfsck_result
* @returns 0 on success, < 0 on error, with the following possible negative
* values from enum jfsck_return: J_ENOENT if there was no such file with
* the given name, J_ENOJOURNAL if there was no journal at the given
* jdir, J_ENOMEM if memory could not be allocated, J_ECLEANUP if there
* was an error cleaning the journal, J_EIO if there was an I/O error.
* @ingroup check
*/
enum jfsck_return jfsck(const char *name, const char *jdir,
struct jfsck_result *res, unsigned int flags);
/*
* UNIX API wrappers
*/
/** Read from the file. Works just like UNIX read(2).
*
* @param fs file to read from
* @param buf buffer used to store the data
libjio/libjio/libjio.h view on Meta::CPAN
int jfseek(jfs_t *stream, long offset, int whence);
long jftell(jfs_t *stream);
void jrewind(jfs_t *stream);
FILE *jfsopen(jfs_t *stream, const char *mode);
/*
* jopen() flags.
*
* Internally used also for jtrans_t flags.
*/
/** Don't lock the file before operating on it.
*
* @see jopen()
* @ingroup basic */
#define J_NOLOCK 1
/** No need to read rollback information.
*
libjio/libjio/libjio.h view on Meta::CPAN
/** Marks a file as read-only.
*
* For internal use only, automatically set when O_RDONLY is passed to
* jopen().
*
* @internal */
#define J_RDONLY 512
/*
* jtrans_t flags.
*
* For internal use only, but must be in the same space as the jopen() flags.
*/
/** Marks a transaction as committed.
* @internal */
#define J_COMMITTED 1024
/** Marks a transaction as rollbacked.
* @internal */
#define J_ROLLBACKED 2048
/** Marks a transaction as rollbacking.
* @internal */
#define J_ROLLBACKING 4096
/*
* jfsck() flags
*/
/** Perform a journal cleanup. Used in jfsck().
*
* @see jfsck()
* @ingroup check */
#define J_CLEANUP 1
#endif
libjio/libjio/libjio.pc.in view on Meta::CPAN
prefix=++PREFIX++
libdir=${prefix}/lib
includedir=${prefix}/include
Name: libjio
Description: A library for Journaled I/O
URL: http://blitiri.com.ar/p/libjio/
Version: ++VERSION++
Libs: -L${libdir} -ljio
Cflags: -I${includedir} ++CFLAGS++
libjio/libjio/trans.c view on Meta::CPAN
#include "compat.h"
#include "journal.h"
#include "trans.h"
/*
* Transaction functions
*/
/* Initialize a transaction structure */
struct jtrans *jtrans_new(struct jfs *fs, unsigned int flags)
{
pthread_mutexattr_t attr;
struct jtrans *ts;
ts = malloc(sizeof(struct jtrans));
if (ts == NULL)
return NULL;
ts->fs = fs;
ts->id = 0;
ts->flags = fs->flags | flags;
ts->op = NULL;
ts->numops_r = 0;
ts->numops_w = 0;
ts->len_w = 0;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL);
pthread_mutex_init(&(ts->lock), &attr);
pthread_mutexattr_destroy(&attr);
libjio/libjio/trans.c view on Meta::CPAN
}
/** Lock/unlock the ranges of the file covered by the transaction. mode must
* be either F_LOCKW or F_UNLOCK. Returns 0 on success, -1 on error. */
static int lock_file_ranges(struct jtrans *ts, int mode)
{
unsigned int nops;
off_t lr, min_offset;
struct operation *op, *start_op;
if (ts->flags & J_NOLOCK)
return 0;
/* Lock/unlock always in the same order to avoid deadlocks. We will
* begin with the operation that has the smallest start offset, and go
* from there.
* Note that this is O(n^2), but n is usually (very) small, and we're
* about to do synchronous I/O, so it's not really worrying. It has a
* small optimization to help when the operations tend to be in the
* right order. */
nops = 0;
libjio/libjio/trans.c view on Meta::CPAN
static int jtrans_add_common(struct jtrans *ts, const void *buf, size_t count,
off_t offset, enum op_direction direction)
{
struct operation *op, *tmpop;
op = tmpop = NULL;
pthread_mutex_lock(&(ts->lock));
/* Writes are not allowed in read-only mode, they fail early */
if ((ts->flags & J_RDONLY) && direction == D_WRITE)
goto error;
if (count == 0)
goto error;
if ((long long) ts->len_w + count > MAX_TSIZE)
goto error;
op = malloc(sizeof(struct operation));
if (op == NULL)
libjio/libjio/trans.c view on Meta::CPAN
op->len = count;
op->offset = offset;
op->plen = 0;
op->pdata = NULL;
op->locked = 0;
op->direction = direction;
if (direction == D_WRITE) {
memcpy(op->buf, buf, count);
if (!(ts->flags & J_NOROLLBACK)) {
/* jtrans_commit() will want to read the current data,
* so we tell the kernel about that */
posix_fadvise(ts->fs->fd, offset, count,
POSIX_FADV_WILLNEED);
}
} else {
/* this casts the const away, which is ugly but let us have a
* common read/write path and avoid useless code repetition
* just to handle it */
op->buf = (void *) buf;
libjio/libjio/trans.c view on Meta::CPAN
ssize_t jtrans_commit(struct jtrans *ts)
{
ssize_t r, retval = -1;
struct operation *op;
struct jlinger *linger;
jop_t *jop = NULL;
size_t written = 0;
pthread_mutex_lock(&(ts->lock));
/* clear the flags */
ts->flags = ts->flags & ~J_COMMITTED;
ts->flags = ts->flags & ~J_ROLLBACKED;
if (ts->numops_r + ts->numops_w == 0)
goto exit;
/* fail for read-only accesses if we have write operations */
if (ts->numops_w && (ts->flags & J_RDONLY))
goto exit;
/* Lock all the regions we're going to work with; otherwise there
* could be another transaction trying to write the same spots and we
* could end up with interleaved writes, that could break atomicity
* warantees if we need to rollback.
* Note we do this before creating a new transaction, so we know it's
* not possible to have two overlapping transactions on disk at the
* same time. */
if (lock_file_ranges(ts, F_LOCKW) != 0)
goto unlock_exit;
/* create and fill the transaction file only if we have at least one
* write operation */
if (ts->numops_w) {
jop = journal_new(ts->fs, ts->flags);
if (jop == NULL)
goto unlock_exit;
}
for (op = ts->op; op != NULL; op = op->next) {
if (op->direction == D_READ)
continue;
r = journal_add_op(jop, op->buf, op->len, op->offset);
if (r != 0)
goto unlink_exit;
fiu_exit_on("jio/commit/tf_opdata");
}
if (jop)
journal_pre_commit(jop);
fiu_exit_on("jio/commit/tf_data");
if (!(ts->flags & J_NOROLLBACK)) {
for (op = ts->op; op != NULL; op = op->next) {
if (op->direction == D_READ)
continue;
r = operation_read_prev(ts, op);
if (r < 0)
goto unlink_exit;
}
}
libjio/libjio/trans.c view on Meta::CPAN
}
/* from now on, write ops (which are more interesting) */
r = spwrite(ts->fs->fd, op->buf, op->len, op->offset);
if (r != op->len)
goto rollback_exit;
written += r;
if (have_sync_range && !(ts->flags & J_LINGER)) {
r = sync_range_submit(ts->fs->fd, op->len,
op->offset);
if (r != 0)
goto rollback_exit;
}
fiu_exit_on("jio/commit/wrote_op");
}
fiu_exit_on("jio/commit/wrote_all_ops");
if (jop && (ts->flags & J_LINGER)) {
struct jlinger *lp;
linger = malloc(sizeof(struct jlinger));
if (linger == NULL)
goto rollback_exit;
linger->jop = jop;
linger->next = NULL;
pthread_mutex_lock(&(ts->fs->ltlock));
libjio/libjio/trans.c view on Meta::CPAN
if (r != 0)
goto rollback_exit;
}
} else {
if (fdatasync(ts->fs->fd) != 0)
goto rollback_exit;
}
}
/* mark the transaction as committed */
ts->flags = ts->flags | J_COMMITTED;
retval = 1;
rollback_exit:
/* If the transaction failed we try to recover by rolling it back.
* Only used if it has at least one write operation.
*
* NOTE: on extreme conditions (ENOSPC/disk failure) this can fail
* too! There's nothing much we can do in that case, the caller should
* take care of it by itself.
*
* Transactions that were successfuly recovered by rolling them back
* will have J_ROLLBACKED in their flags. */
if (jop && !(ts->flags & J_COMMITTED) &&
!(ts->flags & J_ROLLBACKING)) {
r = ts->flags;
ts->flags = ts->flags | J_NOLOCK | J_ROLLBACKING;
if (jtrans_rollback(ts) >= 0) {
ts->flags = r | J_ROLLBACKED;
retval = -1;
} else {
ts->flags = r;
retval = -2;
}
}
unlink_exit:
/* If the journal operation is no longer needed, we remove it from the
* disk.
*
* Extreme conditions (filesystem just got read-only, for example) can
* cause journal_free() to fail, but there's not much left to do at
* that point, and the caller will have to be careful and stop its
* operations. In that case, we will return -2, and the transaction
* will be marked as J_COMMITTED to indicate that the data was
* effectively written to disk. */
if (jop) {
/* Note we only unlink if we've written down the real data, or
* at least rolled it back properly */
int data_is_safe = (ts->flags & J_COMMITTED) ||
(ts->flags & J_ROLLBACKED);
r = journal_free(jop, data_is_safe ? 1 : 0);
if (r != 0)
retval = -2;
jop = NULL;
}
unlock_exit:
/* always unlock everything at the end; otherwise we could have
* half-overlapping transactions applying simultaneously, and if
libjio/libjio/trans.c view on Meta::CPAN
ssize_t jtrans_rollback(struct jtrans *ts)
{
ssize_t rv;
struct jtrans *newts;
struct operation *op, *curop, *lop;
newts = jtrans_new(ts->fs, 0);
if (newts == NULL)
return -1;
newts->flags = ts->flags;
newts->numops_r = 0;
newts->numops_w = 0;
newts->len_w = 0;
if (ts->op == NULL || ts->flags & J_NOROLLBACK) {
rv = -1;
goto exit;
}
/* find the last operation */
for (op = ts->op; op->next != NULL; op = op->next)
;
/* and traverse the list backwards, skipping read operations */
for ( ; op != NULL; op = op->prev) {
libjio/libjio/trans.c view on Meta::CPAN
return rv;
}
/*
* Basic operations
*/
/* Open a file */
struct jfs *jopen(const char *name, int flags, int mode, unsigned int jflags)
{
int jfd, rv;
unsigned int t;
char jdir[PATH_MAX], jlockfile[PATH_MAX];
struct stat sinfo;
pthread_mutexattr_t attr;
struct jfs *fs;
fs = malloc(sizeof(struct jfs));
if (fs == NULL)
libjio/libjio/trans.c view on Meta::CPAN
fs->fd = -1;
fs->jfd = -1;
fs->jdir = NULL;
fs->jdirfd = -1;
fs->jmap = MAP_FAILED;
fs->as_cfg = NULL;
/* we provide either read-only or read-write access, because when we
* commit a transaction we read the current contents before applying,
* and write access is needed for locking with fcntl; the test is done
* this way because O_RDONLY is usually 0, so "if (flags & O_RDONLY)"
* will fail. */
if ((flags & O_WRONLY) || (flags & O_RDWR)) {
flags = flags & ~O_WRONLY;
flags = flags & ~O_RDONLY;
flags = flags | O_RDWR;
} else {
jflags = jflags | J_RDONLY;
}
fs->name = strdup(name);
fs->flags = jflags;
fs->open_flags = flags;
fs->ltrans = NULL;
fs->ltrans_len = 0;
/* Note on fs->lock usage: this lock is used only to protect the file
* pointer. This means that it must only be held while performing
* operations that depend or alter the file pointer (jread, jreadv,
* jwrite, jwritev), but the others (jpread, jpwrite) are left
* unprotected because they can be performed in parallel as long as
* they don't affect the same portion of the file (this is protected
* by lockf). The lock doesn't slow things down tho: any threaded app
libjio/libjio/trans.c view on Meta::CPAN
* it here. If performance is essential, the jpread/jpwrite functions
* should be used, just as real life.
* About fs->ltlock, it's used to protect the lingering transactions
* list, fs->ltrans. */
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL);
pthread_mutex_init( &(fs->lock), &attr);
pthread_mutex_init( &(fs->ltlock), &attr);
pthread_mutexattr_destroy(&attr);
fs->fd = open(name, flags, mode);
if (fs->fd < 0)
goto error_exit;
/* nothing else to do for read-only access */
if (jflags & J_RDONLY) {
return fs;
}
if (!get_jdir(name, jdir))
goto error_exit;
rv = mkdir(jdir, 0750);
rv = lstat(jdir, &sinfo);
if (rv < 0 || !S_ISDIR(sinfo.st_mode))
goto error_exit;
libjio/libjio/trans.c view on Meta::CPAN
/* Close a file opened with jopen() */
int jclose(struct jfs *fs)
{
int ret;
ret = 0;
if (jfs_autosync_stop(fs))
ret = -1;
if (! (fs->flags & J_RDONLY)) {
if (jsync(fs))
ret = -1;
if (fs->jfd < 0 || close(fs->jfd))
ret = -1;
if (fs->jdirfd < 0 || close(fs->jdirfd))
ret = -1;
if (fs->jmap != MAP_FAILED)
munmap(fs->jmap, sizeof(unsigned int));
}
libjio/libjio/trans.h view on Meta::CPAN
struct operation;
/** A transaction */
struct jtrans {
/** Journal file structure to operate on */
struct jfs *fs;
/** Transaction id */
int id;
/** Transaction flags */
uint32_t flags;
/** Number of read operations in the list */
unsigned int numops_r;
/** Number of write operations in the list */
unsigned int numops_w;
/** Sum of the lengths of the write operations */
size_t len_w;
libjio/libjio/unix.c view on Meta::CPAN
ssize_t rv;
off_t pos;
struct jtrans *ts;
ts = jtrans_new(fs, 0);
if (ts == NULL)
return -1;
pthread_mutex_lock(&(fs->lock));
if (fs->open_flags & O_APPEND)
pos = lseek(fs->fd, 0, SEEK_END);
else
pos = lseek(fs->fd, 0, SEEK_CUR);
rv = jtrans_add_w(ts, buf, count, pos);
if (rv < 0)
goto exit;
rv = jtrans_commit(ts);
libjio/libjio/unix.c view on Meta::CPAN
ssize_t rv;
off_t ipos, t;
struct jtrans *ts;
ts = jtrans_new(fs, 0);
if (ts == NULL)
return -1;
pthread_mutex_lock(&(fs->lock));
if (fs->open_flags & O_APPEND)
ipos = lseek(fs->fd, 0, SEEK_END);
else
ipos = lseek(fs->fd, 0, SEEK_CUR);
t = ipos;
sum = 0;
for (i = 0; i < count; i++) {
rv = jtrans_add_w(ts, vector[i].iov_base,
vector[i].iov_len, t);
libjio/tests/behaviour/t_corruption.py view on Meta::CPAN
tf.numops = 55
tf.save()
fsck_verify(n, broken = 1)
assert content(n) == ''
cleanup(n)
def test_c08():
"broken journal"
c = gencontent()
f, jf = bitmp(jflags = 0)
n = f.name
def f1(f, jf):
fiu.enable("jio/commit/tf_sync")
jf.write(c)
run_forked(f1, f, jf)
assert content(n) == ''
open(jiodir(n) + '/broken', 'w+')
libjio/tests/behaviour/t_normal.py view on Meta::CPAN
jf.write('x')
jf.jmove_journal(p)
jf.write('y')
del jf
assert libjio.jfsck(n, p)['total'] == 0
os.unlink(n)
def test_n14():
"autosync"
f, jf = bitmp(jflags = libjio.J_LINGER)
n = f.name
jf.autosync_start(1, 10)
jf.write('x' * 200)
jf.write('x' * 200)
jf.autosync_stop()
del jf
fsck_verify(n)
cleanup(n)
def test_n15():
"jpread/jpwrite"
c = gencontent()
f, jf = bitmp(jflags = libjio.J_LINGER)
n = f.name
jf.pwrite(c, 2000)
assert content(n) == '\0' * 2000 + c
assert jf.pread(len(c), 2000) == c
del jf
fsck_verify(n)
cleanup(n)
libjio/tests/behaviour/t_normal.py view on Meta::CPAN
jf.write('y')
del jf
os.unlink(p + '/x')
assert libjio.jfsck(n, p)['total'] == 0
os.unlink(n)
def test_n18():
"jtrans_rollback with norollback"
c = gencontent()
f, jf = bitmp(jflags = libjio.J_NOROLLBACK)
n = f.name
t = jf.new_trans()
t.add_w(c, 80)
t.commit()
try:
t.rollback()
except IOError:
pass
else:
libjio/tests/behaviour/t_normal.py view on Meta::CPAN
assert content(n) == "hello world"
assert l[0] == "hello " and l[1] == "world"
fsck_verify(n)
cleanup(n)
def test_n22():
"jpread/jpwrite ~2mb"
c = gencontent(2 * 1024 * 1024 + 1465)
f, jf = bitmp(jflags = libjio.J_LINGER)
n = f.name
jf.pwrite(c, 2000)
assert content(n) == '\0' * 2000 + c
assert jf.pread(len(c), 2000) == c
del jf
fsck_verify(n)
cleanup(n)
libjio/tests/behaviour/tf.py view on Meta::CPAN
s += a
s = s[:size]
return s
def content(path):
"Returns the content of the given path."
f = open(path)
return f.read()
def biopen(path, mode = 'w+', jflags = 0):
"Returns (open(path), libjio.open(path))."
if 'r' in mode:
flags = os.O_RDONLY
if '+' in mode:
flags = os.O_RDWR
elif 'w' in mode:
flags = os.O_RDWR
if '+' in mode:
flags = flags | os.O_CREAT | os.O_TRUNC
elif 'a' in mode:
flags = os.O_RDWR | os.O_APPEND
else:
raise RuntimeError
return open(path, mode), libjio.open(path, flags, 0400, jflags)
def bitmp(mode = 'w+', jflags = 0):
"Opens a temporary file with biopen()."
path = tmppath()
return biopen(path, mode, jflags)
def run_with_tmp(func, jflags = 0):
"""Runs the given function, that takes a file and a jfile as
parameters, using a temporary file. Returns the path of the temporary
file. The function runs in a new process that exits afterwards."""
f, jf = bitmp(jflags = jflags)
run_forked(func, f, jf)
return f.name
def jiodir(path):
return os.path.dirname(path) + '/.' + os.path.basename(path) + '.jio'
def transpath(path, ntrans):
jpath = jiodir(path)
return jpath + '/' + str(ntrans)
def fsck(path, flags = 0):
"Calls libjio's jfsck()."
res = libjio.jfsck(path, flags = flags)
return res
def fsck_verify(n, **kwargs):
"""Runs fsck(n), and verifies that the fsck result matches the given
values. The default is to match all elements except total to 0 (total
is calculated automatically from the sum of the others). Raises an
AssertionError if the given results were not the ones expected."""
expected = {
'invalid': 0,
'broken': 0,
'reapplied': 0,
'corrupt': 0,
'in_progress': 0,
}
expected.update(kwargs)
expected['total'] = sum(expected.values())
res = fsck(n, flags = libjio.J_CLEANUP)
for k in expected:
if k not in res:
raise AssertionError, k + ' not in res'
if res[k] != expected[k]:
raise AssertionError, k + ' does not match: ' + \
str(res)
def cleanup(path):
"""Unlinks the path and its temporary libjio directory. The libjio
libjio/tests/behaviour/tf.py view on Meta::CPAN
def __setattr__(self, name, value):
self[name] = value
def __delattr__(self, name):
del self[name]
class TransFile (object):
def __init__(self, path = ''):
self.ver = 1
self.id = -1
self.flags = 0
self.numops = -1
self.checksum = -1
self.ops = []
self.path = path
if path:
self.load()
def load(self):
fd = open(self.path)
# header
hdrfmt = "!HHI"
self.ver, self.flags, self.id = struct.unpack(hdrfmt,
fd.read(struct.calcsize(hdrfmt)))
# operations (header only)
opfmt = "!IQ"
self.ops = []
while True:
tlen, offset = struct.unpack(opfmt,
fd.read(struct.calcsize(opfmt)))
if tlen == offset == 0:
break
libjio/tests/behaviour/tf.py view on Meta::CPAN
# trailer
trailerfmt = "!II"
self.numops, self.checksum = struct.unpack(trailerfmt,
fd.read(struct.calcsize(trailerfmt)))
def save(self):
# the lack of integrity checking in this function is
# intentional, so we can write broken transactions and see how
# jfsck() copes with them
fd = open(self.path, 'w')
fd.write(struct.pack("!HHI", self.ver, self.flags, self.id))
for o in self.ops:
fd.write(struct.pack("!IQ", o.tlen, o.offset,))
fd.write(o.payload)
fd.write(struct.pack("!IQ", 0, 0))
fd.write(struct.pack("!II", self.numops, self.checksum))
def __repr__(self):
return '<TransFile %s: id:%d f:%s n:%d ops:%s>' % \
(self.path, self.id, hex(self.flags), self.numops,
self.ops)
def gen_ret_seq(seq):
"""Returns a function that each time it is called returns a value of
the given sequence, in order. When the sequence is exhausted, returns
the last value."""
it = iter(seq)
last = [0]
def newf(*args, **kwargs):
libjio/tests/stress/jiostress view on Meta::CPAN
size = random.randint(0, (maxend - 1) - start) % maxsize
return start, start + size
def randfloat(min, max):
return min + random.random() % (max - min)
class ConsistencyError (Exception):
pass
def jfsck(fname, cleanup = False):
flags = 0
if cleanup:
flags = libjio.J_CLEANUP
try:
r = libjio.jfsck(fname, flags = flags)
return r
except IOError as e:
if e.args[0] == libjio.J_ENOJOURNAL:
return { 'total': 0 }
else:
raise
def comp_cont(bytes):
"'aaaabbcc' -> [ ('a', 4), ('b', 2), ('c', 2) ]"
l = []
libjio/tests/stress/jiostress view on Meta::CPAN
lockmgr, do_verify):
self.fname = fname
self.fsize = fsize
self.nops = nops
self.use_fi = use_fi
self.use_as = use_as
self.output = output
self.lockmgr = lockmgr
self.do_verify = do_verify
jflags = 0
if use_as:
jflags = libjio.J_LINGER
self.jf = libjio.open(fname, libjio.O_RDWR | libjio.O_CREAT,
0o600, jflags)
self.f = open(fname, mode = 'rb')
self.jf.truncate(fsize)
if use_as:
self.jf.autosync_start(5, 10 * 1024 * 1024)
def apply(self, trans):
trans.prepare()
trans.apply()
unless $obj->installed;
# If we got our config from pkg-config, do it again with ExtUtils::Liblist
# so we can test that method too.
$obj->_try_liblist() if $obj->how eq 'pkg-config';
# Now that we've done liblist, our method should be 'ExtUtils::Liblist'
is($obj->method, 'ExtUtils::Liblist', 'Detection method is correct');
# Everything should still be defined
is(ref $obj->cflags, 'ARRAY', '->cflags returns an ARRAY ref');
is(ref $obj->ldflags, 'ARRAY', '->ldflags returns an ARRAY ref');
# Returns an array if calling in list context
my @a = $obj->cflags;
ok(scalar(@a) > 0, '->cflags returns a LIST');
@a = $obj->ldflags;
ok(scalar(@a) > 0, '->ldflags returns a LIST');
SKIP: {
skip('version is only returned by pkg-config', 1)
unless $obj->how eq 'pkg-config';
ok(defined $obj->version, 'Version is defined');
}
}
# Make sure the returned values are false
SKIP: {
skip('these tests are for when libjio is not installed', 2)
if $obj->installed;
ok(!$obj->cflags, '->cflags is false');
ok(!$obj->ldflags, '->ldflags is false');
}
# Make sure we try them in void context
$obj->ldflags;
$obj->cflags;
t/03exceptions.t view on Meta::CPAN
my $obj = Alien::Libjio->new();
eval { $obj->new(); };
ok($@, '->new called as an object method');
eval { Alien::Libjio->installed; };
ok($@, '->installed called as a class method');
eval { Alien::Libjio->version; };
ok($@, '->version called as a class method');
eval { Alien::Libjio->ldflags; };
ok($@, '->ldflags called as a class method');
eval { Alien::Libjio->cflags; };
ok($@, '->cflags called as a class method');
}
( run in 1.513 second using v1.01-cache-2.11-cpan-94b05bcf43c )