view release on metacpan or  search on metacpan

libjio/tests/stress/jiostress  view on Meta::CPAN

	except IOError as e:
		if e.args[0] == libjio.J_ENOJOURNAL:
			return { 'total': 0 }
		else:
			raise

def comp_cont(bytes):
	"'aaaabbcc' -> [ ('a', 4), ('b', 2), ('c', 2) ]"
	l = []
	prev = bytes[0]
	c = 1
	for b in bytes[1:]:
		if (b == prev):
			c += 1
			continue

		l.append((prev, c))
		prev = b
		c = 1
	l.append((b, c))
	return l

def pread(fd, start, end):
	ppos = fd.tell()
	fd.seek(start, 0)
	r = bytes()
	c = 0
	total = end - start
	while c < total:
		n = fd.read(total - c)
		if (n == ''):
			break
		c += len(n)
		r += n
	fd.seek(ppos, 0)
	assert c == end - start
	return r

#
# Output handler, used to get a nice output when using multiple processes
#

class OutputHandler:
	def __init__(self, every):
		# fds to read from
		self.rs = []

		# we will report every this number of seconds
		self.every = every

		# how many transactions has each child processed; we use the
		# read end of the pipe to identify them
		self.ntrans = {}

		# like self.ntrans but counts only the failed ones
		self.nfailures = {}

		# fd to write to, only relevant in the child
		self.w = None

		# p = parent, c = child
		self.end = 'p'

		# last transaction number print
		self.last_print = 0

		# time of the last print
		self.last_print_time = 0

	def prefork(self):
		r, w = os.pipe()
		self.rs.append(r)
		self.ntrans[r] = 0
		self.nfailures[r] = 0
		self.w = w

	def child(self):
		self.end = 'c'
		os.close(self.rs[-1])
		self.rs = []

	def parent(self):
		os.close(self.w)
		self.w = None

	SUCCESS = bytes('1', encoding = 'ascii')
	FAILURE = bytes('0', encoding = 'ascii')

	def feed(self, success = True):
		if success:
			os.write(self.w, OutputHandler.SUCCESS)
		else:
			os.write(self.w, OutputHandler.FAILURE)

	def output_loop(self):
		while self.rs:
			rr, rw, rx = select.select(self.rs, [], [], 1)
			for r in rr:
				d = os.read(r, 1)
				if not d:
					self.rs.remove(r)
				else:
					self.ntrans[r] += 1
					if d == OutputHandler.FAILURE:
						self.nfailures[r] += 1

			self.cond_print()
		self.print()
		return sum(self.ntrans.values()), sum(self.nfailures.values())

	def cond_print(self):
		if time.time() - self.last_print_time >= self.every:
			self.print()

	def print(self):
		self.last_print_time = time.time()
		for r in sorted(self.ntrans):
			print("%4d" % self.ntrans[r], end = ' ')
		print()


#
# Lock manager, used to lock ranges between multiple processes
#
# We can't lock the real file because that would ruin libjio's locking, so we
# create a new file, remove it, and use fcntl locking. Not very elegant but it
# does the trick.
#

class VoidLockManager:
	def __init__(self):
		pass

	def lock(self, start, end):
		pass

	def unlock(self, start, end):
		pass

class LockManager:
	def __init__(self):
		fname = "/tmp/js-lock-tmp." + str(os.getpid())

libjio/tests/stress/jiostress  view on Meta::CPAN

		self.fname = fname
		self.fsize = fsize
		self.nops = nops
		self.use_fi = use_fi
		self.use_as = use_as
		self.output = output
		self.lockmgr = lockmgr
		self.do_verify = do_verify

		jflags = 0
		if use_as:
			jflags = libjio.J_LINGER

		self.jf = libjio.open(fname, libjio.O_RDWR | libjio.O_CREAT,
				0o600, jflags)
		self.f = open(fname, mode = 'rb')

		self.jf.truncate(fsize)

		if use_as:
			self.jf.autosync_start(5, 10 * 1024 * 1024)

	def apply(self, trans):
		trans.prepare()
		trans.apply()
		trans.verify()
		return True

	def apply_fork(self, trans):
		# do the prep before the fork so we can verify() afterwards
		trans.prepare()

		sys.stdout.flush()
		pid = os.fork()
		if pid == 0:
			# child
			try:
				self.fiu_enable()
				trans.apply()
				self.fiu_disable()
			except (IOError, MemoryError):
				try:
					self.reopen(trans)
				except (IOError, MemoryError):
					pass
				except:
					self.fiu_disable()
					traceback.print_exc()
				self.fiu_disable()
				sys.exit(1)
			except MemoryError:
				self.fiu_disable()
				sys.exit(1)
			except:
				self.fiu_disable()
				traceback.print_exc()
				sys.exit(1)
			trans.verify()
			sys.exit(0)
		else:
			# parent
			id, status = os.waitpid(pid, 0)
			if not os.WIFEXITED(status):
				i = (status,
					os.WIFSIGNALED(status),
					os.WTERMSIG(status))
				raise RuntimeError(i)

			if os.WEXITSTATUS(status) != 0:
				return False
			return True

	def reopen(self, trans):
		self.jf = None
		r = jfsck(self.fname)

		trans.verify(write_only = True)

		self.jf = libjio.open(self.fname,
			libjio.O_RDWR | libjio.O_CREAT, 0o600)
		return r

	def fiu_enable(self):
		if not self.use_fi:
			return

		# To improve code coverage, we randomize the probability each
		# time we enable failure points
		fiu.enable_random('jio/*',
				probability = randfloat(0.0005, 0.005))
		fiu.enable_random('linux/*',
				probability = randfloat(0.005, 0.03))
		fiu.enable_random('posix/*',
			probability = randfloat(0.005, 0.03))
		fiu.enable_random('libc/mm/*',
			probability = randfloat(0.003, 0.07))
		fiu.enable_random('libc/str/*',
			probability = randfloat(0.005, 0.07))

	def fiu_disable(self):
		if self.use_fi:
			fiu.disable('libc/mm/*')
			fiu.disable('posix/*')
			fiu.disable('jio/*')
			fiu.disable('linux/*')

	def run(self):
		nfailures = 0

		for i in range(1, self.nops + 1):
			trans = random.choice(t_list)(self.f, self.jf,
					self.fsize, self.lockmgr,
					self.do_verify)

			if self.use_fi:
				r = self.apply_fork(trans)
			else:
				r = self.apply(trans)

			if r:
				self.output.feed(success = True)
			else:
				self.output.feed(success = False)
				nfailures += 1
				r = self.reopen(trans)
				trans.verify(write_only = True)

		return nfailures


#
# Main
#

def run_stressers(nproc, fname, fsize, nops, use_fi, use_as, output, lockmgr,
		do_verify):
	pids = []
	print("Launching stress test")
	for i in range(nproc):
		# Calculate how many operations will this child perform. The
		# last one will work a little more so we get exactly nops.
		# Note we prefer to work extra in the end rather than having
		# the last process with 0 child_nops, that's why we use int()
		# instead of round() or ceil().
		child_nops = int(nops / nproc)
		if i == nproc - 1:
			child_nops = nops - int(nops / nproc) * i

		output.prefork()
		sys.stdout.flush()
		pid = os.fork()
		if pid == 0:
			# child
			output.child()
			s = Stresser(fname, fsize, child_nops, use_fi, use_as,
					output, lockmgr, do_verify)
			s.run()
			sys.exit(0)
		else:
			output.parent()
			pids.append(pid)

	print("Launched stress tests")
	totalops, nfailures = output.output_loop()
	print("Stress test completed, waiting for children")
	nerrors = 0
	for pid in pids:
		rpid, status = os.waitpid(pid, 0)
		if os.WEXITSTATUS(status) != 0:
			nerrors += 1

	print("  %d operations" % totalops)
	print("  %d simulated failures" % nfailures)
	print("  %d processes ended with errors" % nerrors)
	if nerrors:
		return False
	return True

def main():
	usage = "Use: %prog [options] <file name> <file size in Mb>"
	parser = OptionParser(usage = usage)
	parser.add_option("-n", "--nops", dest = "nops", type = "int",
		default = 100,
		help = "number of operations (defaults to %default)")
	parser.add_option("-p", "--nproc", dest = "nproc", type = "int",
		default = 1,
		help = "number of processes (defaults to %default)")
	parser.add_option("", "--fi", dest = "use_fi",
		action = "store_true", default = False,
		help = "use fault injection (conflicts with --as and -p > 1)")
	parser.add_option("", "--as", dest = "use_as",
		action = "store_true", default = False,
		help = "use J_LINGER + autosync (conflicts with --fi)")
	parser.add_option("", "--no-internal-lock",
		dest = "use_internal_locks", action = "store_false",
		default = True,
		help = "do not lock internally, disables verification")
	parser.add_option("", "--no-verify", dest = "do_verify",
		action = "store_false", default = True,
		help = "do not perform verifications")
	parser.add_option("", "--keep", dest = "keep",
		action = "store_true", default = False,
		help = "keep the file after completing the test")
	parser.add_option("", "--force", dest = "force",
		action = "store_true", default = False,
		help = "force the tests to run, even if conflicting"
			+ " options are selected")

	options, args = parser.parse_args()

	if len(args) != 2:
		parser.print_help()
		return 1

	fname = args[0]
	try:
		fsize = int(args[1]) * 1024 * 1024
	except ValueError:
		print("Error: the size of the file must be numeric")
		return 1