view release on metacpan or  search on metacpan

libjio/tests/stress/jiostress  view on Meta::CPAN

	return bytes(str(gbcount) * n, 'ascii')

def randfrange(maxend, maxsize):
	start = random.randint(0, maxend - 1)
	size = random.randint(0, (maxend - 1) - start) % maxsize
	return start, start + size

def randfloat(min, max):
	return min + random.random() % (max - min)

class ConsistencyError (Exception):
	pass

def jfsck(fname, cleanup = False):
	flags = 0
	if cleanup:
		flags = libjio.J_CLEANUP

	try:
		r = libjio.jfsck(fname, flags = flags)
		return r
	except IOError as e:
		if e.args[0] == libjio.J_ENOJOURNAL:
			return { 'total': 0 }
		else:
			raise

def comp_cont(bytes):
	"'aaaabbcc' -> [ ('a', 4), ('b', 2), ('c', 2) ]"
	l = []
	prev = bytes[0]
	c = 1
	for b in bytes[1:]:
		if (b == prev):
			c += 1
			continue

		l.append((prev, c))
		prev = b
		c = 1
	l.append((b, c))
	return l

def pread(fd, start, end):
	ppos = fd.tell()
	fd.seek(start, 0)
	r = bytes()
	c = 0
	total = end - start
	while c < total:
		n = fd.read(total - c)
		if (n == ''):
			break
		c += len(n)
		r += n
	fd.seek(ppos, 0)
	assert c == end - start
	return r

#
# Output handler, used to get a nice output when using multiple processes
#

class OutputHandler:
	def __init__(self, every):
		# fds to read from
		self.rs = []

		# we will report every this number of seconds
		self.every = every

		# how many transactions has each child processed; we use the
		# read end of the pipe to identify them
		self.ntrans = {}

		# like self.ntrans but counts only the failed ones
		self.nfailures = {}

		# fd to write to, only relevant in the child
		self.w = None

		# p = parent, c = child
		self.end = 'p'

		# last transaction number print
		self.last_print = 0

		# time of the last print
		self.last_print_time = 0

	def prefork(self):
		r, w = os.pipe()
		self.rs.append(r)
		self.ntrans[r] = 0
		self.nfailures[r] = 0
		self.w = w

	def child(self):
		self.end = 'c'
		os.close(self.rs[-1])
		self.rs = []

	def parent(self):
		os.close(self.w)
		self.w = None

	SUCCESS = bytes('1', encoding = 'ascii')
	FAILURE = bytes('0', encoding = 'ascii')

	def feed(self, success = True):
		if success:
			os.write(self.w, OutputHandler.SUCCESS)
		else:
			os.write(self.w, OutputHandler.FAILURE)

	def output_loop(self):
		while self.rs:
			rr, rw, rx = select.select(self.rs, [], [], 1)
			for r in rr:
				d = os.read(r, 1)
				if not d:
					self.rs.remove(r)
				else:
					self.ntrans[r] += 1
					if d == OutputHandler.FAILURE:
						self.nfailures[r] += 1

			self.cond_print()
		self.print()
		return sum(self.ntrans.values()), sum(self.nfailures.values())

	def cond_print(self):
		if time.time() - self.last_print_time >= self.every:
			self.print()

	def print(self):
		self.last_print_time = time.time()
		for r in sorted(self.ntrans):
			print("%4d" % self.ntrans[r], end = ' ')
		print()


#
# Lock manager, used to lock ranges between multiple processes
#
# We can't lock the real file because that would ruin libjio's locking, so we
# create a new file, remove it, and use fcntl locking. Not very elegant but it
# does the trick.
#

class VoidLockManager:
	def __init__(self):
		pass

	def lock(self, start, end):
		pass

	def unlock(self, start, end):
		pass

class LockManager:
	def __init__(self):
		fname = "/tmp/js-lock-tmp." + str(os.getpid())
		self.fd = open(fname, 'w+')
		os.unlink(fname)

	def lock(self, start, end):
		#print(os.getpid(), '\tlock:', start, end)
		#sys.stdout.flush()
		fcntl.lockf(self.fd, fcntl.LOCK_EX, end - start, start)

	def unlock(self, start, end):
		#print(os.getpid(), '\tunlock:', start, end)
		#sys.stdout.flush()
		fcntl.lockf(self.fd, fcntl.LOCK_UN, end - start, start)


#
# A range of bytes inside a file, used inside the transactions
#
# Note it can't "remember" the fd as it may change between prepare() and
# verify().
#

class Range:
	def __init__(self, fsize, maxlen, lockmgr):
		# public
		self.start, self.end = randfrange(fsize, maxlen)
		self.new_data = None
		self.type = 'r'

		# private
		self.prev_data = None
		self.new_data_ctx = None
		self.read_buf = None
		self.lockmgr = lockmgr
		self.locked = False

		# read an extended range so we can check we
		# only wrote what we were supposed to
		self.ext_start = max(0, self.start - 32)
		self.ext_end = min(fsize, self.end + 32)

	def __lt__(self, other):

libjio/tests/stress/jiostress  view on Meta::CPAN

	def fiu_enable(self):
		if not self.use_fi:
			return

		# To improve code coverage, we randomize the probability each
		# time we enable failure points
		fiu.enable_random('jio/*',
				probability = randfloat(0.0005, 0.005))
		fiu.enable_random('linux/*',
				probability = randfloat(0.005, 0.03))
		fiu.enable_random('posix/*',
			probability = randfloat(0.005, 0.03))
		fiu.enable_random('libc/mm/*',
			probability = randfloat(0.003, 0.07))
		fiu.enable_random('libc/str/*',
			probability = randfloat(0.005, 0.07))

	def fiu_disable(self):
		if self.use_fi:
			fiu.disable('libc/mm/*')
			fiu.disable('posix/*')
			fiu.disable('jio/*')
			fiu.disable('linux/*')

	def run(self):
		nfailures = 0

		for i in range(1, self.nops + 1):
			trans = random.choice(t_list)(self.f, self.jf,
					self.fsize, self.lockmgr,
					self.do_verify)

			if self.use_fi:
				r = self.apply_fork(trans)
			else:
				r = self.apply(trans)

			if r:
				self.output.feed(success = True)
			else:
				self.output.feed(success = False)
				nfailures += 1
				r = self.reopen(trans)
				trans.verify(write_only = True)

		return nfailures


#
# Main
#

def run_stressers(nproc, fname, fsize, nops, use_fi, use_as, output, lockmgr,
		do_verify):
	pids = []
	print("Launching stress test")
	for i in range(nproc):
		# Calculate how many operations will this child perform. The
		# last one will work a little more so we get exactly nops.
		# Note we prefer to work extra in the end rather than having
		# the last process with 0 child_nops, that's why we use int()
		# instead of round() or ceil().
		child_nops = int(nops / nproc)
		if i == nproc - 1:
			child_nops = nops - int(nops / nproc) * i

		output.prefork()
		sys.stdout.flush()
		pid = os.fork()
		if pid == 0:
			# child
			output.child()
			s = Stresser(fname, fsize, child_nops, use_fi, use_as,
					output, lockmgr, do_verify)
			s.run()
			sys.exit(0)
		else:
			output.parent()
			pids.append(pid)

	print("Launched stress tests")
	totalops, nfailures = output.output_loop()
	print("Stress test completed, waiting for children")
	nerrors = 0
	for pid in pids:
		rpid, status = os.waitpid(pid, 0)
		if os.WEXITSTATUS(status) != 0:
			nerrors += 1

	print("  %d operations" % totalops)
	print("  %d simulated failures" % nfailures)
	print("  %d processes ended with errors" % nerrors)
	if nerrors:
		return False
	return True

def main():
	usage = "Use: %prog [options] <file name> <file size in Mb>"
	parser = OptionParser(usage = usage)
	parser.add_option("-n", "--nops", dest = "nops", type = "int",
		default = 100,
		help = "number of operations (defaults to %default)")
	parser.add_option("-p", "--nproc", dest = "nproc", type = "int",
		default = 1,
		help = "number of processes (defaults to %default)")
	parser.add_option("", "--fi", dest = "use_fi",
		action = "store_true", default = False,
		help = "use fault injection (conflicts with --as and -p > 1)")
	parser.add_option("", "--as", dest = "use_as",
		action = "store_true", default = False,
		help = "use J_LINGER + autosync (conflicts with --fi)")
	parser.add_option("", "--no-internal-lock",
		dest = "use_internal_locks", action = "store_false",
		default = True,
		help = "do not lock internally, disables verification")
	parser.add_option("", "--no-verify", dest = "do_verify",
		action = "store_false", default = True,
		help = "do not perform verifications")
	parser.add_option("", "--keep", dest = "keep",
		action = "store_true", default = False,
		help = "keep the file after completing the test")
	parser.add_option("", "--force", dest = "force",
		action = "store_true", default = False,
		help = "force the tests to run, even if conflicting"
			+ " options are selected")

	options, args = parser.parse_args()

	if len(args) != 2:
		parser.print_help()
		return 1

	fname = args[0]
	try:
		fsize = int(args[1]) * 1024 * 1024
	except ValueError:
		print("Error: the size of the file must be numeric")
		return 1

	if not options.force:
		if options.use_fi and options.use_as:
			print("Error: --fi and --as cannot be used together")
			return 1

		if options.use_fi and options.nproc > 1:
			print("Error: --fi cannot be used with multiple processes")
			return 1

	if not options.use_internal_locks:
		options.do_verify = False

	output = OutputHandler(every = 2)
	if options.use_internal_locks:
		lockmgr = LockManager()
	else:
		lockmgr = VoidLockManager()

	success = run_stressers(options.nproc, fname, fsize, options.nops,
			options.use_fi, options.use_as, output, lockmgr,
			options.do_verify)

	r = jfsck(fname)
	print("Final check completed")
	if success and not options.keep:
		jfsck(fname, cleanup = True)
		os.unlink(fname)

	if not success:
		print("Test failed")
		return 1
	return 0


if __name__ == '__main__':
	sys.exit(main())