diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py new file mode 100644 index 00000000..a69b1b75 --- /dev/null +++ b/tests/test-zstd-speed.py @@ -0,0 +1,222 @@ +#! /usr/bin/env python +# execute(), fetch(), notify() are based on https://github.com/getlantern/build-automation/blob/master/build.py + +import argparse +import os +import string +import time +import traceback +from subprocess import Popen, PIPE + +repo_url = 'https://github.com/Cyan4973/zstd.git' +test_dir_name = 'speedTest' + + +def log(text): + print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text + +def execute(command, print_output=True): + log("> " + command) + popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True, cwd=execute.cwd) + itout = iter(popen.stdout.readline, b"") + iterr = iter(popen.stderr.readline, b"") + stdout_lines = list(itout) + if print_output: + print ''.join(stdout_lines) + stderr_lines = list(iterr) + if stderr_lines: + print ''.join(stderr_lines) + popen.communicate() + if popen.returncode is not None and popen.returncode != 0: + raise RuntimeError(''.join(stderr_lines)) + return stdout_lines + stderr_lines +execute.cwd = None + + +def fetch(): + execute('git fetch -p') + output = execute('git branch -rl') + for line in output: + if "HEAD" in line: + output.remove(line) # remove "origin/HEAD -> origin/dev" + branches = map(lambda l: l.strip(), output) + return map(lambda b: (b, execute('git show -s --format=%h ' + b)[0].strip()), branches) + + +def notify(branch, commit, last_commit): + text_tmpl = string.Template('Changes since $last_commit:\r\n$commits') + branch = branch.split('/')[1] + fmt = '--format="%h: (%an) %s, %ar"' + if last_commit is None: + commits = execute('git log -n 10 %s %s' % (fmt, commit)) + else: + commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit)) + + text = text_tmpl.substitute({'last_commit': last_commit, 'commits': ''.join(commits)}) + print str("commits for %s: %s" % (commit, text)) + + +def compile(branch, commit, dry_run): + local_branch = string.split(branch, '/')[1] + version = local_branch.rpartition('-')[2] + version = version + '_' + commit + execute('git checkout -- . && git checkout ' + branch) + if not dry_run: + execute('VERSION=' + version + '; make clean zstdprogram') + + +def get_last_commit(resultsFileName): + if not os.path.isfile(resultsFileName): + return None, None, None + commit = None + cspeed = [] + dspeed = [] + with open(resultsFileName,'r') as f: + for line in f: + words = line.split() + if len(words) == 2: # branch + commit + commit = words[1]; + cspeed = [] + dspeed = [] + if (len(words) == 8): + cspeed.append(float(words[3])) + dspeed.append(float(words[5])) + #if commit != None: + # print "commit=%s cspeed=%s dspeed=%s" % (commit, cspeed, dspeed) + return commit, cspeed, dspeed + + +def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFilePath, fileName, last_cspeed, last_dspeed, lower_limit, maxLoadAvg): + while os.getloadavg()[0] > maxLoadAvg: + print "bench loadavg=%.2f is higher than %s" % (os.getloadavg()[0], maxLoadAvg) + time.sleep(30) + start_load = str(os.getloadavg()) + result = execute('programs/zstd -qb1e' + str(lastCLevel) + ' ' + testFilePath) + end_load = str(os.getloadavg()) + linesExpected = lastCLevel + 2; + if len(result) != linesExpected: + print "len(result)=%d is different that expected %d" % (len(result), linesExpected) + return "" + with open(resultsFileName, "a") as myfile: + myfile.write(branch + " " + commit + "\n") + myfile.writelines(result) + myfile.close() + if (last_cspeed == None): + return "" + commit, cspeed, dspeed = get_last_commit(resultsFileName) + text = "" + for i in range(0, min(len(cspeed), len(last_cspeed))): + if (cspeed[i]/last_cspeed[i] < lower_limit): + text += "WARNING: File=%s level=%d cspeed=%s last=%s diff=%s\n" % (fileName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i]) + if (dspeed[i]/last_dspeed[i] < lower_limit): + text += "WARNING: File=%s level=%d dspeed=%s last=%s diff=%s\n" % (fileName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i]) + if text: + text += "maxLoadAvg=%s load average at start=%s end=%s\n" % (maxLoadAvg, start_load, end_load) + return text + + +def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit): + with open(logFileName, "w") as myfile: + myfile.writelines(text) + myfile.close() + execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName) + + +def main(args, test_path, clone_path, testFilePaths): + print "test_path=%s" % test_path + print "clone_path=%s" % clone_path + print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths) + print "emails=%s" % args.emails + print "maxLoadAvg=%s" % args.maxLoadAvg + print "lowerLimit=%s" % args.lowerLimit + print "lastCLevel=%s" % args.lastCLevel + print "sleepTime=%s" % args.sleepTime + print "dry_run=%s" % args.dry_run + + for branch, commit in fetch(): + log("checking branch %s: head %s" % (branch, commit)) + try: + commitFileName = test_path + "/commit_" + branch.replace("/", "_") + if os.path.isfile(commitFileName): + last_commit = file(commitFileName, 'r').read() + else: + last_commit = None + file(commitFileName, 'w').write(commit) + + if commit == last_commit: + log("skipping branch %s: head %s already processed" % (branch, commit)) + else: + log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit)) + compile(branch, commit, args.dry_run) + + logFileName = test_path + "/log_" + branch.replace("/", "_") + text_to_send = [] + results_files = "" + for filePath in testFilePaths: + fileName = filePath.rpartition('/')[2] + resultsFileName = test_path + "/results_" + branch.replace("/", "_") + "_" + fileName + last_commit, cspeed, dspeed = get_last_commit(resultsFileName) + + if not args.dry_run: + text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg) + if text: + text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg) + if text: + text_to_send.append(text) + results_files += resultsFileName + " " + if text_to_send: + send_email(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit) + notify(branch, commit, last_commit) + except Exception as e: + stack = traceback.format_exc() + log("Error build %s, error %s" % (branch, str(e)) ) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('testFileNames', help='file names list for speed test') + parser.add_argument('emails', help='e-mails to send warnings') + parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit e.g. 0.98', default=0.98) + parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75) + parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5) + parser.add_argument('--sleepTime', type=int, help='frequency of checking in seconds', default=60) + parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False) + args = parser.parse_args() + + # check if test files are accessible + testFileNames = args.testFileNames.split() + testFilePaths = [] + for fileName in testFileNames: + if os.path.isfile(fileName): + testFilePaths.append(os.path.abspath(fileName)) + else: + raise RuntimeError("File not found: " + fileName) + + test_path = os.getcwd() + '/' + test_dir_name # /path/to/zstd/tests/speedTest + clone_path = test_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd + execute.cwd = clone_path + + # clone ZSTD repo if needed + if not os.path.isdir(test_path): + os.mkdir(test_path) + if not os.path.isdir(clone_path): + execute('git clone ' + repo_url) + if not os.path.isdir(clone_path): + raise RuntimeError("ZSTD clone not found: " + clone_path) + + while True: + pid = str(os.getpid()) + pidfile = "./speedTest.pid" + if os.path.isfile(pidfile): + print "%s already exists, exiting" % pidfile + else: + file(pidfile, 'w').write(pid) + try: + loadavg = os.getloadavg()[0] + if (loadavg <= args.maxLoadAvg): + main(args, test_path, clone_path, testFilePaths) + else: + print "loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg) + finally: + os.unlink(pidfile) + time.sleep(args.sleepTime) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index 19e083a8..437cd4c0 100644 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -11,7 +11,7 @@ import subprocess import sys repo_url = 'https://github.com/Cyan4973/zstd.git' -tmp_dir_name = 'versionsTest/zstdtest' +tmp_dir_name = 'tests/versionsTest' make_cmd = 'make' git_cmd = 'git' test_dat_src = 'README.md' @@ -110,8 +110,8 @@ def decompress_zst(tag): if __name__ == '__main__': error_code = 0 base_dir = os.getcwd() + '/..' # /path/to/zstd - tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/versionsTest/zstdtest - clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/versionsTest/zstdtest/zstd + tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest + clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd programs_dir = base_dir + '/programs' # /path/to/zstd/programs os.makedirs(tmp_dir, exist_ok=True) @@ -130,14 +130,14 @@ if __name__ == '__main__': # Build all release zstd for tag in tags: os.chdir(base_dir) - dst_zstd = '{}/zstd.{}' .format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/zstd. + dst_zstd = '{}/zstd.{}' .format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/zstd. if not os.path.isfile(dst_zstd) or tag == head: if tag != head: - r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/ + r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/ os.makedirs(r_dir, exist_ok=True) os.chdir(clone_dir) git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) - os.chdir(r_dir + '/programs') # /path/to/zstd/zstdtest//programs + os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest//programs make(['clean', 'zstd'], False) else: os.chdir(programs_dir)