AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #221 from inikep/dev

Browse Source 
test-zstd-speed.py: added --message in start and finish e-mail
This commit is contained in:
Yann Collet 2016-06-22 23:34:19 +02:00 committed by GitHub
commit 91d67f0226
1 changed files with 45 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
tests/test-zstd-speed.py
View File

@ -88,16 +88,6 @@ def git_get_changes(branch, commit, last_commit):
return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits) return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits)
def compile(branch, commit, last_commit, dry_run):
local_branch = string.split(branch, '/')[1]
version = local_branch.rpartition('-')[2]
version = version + '_' + commit
execute('git checkout -- . && git checkout ' + branch)
print(git_get_changes(branch, commit, last_commit))
if not dry_run:
execute('make clean zstdprogram MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
def get_last_results(resultsFileName): def get_last_results(resultsFileName):
if not os.path.isfile(resultsFileName): if not os.path.isfile(resultsFileName):
return None, None, None return None, None, None
@ -123,7 +113,7 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP
log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" % (os.getloadavg()[0], maxLoadAvg, sleepTime)) log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" % (os.getloadavg()[0], maxLoadAvg, sleepTime))
time.sleep(sleepTime) time.sleep(sleepTime)
start_load = str(os.getloadavg()) start_load = str(os.getloadavg())
result = execute('programs/zstd -qi5b1e' + str(lastCLevel) + ' ' + testFilePath, print_output=True) result = execute('programs/zstd -qi5b1e%s %s' % (lastCLevel, testFilePath), print_output=True)
end_load = str(os.getloadavg()) end_load = str(os.getloadavg())
linesExpected = lastCLevel + 2; linesExpected = lastCLevel + 2;
if len(result) != linesExpected: if len(result) != linesExpected:
@ -148,44 +138,37 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP
return text return text
def check_commit(branch, commit, args, testFilePaths, have_mutt, have_mail): def update_config_file(branch, commit):
try: last_commit = None
last_commit = None commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
commitFileName = working_path + "/commit_" + branch.replace("/", "_") if os.path.isfile(commitFileName):
if os.path.isfile(commitFileName): last_commit = file(commitFileName, 'r').read()
last_commit = file(commitFileName, 'r').read() file(commitFileName, 'w').write(commit)
file(commitFileName, 'w').write(commit) return last_commit
if commit == last_commit:
log("skipping branch %s: head %s already processed" % (branch, commit))
return
log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit)) def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
compile(branch, commit, last_commit, args.dry_run) local_branch = string.split(branch, '/')[1]
version = local_branch.rpartition('-')[2] + '_' + commit
logFileName = working_path + "/log_" + branch.replace("/", "_") if not args.dry_run:
text_to_send = [] execute('make clean zstdprogram MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
results_files = "" logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
for filePath in testFilePaths: text_to_send = []
fileName = filePath.rpartition('/')[2] results_files = ""
resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName for filePath in testFilePaths:
last_commit, cspeed, dspeed = get_last_results(resultsFileName) fileName = filePath.rpartition('/')[2]
resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
if not args.dry_run: last_commit, cspeed, dspeed = get_last_results(resultsFileName)
if not args.dry_run:
text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
if text:
log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message) text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
if text: if text:
log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit)) text_to_send.append(text)
text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message) results_files += resultsFileName + " "
if text: if text_to_send:
text_to_send.append(text) send_email_with_attachments(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail)
results_files += resultsFileName + " "
if text_to_send:
send_email_with_attachments(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail)
except Exception as e:
stack = traceback.format_exc()
email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit)
send_email(args.emails, email_topic, stack, have_mutt, have_mail)
print(stack)
if __name__ == '__main__': if __name__ == '__main__':
@ -205,6 +188,7 @@ if __name__ == '__main__':
testFileNames = args.testFileNames.split() testFileNames = args.testFileNames.split()
testFilePaths = [] testFilePaths = []
for fileName in testFileNames: for fileName in testFileNames:
fileName = os.path.expanduser(fileName)
if os.path.isfile(fileName): if os.path.isfile(fileName):
testFilePaths.append(os.path.abspath(fileName)) testFilePaths.append(os.path.abspath(fileName))
else: else:
@ -248,7 +232,7 @@ if __name__ == '__main__':
log("ERROR: %s already exists, exiting" % pidfile) log("ERROR: %s already exists, exiting" % pidfile)
exit(1) exit(1)
send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, '', have_mutt, have_mail) send_email(args.emails, email_header + ':%s test-zstd-speed.py has been started' % pid, args.message, have_mutt, have_mail)
file(pidfile, 'w').write(pid) file(pidfile, 'w').write(pid)
while True: while True:
@ -257,14 +241,25 @@ if __name__ == '__main__':
if (loadavg <= args.maxLoadAvg): if (loadavg <= args.maxLoadAvg):
branches = git_get_branches() branches = git_get_branches()
for branch in branches: for branch in branches:
commits = execute('git show -s --format=%h ' + branch)[0] commit = execute('git show -s --format=%h ' + branch)[0]
for commit in [commits]: last_commit = update_config_file(branch, commit)
check_commit(branch, commit, args, testFilePaths, have_mutt, have_mail) if commit == last_commit:
log("skipping branch %s: head %s already processed" % (branch, commit))
else:
log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
execute('git checkout -- . && git checkout ' + branch)
print(git_get_changes(branch, commit, last_commit))
test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail)
else: else:
log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg)) log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
log("sleep for %s seconds" % args.sleepTime) log("sleep for %s seconds" % args.sleepTime)
time.sleep(args.sleepTime) time.sleep(args.sleepTime)
except Exception as e:
stack = traceback.format_exc()
email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit)
send_email(args.emails, email_topic, stack, have_mutt, have_mail)
print(stack)
except KeyboardInterrupt: except KeyboardInterrupt:
os.unlink(pidfile) os.unlink(pidfile)
send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, '', have_mutt, have_mail) send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail)
exit(0) exit(0)