added test-zstd-speed.py

tests/test-zstd-speed.py

@@ -0,0 +1,222 @@
+#! /usr/bin/env python
+# execute(), fetch(), notify() are based on https://github.com/getlantern/build-automation/blob/master/build.py
+
+import argparse
+import os
+import string
+import time
+import traceback
+from subprocess import Popen, PIPE
+
+repo_url = 'https://github.com/Cyan4973/zstd.git'
+test_dir_name = 'speedTest'
+
+
+def log(text):
+    print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text
+
+def execute(command, print_output=True):
+    log("> " + command)
+    popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True, cwd=execute.cwd)
+    itout = iter(popen.stdout.readline, b"")
+    iterr = iter(popen.stderr.readline, b"")
+    stdout_lines = list(itout)
+    if print_output:
+        print ''.join(stdout_lines)
+    stderr_lines = list(iterr)
+    if stderr_lines:
+        print ''.join(stderr_lines)
+    popen.communicate()
+    if popen.returncode is not None and popen.returncode != 0:
+        raise RuntimeError(''.join(stderr_lines))
+    return stdout_lines + stderr_lines
+execute.cwd = None
+
+
+def fetch():
+    execute('git fetch -p')
+    output = execute('git branch -rl')
+    for line in output:
+        if "HEAD" in line: 
+            output.remove(line) # remove "origin/HEAD -> origin/dev"
+    branches = map(lambda l: l.strip(), output)
+    return map(lambda b: (b, execute('git show -s --format=%h ' + b)[0].strip()), branches)
+
+
+def notify(branch, commit, last_commit):
+    text_tmpl = string.Template('Changes since $last_commit:\r\n$commits')
+    branch = branch.split('/')[1]
+    fmt = '--format="%h: (%an) %s, %ar"'
+    if last_commit is None:
+        commits = execute('git log -n 10 %s %s' % (fmt, commit))
+    else:
+        commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
+
+    text = text_tmpl.substitute({'last_commit': last_commit, 'commits': ''.join(commits)})
+    print str("commits for %s: %s" % (commit, text))
+
+
+def compile(branch, commit, dry_run):
+    local_branch = string.split(branch, '/')[1]
+    version = local_branch.rpartition('-')[2]
+    version = version + '_' + commit
+    execute('git checkout -- . && git checkout ' + branch)
+    if not dry_run:
+        execute('VERSION=' + version + '; make clean zstdprogram')
+
+
+def get_last_commit(resultsFileName):
+    if not os.path.isfile(resultsFileName):
+        return None, None, None
+    commit = None
+    cspeed = []
+    dspeed = []
+    with open(resultsFileName,'r') as f:
+        for line in f:
+            words = line.split()
+            if len(words) == 2: # branch + commit
+                commit = words[1];
+                cspeed = []
+                dspeed = []
+            if (len(words) == 8): 
+                cspeed.append(float(words[3]))
+                dspeed.append(float(words[5]))
+        #if commit != None:
+        #    print "commit=%s cspeed=%s dspeed=%s" % (commit, cspeed, dspeed)
+    return commit, cspeed, dspeed
+
+
+def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFilePath, fileName, last_cspeed, last_dspeed, lower_limit, maxLoadAvg):
+    while os.getloadavg()[0] > maxLoadAvg:
+        print "bench loadavg=%.2f is higher than %s" % (os.getloadavg()[0], maxLoadAvg)
+        time.sleep(30)
+    start_load = str(os.getloadavg())
+    result = execute('programs/zstd -qb1e' + str(lastCLevel) + ' ' + testFilePath)
+    end_load = str(os.getloadavg())
+    linesExpected = lastCLevel + 2;
+    if len(result) != linesExpected:
+        print "len(result)=%d is different that expected %d" % (len(result), linesExpected)
+        return ""
+    with open(resultsFileName, "a") as myfile:
+        myfile.write(branch + " " + commit + "\n")
+        myfile.writelines(result)
+        myfile.close()
+        if (last_cspeed == None):
+            return ""
+        commit, cspeed, dspeed = get_last_commit(resultsFileName)
+        text = ""
+        for i in range(0, min(len(cspeed), len(last_cspeed))):
+            if (cspeed[i]/last_cspeed[i] < lower_limit):
+                text += "WARNING: File=%s level=%d cspeed=%s last=%s diff=%s\n" % (fileName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i])
+            if (dspeed[i]/last_dspeed[i] < lower_limit):
+                text += "WARNING: File=%s level=%d dspeed=%s last=%s diff=%s\n" % (fileName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i])
+        if text:
+            text += "maxLoadAvg=%s  load average at start=%s end=%s\n" % (maxLoadAvg, start_load, end_load)
+        return text
+
+
+def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit):
+    with open(logFileName, "w") as myfile:
+        myfile.writelines(text)
+        myfile.close()
+        execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName)
+
+
+def main(args, test_path, clone_path, testFilePaths):
+    print "test_path=%s" % test_path
+    print "clone_path=%s" % clone_path
+    print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)
+    print "emails=%s" % args.emails
+    print "maxLoadAvg=%s" % args.maxLoadAvg
+    print "lowerLimit=%s" % args.lowerLimit
+    print "lastCLevel=%s" % args.lastCLevel
+    print "sleepTime=%s" % args.sleepTime
+    print "dry_run=%s" % args.dry_run
+
+    for branch, commit in fetch():
+        log("checking branch %s: head %s" % (branch, commit))
+        try:
+            commitFileName = test_path + "/commit_" + branch.replace("/", "_")
+            if os.path.isfile(commitFileName):
+                last_commit = file(commitFileName, 'r').read()
+            else:
+                last_commit = None
+            file(commitFileName, 'w').write(commit)
+
+            if commit == last_commit:
+                log("skipping branch %s: head %s already processed" % (branch, commit))
+            else:
+                log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
+                compile(branch, commit, args.dry_run)
+
+                logFileName = test_path + "/log_" + branch.replace("/", "_")
+                text_to_send = []
+                results_files = ""
+                for filePath in testFilePaths:
+                    fileName = filePath.rpartition('/')[2]
+                    resultsFileName = test_path + "/results_" + branch.replace("/", "_") + "_" + fileName
+                    last_commit, cspeed, dspeed = get_last_commit(resultsFileName)
+
+                    if not args.dry_run:
+                        text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg)
+                        if text:
+                            text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg)
+                            if text:
+                                text_to_send.append(text)
+                                results_files += resultsFileName + " "
+                if text_to_send:
+                    send_email(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit)
+                notify(branch, commit, last_commit)
+        except Exception as e:
+            stack = traceback.format_exc()
+            log("Error build %s, error %s" % (branch, str(e)) )
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('testFileNames', help='file names list for speed test')
+    parser.add_argument('emails', help='e-mails to send warnings')
+    parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit e.g. 0.98', default=0.98)
+    parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
+    parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
+    parser.add_argument('--sleepTime', type=int, help='frequency of checking in seconds', default=60)
+    parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False)
+    args = parser.parse_args()
+
+    # check if test files are accessible
+    testFileNames = args.testFileNames.split()
+    testFilePaths = []
+    for fileName in testFileNames:
+        if os.path.isfile(fileName):
+            testFilePaths.append(os.path.abspath(fileName))
+        else:
+            raise RuntimeError("File not found: " + fileName)
+
+    test_path = os.getcwd() + '/' + test_dir_name     # /path/to/zstd/tests/speedTest 
+    clone_path = test_path + '/' + 'zstd'             # /path/to/zstd/tests/speedTest/zstd 
+    execute.cwd = clone_path
+
+    # clone ZSTD repo if needed
+    if not os.path.isdir(test_path):
+        os.mkdir(test_path)
+    if not os.path.isdir(clone_path):
+        execute('git clone ' + repo_url)
+    if not os.path.isdir(clone_path):
+        raise RuntimeError("ZSTD clone not found: " + clone_path)
+
+    while True:
+        pid = str(os.getpid())
+        pidfile = "./speedTest.pid"
+        if os.path.isfile(pidfile):
+            print "%s already exists, exiting" % pidfile
+        else:
+            file(pidfile, 'w').write(pid)
+            try:
+                loadavg = os.getloadavg()[0]
+                if (loadavg <= args.maxLoadAvg):
+                    main(args, test_path, clone_path, testFilePaths)
+                else:
+                    print "loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg)
+            finally:
+                os.unlink(pidfile)
+        time.sleep(args.sleepTime)

tests/test-zstd-versions.py

@@ -11,7 +11,7 @@ import subprocess
 import sys
 
 repo_url = 'https://github.com/Cyan4973/zstd.git'
-tmp_dir_name = 'versionsTest/zstdtest'
+tmp_dir_name = 'tests/versionsTest'
 make_cmd = 'make'
 git_cmd = 'git'
 test_dat_src = 'README.md'
@@ -110,8 +110,8 @@ def decompress_zst(tag):
 if __name__ == '__main__':
     error_code = 0
     base_dir = os.getcwd() + '/..'           # /path/to/zstd
-    tmp_dir = base_dir + '/' + tmp_dir_name  # /path/to/zstd/versionsTest/zstdtest
-    clone_dir = tmp_dir + '/' + 'zstd'       # /path/to/zstd/versionsTest/zstdtest/zstd
+    tmp_dir = base_dir + '/' + tmp_dir_name  # /path/to/zstd/tests/versionsTest
+    clone_dir = tmp_dir + '/' + 'zstd'       # /path/to/zstd/tests/versionsTest/zstd
     programs_dir = base_dir + '/programs'    # /path/to/zstd/programs
     os.makedirs(tmp_dir, exist_ok=True)
 
@@ -130,14 +130,14 @@ if __name__ == '__main__':
     # Build all release zstd
     for tag in tags:
         os.chdir(base_dir)
-        dst_zstd = '{}/zstd.{}'  .format(tmp_dir, tag)  # /path/to/zstd/test/zstdtest/zstd.<TAG>
+        dst_zstd = '{}/zstd.{}'  .format(tmp_dir, tag)  # /path/to/zstd/tests/versionsTest/zstd.<TAG>
         if not os.path.isfile(dst_zstd) or tag == head:
             if tag != head:
-                r_dir = '{}/{}'.format(tmp_dir, tag)  # /path/to/zstd/test/zstdtest/<TAG>
+                r_dir = '{}/{}'.format(tmp_dir, tag)  # /path/to/zstd/tests/versionsTest/<TAG>
                 os.makedirs(r_dir, exist_ok=True)
                 os.chdir(clone_dir)
                 git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
-                os.chdir(r_dir + '/programs')  # /path/to/zstd/zstdtest/<TAG>/programs
+                os.chdir(r_dir + '/programs')  # /path/to/zstd/tests/versionsTest/<TAG>/programs
                 make(['clean', 'zstd'], False)
             else:
                 os.chdir(programs_dir)