2016-10-28 17:24:16 +00:00
#! /usr/bin/env python3
2016-06-09 10:54:06 +00:00
2016-08-30 17:04:33 +00:00
#
2016-09-12 12:17:47 +00:00
# Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
2016-08-30 17:04:33 +00:00
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
#
2016-09-15 18:31:29 +00:00
# Limitations:
# - doesn't support filenames with spaces
# - dir1/zstd and dir2/zstd will be merged in a single results file
2016-06-09 10:54:06 +00:00
import argparse
import os
import string
2016-07-30 23:59:23 +00:00
import subprocess
2016-06-09 10:54:06 +00:00
import time
import traceback
2016-08-10 12:14:01 +00:00
import hashlib
2016-07-22 03:02:27 +00:00
2016-10-28 17:24:16 +00:00
script_version = ' v1.1.1 (2016-10-28) '
2016-09-03 05:11:49 +00:00
default_repo_url = ' https://github.com/facebook/zstd.git '
2016-06-22 10:12:35 +00:00
working_dir_name = ' speedTest '
2016-07-22 03:02:27 +00:00
working_path = os . getcwd ( ) + ' / ' + working_dir_name # /path/to/zstd/tests/speedTest
clone_path = working_path + ' / ' + ' zstd ' # /path/to/zstd/tests/speedTest/zstd
2016-08-10 12:14:01 +00:00
email_header = ' ZSTD_speedTest '
2016-06-22 10:12:35 +00:00
pid = str ( os . getpid ( ) )
2016-07-19 13:49:14 +00:00
verbose = False
2016-09-12 12:17:47 +00:00
clang_version = " unknown "
gcc_version = " unknown "
2016-10-28 17:24:16 +00:00
args = None
2016-06-09 10:54:06 +00:00
2016-08-10 12:14:01 +00:00
def hashfile ( hasher , fname , blocksize = 65536 ) :
with open ( fname , " rb " ) as f :
for chunk in iter ( lambda : f . read ( blocksize ) , b " " ) :
hasher . update ( chunk )
return hasher . hexdigest ( )
2016-06-09 10:54:06 +00:00
def log ( text ) :
2016-06-21 17:28:51 +00:00
print ( time . strftime ( " % Y/ % m/ %d % H: % M: % S " ) + ' - ' + text )
2016-06-09 10:54:06 +00:00
2016-06-21 17:28:51 +00:00
2016-07-19 13:49:14 +00:00
def execute ( command , print_command = True , print_output = False , print_error = True , param_shell = True ) :
if print_command :
log ( " > " + command )
2016-10-28 17:24:16 +00:00
popen = subprocess . Popen ( command , stdout = subprocess . PIPE , stderr = subprocess . PIPE , shell = param_shell , cwd = execute . cwd )
stdout_lines , stderr_lines = popen . communicate ( timeout = args . timeout )
stderr_lines = stderr_lines . decode ( " utf-8 " )
stdout_lines = stdout_lines . decode ( " utf-8 " )
2016-06-21 17:28:51 +00:00
if print_output :
2016-10-28 17:24:16 +00:00
if stdout_lines :
print ( stdout_lines )
if stderr_lines :
print ( stderr_lines )
2016-06-09 10:54:06 +00:00
if popen . returncode is not None and popen . returncode != 0 :
2016-10-28 17:24:16 +00:00
if stderr_lines and not print_output and print_error :
print ( stderr_lines )
raise RuntimeError ( stdout_lines + stderr_lines )
return ( stdout_lines + stderr_lines ) . splitlines ( )
2016-06-09 10:54:06 +00:00
execute . cwd = None
2016-06-10 10:53:12 +00:00
def does_command_exist ( command ) :
2016-06-22 10:12:35 +00:00
try :
2016-07-30 23:59:23 +00:00
execute ( command , verbose , False , False )
except Exception :
2016-06-22 10:12:35 +00:00
return False
return True
2016-06-10 10:53:12 +00:00
2016-06-22 11:07:58 +00:00
def send_email ( emails , topic , text , have_mutt , have_mail ) :
logFileName = working_path + ' / ' + ' tmpEmailContent '
with open ( logFileName , " w " ) as myfile :
myfile . writelines ( text )
myfile . close ( )
if have_mutt :
2016-07-19 13:49:14 +00:00
execute ( ' mutt -s " ' + topic + ' " ' + emails + ' < ' + logFileName , verbose )
2016-06-22 11:07:58 +00:00
elif have_mail :
2016-07-19 13:49:14 +00:00
execute ( ' mail -s " ' + topic + ' " ' + emails + ' < ' + logFileName , verbose )
2016-06-22 11:07:58 +00:00
else :
log ( " e-mail cannot be sent (mail or mutt not found) " )
2016-07-30 23:59:23 +00:00
def send_email_with_attachments ( branch , commit , last_commit , args , text , results_files ,
logFileName , have_mutt , have_mail ) :
2016-06-22 11:07:58 +00:00
with open ( logFileName , " w " ) as myfile :
myfile . writelines ( text )
myfile . close ( )
2016-08-10 12:14:01 +00:00
email_topic = ' [ %s : %s ] Warning for %s : %s last_commit= %s speed< %s ratio< %s ' \
2016-07-30 23:59:23 +00:00
% ( email_header , pid , branch , commit , last_commit ,
args . lowerLimit , args . ratioLimit )
2016-06-22 11:07:58 +00:00
if have_mutt :
2016-07-30 23:59:23 +00:00
execute ( ' mutt -s " ' + email_topic + ' " ' + args . emails + ' -a ' + results_files
+ ' < ' + logFileName )
2016-06-22 11:07:58 +00:00
elif have_mail :
2016-07-19 15:59:53 +00:00
execute ( ' mail -s " ' + email_topic + ' " ' + args . emails + ' < ' + logFileName )
2016-06-22 11:07:58 +00:00
else :
log ( " e-mail cannot be sent (mail or mutt not found) " )
def git_get_branches ( ) :
2016-07-19 13:49:14 +00:00
execute ( ' git fetch -p ' , verbose )
branches = execute ( ' git branch -rl ' , verbose )
2016-07-19 11:09:00 +00:00
output = [ ]
for line in branches :
if ( " HEAD " not in line ) and ( " coverity_scan " not in line ) and ( " gh-pages " not in line ) :
output . append ( line . strip ( ) )
return output
2016-06-09 10:54:06 +00:00
2016-06-22 13:42:26 +00:00
def git_get_changes ( branch , commit , last_commit ) :
2016-06-09 10:54:06 +00:00
fmt = ' --format= " % h: ( %a n) %s , %a r " '
if last_commit is None :
2016-06-22 10:12:35 +00:00
commits = execute ( ' git log -n 10 %s %s ' % ( fmt , commit ) )
2016-06-09 10:54:06 +00:00
else :
2016-06-22 10:12:35 +00:00
commits = execute ( ' git --no-pager log %s %s .. %s ' % ( fmt , last_commit , commit ) )
2016-06-22 13:42:26 +00:00
return str ( ' Changes in %s since %s : \n ' % ( branch , last_commit ) ) + ' \n ' . join ( commits )
2016-06-22 11:07:58 +00:00
2016-06-09 10:54:06 +00:00
2016-06-22 12:01:53 +00:00
def get_last_results ( resultsFileName ) :
2016-06-09 10:54:06 +00:00
if not os . path . isfile ( resultsFileName ) :
2016-07-19 15:59:53 +00:00
return None , None , None , None
2016-06-09 10:54:06 +00:00
commit = None
2016-07-19 15:59:53 +00:00
csize = [ ]
2016-06-09 10:54:06 +00:00
cspeed = [ ]
dspeed = [ ]
2016-07-30 23:59:23 +00:00
with open ( resultsFileName , ' r ' ) as f :
2016-06-09 10:54:06 +00:00
for line in f :
words = line . split ( )
2016-09-12 12:17:47 +00:00
if len ( words ) < = 4 : # branch + commit + compilerVer + md5
2016-07-30 23:59:23 +00:00
commit = words [ 1 ]
2016-07-19 15:59:53 +00:00
csize = [ ]
2016-06-09 10:54:06 +00:00
cspeed = [ ]
dspeed = [ ]
2016-09-15 17:56:04 +00:00
if ( len ( words ) == 8 ) or ( len ( words ) == 9 ) : # results: "filename" or "XX files"
2016-07-19 15:59:53 +00:00
csize . append ( int ( words [ 1 ] ) )
2016-06-09 10:54:06 +00:00
cspeed . append ( float ( words [ 3 ] ) )
dspeed . append ( float ( words [ 5 ] ) )
2016-07-19 15:59:53 +00:00
return commit , csize , cspeed , dspeed
2016-06-09 10:54:06 +00:00
2016-09-12 12:17:47 +00:00
def benchmark_and_compare ( branch , commit , last_commit , args , executableName , md5sum , compilerVersion , resultsFileName ,
2016-07-30 23:59:23 +00:00
testFilePath , fileName , last_csize , last_cspeed , last_dspeed ) :
2016-06-10 10:53:12 +00:00
sleepTime = 30
2016-07-19 15:59:53 +00:00
while os . getloadavg ( ) [ 0 ] > args . maxLoadAvg :
2016-07-30 23:59:23 +00:00
log ( " WARNING: bench loadavg= %.2f is higher than %s , sleeping for %s seconds "
% ( os . getloadavg ( ) [ 0 ] , args . maxLoadAvg , sleepTime ) )
2016-06-10 10:53:12 +00:00
time . sleep ( sleepTime )
2016-06-09 10:54:06 +00:00
start_load = str ( os . getloadavg ( ) )
2016-09-15 18:31:29 +00:00
if args . dictionary :
result = execute ( ' programs/ %s -rqi5b1e %s -D %s %s ' % ( executableName , args . lastCLevel , args . dictionary , testFilePath ) , print_output = True )
else :
result = execute ( ' programs/ %s -rqi5b1e %s %s ' % ( executableName , args . lastCLevel , testFilePath ) , print_output = True )
2016-06-09 10:54:06 +00:00
end_load = str ( os . getloadavg ( ) )
2016-07-30 23:59:23 +00:00
linesExpected = args . lastCLevel + 1
2016-06-09 10:54:06 +00:00
if len ( result ) != linesExpected :
2016-06-22 11:07:58 +00:00
raise RuntimeError ( " ERROR: number of result lines= %d is different that expected %d \n %s " % ( len ( result ) , linesExpected , ' \n ' . join ( result ) ) )
2016-06-09 10:54:06 +00:00
with open ( resultsFileName , " a " ) as myfile :
2016-09-12 12:17:47 +00:00
myfile . write ( ' %s %s %s md5= %s \n ' % ( branch , commit , compilerVersion , md5sum ) )
2016-06-22 10:12:35 +00:00
myfile . write ( ' \n ' . join ( result ) + ' \n ' )
2016-06-09 10:54:06 +00:00
myfile . close ( )
if ( last_cspeed == None ) :
2016-06-22 10:12:35 +00:00
log ( " WARNING: No data for comparison for branch= %s file= %s " % ( branch , fileName ) )
2016-06-09 10:54:06 +00:00
return " "
2016-07-19 15:59:53 +00:00
commit , csize , cspeed , dspeed = get_last_results ( resultsFileName )
2016-06-09 10:54:06 +00:00
text = " "
for i in range ( 0 , min ( len ( cspeed ) , len ( last_cspeed ) ) ) :
2016-07-25 08:35:53 +00:00
print ( " %s : %s - %d cSpeed= %6.2f cLast= %6.2f cDiff= %1.4f dSpeed= %6.2f dLast= %6.2f dDiff= %1.4f ratioDiff= %1.4f %s " % ( branch , commit , i + 1 , cspeed [ i ] , last_cspeed [ i ] , cspeed [ i ] / last_cspeed [ i ] , dspeed [ i ] , last_dspeed [ i ] , dspeed [ i ] / last_dspeed [ i ] , float ( last_csize [ i ] ) / csize [ i ] , fileName ) )
2016-07-19 15:59:53 +00:00
if ( cspeed [ i ] / last_cspeed [ i ] < args . lowerLimit ) :
2016-07-26 11:05:01 +00:00
text + = " WARNING: %s - %d cSpeed= %.2f cLast= %.2f cDiff= %.4f %s \n " % ( executableName , i + 1 , cspeed [ i ] , last_cspeed [ i ] , cspeed [ i ] / last_cspeed [ i ] , fileName )
2016-07-19 15:59:53 +00:00
if ( dspeed [ i ] / last_dspeed [ i ] < args . lowerLimit ) :
2016-07-26 11:05:01 +00:00
text + = " WARNING: %s - %d dSpeed= %.2f dLast= %.2f dDiff= %.4f %s \n " % ( executableName , i + 1 , dspeed [ i ] , last_dspeed [ i ] , dspeed [ i ] / last_dspeed [ i ] , fileName )
2016-07-25 08:35:53 +00:00
if ( float ( last_csize [ i ] ) / csize [ i ] < args . ratioLimit ) :
2016-07-26 11:05:01 +00:00
text + = " WARNING: %s - %d cSize= %d last_cSize= %d diff= %.4f %s \n " % ( executableName , i + 1 , csize [ i ] , last_csize [ i ] , float ( last_csize [ i ] ) / csize [ i ] , fileName )
2016-06-09 10:54:06 +00:00
if text :
2016-09-12 12:17:47 +00:00
text = args . message + ( " \n maxLoadAvg= %s load average at start= %s end= %s \n %s last_commit= %s md5= %s \n " % ( args . maxLoadAvg , start_load , end_load , compilerVersion , last_commit , md5sum ) ) + text
2016-06-09 10:54:06 +00:00
return text
2016-06-22 16:12:57 +00:00
def update_config_file ( branch , commit ) :
last_commit = None
commitFileName = working_path + " /commit_ " + branch . replace ( " / " , " _ " ) + " .txt "
if os . path . isfile ( commitFileName ) :
2016-10-28 17:24:16 +00:00
with open ( commitFileName , ' r ' ) as infile :
last_commit = infile . read ( )
with open ( commitFileName , ' w ' ) as outfile :
outfile . write ( commit )
2016-06-22 16:12:57 +00:00
return last_commit
2016-09-12 12:17:47 +00:00
def double_check ( branch , commit , args , executableName , md5sum , compilerVersion , resultsFileName , filePath , fileName ) :
2016-07-26 11:05:01 +00:00
last_commit , csize , cspeed , dspeed = get_last_results ( resultsFileName )
if not args . dry_run :
2016-09-12 12:17:47 +00:00
text = benchmark_and_compare ( branch , commit , last_commit , args , executableName , md5sum , compilerVersion , resultsFileName , filePath , fileName , csize , cspeed , dspeed )
2016-07-26 11:05:01 +00:00
if text :
log ( " WARNING: redoing tests for branch %s : commit %s " % ( branch , commit ) )
2016-09-12 12:17:47 +00:00
text = benchmark_and_compare ( branch , commit , last_commit , args , executableName , md5sum , compilerVersion , resultsFileName , filePath , fileName , csize , cspeed , dspeed )
2016-07-26 11:05:01 +00:00
return text
2016-06-22 16:12:57 +00:00
def test_commit ( branch , commit , last_commit , args , testFilePaths , have_mutt , have_mail ) :
2016-10-28 17:24:16 +00:00
local_branch = branch . split ( ' / ' ) [ 1 ]
2016-06-22 18:06:42 +00:00
version = local_branch . rpartition ( ' - ' ) [ 2 ] + ' _ ' + commit
if not args . dry_run :
2016-08-10 12:14:01 +00:00
execute ( ' make -C programs clean zstd CC=clang MOREFLAGS= " -Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT= %s " && ' % version +
' mv programs/zstd programs/zstd_clang && ' +
2016-09-15 17:56:04 +00:00
' make -C programs clean zstd zstd32 MOREFLAGS= " -DZSTD_GIT_COMMIT= %s " ' % version )
2016-08-10 12:14:01 +00:00
md5_zstd = hashfile ( hashlib . md5 ( ) , clone_path + ' /programs/zstd ' )
md5_zstd32 = hashfile ( hashlib . md5 ( ) , clone_path + ' /programs/zstd32 ' )
md5_zstd_clang = hashfile ( hashlib . md5 ( ) , clone_path + ' /programs/zstd_clang ' )
2016-08-23 11:54:37 +00:00
print ( " md5(zstd)= %s \n md5(zstd32)= %s \n md5(zstd_clang)= %s " % ( md5_zstd , md5_zstd32 , md5_zstd_clang ) )
2016-09-12 12:17:47 +00:00
print ( " gcc_version= %s clang_version= %s " % ( gcc_version , clang_version ) )
2016-06-22 16:12:57 +00:00
logFileName = working_path + " /log_ " + branch . replace ( " / " , " _ " ) + " .txt "
text_to_send = [ ]
results_files = " "
2016-09-15 18:31:29 +00:00
if args . dictionary :
dictName = args . dictionary . rpartition ( ' / ' ) [ 2 ]
else :
dictName = None
2016-06-22 16:12:57 +00:00
for filePath in testFilePaths :
fileName = filePath . rpartition ( ' / ' ) [ 2 ]
2016-09-15 18:31:29 +00:00
if dictName :
resultsFileName = working_path + " / " + dictName . replace ( " . " , " _ " ) + " _ " + branch . replace ( " / " , " _ " ) + " _ " + fileName . replace ( " . " , " _ " ) + " .txt "
else :
resultsFileName = working_path + " /results_ " + branch . replace ( " / " , " _ " ) + " _ " + fileName . replace ( " . " , " _ " ) + " .txt "
2016-09-12 12:17:47 +00:00
text = double_check ( branch , commit , args , ' zstd ' , md5_zstd , ' gcc_version= ' + gcc_version , resultsFileName , filePath , fileName )
2016-07-26 11:05:01 +00:00
if text :
text_to_send . append ( text )
results_files + = resultsFileName + " "
resultsFileName = working_path + " /results32_ " + branch . replace ( " / " , " _ " ) + " _ " + fileName . replace ( " . " , " _ " ) + " .txt "
2016-09-12 12:17:47 +00:00
text = double_check ( branch , commit , args , ' zstd32 ' , md5_zstd32 , ' gcc_version= ' + gcc_version , resultsFileName , filePath , fileName )
2016-08-10 12:14:01 +00:00
if text :
text_to_send . append ( text )
results_files + = resultsFileName + " "
resultsFileName = working_path + " /resultsClang_ " + branch . replace ( " / " , " _ " ) + " _ " + fileName . replace ( " . " , " _ " ) + " .txt "
2016-09-12 12:17:47 +00:00
text = double_check ( branch , commit , args , ' zstd_clang ' , md5_zstd_clang , ' clang_version= ' + clang_version , resultsFileName , filePath , fileName )
2016-07-26 11:05:01 +00:00
if text :
text_to_send . append ( text )
results_files + = resultsFileName + " "
2016-06-22 16:12:57 +00:00
if text_to_send :
2016-07-19 15:59:53 +00:00
send_email_with_attachments ( branch , commit , last_commit , args , text_to_send , results_files , logFileName , have_mutt , have_mail )
2016-06-09 10:54:06 +00:00
if __name__ == ' __main__ ' :
parser = argparse . ArgumentParser ( )
2016-09-15 18:41:37 +00:00
parser . add_argument ( ' testFileNames ' , help = ' file or directory names list for speed benchmark ' )
2016-06-10 10:53:12 +00:00
parser . add_argument ( ' emails ' , help = ' list of e-mail addresses to send warnings ' )
2016-09-15 18:31:29 +00:00
parser . add_argument ( ' --dictionary ' , ' -D ' , help = ' path to the dictionary ' )
parser . add_argument ( ' --message ' , ' -m ' , help = ' attach an additional message to e-mail ' , default = " " )
2016-06-21 09:26:17 +00:00
parser . add_argument ( ' --repoURL ' , help = ' changes default repository URL ' , default = default_repo_url )
2016-09-15 18:31:29 +00:00
parser . add_argument ( ' --lowerLimit ' , ' -l ' , type = float , help = ' send email if speed is lower than given limit ' , default = 0.98 )
parser . add_argument ( ' --ratioLimit ' , ' -r ' , type = float , help = ' send email if ratio is lower than given limit ' , default = 0.999 )
2016-06-09 10:54:06 +00:00
parser . add_argument ( ' --maxLoadAvg ' , type = float , help = ' maximum load average to start testing ' , default = 0.75 )
parser . add_argument ( ' --lastCLevel ' , type = int , help = ' last compression level for testing ' , default = 5 )
2016-09-15 18:31:29 +00:00
parser . add_argument ( ' --sleepTime ' , ' -s ' , type = int , help = ' frequency of repository checking in seconds ' , default = 300 )
2016-10-28 18:40:21 +00:00
parser . add_argument ( ' --timeout ' , ' -t ' , type = int , help = ' timeout for executing shell commands ' , default = 1800 )
2016-06-09 10:54:06 +00:00
parser . add_argument ( ' --dry-run ' , dest = ' dry_run ' , action = ' store_true ' , help = ' not build ' , default = False )
2016-09-15 18:31:29 +00:00
parser . add_argument ( ' --verbose ' , ' -v ' , action = ' store_true ' , help = ' more verbose logs ' , default = False )
2016-06-09 10:54:06 +00:00
args = parser . parse_args ( )
2016-07-19 13:49:14 +00:00
verbose = args . verbose
2016-06-09 10:54:06 +00:00
# check if test files are accessible
testFileNames = args . testFileNames . split ( )
testFilePaths = [ ]
for fileName in testFileNames :
2016-06-22 15:11:01 +00:00
fileName = os . path . expanduser ( fileName )
2016-09-15 17:56:04 +00:00
if os . path . isfile ( fileName ) or os . path . isdir ( fileName ) :
2016-06-09 10:54:06 +00:00
testFilePaths . append ( os . path . abspath ( fileName ) )
else :
2016-09-15 17:56:04 +00:00
log ( " ERROR: File/directory not found: " + fileName )
2016-06-21 09:26:17 +00:00
exit ( 1 )
2016-06-09 10:54:06 +00:00
2016-09-15 18:31:29 +00:00
# check if dictionary is accessible
if args . dictionary :
args . dictionary = os . path . abspath ( os . path . expanduser ( args . dictionary ) )
if not os . path . isfile ( args . dictionary ) :
log ( " ERROR: Dictionary not found: " + args . dictionary )
exit ( 1 )
2016-06-10 10:53:12 +00:00
# check availability of e-mail senders
2016-07-30 23:59:23 +00:00
have_mutt = does_command_exist ( " mutt -h " )
have_mail = does_command_exist ( " mail -V " )
2016-06-10 11:59:08 +00:00
if not have_mutt and not have_mail :
2016-06-21 09:26:17 +00:00
log ( " ERROR: e-mail senders ' mail ' or ' mutt ' not found " )
exit ( 1 )
2016-06-09 10:54:06 +00:00
2016-09-12 12:17:47 +00:00
clang_version = execute ( " clang -v 2>&1 | grep ' clang version ' | sed -e ' s:.*version \\ ([0-9.]* \\ ).*: \\ 1: ' -e ' s: \\ . \\ ([0-9][0-9] \\ ): \\ 1:g ' " , verbose ) [ 0 ] ;
gcc_version = execute ( " gcc -dumpversion " , verbose ) [ 0 ] ;
2016-09-15 18:31:29 +00:00
2016-07-19 13:49:14 +00:00
if verbose :
print ( " PARAMETERS: \n repoURL= %s " % args . repoURL )
print ( " working_path= %s " % working_path )
print ( " clone_path= %s " % clone_path )
print ( " testFilePath( %s )= %s " % ( len ( testFilePaths ) , testFilePaths ) )
print ( " message= %s " % args . message )
print ( " emails= %s " % args . emails )
2016-09-15 18:31:29 +00:00
print ( " dictionary= %s " % args . dictionary )
2016-07-19 13:49:14 +00:00
print ( " maxLoadAvg= %s " % args . maxLoadAvg )
print ( " lowerLimit= %s " % args . lowerLimit )
2016-07-19 15:59:53 +00:00
print ( " ratioLimit= %s " % args . ratioLimit )
2016-07-19 13:49:14 +00:00
print ( " lastCLevel= %s " % args . lastCLevel )
print ( " sleepTime= %s " % args . sleepTime )
2016-10-28 17:24:16 +00:00
print ( " timeout= %s " % args . timeout )
2016-07-19 13:49:14 +00:00
print ( " dry_run= %s " % args . dry_run )
print ( " verbose= %s " % args . verbose )
print ( " have_mutt= %s have_mail= %s " % ( have_mutt , have_mail ) )
2016-06-10 10:53:12 +00:00
2016-06-21 09:26:17 +00:00
# clone ZSTD repo if needed
2016-06-22 10:12:35 +00:00
if not os . path . isdir ( working_path ) :
os . mkdir ( working_path )
2016-06-21 09:26:17 +00:00
if not os . path . isdir ( clone_path ) :
2016-06-22 10:12:35 +00:00
execute . cwd = working_path
2016-06-21 09:26:17 +00:00
execute ( ' git clone ' + args . repoURL )
if not os . path . isdir ( clone_path ) :
log ( " ERROR: ZSTD clone not found: " + clone_path )
exit ( 1 )
execute . cwd = clone_path
# check if speedTest.pid already exists
pidfile = " ./speedTest.pid "
if os . path . isfile ( pidfile ) :
log ( " ERROR: %s already exists, exiting " % pidfile )
exit ( 1 )
2016-08-10 12:14:01 +00:00
send_email ( args . emails , ' [ %s : %s ] test-zstd-speed.py %s has been started ' % ( email_header , pid , script_version ) , args . message , have_mutt , have_mail )
2016-10-28 17:24:16 +00:00
with open ( pidfile , ' w ' ) as the_file :
the_file . write ( pid )
2016-06-22 12:01:53 +00:00
2016-10-28 17:24:16 +00:00
branch = " "
commit = " "
first_time = True
2016-06-09 10:54:06 +00:00
while True :
2016-06-21 09:26:17 +00:00
try :
2016-10-28 17:24:16 +00:00
if first_time :
first_time = False
else :
time . sleep ( args . sleepTime )
2016-06-21 09:26:17 +00:00
loadavg = os . getloadavg ( ) [ 0 ]
if ( loadavg < = args . maxLoadAvg ) :
2016-06-22 11:07:58 +00:00
branches = git_get_branches ( )
2016-06-22 10:12:35 +00:00
for branch in branches :
2016-07-19 13:49:14 +00:00
commit = execute ( ' git show -s --format= % h ' + branch , verbose ) [ 0 ]
2016-06-22 16:12:57 +00:00
last_commit = update_config_file ( branch , commit )
if commit == last_commit :
log ( " skipping branch %s : head %s already processed " % ( branch , commit ) )
else :
log ( " build branch %s : head %s is different from prev %s " % ( branch , commit , last_commit ) )
2016-06-22 18:06:42 +00:00
execute ( ' git checkout -- . && git checkout ' + branch )
print ( git_get_changes ( branch , commit , last_commit ) )
2016-06-22 16:12:57 +00:00
test_commit ( branch , commit , last_commit , args , testFilePaths , have_mutt , have_mail )
2016-06-21 09:26:17 +00:00
else :
log ( " WARNING: main loadavg= %.2f is higher than %s " % ( loadavg , args . maxLoadAvg ) )
2016-07-19 13:49:14 +00:00
if verbose :
log ( " sleep for %s seconds " % args . sleepTime )
2016-06-22 16:12:57 +00:00
except Exception as e :
stack = traceback . format_exc ( )
2016-08-10 12:14:01 +00:00
email_topic = ' [ %s : %s ] ERROR in %s : %s ' % ( email_header , pid , branch , commit )
2016-06-22 16:12:57 +00:00
send_email ( args . emails , email_topic , stack , have_mutt , have_mail )
print ( stack )
2016-06-22 12:01:53 +00:00
except KeyboardInterrupt :
2016-06-21 09:26:17 +00:00
os . unlink ( pidfile )
2016-08-10 12:14:01 +00:00
send_email ( args . emails , ' [ %s : %s ] test-zstd-speed.py %s has been stopped ' % ( email_header , pid , script_version ) , args . message , have_mutt , have_mail )
2016-06-22 12:01:53 +00:00
exit ( 0 )