diff --git a/build/VS2010/zstd/generate_res/generate_res.bat b/build/VS2010/zstd/generate_res/generate_res.bat deleted file mode 100644 index b552dcc3..00000000 --- a/build/VS2010/zstd/generate_res/generate_res.bat +++ /dev/null @@ -1,3 +0,0 @@ -REM http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable -REM copy "c:\Program Files (x86)\Windows Kits\8.1\Include\um\verrsrc.h" . -windres -I ..\..\..\..\lib -O coff -I . -i ..\zstd.rc -o zstd.res diff --git a/build/VS2010/zstd/generate_res/verrsrc.h b/build/VS2010/zstd/generate_res/verrsrc.h deleted file mode 100644 index 37e48d30..00000000 --- a/build/VS2010/zstd/generate_res/verrsrc.h +++ /dev/null @@ -1,172 +0,0 @@ -#include - -/*****************************************************************************\ -* * -* verrsrc.h - Version Resource definitions * -* * -* Include file declaring version resources in rc files * -* * -* Copyright (c) Microsoft Corporation. All rights reserved. * -* * -\*****************************************************************************/ - -#pragma region Application Family -#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) - -/* ----- Symbols ----- */ -#define VS_FILE_INFO RT_VERSION -#define VS_VERSION_INFO 1 -#define VS_USER_DEFINED 100 - -/* ----- VS_VERSION.dwFileFlags ----- */ -#ifndef _MAC -#define VS_FFI_SIGNATURE 0xFEEF04BDL -#else -#define VS_FFI_SIGNATURE 0xBD04EFFEL -#endif -#define VS_FFI_STRUCVERSION 0x00010000L -#define VS_FFI_FILEFLAGSMASK 0x0000003FL - -/* ----- VS_VERSION.dwFileFlags ----- */ -#define VS_FF_DEBUG 0x00000001L -#define VS_FF_PRERELEASE 0x00000002L -#define VS_FF_PATCHED 0x00000004L -#define VS_FF_PRIVATEBUILD 0x00000008L -#define VS_FF_INFOINFERRED 0x00000010L -#define VS_FF_SPECIALBUILD 0x00000020L - -/* ----- VS_VERSION.dwFileOS ----- */ -#define VOS_UNKNOWN 0x00000000L -#define VOS_DOS 0x00010000L -#define VOS_OS216 0x00020000L -#define VOS_OS232 0x00030000L -#define VOS_NT 0x00040000L -#define VOS_WINCE 0x00050000L - -#define VOS__BASE 0x00000000L -#define VOS__WINDOWS16 0x00000001L -#define VOS__PM16 0x00000002L -#define VOS__PM32 0x00000003L -#define VOS__WINDOWS32 0x00000004L - -#define VOS_DOS_WINDOWS16 0x00010001L -#define VOS_DOS_WINDOWS32 0x00010004L -#define VOS_OS216_PM16 0x00020002L -#define VOS_OS232_PM32 0x00030003L -#define VOS_NT_WINDOWS32 0x00040004L - -/* ----- VS_VERSION.dwFileType ----- */ -#define VFT_UNKNOWN 0x00000000L -#define VFT_APP 0x00000001L -#define VFT_DLL 0x00000002L -#define VFT_DRV 0x00000003L -#define VFT_FONT 0x00000004L -#define VFT_VXD 0x00000005L -#define VFT_STATIC_LIB 0x00000007L - -/* ----- VS_VERSION.dwFileSubtype for VFT_WINDOWS_DRV ----- */ -#define VFT2_UNKNOWN 0x00000000L -#define VFT2_DRV_PRINTER 0x00000001L -#define VFT2_DRV_KEYBOARD 0x00000002L -#define VFT2_DRV_LANGUAGE 0x00000003L -#define VFT2_DRV_DISPLAY 0x00000004L -#define VFT2_DRV_MOUSE 0x00000005L -#define VFT2_DRV_NETWORK 0x00000006L -#define VFT2_DRV_SYSTEM 0x00000007L -#define VFT2_DRV_INSTALLABLE 0x00000008L -#define VFT2_DRV_SOUND 0x00000009L -#define VFT2_DRV_COMM 0x0000000AL -#define VFT2_DRV_INPUTMETHOD 0x0000000BL -#define VFT2_DRV_VERSIONED_PRINTER 0x0000000CL - -/* ----- VS_VERSION.dwFileSubtype for VFT_WINDOWS_FONT ----- */ -#define VFT2_FONT_RASTER 0x00000001L -#define VFT2_FONT_VECTOR 0x00000002L -#define VFT2_FONT_TRUETYPE 0x00000003L - -#endif /* WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) */ -#pragma endregion - -#pragma region Desktop Family -#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) - -/* ----- VerFindFile() flags ----- */ -#define VFFF_ISSHAREDFILE 0x0001 - -#define VFF_CURNEDEST 0x0001 -#define VFF_FILEINUSE 0x0002 -#define VFF_BUFFTOOSMALL 0x0004 - -/* ----- VerInstallFile() flags ----- */ -#define VIFF_FORCEINSTALL 0x0001 -#define VIFF_DONTDELETEOLD 0x0002 - -#define VIF_TEMPFILE 0x00000001L -#define VIF_MISMATCH 0x00000002L -#define VIF_SRCOLD 0x00000004L - -#define VIF_DIFFLANG 0x00000008L -#define VIF_DIFFCODEPG 0x00000010L -#define VIF_DIFFTYPE 0x00000020L - -#define VIF_WRITEPROT 0x00000040L -#define VIF_FILEINUSE 0x00000080L -#define VIF_OUTOFSPACE 0x00000100L -#define VIF_ACCESSVIOLATION 0x00000200L -#define VIF_SHARINGVIOLATION 0x00000400L -#define VIF_CANNOTCREATE 0x00000800L -#define VIF_CANNOTDELETE 0x00001000L -#define VIF_CANNOTRENAME 0x00002000L -#define VIF_CANNOTDELETECUR 0x00004000L -#define VIF_OUTOFMEMORY 0x00008000L - -#define VIF_CANNOTREADSRC 0x00010000L -#define VIF_CANNOTREADDST 0x00020000L - -#define VIF_BUFFTOOSMALL 0x00040000L -#define VIF_CANNOTLOADLZ32 0x00080000L -#define VIF_CANNOTLOADCABINET 0x00100000L - -#ifndef RC_INVOKED /* RC doesn't need to see the rest of this */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* - FILE_VER_GET_... flags are for use by - GetFileVersionInfoSizeEx - GetFileVersionInfoExW -*/ -#define FILE_VER_GET_LOCALISED 0x01 -#define FILE_VER_GET_NEUTRAL 0x02 -#define FILE_VER_GET_PREFETCHED 0x04 - -/* ----- Types and structures ----- */ - -typedef struct tagVS_FIXEDFILEINFO -{ - DWORD dwSignature; /* e.g. 0xfeef04bd */ - DWORD dwStrucVersion; /* e.g. 0x00000042 = "0.42" */ - DWORD dwFileVersionMS; /* e.g. 0x00030075 = "3.75" */ - DWORD dwFileVersionLS; /* e.g. 0x00000031 = "0.31" */ - DWORD dwProductVersionMS; /* e.g. 0x00030010 = "3.10" */ - DWORD dwProductVersionLS; /* e.g. 0x00000031 = "0.31" */ - DWORD dwFileFlagsMask; /* = 0x3F for version "0.42" */ - DWORD dwFileFlags; /* e.g. VFF_DEBUG | VFF_PRERELEASE */ - DWORD dwFileOS; /* e.g. VOS_DOS_WINDOWS16 */ - DWORD dwFileType; /* e.g. VFT_DRIVER */ - DWORD dwFileSubtype; /* e.g. VFT2_DRV_KEYBOARD */ - DWORD dwFileDateMS; /* e.g. 0 */ - DWORD dwFileDateLS; /* e.g. 0 */ -} VS_FIXEDFILEINFO; - -#ifdef __cplusplus -} -#endif - -#endif /* !RC_INVOKED */ - -#endif /* WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) */ -#pragma endregion - diff --git a/programs/Makefile b/programs/Makefile index 013f5435..f5e625f0 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -70,7 +70,7 @@ VOID = /dev/null endif -.PHONY: default all clean clean_decomp_o install uninstall +.PHONY: default all clean clean_decomp_o install uninstall generate_res default: zstd @@ -82,6 +82,9 @@ $(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c +ifneq (,$(filter Windows%,$(OS))) + windres\generate_res.bat +endif $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT) @@ -90,6 +93,9 @@ $(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c +ifneq (,$(filter Windows%,$(OS))) + windres\generate_res.bat +endif $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT) @@ -123,6 +129,8 @@ zstd-decompress: clean_decomp_o zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) +generate_res: + windres\generate_res.bat clean: $(MAKE) -C ../lib clean diff --git a/programs/windres/generate_res.bat b/programs/windres/generate_res.bat new file mode 100644 index 00000000..76e47fa0 --- /dev/null +++ b/programs/windres/generate_res.bat @@ -0,0 +1,11 @@ +@echo off +REM http://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable + +where /q windres.exe +IF ERRORLEVEL 1 ( + ECHO The windres.exe is missing. Ensure it is installed and placed in your PATH. + EXIT /B +) ELSE ( + windres.exe -I ..\lib -I windres -i windres\zstd.rc -O coff -F pe-x86-64 -o windres\zstd64.res + windres.exe -I ..\lib -I windres -i windres\zstd.rc -O coff -F pe-i386 -o windres\zstd32.res +) diff --git a/programs/windres/verrsrc.h b/programs/windres/verrsrc.h new file mode 100644 index 00000000..e282add0 --- /dev/null +++ b/programs/windres/verrsrc.h @@ -0,0 +1,8 @@ +/* minimal set of defines required to generate zstd.res from zstd.rc */ + +#define VS_VERSION_INFO 1 + +#define VS_FFI_FILEFLAGSMASK 0x0000003FL +#define VOS_NT_WINDOWS32 0x00040004L +#define VFT_DLL 0x00000002L +#define VFT2_UNKNOWN 0x00000000L diff --git a/programs/windres/zstd.rc b/programs/windres/zstd.rc new file mode 100644 index 00000000..f5e40473 --- /dev/null +++ b/programs/windres/zstd.rc @@ -0,0 +1,51 @@ +// Microsoft Visual C++ generated resource script. +// + +#include "zstd.h" /* ZSTD_VERSION_STRING */ +#define APSTUDIO_READONLY_SYMBOLS +#include "verrsrc.h" +#undef APSTUDIO_READONLY_SYMBOLS + + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +LANGUAGE 9, 1 + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 + PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "Yann Collet, Facebook, Inc." + VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" + VALUE "FileVersion", ZSTD_VERSION_STRING + VALUE "InternalName", "zstd.exe" + VALUE "LegalCopyright", "Copyright (c) 2013-present, Yann Collet, Facebook, Inc." + VALUE "OriginalFilename", "zstd.exe" + VALUE "ProductName", "Zstandard" + VALUE "ProductVersion", ZSTD_VERSION_STRING + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1200 + END +END + +#endif diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index 56b4d46b..a556b033 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # # Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. @@ -21,7 +21,7 @@ import time import traceback import hashlib -script_version = 'v1.0.1 (2016-09-15)' +script_version = 'v1.1.1 (2016-10-28)' default_repo_url = 'https://github.com/facebook/zstd.git' working_dir_name = 'speedTest' working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest @@ -31,7 +31,7 @@ pid = str(os.getpid()) verbose = False clang_version = "unknown" gcc_version = "unknown" - +args = None def hashfile(hasher, fname, blocksize=65536): @@ -48,17 +48,20 @@ def log(text): def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True): if print_command: log("> " + command) - popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - shell=param_shell, cwd=execute.cwd) - stdout = popen.communicate()[0] - stdout_lines = stdout.splitlines() + popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd) + stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout) + stderr_lines = stderr_lines.decode("utf-8") + stdout_lines = stdout_lines.decode("utf-8") if print_output: - print('\n'.join(stdout_lines)) + if stdout_lines: + print(stdout_lines) + if stderr_lines: + print(stderr_lines) if popen.returncode is not None and popen.returncode != 0: - if not print_output and print_error: - print('\n'.join(stdout_lines)) - raise RuntimeError('\n'.join(stdout_lines)) - return stdout_lines + if stderr_lines and not print_output and print_error: + print(stderr_lines) + raise RuntimeError(stdout_lines + stderr_lines) + return (stdout_lines + stderr_lines).splitlines() execute.cwd = None @@ -183,8 +186,10 @@ def update_config_file(branch, commit): last_commit = None commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt" if os.path.isfile(commitFileName): - last_commit = file(commitFileName, 'r').read() - file(commitFileName, 'w').write(commit) + with open(commitFileName, 'r') as infile: + last_commit = infile.read() + with open(commitFileName, 'w') as outfile: + outfile.write(commit) return last_commit @@ -199,7 +204,7 @@ def double_check(branch, commit, args, executableName, md5sum, compilerVersion, def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail): - local_branch = string.split(branch, '/')[1] + local_branch = branch.split('/')[1] version = local_branch.rpartition('-')[2] + '_' + commit if not args.dry_run: execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version + @@ -255,6 +260,7 @@ if __name__ == '__main__': parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75) parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5) parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300) + parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=600) parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False) parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False) args = parser.parse_args() @@ -301,6 +307,7 @@ if __name__ == '__main__': print("ratioLimit=%s" % args.ratioLimit) print("lastCLevel=%s" % args.lastCLevel) print("sleepTime=%s" % args.sleepTime) + print("timeout=%s" % args.timeout) print("dry_run=%s" % args.dry_run) print("verbose=%s" % args.verbose) print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail)) @@ -323,10 +330,18 @@ if __name__ == '__main__': exit(1) send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail) - file(pidfile, 'w').write(pid) + with open(pidfile, 'w') as the_file: + the_file.write(pid) + branch = "" + commit = "" + first_time = True while True: try: + if first_time: + first_time = False + else: + time.sleep(args.sleepTime) loadavg = os.getloadavg()[0] if (loadavg <= args.maxLoadAvg): branches = git_get_branches() @@ -344,13 +359,11 @@ if __name__ == '__main__': log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg)) if verbose: log("sleep for %s seconds" % args.sleepTime) - time.sleep(args.sleepTime) except Exception as e: stack = traceback.format_exc() email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit) send_email(args.emails, email_topic, stack, have_mutt, have_mail) print(stack) - time.sleep(args.sleepTime) except KeyboardInterrupt: os.unlink(pidfile) send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail)