From f89a1d9d8a92d89cb417ae45bf7e741878cbd35c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 20 Sep 2018 14:20:32 -0700 Subject: [PATCH] ICU-20066 add a copyright scan stage (#26) - name the travis build steps - copy cpyskip.txt to /.cpyskip.txt - add one named "lint" which runs cpyscan.pl - remove network access from Cpy.pm (requires installation) --- .cpyskip.txt | 103 +++++++++++++++++++++++++++++ .travis.yml | 26 ++++++-- tools/scripts/cpysearch/Cpy.pm | 14 ++-- tools/scripts/cpysearch/cpyscan.pl | 17 ++++- 4 files changed, 144 insertions(+), 16 deletions(-) create mode 100644 .cpyskip.txt diff --git a/.cpyskip.txt b/.cpyskip.txt new file mode 100644 index 0000000000..c8b824262c --- /dev/null +++ b/.cpyskip.txt @@ -0,0 +1,103 @@ +# +# Copyright (C) 2017 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html +# +# Copyright (c) 2005-2011 International Business Machines +# Corporation and others. All Rights Reserved. +## see NOTE below. +## Also see: http://icu-project.org/copyright-scan.html +# +# git stuff +# .git is ignored by code. +#.git/* +.gitignore +.gitattributes +.travis.yml +.appveyor.yml +# suffix matches - start with '*'. They are turned into as RE, '.brk$' +*.brk +*.bz2 +*.classpath +*.csproj +*.cvsignore +*.dat +*.DS_Store +*.doc +*.gif +*.gz +*.ico +*.icu +*.intaglio +*.jar +*.jpg +*.launch +*.nrm +*.odp +*.otf +*.pdf +*.png +*.ppt +*.prefs +*.project +*.res +*.rtf +*.sln +*.sxd +*.sxg +*.sxw +*.tri2 +*.vcproj +*.vcxproj +*.vcxproj.filters +*.zip + +# UnicodeData.txt does not have any header. +*/UnicodeData.txt + +# +# ICU4C +# +icu4c/source/aclocal.m4 +icu4c/source/config.guess +icu4c/source/config.log +icu4c/source/config.status +icu4c/source/config.sub +icu4c/source/install-sh +icu4c/source/extra/uconv/samples/* +icu4c/source/samples/layout/Sample.txt +icu4c/source/samples/ucnv/data01.txt +icu4c/source/samples/ufortune/resources/res-file-list.txt +icu4c/source/test/testdata/ConverterSelectorTestUTF8.txt +icu4c/source/test/testdata/encoded.utf16be +icu4c/source/test/testdata/idna_conf.txt +icu4c/source/test/testdata/ra.xlf +icu4c/source/test/testdata/re_tests.txt +icu4c/source/test/thaitest/space.txt +icu4c/source/tools/tzcode/asctime.c +icu4c/source/tools/tzcode/ialloc.c +icu4c/source/tools/tzcode/localtime.c +icu4c/source/tools/tzcode/private.h +icu4c/source/tools/tzcode/scheck.c +icu4c/source/tools/tzcode/tzfile.h +icu4c/source/tools/tzcode/tzselect.ksh +icu4c/source/tools/tzcode/zdump.c +icu4c/source/tools/tzcode/zic.c +# +# ICU4J +# +icu4j/eclipse-build/pdebuild/allElements.xml +icu4j/eclipse-build/pdebuild/customTargets.xml +icu4j/main/tests/core/src/com/ibm/icu/dev/data/IDNATestInput.txt +icu4j/main/tests/core/src/com/ibm/icu/dev/data/unicode/confusablesWholeScript.txt +icu4j/main/tests/core/src/com/ibm/icu/dev/data/unicode/UnicodeData.txt +icu4j/main/tests/core/src/com/ibm/icu/dev/test/duration/testdata/* +icu4j/perf-tests/data/conversion/* +# +# tools +# +tools/trac/IcuCodeTools/* +tools/unicodetools/* +# +# vendor +# +vendor/double-conversion/upstream/* diff --git a/.travis.yml b/.travis.yml index e1e4994c09..126236042a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ matrix: include: - - language: java + - name: "j" + language: java env: BUILD=ICU4J before_script: - cd icu4j @@ -10,12 +11,20 @@ matrix: after_failure: - cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;` - - language: cpp + - name: "c: linux gcc" + language: cpp env: BUILD=ICU4C_GCC compiler: gcc script: cd icu4c/source && ./runConfigureICU Linux && make -j2 check - - language: cpp + - name: "c: linux clang" + language: cpp + env: BUILD=ICU4C_CLANG + compiler: clang + script: cd icu4c/source && ./runConfigureICU Linux && make -j2 check + + - name: "c: osx clang" + language: cpp env: BUILD=MACINTOSH os: osx compiler: clang @@ -24,7 +33,8 @@ matrix: # Clang Linux with address sanitizer. # Note - the 'sudo: true' option forces Travis to use a Virtual machine on GCE instead of # a Container on EC2 or Packet. Asan builds of ICU fail otherwise. - - language: cpp + - name: "c: linux asan" + language: cpp env: BUILD=ICU4C_CLANG_ASAN os: linux dist: trusty @@ -43,7 +53,8 @@ matrix: # Clang Linux with thread sanitizer. # - - language: cpp + - name: "c: linux tsan" + language: cpp env: BUILD=ICU4C_CLANG_THREAD_SAN os: linux dist: trusty @@ -58,3 +69,8 @@ matrix: - clang-5.0 script: - cd icu4c/source && CPPFLAGS="-fsanitize=thread" LDFLAGS="-fsanitize=thread" ./runConfigureICU --enable-debug --disable-release Linux --disable-renaming && make -j2 check + + # copyright scan / future linter + - name "lint" + script: + - perl tools/scripts/cpysearch/cpyscan.pl diff --git a/tools/scripts/cpysearch/Cpy.pm b/tools/scripts/cpysearch/Cpy.pm index 073e5514b1..6d86a49c9d 100644 --- a/tools/scripts/cpysearch/Cpy.pm +++ b/tools/scripts/cpysearch/Cpy.pm @@ -15,7 +15,6 @@ package Cpy; use strict; use warnings; use base 'Exporter'; -use LWP::Simple; our @EXPORT = qw(any glob_to_regex should_ignore); @@ -81,19 +80,16 @@ sub glob_to_regex($) { } # Load cpyskip.txt contents. -# Try local cpyskip.txt first - if not found, try online version -our $cpyskip_file = "cpyskip.txt"; +# Try local .cpyskip.txt +# no support for HTTP fetch. +our $cpyskip_file = ".cpyskip.txt"; our @cpyskip_lines; if (open(our $cpyskip_fh, "<", $cpyskip_file)) { @cpyskip_lines = <$cpyskip_fh>; close $cpyskip_fh; - print "Using local cpyskip.txt\n"; + # print "Using local cpyskip.txt\n"; } else { - our $cpyskip_url = "http://source.icu-project.org/cpyskip.txt"; - our $cpyskip = get($cpyskip_url); - die "Can't get $cpyskip_url" if (! defined $cpyskip); - @cpyskip_lines = split(/\n/, $cpyskip); - print "Using " . $cpyskip_url . "\n"; + die "Could not open $cpyskip_file"; } our @ignore_globs = map { chomp; glob_to_regex($_) } grep { /^\s*[^#\s]+/ } diff --git a/tools/scripts/cpysearch/cpyscan.pl b/tools/scripts/cpysearch/cpyscan.pl index 1e13fa91ea..71a42bd832 100755 --- a/tools/scripts/cpysearch/cpyscan.pl +++ b/tools/scripts/cpysearch/cpyscan.pl @@ -24,10 +24,16 @@ use lib $Bin; use Cpy; my $icu_src = $ARGV[0] || "."; +my $exitStatus = 0; my $icu_src_len = length($icu_src); die "Can't open ICU directory: $icu_src" unless -d $icu_src; find({ wanted => sub { + # save a little bit of time. + if ($_ eq './.git') { + $File::Find::prune = 1; + return; + } return unless -f; my $relpath = substr($_, $icu_src_len + 1); return if should_ignore($relpath); @@ -36,8 +42,15 @@ find({ my $result = any { $_ =~ /(Copyright|©).*Unicode/i } ; close F; - - print "$relpath\n" unless $result; + if (not $result) { + print "$relpath\n"; + $exitStatus = 1; + } }, no_chdir => 1, }, $icu_src); + +if ($exitStatus) { + die "Above files did not contain the correct copyright notice."; +} +exit $exitStatus; \ No newline at end of file