From 0c2add016de218e3e38658fbed9dd3bb79733f56 Mon Sep 17 00:00:00 2001 From: Florin Malita Date: Wed, 6 Feb 2019 02:28:24 +0000 Subject: [PATCH] Revert "third_party/icu: use upstream" This reverts commit e6cfe7718592468518f15d4a95dfa194bd8ab57a. Reason for revert: Android & Win7 perf/test failures. Original change's description: > third_party/icu: use upstream > > Change DEPS to point at upstream, not chromium's fork. > `skia_use_icu` defaults to true everywhere but iOS > Clean up SkLoadICU: use U_ICUDATA_ENTRY_POINT, no DLL. > Add data dep to icu/icu.gni > Scripts: icu/{make_data_assembly.py,make_data_obj_win.py} > Scripts: icu/{update_icu.sh,build_icu_data_file.py} for rebuilding > document process in icu/README.md > > Bug: skia:8702 > Change-Id: I99789749ba84ae737f6c62475d0d676cd36715bb > Reviewed-on: https://skia-review.googlesource.com/c/186870 > Commit-Queue: Hal Canary > Reviewed-by: Florin Malita > Reviewed-by: Ben Wagner TBR=halcanary@google.com,bungeman@google.com,fmalita@chromium.org,fmalita@google.com Change-Id: Id379bc2be7d53eca67b2c557ac20bd44073a6258 No-Presubmit: true No-Tree-Checks: true No-Try: true Bug: skia:8702 Reviewed-on: https://skia-review.googlesource.com/c/189647 Reviewed-by: Florin Malita Commit-Queue: Florin Malita --- DEPS | 2 +- gn/skia.gni | 3 +- third_party/icu/BUILD.gn | 108 +++++++++++-------------- third_party/icu/README.md | 47 ----------- third_party/icu/SkLoadICU.h | 38 ++++++--- third_party/icu/build_icu_data_file.py | 53 ------------ third_party/icu/download_file.py | 26 ------ third_party/icu/hasher.py | 26 ------ third_party/icu/icu.gni | 5 +- third_party/icu/make_data_assembly.py | 57 ------------- third_party/icu/make_data_obj_win.py | 55 ------------- third_party/icu/update_icu.sh | 48 ----------- 12 files changed, 82 insertions(+), 386 deletions(-) delete mode 100644 third_party/icu/README.md delete mode 100755 third_party/icu/build_icu_data_file.py delete mode 100755 third_party/icu/download_file.py delete mode 100755 third_party/icu/hasher.py delete mode 100755 third_party/icu/make_data_assembly.py delete mode 100755 third_party/icu/make_data_obj_win.py delete mode 100755 third_party/icu/update_icu.sh diff --git a/DEPS b/DEPS index b3060f7dc9..841df8f38f 100644 --- a/DEPS +++ b/DEPS @@ -13,7 +13,7 @@ deps = { "third_party/externals/expat" : "https://android.googlesource.com/platform/external/expat.git@android-6.0.1_r55", "third_party/externals/freetype" : "https://skia.googlesource.com/third_party/freetype2.git@7edc937fe679d14d66f55cf6f7fa607925d38f3c", "third_party/externals/harfbuzz" : "https://skia.googlesource.com/third_party/harfbuzz.git@8be74d85534534dbdd39a0a6f496e26e9f3e661d", - "third_party/externals/icu" : "https://skia.googlesource.com/external/github.com/unicode-org/icu@a96dc7faa584c2408c6fa1e9c61d121d45a33563", + "third_party/externals/icu" : "https://chromium.googlesource.com/chromium/deps/icu.git@407b39301e71006b68bd38e770f35d32398a7b14", "third_party/externals/imgui" : "https://skia.googlesource.com/external/github.com/ocornut/imgui.git@d38d7c6628bebd02692cfdd6fa76b4d992a35b75", # TODO: remove jsoncpp after migrating clients to SkJSON "third_party/externals/jsoncpp" : "https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git@1.0.0", diff --git a/gn/skia.gni b/gn/skia.gni index 5123b10617..a7450aa3cb 100644 --- a/gn/skia.gni +++ b/gn/skia.gni @@ -3,5 +3,6 @@ # found in the LICENSE file. declare_args() { - skia_use_icu = !is_ios + skia_use_icu = !is_fuchsia && !is_ios && !is_win # TODO: Windows } + diff --git a/third_party/icu/BUILD.gn b/third_party/icu/BUILD.gn index 73cdd1e89c..e3f5a12fa0 100644 --- a/third_party/icu/BUILD.gn +++ b/third_party/icu/BUILD.gn @@ -5,7 +5,6 @@ declare_args() { skia_use_system_icu = is_official_build - skia_icu_use_prebuilt_data = true } import("../../gn/skia.gni") @@ -19,86 +18,77 @@ if (skia_use_icu) { defines = [ "U_USING_ICU_NAMESPACE=0" ] } } else { - if (target_cpu == "wasm") { - _data = "$target_gen_dir/icudtl_dat.cpp" - _data_script = "make_data_assembly.py" - } else if (is_win) { - _data = "$target_gen_dir/icudtl_dat.obj" - _data_script = "make_data_obj_win.py" + data_assembly = "$target_gen_dir/icudtl_dat.S" + data_dir = "../externals/icu/" + if (is_android) { + data_dir += "android" + } else if (is_ios) { + data_dir += "ios" } else { - _data = "$target_gen_dir/icudtl_dat.S" - _data_script = "make_data_assembly.py" + data_dir += "common" } + action("make_data_assembly") { + script = "../externals/icu/scripts/make_data_assembly.py" + inputs = [ + "$data_dir/icudtl.dat", + ] + outputs = [ + "$data_assembly", + ] + args = [ + rebase_path(inputs[0], root_build_dir), + rebase_path(data_assembly, root_build_dir), + ] + if (is_mac || is_ios) { + args += [ "--mac" ] + } + } + third_party("icu") { public_include_dirs = [ - "../externals/icu/icu4c/source/common", - "../externals/icu/icu4c/source/i18n", + "../externals/icu/source/common", + "../externals/icu/source/i18n", ".", ] public_defines = [ "U_USING_ICU_NAMESPACE=0", - "U_STATIC_IMPLEMENTATION", "SK_USING_THIRD_PARTY_ICU", ] configs -= [ "//gn:no_rtti" ] - deps = [ - ":assemble_data", - ] - sources = icu_sources - sources += [ _data ] defines = [ # http://userguide.icu-project.org/howtouseicu - "U_ENABLE_DYLOAD=0", - "U_NOEXCEPT=", "U_COMMON_IMPLEMENTATION", + "U_STATIC_IMPLEMENTATION", + "U_ENABLE_DYLOAD=0", "U_I18N_IMPLEMENTATION", ] + sources = icu_sources if (is_win) { - defines += [ "U_PLATFORM_HAS_WINUWP_API=1" ] - } - } - _dat_file = "$target_gen_dir/icu.dat" - action("assemble_data") { - script = _data_script - sources = [ - _dat_file, - ] - outputs = [ - _data, - ] - args = [ - icu_dtname, - current_os, - target_cpu, - rebase_path(_dat_file, root_build_dir), - rebase_path(_data, root_build_dir), - ] - deps = [ - ":get_data", - ] - } - if (skia_icu_use_prebuilt_data) { - action("get_data") { - script = "download_file.py" - outputs = [ - _dat_file, + deps = [ + ":icudata", ] - args = [ - "https://storage.googleapis.com/$icu_dat_bucket/$icu_dat_md5", - icu_dat_md5, - rebase_path(_dat_file, root_build_dir), + public_defines += [ + "U_NOEXCEPT=", + "U_STATIC_IMPLEMENTATION", + ] + libs = [ "Advapi32.lib" ] + sources += [ "../externals/icu/source/stubdata/stubdata.cpp" ] + } else { + sources += [ "$data_assembly" ] + deps = [ + ":make_data_assembly", ] } - } else { - action("get_data") { - script = "build_icu_data_file.py" + } + if (is_win) { + copy("icudata") { + sources = [ + "../externals/icu/windows/icudt.dll", + ] outputs = [ - _dat_file, - ] - args = [ - rebase_path("../externals/icu", root_build_dir), - rebase_path(_dat_file, root_build_dir), + "$root_out_dir/icudt.dll", ] + data = outputs } } } diff --git a/third_party/icu/README.md b/third_party/icu/README.md deleted file mode 100644 index 4c1c57f0ef..0000000000 --- a/third_party/icu/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# ICU - -If the GN variable `use_icu` is set and `skia_use_system_icu` is unset, then -the build system will build ICU from the source. The repository in -`../externals/icu` is a mirror of the Unicode Consortium's repository. - -ICU requires a platform-independent data file, which it does not provide in its -repository. The data file can be built by first building the ICU tools to -process the provided source data, but this is a giant hassle for building ICU -in a cross-platform way (which might necessitate building ICU twice). - -icu/BUILD.gn has two action build rules: `get_data` and `assemble_data`. -`get_data` downloads the `icu.dat` file if it is not already downloaded. The -Google Cloud Storage location of the file is content-addressed with the -`icu_dat_md5` GN variable found in `icu.gni`. - -`assemble_data` will produce either a C++ source file (for our WASM build), an -object file (for our Windows builds), or a assembly file (for everything else). -These files will contain a single exported symbol, `icudtNN_dat` which points -at a block of data containing the contents of the `icu.dat` file. - -This build also provides the `SkLoadICU` function, which must be called on -Windows for some reason. - -## How To Update ICU Dependency - -When we update the ICU branch, we will use the `update_icu.sh` script to -update the `icu.gni` file and produce a new `.dat` file for upload. - -1) Edit `DEPS` file. - -2) Run update script: - - third_party/icu/update_icu.sh - -3) Upload new data file. `DATA_FILE` and `GS_URL` will be provided by the - output of the previous step. - - gsutil cp DATA_FILE GS_URL - -4) Commit: - - git add DEPS third_party/icu/icu.gni - git commit -m 'DEPS: update ICU' - git push origin @:refs/for/master - bin/sysopen https://review.skia.org/$(bin/gerrit-number @) - diff --git a/third_party/icu/SkLoadICU.h b/third_party/icu/SkLoadICU.h index 94cad21f81..c443347290 100644 --- a/third_party/icu/SkLoadICU.h +++ b/third_party/icu/SkLoadICU.h @@ -11,20 +11,40 @@ #ifdef SK_BUILD_FOR_WIN +#include "../private/SkLeanWindows.h" + +#include "unicode/uvernum.h" #include "unicode/udata.h" -extern "C" const char U_ICUDATA_ENTRY_POINT[]; +#define ICU_UTIL_DATA_SYMBOL "icudt" U_ICU_VERSION_SHORT "_dat" +#define ICU_UTIL_DATA_SHARED_MODULE_NAME "icudt.dll" -static inline void SkLoadICU() { - UErrorCode udata_setCommonData_error = U_ZERO_ERROR; - udata_setCommonData(&U_ICUDATA_ENTRY_POINT, &udata_setCommonData_error); - SkASSERT_RELEASE(udata_setCommonData_error == U_ZERO_ERROR); - - UErrorCode udata_setFileAccess_error = U_ZERO_ERROR; - udata_setFileAccess(UDATA_NO_FILES, &udata_setFileAccess_error); - SkASSERT_RELEASE(udata_setFileAccess_error == U_ZERO_ERROR); +inline void SkLoadICU() { + HMODULE module = LoadLibraryA(ICU_UTIL_DATA_SHARED_MODULE_NAME); + if (!module) { + SK_ABORT("Failed to load " ICU_UTIL_DATA_SHARED_MODULE_NAME "\n"); + } + FARPROC addr = GetProcAddress(module, ICU_UTIL_DATA_SYMBOL); + if (!addr) { + SK_ABORT("Symbol " ICU_UTIL_DATA_SYMBOL " missing in " + ICU_UTIL_DATA_SHARED_MODULE_NAME ".\n"); + } + UErrorCode err = U_ZERO_ERROR; + udata_setCommonData(reinterpret_cast(addr), &err); + if (err != U_ZERO_ERROR) { + SkDebugf("udata_setCommonData() returned %d.\n", (int)err); + SK_ABORT(""); + } + udata_setFileAccess(UDATA_ONLY_PACKAGES, &err); + if (err != U_ZERO_ERROR) { + SkDebugf("udata_setFileAccess() returned %d.\n", (int)err); + SK_ABORT(""); + } } +#undef ICU_UTIL_DATA_SHARED_MODULE_NAME +#undef ICU_UTIL_DATA_SYMBOL + #else inline void SkLoadICU() {} #endif // SK_BUILD_FOR_WIN diff --git a/third_party/icu/build_icu_data_file.py b/third_party/icu/build_icu_data_file.py deleted file mode 100755 index 191b856354..0000000000 --- a/third_party/icu/build_icu_data_file.py +++ /dev/null @@ -1,53 +0,0 @@ -#! /usr/bin/env python -# Copyright 2019 Google LLC. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -''' -Builds ICU from a clean repository and saves the data file. -''' - -import glob -import os -import shutil -import subprocess -import sys -import tempfile - -def build_icu(icu_source_repository, destination): - if sys.platform.startswith('linux'): - cur_os = 'Linux' - elif sys.platform.startswith('darwin'): - cur_os = 'MacOSX' - else: - sys.exit(1) - - cwd = os.getcwd() - assert os.path.exists(icu_source_repository) - icu_source_repository = os.path.abspath(icu_source_repository) - configure = icu_source_repository + '/icu4c/source/runConfigureICU' - assert os.path.exists(configure) - destination = os.path.abspath(destination) - tmp = tempfile.mkdtemp() - os.chdir(tmp) - icuflags = ['-DU_CHARSET_IS_UTF8=1', - '-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1', - '-DU_HIDE_OBSOLETE_UTF_OLD_H=1'] - env_copy = os.environ.copy() - env_copy['CPPFLAGS'] = ' '.join(icuflags) - subprocess.check_call([configure, - cur_os, - '--enable-static', - '--disable-shared', - '--with-data-packaging=archive'], - env=env_copy) - subprocess.check_call(['make', '-j']) - data_files = glob.glob('data/out/icudt*.dat') - assert len(data_files) == 1 - shutil.move(data_files[0], destination) - os.chdir(cwd) - shutil.rmtree(tmp) - -if __name__ == '__main__': - print '\n'.join('>>> %r' % a for a in sys.argv) - build_icu(sys.argv[1], sys.argv[2]) diff --git a/third_party/icu/download_file.py b/third_party/icu/download_file.py deleted file mode 100755 index 672ae426b9..0000000000 --- a/third_party/icu/download_file.py +++ /dev/null @@ -1,26 +0,0 @@ -#! /usr/bin/env python -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -''' -Download from URL to PATH only if the MD5 checksum does not match. -''' - -import os -import sys -import urllib - -import hasher - -def download(url, md5, path): - if md5 == hasher.hash_file('md5', path): - return - dirname = os.path.dirname(path) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - urllib.urlretrieve(url, path) - assert md5 == hasher.hash_file('md5', path) - -if __name__ == '__main__': - print '\n'.join('>>> %r' % a for a in sys.argv) - download(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/third_party/icu/hasher.py b/third_party/icu/hasher.py deleted file mode 100755 index adf515ff8e..0000000000 --- a/third_party/icu/hasher.py +++ /dev/null @@ -1,26 +0,0 @@ -#! /usr/bin/env python -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -''' -Return the checksum of FILENAME, using the given ALGORITHM -''' - -import hashlib -import mmap -import os -import sys - -def hash_file(algorithm, filename): - assert algorithm in hashlib.algorithms - if not os.path.exists(filename): - return '' - h = hashlib.new(algorithm) - with open(filename, 'rb') as f: - m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) - h.update(m) - m.close() - return h.hexdigest() - -if __name__ == '__main__': - sys.stdout.write(hash_file(sys.argv[1], sys.argv[2]) + '\n') diff --git a/third_party/icu/icu.gni b/third_party/icu/icu.gni index 9a881bf0d7..2c32fa9f92 100644 --- a/third_party/icu/icu.gni +++ b/third_party/icu/icu.gni @@ -1,10 +1,7 @@ # Copyright 2019 Google LLC. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -icu_dtname = "icudt63_dat" -icu_dat_md5 = "6217818389487f8540fb0adff2c3e5b5" -icu_dat_bucket = "skia-icu-data" -_src = "../externals/icu/icu4c/source" +_src = "../externals/icu/source" icu_sources = [ "$_src/common/appendable.cpp", "$_src/common/bmpset.cpp", diff --git a/third_party/icu/make_data_assembly.py b/third_party/icu/make_data_assembly.py deleted file mode 100755 index a4a4870210..0000000000 --- a/third_party/icu/make_data_assembly.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 Google LLC. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -''' -Generate a source file containing the given binary data. - -File type is either assembly or C++, depending on platform. -''' - -import os -import struct -import sys -import mmap - -def iterate_as_uint32(path): - with open(path, 'rb') as f: - s = struct.Struct('@I') - assert s.size == 4 - mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) - assert (len(mm) % s.size) == 0 - for offset in xrange(0, len(mm), s.size): - yield s.unpack_from(mm, offset)[0] - mm.close() - - -def convert(fmt, name, src_path, dst_path): - header, line_begin, line_end, footer = fmt - assert os.path.exists(src_path) - src = iterate_as_uint32(src_path) - with open(dst_path, 'w') as o: - o.write(header.format(name)) - while True: - line = ','.join('%d' % v for _, v in zip(range(8), src)) - if not line: - break - o.write('%s%s%s\n' % (line_begin, line, line_end)) - o.write(footer.format(name)) - - -gcc_asm = ('.globl {0}\n.balign 16\n{0}:\n', '.long ', '', '') - -cpp = ('#include \nextern "C" uint32_t {0}[] __attribute__((aligned(16))) = {{\n', - '', ',', '}};\n') - -def main(name, current_os, target_cpu, src, dst): - wasm = target_cpu == 'wasm' - fmt = gcc_asm if not wasm else cpp - if not wasm and current_os in ['mac', 'ios', 'tvos']: - name = '_' + name - convert(fmt, name, src, dst) - -if __name__ == '__main__': - print '\n'.join('>>> %r' % x for x in sys.argv) - main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) diff --git a/third_party/icu/make_data_obj_win.py b/third_party/icu/make_data_obj_win.py deleted file mode 100755 index 020d928dc7..0000000000 --- a/third_party/icu/make_data_obj_win.py +++ /dev/null @@ -1,55 +0,0 @@ -#! /usr/bin/env python - -# Copyright 2019 Google LLC. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -''' -Generate a Windows object file containing the given binary data. -''' - -import sys -import mmap -import struct -import time - -def write_windows_obj(symbol_name, target_cpu, src_path, dst_path): - machine_codes = {'x86' : 0x014C, - 'x64' : 0x8664, - 'arm' : 0x01C0, - 'arm64' : 0xAA64} - if target_cpu == 'x86': - symbol_name = '_' + symbol_name - assert target_cpu in machine_codes - src_file = open(src_path, 'rb') - src = mmap.mmap(src_file.fileno(), 0, access=mmap.ACCESS_READ) - image_file_header = struct.Struct('HHIIIHH') - image_section_header = struct.Struct('8sIIIIIIHHI') - linker_options = '-export:' + symbol_name + ',data ' - header_len = image_file_header.size + 2 * image_section_header.size - opts_length = len(linker_options) - file_header = image_file_header.pack(machine_codes[target_cpu], 2, - int(time.time()), header_len + opts_length + len(src), 1, 0, 0) - section1_header = image_section_header.pack( - '.drectve', 0, 0, opts_length, header_len, - 0, 0, 0, 0, 0x00100A00) - section2_header = image_section_header.pack( - '.rdata', 0, 0, len(src), header_len + opts_length, - 0, 0, 0, 0, 0x40100000) - symbols = struct.pack('IIIHHBB', 0, 4, 0, 2, 0, 2, 0) - symbol_names = struct.pack('I', 5 + len(symbol_name)) + symbol_name + '\000' - with open(dst_path, 'wb') as o: - o.write(file_header) - o.write(section1_header) - o.write(section2_header) - o.write(linker_options) - o.write(src) - o.write(symbols) - o.write(symbol_names) - src.close() - src_file.close() - - -if __name__ == '__main__': - print '\n'.join('>>> %r' % a for a in sys.argv) - write_windows_obj(sys.argv[1], sys.argv[3], sys.argv[4], sys.argv[5]) diff --git a/third_party/icu/update_icu.sh b/third_party/icu/update_icu.sh deleted file mode 100755 index 6c600289d5..0000000000 --- a/third_party/icu/update_icu.sh +++ /dev/null @@ -1,48 +0,0 @@ -#! /bin/sh - -# Copyright 2019 Google LLC. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Run this script after updating ../../DEPS to point at a new revision of ICU. - -set -x -e - -DATA_TMP="$(mktemp "${TMPDIR:-/tmp}/icu_dat.XXXXXXXXXX";)" -cd "$(dirname "$0")" -python ../../tools/git-sync-deps -python ./build_icu_data_file.py ../externals/icu "$DATA_TMP" -UVERNUM_FILE=../externals/icu/icu4c/source/common/unicode/uvernum.h -UVERNUM_SEDCMD='s/^#define U_ICU_VERSION_MAJOR_NUM \([1-9][0-9]*\)$/\1/p' -SYMBOL="icudt$(sed -n "$UVERNUM_SEDCMD" "$UVERNUM_FILE")_dat" -CHECKSUM="$(python ./hasher.py 'md5' "$DATA_TMP")" - -BUCKET='skia-icu-data' -SRC='../externals/icu/icu4c/source' -{ - cat <<- EOM - # Copyright $(date +%Y) Google LLC. - # Use of this source code is governed by a BSD-style license that can be - # found in the LICENSE file. - icu_dtname = "${SYMBOL}" - icu_dat_md5 = "${CHECKSUM}" - icu_dat_bucket = "${BUCKET}" - _src = "$SRC" - icu_sources = [ - EOM - find $SRC/common $SRC/i18n -name '*.cpp' -o -name '*.h' -o -name '*.c' | \ - LANG= sort | \ - sed "s#^${SRC}/\(.*\)\$# \"\$_src/\1\",#" - printf ']\n' -} > icu.gni - -set +x +e -URL="https://storage.googleapis.com/${BUCKET}/$CHECKSUM" -if [ "$(curl -s -I "$URL" | head -n 1|cut -d' ' -f2)" != 200 ]; then - echo "TODO: upload the dat file:" - echo " gsutil cp '$DATA_TMP' 'gs://${BUCKET}/$CHECKSUM'" -else - echo "DAT file is already here: $URL" -fi -echo SUCCESS -echo '( git add third_party/icu/icu.gni )'