Revert "third_party/icu: use upstream"

This reverts commit e6cfe77185.

Reason for revert: Android & Win7 perf/test failures.


Original change's description:
> third_party/icu:  use upstream
> 
>     Change DEPS to point at upstream, not chromium's fork.
>     `skia_use_icu` defaults to true everywhere but iOS
>     Clean up SkLoadICU:  use U_ICUDATA_ENTRY_POINT, no DLL.
>     Add data dep to icu/icu.gni
>     Scripts: icu/{make_data_assembly.py,make_data_obj_win.py}
>     Scripts: icu/{update_icu.sh,build_icu_data_file.py} for rebuilding
>     document process in icu/README.md
> 
> Bug: skia:8702
> Change-Id: I99789749ba84ae737f6c62475d0d676cd36715bb
> Reviewed-on: https://skia-review.googlesource.com/c/186870
> Commit-Queue: Hal Canary <halcanary@google.com>
> Reviewed-by: Florin Malita <fmalita@chromium.org>
> Reviewed-by: Ben Wagner <bungeman@google.com>

TBR=halcanary@google.com,bungeman@google.com,fmalita@chromium.org,fmalita@google.com

Change-Id: Id379bc2be7d53eca67b2c557ac20bd44073a6258
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: skia:8702
Reviewed-on: https://skia-review.googlesource.com/c/189647
Reviewed-by: Florin Malita <fmalita@chromium.org>
Commit-Queue: Florin Malita <fmalita@chromium.org>
This commit is contained in:
Florin Malita 2019-02-06 02:28:24 +00:00 committed by Skia Commit-Bot
parent 7f993d52ff
commit 0c2add016d
12 changed files with 82 additions and 386 deletions

2
DEPS
View File

@ -13,7 +13,7 @@ deps = {
"third_party/externals/expat" : "https://android.googlesource.com/platform/external/expat.git@android-6.0.1_r55",
"third_party/externals/freetype" : "https://skia.googlesource.com/third_party/freetype2.git@7edc937fe679d14d66f55cf6f7fa607925d38f3c",
"third_party/externals/harfbuzz" : "https://skia.googlesource.com/third_party/harfbuzz.git@8be74d85534534dbdd39a0a6f496e26e9f3e661d",
"third_party/externals/icu" : "https://skia.googlesource.com/external/github.com/unicode-org/icu@a96dc7faa584c2408c6fa1e9c61d121d45a33563",
"third_party/externals/icu" : "https://chromium.googlesource.com/chromium/deps/icu.git@407b39301e71006b68bd38e770f35d32398a7b14",
"third_party/externals/imgui" : "https://skia.googlesource.com/external/github.com/ocornut/imgui.git@d38d7c6628bebd02692cfdd6fa76b4d992a35b75",
# TODO: remove jsoncpp after migrating clients to SkJSON
"third_party/externals/jsoncpp" : "https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git@1.0.0",

View File

@ -3,5 +3,6 @@
# found in the LICENSE file.
declare_args() {
skia_use_icu = !is_ios
skia_use_icu = !is_fuchsia && !is_ios && !is_win # TODO: Windows
}

View File

@ -5,7 +5,6 @@
declare_args() {
skia_use_system_icu = is_official_build
skia_icu_use_prebuilt_data = true
}
import("../../gn/skia.gni")
@ -19,86 +18,77 @@ if (skia_use_icu) {
defines = [ "U_USING_ICU_NAMESPACE=0" ]
}
} else {
if (target_cpu == "wasm") {
_data = "$target_gen_dir/icudtl_dat.cpp"
_data_script = "make_data_assembly.py"
} else if (is_win) {
_data = "$target_gen_dir/icudtl_dat.obj"
_data_script = "make_data_obj_win.py"
data_assembly = "$target_gen_dir/icudtl_dat.S"
data_dir = "../externals/icu/"
if (is_android) {
data_dir += "android"
} else if (is_ios) {
data_dir += "ios"
} else {
_data = "$target_gen_dir/icudtl_dat.S"
_data_script = "make_data_assembly.py"
data_dir += "common"
}
action("make_data_assembly") {
script = "../externals/icu/scripts/make_data_assembly.py"
inputs = [
"$data_dir/icudtl.dat",
]
outputs = [
"$data_assembly",
]
args = [
rebase_path(inputs[0], root_build_dir),
rebase_path(data_assembly, root_build_dir),
]
if (is_mac || is_ios) {
args += [ "--mac" ]
}
}
third_party("icu") {
public_include_dirs = [
"../externals/icu/icu4c/source/common",
"../externals/icu/icu4c/source/i18n",
"../externals/icu/source/common",
"../externals/icu/source/i18n",
".",
]
public_defines = [
"U_USING_ICU_NAMESPACE=0",
"U_STATIC_IMPLEMENTATION",
"SK_USING_THIRD_PARTY_ICU",
]
configs -= [ "//gn:no_rtti" ]
deps = [
":assemble_data",
]
sources = icu_sources
sources += [ _data ]
defines = [
# http://userguide.icu-project.org/howtouseicu
"U_ENABLE_DYLOAD=0",
"U_NOEXCEPT=",
"U_COMMON_IMPLEMENTATION",
"U_STATIC_IMPLEMENTATION",
"U_ENABLE_DYLOAD=0",
"U_I18N_IMPLEMENTATION",
]
sources = icu_sources
if (is_win) {
defines += [ "U_PLATFORM_HAS_WINUWP_API=1" ]
}
}
_dat_file = "$target_gen_dir/icu.dat"
action("assemble_data") {
script = _data_script
sources = [
_dat_file,
]
outputs = [
_data,
]
args = [
icu_dtname,
current_os,
target_cpu,
rebase_path(_dat_file, root_build_dir),
rebase_path(_data, root_build_dir),
]
deps = [
":get_data",
]
}
if (skia_icu_use_prebuilt_data) {
action("get_data") {
script = "download_file.py"
outputs = [
_dat_file,
deps = [
":icudata",
]
args = [
"https://storage.googleapis.com/$icu_dat_bucket/$icu_dat_md5",
icu_dat_md5,
rebase_path(_dat_file, root_build_dir),
public_defines += [
"U_NOEXCEPT=",
"U_STATIC_IMPLEMENTATION",
]
libs = [ "Advapi32.lib" ]
sources += [ "../externals/icu/source/stubdata/stubdata.cpp" ]
} else {
sources += [ "$data_assembly" ]
deps = [
":make_data_assembly",
]
}
} else {
action("get_data") {
script = "build_icu_data_file.py"
}
if (is_win) {
copy("icudata") {
sources = [
"../externals/icu/windows/icudt.dll",
]
outputs = [
_dat_file,
]
args = [
rebase_path("../externals/icu", root_build_dir),
rebase_path(_dat_file, root_build_dir),
"$root_out_dir/icudt.dll",
]
data = outputs
}
}
}

View File

@ -1,47 +0,0 @@
# ICU
If the GN variable `use_icu` is set and `skia_use_system_icu` is unset, then
the build system will build ICU from the source. The repository in
`../externals/icu` is a mirror of the Unicode Consortium's repository.
ICU requires a platform-independent data file, which it does not provide in its
repository. The data file can be built by first building the ICU tools to
process the provided source data, but this is a giant hassle for building ICU
in a cross-platform way (which might necessitate building ICU twice).
icu/BUILD.gn has two action build rules: `get_data` and `assemble_data`.
`get_data` downloads the `icu.dat` file if it is not already downloaded. The
Google Cloud Storage location of the file is content-addressed with the
`icu_dat_md5` GN variable found in `icu.gni`.
`assemble_data` will produce either a C++ source file (for our WASM build), an
object file (for our Windows builds), or a assembly file (for everything else).
These files will contain a single exported symbol, `icudtNN_dat` which points
at a block of data containing the contents of the `icu.dat` file.
This build also provides the `SkLoadICU` function, which must be called on
Windows for some reason.
## How To Update ICU Dependency
When we update the ICU branch, we will use the `update_icu.sh` script to
update the `icu.gni` file and produce a new `.dat` file for upload.
1) Edit `DEPS` file.
2) Run update script:
third_party/icu/update_icu.sh
3) Upload new data file. `DATA_FILE` and `GS_URL` will be provided by the
output of the previous step.
gsutil cp DATA_FILE GS_URL
4) Commit:
git add DEPS third_party/icu/icu.gni
git commit -m 'DEPS: update ICU'
git push origin @:refs/for/master
bin/sysopen https://review.skia.org/$(bin/gerrit-number @)

View File

@ -11,20 +11,40 @@
#ifdef SK_BUILD_FOR_WIN
#include "../private/SkLeanWindows.h"
#include "unicode/uvernum.h"
#include "unicode/udata.h"
extern "C" const char U_ICUDATA_ENTRY_POINT[];
#define ICU_UTIL_DATA_SYMBOL "icudt" U_ICU_VERSION_SHORT "_dat"
#define ICU_UTIL_DATA_SHARED_MODULE_NAME "icudt.dll"
static inline void SkLoadICU() {
UErrorCode udata_setCommonData_error = U_ZERO_ERROR;
udata_setCommonData(&U_ICUDATA_ENTRY_POINT, &udata_setCommonData_error);
SkASSERT_RELEASE(udata_setCommonData_error == U_ZERO_ERROR);
UErrorCode udata_setFileAccess_error = U_ZERO_ERROR;
udata_setFileAccess(UDATA_NO_FILES, &udata_setFileAccess_error);
SkASSERT_RELEASE(udata_setFileAccess_error == U_ZERO_ERROR);
inline void SkLoadICU() {
HMODULE module = LoadLibraryA(ICU_UTIL_DATA_SHARED_MODULE_NAME);
if (!module) {
SK_ABORT("Failed to load " ICU_UTIL_DATA_SHARED_MODULE_NAME "\n");
}
FARPROC addr = GetProcAddress(module, ICU_UTIL_DATA_SYMBOL);
if (!addr) {
SK_ABORT("Symbol " ICU_UTIL_DATA_SYMBOL " missing in "
ICU_UTIL_DATA_SHARED_MODULE_NAME ".\n");
}
UErrorCode err = U_ZERO_ERROR;
udata_setCommonData(reinterpret_cast<void*>(addr), &err);
if (err != U_ZERO_ERROR) {
SkDebugf("udata_setCommonData() returned %d.\n", (int)err);
SK_ABORT("");
}
udata_setFileAccess(UDATA_ONLY_PACKAGES, &err);
if (err != U_ZERO_ERROR) {
SkDebugf("udata_setFileAccess() returned %d.\n", (int)err);
SK_ABORT("");
}
}
#undef ICU_UTIL_DATA_SHARED_MODULE_NAME
#undef ICU_UTIL_DATA_SYMBOL
#else
inline void SkLoadICU() {}
#endif // SK_BUILD_FOR_WIN

View File

@ -1,53 +0,0 @@
#! /usr/bin/env python
# Copyright 2019 Google LLC.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
'''
Builds ICU from a clean repository and saves the data file.
'''
import glob
import os
import shutil
import subprocess
import sys
import tempfile
def build_icu(icu_source_repository, destination):
if sys.platform.startswith('linux'):
cur_os = 'Linux'
elif sys.platform.startswith('darwin'):
cur_os = 'MacOSX'
else:
sys.exit(1)
cwd = os.getcwd()
assert os.path.exists(icu_source_repository)
icu_source_repository = os.path.abspath(icu_source_repository)
configure = icu_source_repository + '/icu4c/source/runConfigureICU'
assert os.path.exists(configure)
destination = os.path.abspath(destination)
tmp = tempfile.mkdtemp()
os.chdir(tmp)
icuflags = ['-DU_CHARSET_IS_UTF8=1',
'-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1',
'-DU_HIDE_OBSOLETE_UTF_OLD_H=1']
env_copy = os.environ.copy()
env_copy['CPPFLAGS'] = ' '.join(icuflags)
subprocess.check_call([configure,
cur_os,
'--enable-static',
'--disable-shared',
'--with-data-packaging=archive'],
env=env_copy)
subprocess.check_call(['make', '-j'])
data_files = glob.glob('data/out/icudt*.dat')
assert len(data_files) == 1
shutil.move(data_files[0], destination)
os.chdir(cwd)
shutil.rmtree(tmp)
if __name__ == '__main__':
print '\n'.join('>>> %r' % a for a in sys.argv)
build_icu(sys.argv[1], sys.argv[2])

View File

@ -1,26 +0,0 @@
#! /usr/bin/env python
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
'''
Download from URL to PATH only if the MD5 checksum does not match.
'''
import os
import sys
import urllib
import hasher
def download(url, md5, path):
if md5 == hasher.hash_file('md5', path):
return
dirname = os.path.dirname(path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
urllib.urlretrieve(url, path)
assert md5 == hasher.hash_file('md5', path)
if __name__ == '__main__':
print '\n'.join('>>> %r' % a for a in sys.argv)
download(sys.argv[1], sys.argv[2], sys.argv[3])

View File

@ -1,26 +0,0 @@
#! /usr/bin/env python
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
'''
Return the checksum of FILENAME, using the given ALGORITHM
'''
import hashlib
import mmap
import os
import sys
def hash_file(algorithm, filename):
assert algorithm in hashlib.algorithms
if not os.path.exists(filename):
return ''
h = hashlib.new(algorithm)
with open(filename, 'rb') as f:
m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
h.update(m)
m.close()
return h.hexdigest()
if __name__ == '__main__':
sys.stdout.write(hash_file(sys.argv[1], sys.argv[2]) + '\n')

View File

@ -1,10 +1,7 @@
# Copyright 2019 Google LLC.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
icu_dtname = "icudt63_dat"
icu_dat_md5 = "6217818389487f8540fb0adff2c3e5b5"
icu_dat_bucket = "skia-icu-data"
_src = "../externals/icu/icu4c/source"
_src = "../externals/icu/source"
icu_sources = [
"$_src/common/appendable.cpp",
"$_src/common/bmpset.cpp",

View File

@ -1,57 +0,0 @@
#!/usr/bin/env python
# Copyright 2019 Google LLC.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
'''
Generate a source file containing the given binary data.
File type is either assembly or C++, depending on platform.
'''
import os
import struct
import sys
import mmap
def iterate_as_uint32(path):
with open(path, 'rb') as f:
s = struct.Struct('@I')
assert s.size == 4
mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
assert (len(mm) % s.size) == 0
for offset in xrange(0, len(mm), s.size):
yield s.unpack_from(mm, offset)[0]
mm.close()
def convert(fmt, name, src_path, dst_path):
header, line_begin, line_end, footer = fmt
assert os.path.exists(src_path)
src = iterate_as_uint32(src_path)
with open(dst_path, 'w') as o:
o.write(header.format(name))
while True:
line = ','.join('%d' % v for _, v in zip(range(8), src))
if not line:
break
o.write('%s%s%s\n' % (line_begin, line, line_end))
o.write(footer.format(name))
gcc_asm = ('.globl {0}\n.balign 16\n{0}:\n', '.long ', '', '')
cpp = ('#include <cstdint>\nextern "C" uint32_t {0}[] __attribute__((aligned(16))) = {{\n',
'', ',', '}};\n')
def main(name, current_os, target_cpu, src, dst):
wasm = target_cpu == 'wasm'
fmt = gcc_asm if not wasm else cpp
if not wasm and current_os in ['mac', 'ios', 'tvos']:
name = '_' + name
convert(fmt, name, src, dst)
if __name__ == '__main__':
print '\n'.join('>>> %r' % x for x in sys.argv)
main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5])

View File

@ -1,55 +0,0 @@
#! /usr/bin/env python
# Copyright 2019 Google LLC.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
'''
Generate a Windows object file containing the given binary data.
'''
import sys
import mmap
import struct
import time
def write_windows_obj(symbol_name, target_cpu, src_path, dst_path):
machine_codes = {'x86' : 0x014C,
'x64' : 0x8664,
'arm' : 0x01C0,
'arm64' : 0xAA64}
if target_cpu == 'x86':
symbol_name = '_' + symbol_name
assert target_cpu in machine_codes
src_file = open(src_path, 'rb')
src = mmap.mmap(src_file.fileno(), 0, access=mmap.ACCESS_READ)
image_file_header = struct.Struct('HHIIIHH')
image_section_header = struct.Struct('8sIIIIIIHHI')
linker_options = '-export:' + symbol_name + ',data '
header_len = image_file_header.size + 2 * image_section_header.size
opts_length = len(linker_options)
file_header = image_file_header.pack(machine_codes[target_cpu], 2,
int(time.time()), header_len + opts_length + len(src), 1, 0, 0)
section1_header = image_section_header.pack(
'.drectve', 0, 0, opts_length, header_len,
0, 0, 0, 0, 0x00100A00)
section2_header = image_section_header.pack(
'.rdata', 0, 0, len(src), header_len + opts_length,
0, 0, 0, 0, 0x40100000)
symbols = struct.pack('IIIHHBB', 0, 4, 0, 2, 0, 2, 0)
symbol_names = struct.pack('I', 5 + len(symbol_name)) + symbol_name + '\000'
with open(dst_path, 'wb') as o:
o.write(file_header)
o.write(section1_header)
o.write(section2_header)
o.write(linker_options)
o.write(src)
o.write(symbols)
o.write(symbol_names)
src.close()
src_file.close()
if __name__ == '__main__':
print '\n'.join('>>> %r' % a for a in sys.argv)
write_windows_obj(sys.argv[1], sys.argv[3], sys.argv[4], sys.argv[5])

View File

@ -1,48 +0,0 @@
#! /bin/sh
# Copyright 2019 Google LLC.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Run this script after updating ../../DEPS to point at a new revision of ICU.
set -x -e
DATA_TMP="$(mktemp "${TMPDIR:-/tmp}/icu_dat.XXXXXXXXXX";)"
cd "$(dirname "$0")"
python ../../tools/git-sync-deps
python ./build_icu_data_file.py ../externals/icu "$DATA_TMP"
UVERNUM_FILE=../externals/icu/icu4c/source/common/unicode/uvernum.h
UVERNUM_SEDCMD='s/^#define U_ICU_VERSION_MAJOR_NUM \([1-9][0-9]*\)$/\1/p'
SYMBOL="icudt$(sed -n "$UVERNUM_SEDCMD" "$UVERNUM_FILE")_dat"
CHECKSUM="$(python ./hasher.py 'md5' "$DATA_TMP")"
BUCKET='skia-icu-data'
SRC='../externals/icu/icu4c/source'
{
cat <<- EOM
# Copyright $(date +%Y) Google LLC.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
icu_dtname = "${SYMBOL}"
icu_dat_md5 = "${CHECKSUM}"
icu_dat_bucket = "${BUCKET}"
_src = "$SRC"
icu_sources = [
EOM
find $SRC/common $SRC/i18n -name '*.cpp' -o -name '*.h' -o -name '*.c' | \
LANG= sort | \
sed "s#^${SRC}/\(.*\)\$# \"\$_src/\1\",#"
printf ']\n'
} > icu.gni
set +x +e
URL="https://storage.googleapis.com/${BUCKET}/$CHECKSUM"
if [ "$(curl -s -I "$URL" | head -n 1|cut -d' ' -f2)" != 200 ]; then
echo "TODO: upload the dat file:"
echo " gsutil cp '$DATA_TMP' 'gs://${BUCKET}/$CHECKSUM'"
else
echo "DAT file is already here: $URL"
fi
echo SUCCESS
echo '( git add third_party/icu/icu.gni )'