AuroraMiddleware/scuffed-code
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ICU-21028 Adding ICU version file generation to tools

Browse Source 
See #1248
This commit is contained in:
David Beaumont 2020-08-12 16:50:27 +00:00 committed by David Beaumont
parent 05d49fd373
commit 95cd3904aa
5 changed files with 105 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tools/cldr/cldr-to-icu/build-icu-data.xml
View File

@ -46,7 +46,14 @@
<!-- The directory in which the additional ICU XML data is stored. -->
<property name="specialsDir" value="${basedir}/../../../icu4c/source/data/xml"/>
<!-- An override for the CLDR version string to be used in generated data. -->
<!-- Default value for ICU version (icuver.txt). Update this for each release. -->
<property name="icuVersion" value="67.1.0.0"/>
<!-- Default value for ICU data version (icuver.txt). Update this for each release. -->
<property name="icuDataVersion" value="67.1.0.0"/>
<!-- An override for the CLDR version string (icuver.txt and others). This will be
extracted from the CLDR library used for building the data if not set here. -->
<property name="cldrVersion" value=""/>
<!-- The minimum draft status for CLDR data to be used in the conversion. See
@ -110,6 +117,8 @@
<arg value="-DoutDir=${outDir}"/>
<arg value="-DspecialsDir=${specialsDir}"/>
<arg value="-DoutputTypes=${outputTypes}"/>
<arg value="-DicuVersion=${icuVersion}"/>
<arg value="-DicuDataVersion=${icuDataVersion}"/>
<arg value="-DcldrVersion=${cldrVersion}"/>
<arg value="-DminDraftStatus=${minDraftStatus}"/>
<arg value="-DlocaleIdFilter=${localeIdFilter}"/>
@ -128,6 +137,7 @@
</taskdef>
<convert cldrDir="${cldrDir}" outputDir="${outDir}" specialsDir="${specialsDir}"
outputTypes="${outputTypes}" cldrVersion="${cldrVersion}"
icuVersion="${icuVersion}" icuDataVersion="${icuDataVersion}"
minimalDraftStatus="${minDraftStatus}" localeIdFilter="${localeIdFilter}"
includePseudoLocales="${includePseudoLocales}" emitReport="${emitReport}">
@ -387,7 +397,6 @@
<retain path="currencyNumericCodes.txt"/>
<retain path="icudata.rc"/>
<retain path="icustd.txt"/>
<retain path="icuver.txt"/>
<retain path="zoneinfo64.txt"/>
<!-- This file should be removed before the next release. -->
<retain path="miscfiles.mk"/>

28
tools/cldr/cldr-to-icu/src/main/java/org/unicode/icu/tool/cldrtoicu/IcuConverterConfig.java
View File

@ -44,6 +44,8 @@ public final class IcuConverterConfig implements LdmlConverterConfig {
private Path specialsDir =
DEFAULT_ICU_DIR.map(d -> d.resolve("icu4c/source/data/xml")).orElse(null);
private ImmutableSet<OutputType> outputTypes = OutputType.ALL;
private Optional<String> icuVersion = Optional.empty();
private Optional<String> icuDataVersion = Optional.empty();
private Optional<String> cldrVersion = Optional.empty();
private CldrDraftStatus minimumDraftStatus = CldrDraftStatus.CONTRIBUTED;
private boolean emitReport = false;
@ -81,6 +83,20 @@ public final class IcuConverterConfig implements LdmlConverterConfig {
return this;
}
public Builder setIcuVersion(String version) {
if (!version.isEmpty()) {
this.icuVersion = Optional.of(version);
}
return this;
}
public Builder setIcuDataVersion(String version) {
if (!version.isEmpty()) {
this.icuDataVersion = Optional.of(version);
}
return this;
}
public Builder setCldrVersion(String version) {
if (!version.isEmpty()) {
this.cldrVersion = Optional.of(version);
@ -121,7 +137,7 @@ public final class IcuConverterConfig implements LdmlConverterConfig {
private final Path outputDir;
private final Path specialsDir;
private final ImmutableSet<OutputType> outputTypes;
private final String cldrVersion;
private final IcuVersionInfo versionInfo;
private final CldrDraftStatus minimumDraftStatus;
private final boolean emitReport;
private final ImmutableSet<String> allLocaleIds;
@ -141,8 +157,10 @@ public final class IcuConverterConfig implements LdmlConverterConfig {
checkArgument(!this.outputTypes.isEmpty(),
"must specify at least one output type to be generated (possible values are: %s)",
Arrays.asList(OutputType.values()));
this.cldrVersion =
builder.cldrVersion.orElse(CldrDataSupplier.getCldrVersionString());
this.versionInfo = new IcuVersionInfo(
builder.icuVersion.orElseThrow(() -> new IllegalStateException("missing ICU version")),
builder.icuDataVersion.orElseThrow(() -> new IllegalStateException("missing ICU data version")),
builder.cldrVersion.orElse(CldrDataSupplier.getCldrVersionString()));
this.minimumDraftStatus = checkNotNull(builder.minimumDraftStatus);
this.emitReport = builder.emitReport;
// getAllLocaleIds() returns the union of all the specified IDs in the map.
@ -172,8 +190,8 @@ public final class IcuConverterConfig implements LdmlConverterConfig {
}
@Override
public String getCldrVersion() {
return cldrVersion;
public IcuVersionInfo getVersionInfo() {
return versionInfo;
}
@Override

45
tools/cldr/cldr-to-icu/src/main/java/org/unicode/icu/tool/cldrtoicu/LdmlConverter.java
View File

@ -28,15 +28,7 @@ import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -47,6 +39,7 @@ import org.unicode.cldr.api.CldrDataType;
import org.unicode.cldr.api.CldrPath;
import org.unicode.cldr.api.PathMatcher;
import org.unicode.icu.tool.cldrtoicu.LdmlConverterConfig.IcuLocaleDir;
import org.unicode.icu.tool.cldrtoicu.LdmlConverterConfig.IcuVersionInfo;
import org.unicode.icu.tool.cldrtoicu.localedistance.LocaleDistanceMapper;
import org.unicode.icu.tool.cldrtoicu.mapper.Bcp47Mapper;
import org.unicode.icu.tool.cldrtoicu.mapper.BreakIteratorMapper;
@ -169,6 +162,7 @@ public final class LdmlConverter {
WINDOWS_ZONES(SUPPLEMENTAL),
TRANSFORMS(SUPPLEMENTAL),
LOCALE_DISTANCE(SUPPLEMENTAL),
VERSION(SUPPLEMENTAL),
KEY_TYPE_DATA(BCP47);
public static final ImmutableSet<OutputType> ALL = ImmutableSet.copyOf(OutputType.values());
@ -283,7 +277,7 @@ public final class LdmlConverter {
return;
}
String cldrVersion = config.getCldrVersion();
String cldrVersion = config.getVersionInfo().getCldrVersion();
Map<IcuLocaleDir, DependencyGraph> graphMetadata = new HashMap<>();
splitDirs.forEach(d -> graphMetadata.put(d, new DependencyGraph(cldrVersion)));
@ -521,6 +515,10 @@ public final class LdmlConverter {
write(TransformsMapper.process(src, transformDir, fileHeader), transformDir, false);
break;
case VERSION:
writeIcuVersionInfo();
break;
case KEY_TYPE_DATA:
Bcp47Mapper.process(src).forEach(d -> write(d, "misc"));
break;
@ -541,7 +539,7 @@ public final class LdmlConverter {
// supplemental data XML files.
if (addCldrVersion) {
// Not the same path as used by "setVersion()"
icuData.add(RB_CLDR_VERSION, config.getCldrVersion());
icuData.add(RB_CLDR_VERSION, config.getVersionInfo().getCldrVersion());
}
write(icuData, dir);
}
@ -563,11 +561,34 @@ public final class LdmlConverter {
} else {
// These empty files only exist because the target of an alias has a parent locale
// which is itself not in the set of written ICU files. An "indirect alias target".
icuData.setVersion(config.getCldrVersion());
icuData.setVersion(config.getVersionInfo().getCldrVersion());
}
write(icuData, dir, false);
}
private void writeIcuVersionInfo() {
IcuVersionInfo versionInfo = config.getVersionInfo();
IcuData versionData = new IcuData("icuver", false);
versionData.add(RbPath.of("ICUVersion"), versionInfo.getIcuVersion());
versionData.add(RbPath.of("DataVersion"), versionInfo.getIcuDataVersion());
versionData.add(RbPath.of("CLDRVersion"), versionInfo.getCldrVersion());
// Write file via non-helper methods since we need to include a legacy copyright.
Path miscDir = config.getOutputDir().resolve("misc");
createDirectory(miscDir);
ImmutableList<String> versionHeader = ImmutableList.<String>builder()
.addAll(fileHeader)
.add(
"***************************************************************************",
"*",
"* Copyright (C) 2010-2016 International Business Machines",
"* Corporation and others. All Rights Reserved.",
"*",
"***************************************************************************")
.build();
IcuTextWriter.writeToFile(versionData, miscDir, versionHeader, false);
}
// Commonest case for writing data files in "normal" directories.
private void write(IcuData icuData, String dir) {
write(icuData, config.getOutputDir().resolve(dir), false);
}

29
tools/cldr/cldr-to-icu/src/main/java/org/unicode/icu/tool/cldrtoicu/LdmlConverterConfig.java
View File

@ -6,11 +6,14 @@ import java.nio.file.Path;
import java.util.Map;
import java.util.Set;
import com.google.common.base.Preconditions;
import org.unicode.cldr.api.CldrDraftStatus;
import org.unicode.icu.tool.cldrtoicu.LdmlConverter.OutputType;
import com.google.common.base.Ascii;
import static com.google.common.base.Preconditions.checkNotNull;
/** API for configuring the LDML converter. */
public interface LdmlConverterConfig {
/** Output directories for ICU locale data (this is not used for supplemental data). */
@ -57,6 +60,30 @@ public interface LdmlConverterConfig {
}
}
final class IcuVersionInfo {
private final String icuVersion;
private final String icuDataVersion;
private final String cldrVersion;
public IcuVersionInfo(String icuVersion, String icuDataVersion, String cldrVersion) {
this.icuVersion = checkNotNull(icuVersion);
this.icuDataVersion = checkNotNull(icuDataVersion);
this.cldrVersion = checkNotNull(cldrVersion);
}
public String getIcuVersion() {
return icuVersion;
}
public String getIcuDataVersion() {
return icuDataVersion;
}
public String getCldrVersion() {
return cldrVersion;
}
}
/**
* Returns the set of output types to be converted. Use {@link OutputType#ALL} to convert
* everything.
@ -80,7 +107,7 @@ public interface LdmlConverterConfig {
* Returns a CLDR version String (e.g. {@code "36.1"}) according to either the specified option
* or (as a fallback) the version specified by the CLDR library against which this code is run.
*/
String getCldrVersion();
IcuVersionInfo getVersionInfo();
/** Returns the minimal draft status for CLDR data to be converted. */
CldrDraftStatus getMinimumDraftStatus();

10
tools/cldr/cldr-to-icu/src/main/java/org/unicode/icu/tool/cldrtoicu/ant/ConvertIcuDataTask.java
View File

@ -88,6 +88,16 @@ public final class ConvertIcuDataTask extends Task {
this.cldrPath = checkNotNull(Paths.get(path));
}
@SuppressWarnings("unused")
public void setIcuVersion(String icuVersion) {
config.setIcuVersion(icuVersion);
}
@SuppressWarnings("unused")
public void setIcuDataVersion(String icuDataVersion) {
config.setIcuDataVersion(icuDataVersion);
}
@SuppressWarnings("unused")
public void setCldrVersion(String cldrVersion) {
config.setCldrVersion(cldrVersion);