Update SVG asset creation to handle images
This CL updates the infra scripts used to create the CIPD asset for SVG's corpus on gold to download and include image resources. Summary of changes: - Change svg_downloader.py input argument to more generic name - Add --keep_common_prefix arg to svg_downloader.py to preserve the directory hierarchy for images, needed for the W3C test suite. - Update infra SVG create.py script to download images - Add svg_images.txt file with a list of the images we need for the W3C test suite already in gold. Actually updating the corpus will happen in a separate CL. Bug: skia:11229 Change-Id: I5fe9be35db247f577bda6040ca3694a428314d0e Reviewed-on: https://skia-review.googlesource.com/c/skia/+/361516 Reviewed-by: Florin Malita <fmalita@chromium.org> Reviewed-by: Kevin Lubick <kjlubick@google.com> Commit-Queue: Tyler Denniston <tdenniston@google.com>
This commit is contained in:
Tyler Denniston
Skia Commit-Bot
parent
3f904db257
commit
96a97497b3
@ -24,34 +24,48 @@ SVG_GS_BUCKET = 'gs://skia-svgs'
|
||||
def create_asset(target_dir):
|
||||
"""Create the asset."""
|
||||
target_dir = os.path.realpath(target_dir)
|
||||
target_svg_dir = os.path.join(target_dir, 'svg')
|
||||
target_image_dir = os.path.join(target_dir, 'images')
|
||||
|
||||
if not os.path.exists(target_dir):
|
||||
os.makedirs(target_dir)
|
||||
if not os.path.exists(target_svg_dir):
|
||||
os.makedirs(target_svg_dir)
|
||||
|
||||
if not os.path.exists(target_image_dir):
|
||||
os.makedirs(target_image_dir)
|
||||
|
||||
# Download the SVGs specified in tools/svg/svgs.txt
|
||||
download_svgs_cmd = [
|
||||
'python', os.path.join(SVG_TOOLS, 'svg_downloader.py'),
|
||||
'--output_dir', target_dir,
|
||||
'--svgs_file', os.path.join(SVG_TOOLS, 'svgs.txt'),
|
||||
'--output_dir', target_svg_dir,
|
||||
'--input_file', os.path.join(SVG_TOOLS, 'svgs.txt'),
|
||||
]
|
||||
subprocess.check_call(download_svgs_cmd)
|
||||
|
||||
# Download the SVGs specified in tools/svg/svgs_parse_only.txt with a prefix.
|
||||
download_svgs_parse_only_cmd = [
|
||||
'python', os.path.join(SVG_TOOLS, 'svg_downloader.py'),
|
||||
'--output_dir', target_dir,
|
||||
'--svgs_file', os.path.join(SVG_TOOLS, 'svgs_parse_only.txt'),
|
||||
'--output_dir', target_svg_dir,
|
||||
'--input_file', os.path.join(SVG_TOOLS, 'svgs_parse_only.txt'),
|
||||
'--prefix', 'svgparse_',
|
||||
]
|
||||
subprocess.check_call(download_svgs_parse_only_cmd)
|
||||
|
||||
# Download the image resources specified in tools/svg/svg_images.txt
|
||||
download_images_cmd = [
|
||||
'python', os.path.join(SVG_TOOLS, 'svg_downloader.py'),
|
||||
'--output_dir', target_image_dir,
|
||||
'--input_file', os.path.join(SVG_TOOLS, 'svg_images.txt'),
|
||||
'--keep_common_prefix',
|
||||
]
|
||||
subprocess.check_call(download_images_cmd)
|
||||
|
||||
# Download SVGs from Google storage.
|
||||
# The Google storage bucket will either contain private SVGs or SVGs which we
|
||||
# cannot download over the internet using svg_downloader.py.
|
||||
for skbug in ['skbug4713', 'skbug6918']:
|
||||
subprocess.check_call([
|
||||
'gsutil', '-m', 'cp', os.path.join(SVG_GS_BUCKET, skbug, '*'),
|
||||
target_dir
|
||||
target_svg_dir
|
||||
])
|
||||
|
||||
|
||||
|
||||
@ -9,6 +9,11 @@ svgs.txt
|
||||
This text file contains an SVG URL per line.
|
||||
It is a list of the SVG files used to test rendering correctness.
|
||||
|
||||
svg_images.txt
|
||||
--------------
|
||||
This text file contains an image URL per line.
|
||||
It is a list of images used by the SVGs in svgs.txt.
|
||||
|
||||
svgs_parse_only.txt
|
||||
-------------------
|
||||
This text file contains an SVG URL per line.
|
||||
@ -16,10 +21,14 @@ It is a list of the SVG files used to exercise the SVG parsing code.
|
||||
|
||||
svg_downloader.py
|
||||
-----------------
|
||||
This python script parses txt files and downloads SVGs into a specified directory.
|
||||
This python script parses txt files and downloads SVGs and images into a specified directory.
|
||||
|
||||
The script can be run by hand:
|
||||
$ python svg_downloader.py --output_dir /tmp/svgs/
|
||||
OR
|
||||
$ python svg_downloader.py --output_dir /tmp/svgs/ --svgs_file svgs_parse_only.txt --prefix svgparse_
|
||||
$ python svg_downloader.py --output_dir /tmp/svgs/ --input_file svgs_parse_only.txt --prefix svgparse_
|
||||
|
||||
If the --keep_common_prefix argument is specified, URL components after the common prefix
|
||||
will be preserved in the destination directory hierarchy. For example, if the input file contains
|
||||
URLs https://example.com/images/a.png and https://example.com/images/subdir/b.png, the downloaded
|
||||
files will go to output_dir/a.png and output_dir/subdir/b.png.
|
||||
@ -8,38 +8,59 @@
|
||||
|
||||
import optparse
|
||||
import os
|
||||
import sys
|
||||
import urllib
|
||||
|
||||
|
||||
PARENT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
|
||||
def downloadSVGs(svgs_file, output_dir, prefix):
|
||||
with open(svgs_file, 'r') as f:
|
||||
for url in f.xreadlines():
|
||||
svg_url = url.strip()
|
||||
dest_file = os.path.join(output_dir, prefix + os.path.basename(svg_url))
|
||||
print 'Downloading %s' % svg_url
|
||||
urllib.urlretrieve(svg_url, dest_file)
|
||||
def download_files(input_file, output_dir, prefix, keep_common_prefix):
|
||||
with open(input_file, 'r') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
if keep_common_prefix:
|
||||
common_prefix = os.path.commonprefix(lines)
|
||||
|
||||
for url in lines:
|
||||
file_url = url.strip()
|
||||
|
||||
if keep_common_prefix:
|
||||
rel_file = file_url.replace(common_prefix, '')
|
||||
dest_dir = os.path.join(output_dir, os.path.dirname(rel_file))
|
||||
else:
|
||||
dest_dir = output_dir
|
||||
|
||||
dest_file = os.path.join(dest_dir, prefix + os.path.basename(file_url))
|
||||
if not os.path.exists(dest_dir):
|
||||
os.makedirs(dest_dir)
|
||||
|
||||
print 'Downloading %s to %s' % (file_url, dest_file)
|
||||
urllib.urlretrieve(file_url, dest_file)
|
||||
|
||||
|
||||
if '__main__' == __name__:
|
||||
option_parser = optparse.OptionParser()
|
||||
option_parser.add_option(
|
||||
'-s', '--svgs_file',
|
||||
help='Path to the text file containing SVGs. Each line should contain a '
|
||||
'-i', '--input_file',
|
||||
help='Path to the text file containing URLs. Each line should contain a '
|
||||
'single URL.',
|
||||
default=os.path.join(PARENT_DIR, 'svgs.txt'))
|
||||
option_parser.add_option(
|
||||
'-o', '--output_dir',
|
||||
help='The output dir where downloaded SVGs will be stored in.')
|
||||
help='The output dir where downloaded SVGs and images will be stored in.')
|
||||
option_parser.add_option(
|
||||
'-p', '--prefix',
|
||||
help='The prefix which downloaded SVG file will begin with.',
|
||||
help='The prefix which downloaded files will begin with.',
|
||||
default='')
|
||||
option_parser.add_option(
|
||||
'-k', '--keep_common_prefix',
|
||||
help='Preserve everything in the URL after the common prefix as directory '
|
||||
'hierarchy.',
|
||||
action='store_true', default=False)
|
||||
options, unused_args = option_parser.parse_args()
|
||||
|
||||
if not options.output_dir:
|
||||
raise Exception('Must specify --output_dir')
|
||||
sys.exit(downloadSVGs(options.svgs_file, options.output_dir, options.prefix))
|
||||
|
||||
download_files(options.input_file, options.output_dir,
|
||||
options.prefix, options.keep_common_prefix)
|
||||
|
||||
73
tools/svg/svg_images.txt
Normal file
73
tools/svg/svg_images.txt
Normal file
@ -0,0 +1,73 @@
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/20x20.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/blue_10x10.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/bluesquidj.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/bumpMap2.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/DisplaceChecker.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/filters-conv-01-f.includeimage.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/galpha.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/gam030.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/gam045.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/gam056.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/gam100.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/gam200.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/image1.jpg
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi0g01.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi0g02.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi0g04.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi0g08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi0g16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi2c08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi2c16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi3p01.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi3p02.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi3p04.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi3p08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi4a08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi4a16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi6a08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basi6a16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn0g01.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn0g02.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn0g04.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn0g08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn0g16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn2c08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn2c16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn3p01.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn3p02.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn3p04.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn3p08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn4a08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn4a16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn6a08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/basn6a16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/bgai4a08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/bgai4a16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/bgan6a08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/bgan6a16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/bgbn4a08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/bggn4a16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/bgwn6a08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/bgyn6a16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tbbn1g04.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tbbn2c16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tbbn3p08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tbgn2c16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tbgn3p08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tbrn2c08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tbwn1g16.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tbwn3p08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tbyn3p08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/PngSuite/tp1n3p08.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/purplesquidj.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/rects.svg
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/rgbalpha.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/rotate20.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/sign.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/smiley.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/sphere.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/stefan_252_tRNS_opti.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/struct-image-01.jpg
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/struct-image-01.png
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/struct-image-02.jpg
|
||||
https://www.w3.org/Graphics/SVG/Test/20110816/images/struct-symbol-01.png
|
||||
Loading…
Reference in New Issue
Block a user