AuroraMiddleware/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2024-11-08 22:30:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make build-many-glibcs.py track component versions requested and used.

Browse Source 
This patch makes build-many-glibcs.py record the component versions
checked out, and whether those versions were explicitly requested or
defaults, in a file versions.json in the source directory.

The preferred version of a component is the first of: one explicitly
specified on the command line; one explicitly specified on the command
line in a previous run of build-many-glibcs.py; a default version for
that component.  Thus, once you've run build-many-glibcs.py checkout
once with the updated script (making sure to specify the right
versions of any components previously checked out with a non-default
version), in future you can just run it without version specifiers and
it will know when a default-version component has changed its default
version and so should be checked out again.

Because you might have local changes and not want a default-version
component checkout replaced, you need to pass the --replace-sources
option to allow the script to delete and replace a component source
directory automatically; otherwise, it will give an error if a version
has changed.  The script does not try to change branches of git or SVN
checkouts without checking out from scratch; if the version number
requested has changed and --replace-sources is used, the relevant
source directory will be removed completely and a new one checked out
from scratch.

Apart from allowing automatic updates of components with default
versions, this also facilitates bots reporting on the versions used in
a given build.  versions.json contains not just information on the
version number and whether that was requested explicitly, but also git
or SVN revision information intended to be used in email reports from
bots.

	* scripts/build-many-glibcs.py: Import json module.
	(Context.__init__): Take replace_sources argument.  Load
	versions.json.
	(Context.load_versions_json): New function.
	(Context.store_json): Likewise.
	(Context.store_versions_json): Likewise.
	(Context.set_component_version): Likewise.
	(Context.checkout): Update versions.json.  Check for and handle
	changes of version.  Prefer previously explicitly specified
	version to default version.
	(Context.checkout_vcs): Return a revision identifier.
	(Context.git_checkout): Likewise.
	(Context.gcc_checkout): Likewise.
	(get_parser): Add --replace-sources option.
	(main): Pass replace_sources argument to Context call.
This commit is contained in:
Joseph Myers 2016-11-24 22:25:58 +00:00
parent c579f48edb
commit 02c78f02a9
2 changed files with 91 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
ChangeLog
View File

@ -1,3 +1,21 @@
2016-11-24 Joseph Myers <joseph@codesourcery.com>
* scripts/build-many-glibcs.py: Import json module.
(Context.__init__): Take replace_sources argument. Load
versions.json.
(Context.load_versions_json): New function.
(Context.store_json): Likewise.
(Context.store_versions_json): Likewise.
(Context.set_component_version): Likewise.
(Context.checkout): Update versions.json. Check for and handle
changes of version. Prefer previously explicitly specified
version to default version.
(Context.checkout_vcs): Return a revision identifier.
(Context.git_checkout): Likewise.
(Context.gcc_checkout): Likewise.
(get_parser): Add --replace-sources option.
(main): Pass replace_sources argument to Context call.
2016-11-24 Adhemerval Zanella <adhemerval.zanella@linaro.org>
* sysdeps/nptl/fork.c (__libc_fork): Remove pid cache setting.

81
scripts/build-many-glibcs.py
View File

@ -32,6 +32,7 @@ configurations for which compilers or glibc are to be built.
"""
import argparse
import json
import os
import re
import shutil
@ -44,12 +45,14 @@ import urllib.request
class Context(object):
"""The global state associated with builds in a given directory."""
def __init__(self, topdir, parallelism, keep, action):
def __init__(self, topdir, parallelism, keep, replace_sources, action):
"""Initialize the context."""
self.topdir = topdir
self.parallelism = parallelism
self.keep = keep
self.replace_sources = replace_sources
self.srcdir = os.path.join(topdir, 'src')
self.versions_json = os.path.join(self.srcdir, 'versions.json')
self.installdir = os.path.join(topdir, 'install')
self.host_libraries_installdir = os.path.join(self.installdir,
'host-libraries')
@ -65,6 +68,7 @@ class Context(object):
self.glibc_configs = {}
self.makefile_pieces = ['.PHONY: all\n']
self.add_all_configs()
self.load_versions_json()
def get_build_triplet(self):
"""Determine the build triplet with config.guess."""
@ -559,6 +563,32 @@ class Context(object):
for c in configs:
self.glibc_configs[c].build()
def load_versions_json(self):
"""Load information about source directory versions."""
if not os.access(self.versions_json, os.F_OK):
self.versions = {}
return
with open(self.versions_json, 'r') as f:
self.versions = json.load(f)
def store_json(self, data, filename):
"""Store information in a JSON file."""
filename_tmp = filename + '.tmp'
with open(filename_tmp, 'w') as f:
json.dump(data, f, indent=2, sort_keys=True)
os.rename(filename_tmp, filename)
def store_versions_json(self):
"""Store information about source directory versions."""
self.store_json(self.versions, self.versions_json)
def set_component_version(self, component, version, explicit, revision):
"""Set the version information for a component."""
self.versions[component] = {'version': version,
'explicit': explicit,
'revision': revision}
self.store_versions_json()
def checkout(self, versions):
"""Check out the desired component versions."""
default_versions = {'binutils': 'vcs-2.27',
@ -569,6 +599,7 @@ class Context(object):
'mpc': '1.0.3',
'mpfr': '3.1.5'}
use_versions = {}
explicit_versions = {}
for v in versions:
found_v = False
for k in default_versions.keys():
@ -579,6 +610,7 @@ class Context(object):
print('error: multiple versions for %s' % k)
exit(1)
use_versions[k] = vx
explicit_versions[k] = True
found_v = True
break
if not found_v:
@ -586,19 +618,36 @@ class Context(object):
exit(1)
for k in default_versions.keys():
if k not in use_versions:
if k in self.versions and self.versions[k]['explicit']:
use_versions[k] = self.versions[k]['version']
explicit_versions[k] = True
else:
use_versions[k] = default_versions[k]
explicit_versions[k] = False
os.makedirs(self.srcdir, exist_ok=True)
for k in sorted(default_versions.keys()):
update = os.access(self.component_srcdir(k), os.F_OK)
v = use_versions[k]
if (update and
k in self.versions and
v != self.versions[k]['version']):
if not self.replace_sources:
print('error: version of %s has changed from %s to %s, '
'use --replace-sources to check out again' %
(k, self.versions[k]['version'], v))
exit(1)
shutil.rmtree(self.component_srcdir(k))
update = False
if v.startswith('vcs-'):
self.checkout_vcs(k, v[4:], update)
revision = self.checkout_vcs(k, v[4:], update)
else:
self.checkout_tar(k, v, update)
revision = v
self.set_component_version(k, v, explicit_versions[k], revision)
def checkout_vcs(self, component, version, update):
"""Check out the given version of the given component from version
control."""
control. Return a revision identifier."""
if component == 'binutils':
git_url = 'git://sourceware.org/git/binutils-gdb.git'
if version == 'mainline':
@ -606,7 +655,7 @@ class Context(object):
else:
trans = str.maketrans({'.': '_'})
git_branch = 'binutils-%s-branch' % version.translate(trans)
self.git_checkout(component, git_url, git_branch, update)
return self.git_checkout(component, git_url, git_branch, update)
elif component == 'gcc':
if version == 'mainline':
branch = 'trunk'
@ -614,21 +663,22 @@ class Context(object):
trans = str.maketrans({'.': '_'})
branch = 'branches/gcc-%s-branch' % version.translate(trans)
svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
self.gcc_checkout(svn_url, update)
return self.gcc_checkout(svn_url, update)
elif component == 'glibc':
git_url = 'git://sourceware.org/git/glibc.git'
if version == 'mainline':
git_branch = 'master'
else:
git_branch = 'release/%s/master' % version
self.git_checkout(component, git_url, git_branch, update)
r = self.git_checkout(component, git_url, git_branch, update)
self.fix_glibc_timestamps()
return r
else:
print('error: component %s coming from VCS' % component)
exit(1)
def git_checkout(self, component, git_url, git_branch, update):
"""Check out a component from git."""
"""Check out a component from git. Return a commit identifier."""
if update:
subprocess.run(['git', 'remote', 'prune', 'origin'],
cwd=self.component_srcdir(component), check=True)
@ -637,6 +687,11 @@ class Context(object):
else:
subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
self.component_srcdir(component)], check=True)
r = subprocess.run(['git', 'rev-parse', 'HEAD'],
cwd=self.component_srcdir(component),
stdout=subprocess.PIPE,
check=True, universal_newlines=True).stdout
return r.rstrip()
def fix_glibc_timestamps(self):
"""Fix timestamps in a glibc checkout."""
@ -652,12 +707,16 @@ class Context(object):
subprocess.run(['touch', to_touch], check=True)
def gcc_checkout(self, svn_url, update):
"""Check out GCC from SVN."""
"""Check out GCC from SVN. Return the revision number."""
if not update:
subprocess.run(['svn', 'co', '-q', svn_url,
self.component_srcdir('gcc')], check=True)
subprocess.run(['contrib/gcc_update', '--silent'],
cwd=self.component_srcdir('gcc'), check=True)
r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
stdout=subprocess.PIPE,
check=True, universal_newlines=True).stdout
return r.rstrip()
def checkout_tar(self, component, version, update):
"""Check out the given version of the given component from a
@ -1125,6 +1184,9 @@ def get_parser():
help='Whether to keep all build directories, '
'none or only those from failed builds',
default='none', choices=('none', 'all', 'failed'))
parser.add_argument('--replace-sources', action='store_true',
help='Remove and replace source directories '
'with the wrong version of a component')
parser.add_argument('topdir',
help='Toplevel working directory')
parser.add_argument('action',
@ -1142,7 +1204,8 @@ def main(argv):
parser = get_parser()
opts = parser.parse_args(argv)
topdir = os.path.abspath(opts.topdir)
ctx = Context(topdir, opts.parallelism, opts.keep, opts.action)
ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
opts.action)
ctx.run_builds(opts.action, opts.configs)