Make build-many-glibcs.py support running as a bot.

This patch makes build-many-glibcs.py support a "bot" action, for
repeatedly running a checkout and build cycle.

Two new configuration variables are used in bot-config.json.  "delay"
indicates the time to sleep after each bot-cycle round (regardless of
whether that round actually ran any builds); "run" is a boolean, which
is false if the bot should just exit (the point of this is that you
can edit bot-config.json to set this to false to cause a running bot
to exit cleanly between builds) and true if the bot should run.  The
bot does not exit if the bot-cycle process exits with error status
(that can occur when sourceware's load limiting means anonymous
version control access fails, for example), just sleeps until it's
time to try again.

The script is changed to flush stdout before running a subprocess in
bot-cycle, so that when output is redirected (as expected for a bot)
the status messages from bot-cycle appear in their proper position in
its redirected output relative to the output from the subprocesses
run, and to copy the logs directory before running builds in bot-cycle
so that the logs from at least one complete build are always available
for looking at how something failed, even while the next build is
running.

	* scripts/build-many-glibcs.py: Add bot to usage message.  Import
	time module.
	(Context.__init__): Initialize self.logsdir_old.
	(Context.run_builds): Handle bot action.
	(Context.bot_cycle): Copy logs directory before running builds.
	(Context.bot_run_self): Take argument for whether to check
	subprocess result.  Flush stdout before running subprocess.
	(Context.bot): New function.
	(get_parser): Allow bot action.

ChangeLog

@@ -1,3 +1,15 @@
+2016-12-01  Joseph Myers  <joseph@codesourcery.com>
+
+	* scripts/build-many-glibcs.py: Add bot to usage message.  Import
+	time module.
+	(Context.__init__): Initialize self.logsdir_old.
+	(Context.run_builds): Handle bot action.
+	(Context.bot_cycle): Copy logs directory before running builds.
+	(Context.bot_run_self): Take argument for whether to check
+	subprocess result.  Flush stdout before running subprocess.
+	(Context.bot): New function.
+	(get_parser): Allow bot action.
+
 2016-11-30  Joseph Myers  <joseph@codesourcery.com>
 
 	* scripts/build-many-glibcs.py: Add bot-cycle to usage message.

scripts/build-many-glibcs.py

@@ -23,13 +23,14 @@ This script takes as arguments a directory name (containing a src
 subdirectory with sources of the relevant toolchain components) and a
 description of what to do: 'checkout', to check out sources into that
 directory, 'bot-cycle', to run a series of checkout and build steps,
-'host-libraries', to build libraries required by the toolchain,
+'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
-'compilers', to build cross-compilers for various configurations, or
+libraries required by the toolchain, 'compilers', to build
-'glibcs', to build glibc for various configurations and run the
+cross-compilers for various configurations, or 'glibcs', to build
-compilation parts of the testsuite.  Subsequent arguments name the
+glibc for various configurations and run the compilation parts of the
-versions of components to check out (<component>-<version), for
+testsuite.  Subsequent arguments name the versions of components to
-'checkout', or, for actions other than 'checkout' and 'bot-cycle',
+check out (<component>-<version), for 'checkout', or, for actions
-name configurations for which compilers or glibc are to be built.
+other than 'checkout' and 'bot-cycle', name configurations for which
+compilers or glibc are to be built.
 
 """
 
@@ -45,6 +46,7 @@ import smtplib
 import stat
 import subprocess
 import sys
+import time
 import urllib.request
 
 
@@ -66,6 +68,7 @@ class Context(object):
                                                       'host-libraries')
         self.builddir = os.path.join(topdir, 'build')
         self.logsdir = os.path.join(topdir, 'logs')
+        self.logsdir_old = os.path.join(topdir, 'logs-old')
         self.makefile = os.path.join(self.builddir, 'Makefile')
         self.wrapper = os.path.join(self.builddir, 'wrapper')
         self.save_logs = os.path.join(self.builddir, 'save-logs')
@@ -404,6 +407,12 @@ class Context(object):
                 exit(1)
             self.bot_cycle()
             return
+        if action == 'bot':
+            if configs:
+                print('error: configurations specified for bot')
+                exit(1)
+            self.bot()
+            return
         if action == 'host-libraries' and configs:
             print('error: configurations specified for host-libraries')
             exit(1)
@@ -937,6 +946,9 @@ class Context(object):
                 self.clear_last_build_state(a)
             else:
                 print('No need to rebuild %s.' % a)
+        if os.access(self.logsdir, os.F_OK):
+            shutil.rmtree(self.logsdir_old, ignore_errors=True)
+            shutil.copytree(self.logsdir, self.logsdir_old)
         for a in actions:
             if must_build[a]:
                 build_time = datetime.datetime.utcnow()
@@ -1004,13 +1016,31 @@ class Context(object):
         with smtplib.SMTP(self.bot_config['email-server']) as s:
             s.send_message(msg)
 
-    def bot_run_self(self, opts, action):
+    def bot_run_self(self, opts, action, check=True):
         """Run a copy of this script with given options."""
         cmd = [sys.executable, sys.argv[0], '--keep=none',
                '-j%d' % self.parallelism]
         cmd.extend(opts)
         cmd.extend([self.topdir, action])
-        subprocess.run(cmd, check=True)
+        sys.stdout.flush()
+        subprocess.run(cmd, check=check)
+
+    def bot(self):
+        """Run repeated rounds of checkout and builds."""
+        while True:
+            self.load_bot_config_json()
+            if not self.bot_config['run']:
+                print('Bot exiting by request.')
+                exit(0)
+            self.bot_run_self([], 'bot-cycle', check=False)
+            self.load_bot_config_json()
+            if not self.bot_config['run']:
+                print('Bot exiting by request.')
+                exit(0)
+            time.sleep(self.bot_config['delay'])
+            if self.get_script_text() != self.script_text:
+                print('Script changed, bot re-execing.')
+                self.exec_self()
 
 
 class Config(object):
@@ -1464,8 +1494,8 @@ def get_parser():
                         help='Toplevel working directory')
     parser.add_argument('action',
                         help='What to do',
-                        choices=('checkout', 'bot-cycle', 'host-libraries',
+                        choices=('checkout', 'bot-cycle', 'bot',
-                                 'compilers', 'glibcs'))
+                                 'host-libraries', 'compilers', 'glibcs'))
     parser.add_argument('configs',
                         help='Versions to check out or configurations to build',
                         nargs='*')