skia2/infra/bots/recipes/upload_dm_results.py

# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


# Recipe for uploading DM results.


DEPS = [
  'build/file',
  'recipe_engine/json',
  'recipe_engine/path',
  'recipe_engine/properties',
  'recipe_engine/shutil',
  'recipe_engine/step',
  'recipe_engine/time',
]


import calendar


DM_JSON = 'dm.json'
GS_BUCKET = 'gs://skia-infra-gm'
UPLOAD_ATTEMPTS = 5
VERBOSE_LOG = 'verbose.log'


def cp(api, name, src, dst, extra_args=None):
  cmd = ['gsutil', 'cp']
  if extra_args:
    cmd.extend(extra_args)
  cmd.extend([src, dst])

  name = 'upload %s' % name
  for i in xrange(UPLOAD_ATTEMPTS):
    step_name = name
    if i > 0:
      step_name += ' (attempt %d)' % (i+1)
    try:
      api.step(step_name, cmd=cmd)
      break
    except api.step.StepFailure:
      if i == UPLOAD_ATTEMPTS - 1:
        raise


def RunSteps(api):
  builder_name = api.properties['buildername']
  revision = api.properties['revision']

  results_dir = api.path['cwd'].join('dm')

  # Move dm.json and verbose.log to their own directory.
  json_file = results_dir.join(DM_JSON)
  log_file = results_dir.join(VERBOSE_LOG)
  tmp_dir = api.path['cwd'].join('tmp_upload')
  api.shutil.makedirs('tmp dir', tmp_dir, infra_step=True)
  api.shutil.copy('copy dm.json', json_file, tmp_dir)
  api.shutil.copy('copy verbose.log', log_file, tmp_dir)
  api.shutil.remove('rm old dm.json', json_file)
  api.shutil.remove('rm old verbose.log', log_file)

  # Upload the images.
  image_dest_path = '/'.join((GS_BUCKET, 'dm-images-v1'))
  files_to_upload = api.file.glob(
      'find images',
      results_dir.join('*'),
      test_data=['someimage.png'],
      infra_step=True)
  if len(files_to_upload) > 0:
    cp(api, 'images', results_dir.join('*'), image_dest_path)

  # Upload the JSON summary and verbose.log.
  now = api.time.utcnow()
  summary_dest_path = '/'.join([
      'dm-json-v1',
      str(now.year ).zfill(4),
      str(now.month).zfill(2),
      str(now.day  ).zfill(2),
      str(now.hour ).zfill(2),
      revision,
      builder_name,
      str(int(calendar.timegm(now.utctimetuple())))])

  # Trybot results are further siloed by issue/patchset.
  issue = str(api.properties.get('issue', ''))
  patchset = str(api.properties.get('patchset', ''))
  if (api.properties.get('patch_storage', '') == 'gerrit' and
      api.properties.get('nobuildbot', '') != 'True'):
    issue = str(api.properties['event.change.number'])
    patchset = str(api.properties['event.patchSet.ref']).split('/')[-1]
  if issue and patchset:
    summary_dest_path = '/'.join((
        'trybot', summary_dest_path, issue, patchset))

  summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path))

  cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path,
     ['-z', 'json,log'])


def GenTests(api):
  builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug'
  yield (
    api.test('normal_bot') +
    api.properties(buildername=builder,
                   revision='abc123',
                   path_config='kitchen')
  )

  yield (
    api.test('failed_once') +
    api.properties(buildername=builder,
                   revision='abc123',
                   path_config='kitchen') +
    api.step_data('upload images', retcode=1)
  )

  yield (
    api.test('failed_all') +
    api.properties(buildername=builder,
                   revision='abc123',
                   path_config='kitchen') +
    api.step_data('upload images', retcode=1) +
    api.step_data('upload images (attempt 2)', retcode=1) +
    api.step_data('upload images (attempt 3)', retcode=1) +
    api.step_data('upload images (attempt 4)', retcode=1) +
    api.step_data('upload images (attempt 5)', retcode=1)
  )

  builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot'
  yield (
    api.test('trybot') +
    api.properties(buildername=builder,
                   revision='abc123',
                   path_config='kitchen',
                   issue='12345',
                   patchset='1002')
  )

  gerrit_kwargs = {
    'patch_storage': 'gerrit',
    'repository': 'skia',
    'event.patchSet.ref': 'refs/changes/00/2100/2',
    'event.change.number': '2100',
  }
  yield (
      api.test('recipe_with_gerrit_patch') +
      api.properties(
          buildername=builder,
          revision='abc123',
          path_config='kitchen',
          **gerrit_kwargs)
  )