skia2/infra/cts/run_testlab.go

/*
 * Copyright 2018 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strings"
	"syscall"
	"time"

	gstorage "google.golang.org/api/storage/v1"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/golden/go/tsuite"
)

// TODO(stephana): Convert the hard coded whitelist to a command line flag that
// loads a file with the whitelisted devices and versions. Make sure to include
// human readable names for the devices.

var (
	// WHITELIST_DEV_IDS contains a mapping from the device id to the list of
	// Android API versions that we should run agains. Usually this will be the
	// latest version. To see available devices and version run with
	// --dryrun flag or run '$ gcloud firebase test android models list'

	WHITELIST_DEV_IDS = map[string][]string{
		"A0001": {"22"},
		// "E5803":       {"22"},    deprecated
		// "F5121":       {"23"},    deprecated
		"G8142":       {"25"},
		"HWMHA":       {"24"},
		"SH-04H":      {"23"},
		"athene":      {"23"},
		"athene_f":    {"23"},
		"hammerhead":  {"23"},
		"harpia":      {"23"},
		"hero2lte":    {"23"},
		"herolte":     {"24v"},
		"j1acevelte":  {"22"},
		"j5lte":       {"23"},
		"j7xelte":     {"23"},
		"lucye":       {"24"},
		"mako":        {"22"},
		"osprey_umts": {"22"},
		"p1":          {"22"},
		"sailfish":    {"26"},
		"shamu":       {"23"},
		"trelte":      {"22"},
		"zeroflte":    {"22"},
		"zerolte":     {"22"},
	}
)

const (
	META_DATA_FILENAME = "meta.json"
)

// Command line flags.
var (
	serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.")
	dryRun             = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.")
)

const (
	RUN_TESTS_TEMPLATE = `gcloud beta firebase test android run
	--type=game-loop
	--app=%s
	--results-bucket=%s
	--results-dir=%s
	--directories-to-pull=/sdcard/Android/data/org.skia.skqpapp
	%s
`
	MODEL_VERSION_TMPL   = "--device model=%s,version=%s,orientation=portrait"
	RESULT_BUCKET        = "skia-firebase-test-lab"
	RESULT_DIR_TMPL      = "testruns/%s/%s"
	RUN_ID_TMPL          = "testrun-%d"
	CMD_AVAILABE_DEVICES = "gcloud firebase test android models list --format json"
)

func main() {
	common.Init()

	// Get the apk.
	args := flag.Args()
	apk_path := args[0]

	// Make sure we can get the service account client.
	client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
	if err != nil {
		sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err)
	}

	// Get list of all available devices.
	devices, ignoredDevices, err := getAvailableDevices(WHITELIST_DEV_IDS)
	if err != nil {
		sklog.Fatalf("Unable to retrieve available devices: %s", err)
	}
	sklog.Infof("---")
	sklog.Infof("Selected devices:")
	logDevices(devices)

	if err := runTests(apk_path, devices, ignoredDevices, client, *dryRun); err != nil {
		sklog.Fatalf("Error triggering tests on Firebase: %s", err)
	}
}

// getAvailableDevices is given a whitelist. It queries Firebase Testlab for all
// available devices and then returns a list of devices to be tested and the list
// of ignored devices.
func getAvailableDevices(whiteList map[string][]string) ([]*tsuite.DeviceVersions, []*tsuite.DeviceVersions, error) {
	// Get the list of all devices in JSON format from Firebase testlab.
	var buf bytes.Buffer
	cmd := parseCommand(CMD_AVAILABE_DEVICES)
	cmd.Stdout = &buf
	cmd.Stderr = os.Stdout
	if err := cmd.Run(); err != nil {
		return nil, nil, err
	}

	// Unmarshal the result.
	foundDevices := []*tsuite.FirebaseDevice{}
	bufBytes := buf.Bytes()
	if err := json.Unmarshal(bufBytes, &foundDevices); err != nil {
		return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes))
	}

	// iterate over the available devices and partition them.
	allDevices := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
	ret := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
	ignored := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
	for _, dev := range foundDevices {
		// Filter out all the virtual devices.
		if dev.Form == "PHYSICAL" {
			// Only include devices that are on the whitelist and have versions defined.
			if foundVersions, ok := whiteList[dev.ID]; ok && (len(foundVersions) > 0) {
				versionSet := util.NewStringSet(dev.VersionIDs)
				reqVersions := util.NewStringSet(foundVersions)
				whiteListVersions := versionSet.Intersect(reqVersions).Keys()
				ignoredVersions := versionSet.Complement(reqVersions).Keys()
				sort.Strings(whiteListVersions)
				sort.Strings(ignoredVersions)
				ret = append(ret, &tsuite.DeviceVersions{Device: dev, Versions: whiteListVersions})
				ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: ignoredVersions})
			} else {
				ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs})
			}
			allDevices = append(allDevices, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs})
		}
	}

	sklog.Infof("All devices:")
	logDevices(allDevices)

	return ret, ignored, nil
}

// runTests runs the given apk on the given list of devices.
func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, client *http.Client, dryRun bool) error {
	// Get the model-version we want to test. Assume on average each model has 5 supported versions.
	modelSelectors := make([]string, 0, len(devices)*5)
	for _, devRec := range devices {
		for _, version := range devRec.Versions {
			modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.Device.ID, version))
		}
	}

	now := time.Now()
	nowMs := now.UnixNano() / int64(time.Millisecond)
	runID := fmt.Sprintf(RUN_ID_TMPL, nowMs)
	resultsDir := fmt.Sprintf(RESULT_DIR_TMPL, now.Format("2006/01/02/15"), runID)
	cmdStr := fmt.Sprintf(RUN_TESTS_TEMPLATE, apk_path, RESULT_BUCKET, resultsDir, strings.Join(modelSelectors, "\n"))
	cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1))

	// Run the command.
	cmd := parseCommand(cmdStr)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stdout
	exitCode := 0

	if dryRun {
		fmt.Printf("[dry run]: Would have run this command: %s\n", cmdStr)
		return nil
	}

	if err := cmd.Run(); err != nil {
		// Get the exit code.
		if exitError, ok := err.(*exec.ExitError); ok {
			ws := exitError.Sys().(syscall.WaitStatus)
			exitCode = ws.ExitStatus()
		}
		sklog.Errorf("Error running tests: %s", err)
		sklog.Errorf("Exit code: %d", exitCode)

		// Exit code 10 means triggering on Testlab succeeded, but but some of the
		// runs on devices failed. We consider it a success for this script.
		if exitCode != 10 {
			return err
		}
	}

	// Store the result in a meta json file.
	meta := &tsuite.TestRunMeta{
		ID:             runID,
		TS:             nowMs,
		Devices:        devices,
		IgnoredDevices: ignoredDevices,
		ExitCode:       exitCode,
	}

	meta.WriteToGCS(RESULT_BUCKET, resultsDir+"/"+META_DATA_FILENAME, client)
	return nil
}

// logDevices logs the given list of devices.
func logDevices(devices []*tsuite.DeviceVersions) {
	sklog.Infof("Found %d devices.", len(devices))
	for _, dev := range devices {
		sklog.Infof("%-15s %-30s %v / %v", dev.Device.ID, dev.Device.Name, dev.Device.VersionIDs, dev.Versions)
	}
}

// parseCommad parses a command line and wraps it in an exec.Command instance.
func parseCommand(cmdStr string) *exec.Cmd {
	cmdArgs := strings.Split(strings.TrimSpace(cmdStr), " ")
	for idx := range cmdArgs {
		cmdArgs[idx] = strings.TrimSpace(cmdArgs[idx])
	}
	return exec.Command(cmdArgs[0], cmdArgs[1:]...)
}