skia2/infra/cts/run_testlab.go
Eric Boren 999d1d7df8 [infra] Update infra dep
I'm hoping this fixes UpdateGoDeps.

Bug: skia:9505
Change-Id: I4493f61f0257176b0e90e50bd9456041dc398a51
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/266843
Reviewed-by: Ravi Mistry <rmistry@google.com>
Commit-Queue: Eric Boren <borenet@google.com>
2020-01-27 19:48:46 +00:00

474 lines
15 KiB
Go

/*
* Copyright 2018 Google Inc.
*
* Use of this source code is governed by a BSD-style license that can be
* found in the LICENSE file.
*/
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
"sort"
"strconv"
"strings"
"syscall"
"time"
"go.skia.org/infra/go/gcs"
"go.skia.org/infra/go/httputils"
"go.skia.org/infra/go/skerr"
"cloud.google.com/go/storage"
"google.golang.org/api/option"
gstorage "google.golang.org/api/storage/v1"
"go.skia.org/infra/go/auth"
"go.skia.org/infra/go/common"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/go/util"
)
const (
META_DATA_FILENAME = "meta.json"
)
// Command line flags.
var (
devicesFile = flag.String("devices", "", "JSON file that maps device ids to versions to run on. Same format as produced by the dump_devices flag.")
dryRun = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.")
dumpDevFile = flag.String("dump_devices", "", "Creates a JSON file with all physical devices that are not deprecated.")
minAPIVersion = flag.Int("min_api", 0, "Minimum API version required by device.")
maxAPIVersion = flag.Int("max_api", 99, "Maximum API version required by device.")
properties = flag.String("properties", "", "Custom meta data to be added to the uploaded APK. Comma separated list of key=value pairs, i.e. 'k1=v1,k2=v2,k3=v3.")
uploadGCSPath = flag.String("upload_path", "", "GCS path (bucket/path) to where the APK should be uploaded to. It's assume to a full path (not a directory).")
)
const (
RUN_TESTS_TEMPLATE = `gcloud beta firebase test android run
--type=game-loop
--app=%s
--results-bucket=%s
--results-dir=%s
--directories-to-pull=/sdcard/Android/data/org.skia.skqp
--timeout 30m
%s
`
MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait"
RESULT_BUCKET = "skia-firebase-test-lab"
RESULT_DIR_TMPL = "testruns/%s/%s"
RUN_ID_TMPL = "testrun-%d"
CMD_AVAILABLE_DEVICES = "gcloud firebase test android models list --format json"
)
func main() {
common.Init()
// Get the path to the APK. It can be empty if we are dumping the device list.
apkPath := flag.Arg(0)
if *dumpDevFile == "" && apkPath == "" {
sklog.Errorf("Missing APK. The APK file needs to be passed as the positional argument.")
os.Exit(1)
}
// Get the available devices.
fbDevices, deviceList, err := getAvailableDevices()
if err != nil {
sklog.Fatalf("Error retrieving available devices: %s", err)
}
// Dump the device list and exit.
if *dumpDevFile != "" {
if err := writeDeviceList(*dumpDevFile, deviceList); err != nil {
sklog.Fatalf("Unable to write devices: %s", err)
}
return
}
// If no devices are explicitly listed. Use all of them.
whiteList := deviceList
if *devicesFile != "" {
whiteList, err = readDeviceList(*devicesFile)
if err != nil {
sklog.Fatalf("Error reading device file: %s", err)
}
}
// Make sure we can authenticate locally and in the cloud.
ts, err := auth.NewDefaultTokenSource(true, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
if err != nil {
sklog.Fatal(err)
}
client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
// Filter the devices according the white list and other parameters.
devices, ignoredDevices := filterDevices(fbDevices, whiteList, *minAPIVersion, *maxAPIVersion)
sklog.Infof("---\nSelected devices:")
logDevices(devices)
if len(devices) == 0 {
sklog.Errorf("No devices selected. Not running tests.")
os.Exit(1)
}
if err := runTests(apkPath, devices, ignoredDevices, client, *dryRun); err != nil {
sklog.Fatalf("Error running tests on Firebase: %s", err)
}
if !*dryRun && (*uploadGCSPath != "") && (*properties != "") {
if err := uploadAPK(apkPath, *uploadGCSPath, *properties, client); err != nil {
sklog.Fatalf("Error uploading APK to '%s': %s", *uploadGCSPath, err)
}
}
}
// getAvailableDevices queries Firebase Testlab for all physical devices that
// are not deprecated. It returns two lists with the same information.
// The first contains all device information as returned by Firebase while
// the second contains the information necessary to use in a whitelist.
func getAvailableDevices() ([]*DeviceVersions, DeviceList, error) {
// Get the list of all devices in JSON format from Firebase testlab.
var buf bytes.Buffer
var errBuf bytes.Buffer
cmd := parseCommand(CMD_AVAILABLE_DEVICES)
cmd.Stdout = &buf
cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf)
if err := cmd.Run(); err != nil {
return nil, nil, skerr.Wrapf(err, "Error running: %s\nStdErr:%s", CMD_AVAILABLE_DEVICES, errBuf.String())
}
// Unmarshal the result.
foundDevices := []*DeviceVersions{}
bufBytes := buf.Bytes()
if err := json.Unmarshal(bufBytes, &foundDevices); err != nil {
return nil, nil, skerr.Wrapf(err, "Unmarshal of device information failed: \nJSON Input: %s\n", string(bufBytes))
}
// Filter the devices and copy them to device list.
devList := DeviceList{}
ret := make([]*DeviceVersions, 0, len(foundDevices))
for _, foundDev := range foundDevices {
// Only consider physical devices and devices that are not deprecated.
if (foundDev.Form == "PHYSICAL") && !util.In("deprecated", foundDev.Tags) {
ret = append(ret, foundDev)
devList = append(devList, &DevInfo{
ID: foundDev.ID,
Name: foundDev.Name,
RunVersions: foundDev.VersionIDs,
})
}
}
return foundDevices, devList, nil
}
// filterDevices filters the given devices by ensuring that they are in the white list
// and within the given api version range.
// It returns two lists: (accepted_devices, ignored_devices)
func filterDevices(foundDevices []*DeviceVersions, whiteList DeviceList, minAPIVersion, maxAPIVersion int) ([]*DeviceVersions, []*DeviceVersions) {
// iterate over the available devices and partition them.
allDevices := make([]*DeviceVersions, 0, len(foundDevices))
ret := make([]*DeviceVersions, 0, len(foundDevices))
ignored := make([]*DeviceVersions, 0, len(foundDevices))
for _, dev := range foundDevices {
// Only include devices that are on the whitelist and have versions defined.
if targetDev := whiteList.find(dev.ID); targetDev != nil && (len(targetDev.RunVersions) > 0) {
versionSet := util.NewStringSet(dev.VersionIDs)
reqVersions := util.NewStringSet(filterVersions(targetDev.RunVersions, minAPIVersion, maxAPIVersion))
whiteListVersions := versionSet.Intersect(reqVersions).Keys()
ignoredVersions := versionSet.Complement(reqVersions).Keys()
sort.Strings(whiteListVersions)
sort.Strings(ignoredVersions)
if len(whiteListVersions) > 0 {
ret = append(ret, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: whiteListVersions})
}
if len(ignoredVersions) > 0 {
ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: ignoredVersions})
}
} else {
ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs})
}
allDevices = append(allDevices, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs})
}
sklog.Infof("All devices:")
logDevices(allDevices)
return ret, ignored
}
// filterVersions returns the elements in versionIDs where minVersion <= element <= maxVersion.
func filterVersions(versionIDs []string, minVersion, maxVersion int) []string {
ret := make([]string, 0, len(versionIDs))
for _, versionID := range versionIDs {
id, err := strconv.Atoi(versionID)
if err != nil {
sklog.Fatalf("Error parsing version id '%s': %s", versionID, err)
}
if (id >= minVersion) && (id <= maxVersion) {
ret = append(ret, versionID)
}
}
return ret
}
// runTests runs the given apk on the given list of devices.
func runTests(apk_path string, devices, ignoredDevices []*DeviceVersions, client *http.Client, dryRun bool) error {
// Get the model-version we want to test. Assume on average each model has 5 supported versions.
modelSelectors := make([]string, 0, len(devices)*5)
for _, devRec := range devices {
for _, version := range devRec.RunVersions {
modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.FirebaseDevice.ID, version))
}
}
now := time.Now()
nowMs := now.UnixNano() / int64(time.Millisecond)
runID := fmt.Sprintf(RUN_ID_TMPL, nowMs)
resultsDir := fmt.Sprintf(RESULT_DIR_TMPL, now.Format("2006/01/02/15"), runID)
cmdStr := fmt.Sprintf(RUN_TESTS_TEMPLATE, apk_path, RESULT_BUCKET, resultsDir, strings.Join(modelSelectors, "\n"))
cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1))
// Run the command.
var errBuf bytes.Buffer
cmd := parseCommand(cmdStr)
cmd.Stdout = os.Stdout
cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf)
exitCode := 0
if dryRun {
fmt.Printf("[dry run]: Would have run this command: %s\n", cmdStr)
return nil
}
if err := cmd.Run(); err != nil {
// Get the exit code.
if exitError, ok := err.(*exec.ExitError); ok {
ws := exitError.Sys().(syscall.WaitStatus)
exitCode = ws.ExitStatus()
}
sklog.Errorf("Error running tests: %s", err)
sklog.Errorf("Exit code: %d", exitCode)
// Exit code 10 means triggering on Testlab succeeded, but but some of the
// runs on devices failed. We consider it a success for this script.
if exitCode != 10 {
return skerr.Wrapf(err, "Error running: %s\nStdErr:%s", cmdStr, errBuf.String())
}
}
// Store the result in a meta json file.
meta := &TestRunMeta{
ID: runID,
TS: nowMs,
Devices: devices,
IgnoredDevices: ignoredDevices,
ExitCode: exitCode,
}
targetPath := fmt.Sprintf("%s/%s/%s", RESULT_BUCKET, resultsDir, META_DATA_FILENAME)
if err := meta.writeToGCS(targetPath, client); err != nil {
return err
}
sklog.Infof("Meta data written to gs://%s", targetPath)
return nil
}
// uploadAPK uploads the APK at the given path to the bucket/path in gcsPath.
// The key-value pairs in propStr are set as custom meta data of the APK.
func uploadAPK(apkPath, gcsPath, propStr string, client *http.Client) error {
properties, err := splitProperties(propStr)
if err != nil {
return err
}
apkFile, err := os.Open(apkPath)
if err != nil {
return err
}
defer util.Close(apkFile)
if err := copyReaderToGCS(gcsPath, apkFile, client, "application/vnd.android.package-archive", properties, true, false); err != nil {
return err
}
sklog.Infof("APK uploaded to gs://%s", gcsPath)
return nil
}
// splitProperties receives a comma separated list of 'key=value' pairs and
// returnes them as a map.
func splitProperties(propStr string) (map[string]string, error) {
splitProps := strings.Split(propStr, ",")
properties := make(map[string]string, len(splitProps))
for _, oneProp := range splitProps {
kv := strings.Split(oneProp, "=")
if len(kv) != 2 {
return nil, skerr.Fmt("Invalid properties format. Unable to parse '%s'", propStr)
}
properties[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
}
return properties, nil
}
// logDevices logs the given list of devices.
func logDevices(devices []*DeviceVersions) {
sklog.Infof("Found %d devices.", len(devices))
for _, dev := range devices {
fbDev := dev.FirebaseDevice
sklog.Infof("%-15s %-30s %v / %v", fbDev.ID, fbDev.Name, fbDev.VersionIDs, dev.RunVersions)
}
}
// parseCommad parses a command line and wraps it in an exec.Command instance.
func parseCommand(cmdStr string) *exec.Cmd {
cmdArgs := strings.Split(strings.TrimSpace(cmdStr), " ")
for idx := range cmdArgs {
cmdArgs[idx] = strings.TrimSpace(cmdArgs[idx])
}
return exec.Command(cmdArgs[0], cmdArgs[1:]...)
}
// DeviceList is a simple list of devices, primarily used to define the
// whitelist of devices we want to run on.
type DeviceList []*DevInfo
type DevInfo struct {
ID string `json:"id"`
Name string `json:"name"`
RunVersions []string `json:"runVersions"`
}
func (d DeviceList) find(id string) *DevInfo {
for _, devInfo := range d {
if devInfo.ID == id {
return devInfo
}
}
return nil
}
func writeDeviceList(fileName string, devList DeviceList) error {
jsonBytes, err := json.MarshalIndent(devList, "", " ")
if err != nil {
return skerr.Wrapf(err, "Unable to encode JSON")
}
if err := ioutil.WriteFile(fileName, jsonBytes, 0644); err != nil {
skerr.Wrapf(err, "Unable to write file '%s'", fileName)
}
return nil
}
func readDeviceList(fileName string) (DeviceList, error) {
inFile, err := os.Open(fileName)
if err != nil {
return nil, skerr.Wrapf(err, "Unable to open file '%s'", fileName)
}
defer util.Close(inFile)
var devList DeviceList
if err := json.NewDecoder(inFile).Decode(&devList); err != nil {
return nil, skerr.Wrapf(err, "Unable to decode JSON from '%s'", fileName)
}
return devList, nil
}
// FirebaseDevice contains the information and JSON tags for device information
// returned by firebase.
type FirebaseDevice struct {
Brand string `json:"brand"`
Form string `json:"form"`
ID string `json:"id"`
Manufacturer string `json:"manufacturer"`
Name string `json:"name"`
VersionIDs []string `json:"supportedVersionIds"`
Tags []string `json:"tags"`
}
// DeviceVersions combines device information from Firebase Testlab with
// a selected list of versions. This is used to define a subset of versions
// used by a devices.
type DeviceVersions struct {
*FirebaseDevice
// RunVersions contains the version ids of interest contained in Device.
RunVersions []string
}
// TestRunMeta contains the meta data of a complete testrun on firebase.
type TestRunMeta struct {
ID string `json:"id"`
TS int64 `json:"timeStamp"`
Devices []*DeviceVersions `json:"devices"`
IgnoredDevices []*DeviceVersions `json:"ignoredDevices"`
ExitCode int `json:"exitCode"`
}
// writeToGCS writes the meta data as JSON to the given bucket and path in
// GCS. It assumes that the provided client has permissions to write to the
// specified location in GCS.
func (t *TestRunMeta) writeToGCS(gcsPath string, client *http.Client) error {
jsonBytes, err := json.Marshal(t)
if err != nil {
return err
}
return copyReaderToGCS(gcsPath, bytes.NewReader(jsonBytes), client, "", nil, false, true)
}
// TODO(stephana): Merge copyReaderToGCS into the go/gcs in
// the infra repository.
// copyReaderToGCS reads all available content from the given reader and writes
// it to the given path in GCS.
func copyReaderToGCS(gcsPath string, reader io.Reader, client *http.Client, contentType string, metaData map[string]string, public bool, gzip bool) error {
storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client))
if err != nil {
return err
}
bucket, path := gcs.SplitGSPath(gcsPath)
w := storageClient.Bucket(bucket).Object(path).NewWriter(context.Background())
// Set the content if requested.
if contentType != "" {
w.ObjectAttrs.ContentType = contentType
}
// Set the meta data if requested
if metaData != nil {
w.Metadata = metaData
}
// Make the object public if requested.
if public {
w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
}
// Write the everything the reader can provide to the GCS object. Either
// gzip'ed or plain.
if gzip {
w.ObjectAttrs.ContentEncoding = "gzip"
err = util.WithGzipWriter(w, func(w io.Writer) error {
_, err := io.Copy(w, reader)
return err
})
} else {
_, err = io.Copy(w, reader)
}
// Make sure we return an error when we close the remote object.
if err != nil {
_ = w.CloseWithError(err)
return err
}
return w.Close()
}