improve QOL of local fm_driver runs

The default Task Driver logging gets in the way on the console, so I've
sent it to /dev/null for local runs.  We control the horizontal and the
vertical.

Instead, print out each isolated failure and how to reproduce it:

    out/fm -i resources -b cpu -s ducky_yuv_blend #failed:
    	Resource "resources/images/ducky.jpg" not found.
    	../tools/fm/fm.cpp:573: fatal error: "Image(s) failed to load."

    	Signal 5:
    	_sigtramp (+0x1d)
    	sk_abort_no_print() (+0x5)
    	std::__1::__function::__func<...
    	_GLOBAL__sub_I_fm.cpp (+0x0)
    	main (+0x12d5)

    out/fm -i resources -b cpu --skvm true -s ducky_yuv_blend #failed:
    	Resource "resources/images/ducky.jpg" not found.
    	../tools/fm/fm.cpp:573: fatal error: "Image(s) failed to load."

    	Signal 5:
    	_sigtramp (+0x1d)
    	sk_abort_no_print() (+0x5)
    	std::__1::__function::__func<...
    	_GLOBAL__sub_I_fm.cpp (+0x0)
    	main (+0x12d5)

    2 runs of out/fm failed after retries.
    exit status 1

Bot runs still look ok,
https://task-driver.skia.org/td/TEaSLB6jtmRq5XDBUIwS?ifNotFound=https%3A%2F%2Fchromium-swarm.appspot.com%2Ftask%3Fid%3D4bea6af25506c010

Change-Id: I56adacdaeed5545785a3096a4e495eb524db442f
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/287017
Reviewed-by: Mike Klein <mtklein@google.com>
Commit-Queue: Mike Klein <mtklein@google.com>

infra/bots/task_drivers/fm_driver/fm_driver.go

@@ -6,8 +6,11 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"flag"
+	"fmt"
 	"math/rand"
+	"os"
 	"runtime"
 	"strings"
 	"sync"
@@ -36,6 +39,18 @@ func main() {
 	ctx := td.StartRun(projectId, taskId, taskName, output, local)
 	defer td.EndRun(ctx)
 
+	actualStderr := os.Stderr
+	if *local {
+		// Task Driver echoes every exec.Run() stdout and stderr to the console,
+		// which makes it hard to find failures (especially stdout).  Send them to /dev/null.
+		f, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0)
+		if err != nil {
+			td.Fatal(ctx, err)
+		}
+		os.Stdout = f
+		os.Stderr = f
+	}
+
 	if flag.NArg() < 1 {
 		td.Fatalf(ctx, "Please pass an fm binary.")
 	}
@@ -43,13 +58,16 @@ func main() {
 
 	// Run fm --flag to find the names of all linked GMs or tests.
 	query := func(flag string) []string {
-		stdout, err := exec.RunCwd(ctx, ".", fm, flag, "-i", *resources)
-		if err != nil {
+		stdout := &bytes.Buffer{}
+		cmd := &exec.Command{Name: fm, Stdout: stdout}
+		cmd.Args = append(cmd.Args, "-i", *resources)
+		cmd.Args = append(cmd.Args, flag)
+		if err := exec.Run(ctx, cmd); err != nil {
 			td.Fatal(ctx, err)
 		}
 
 		lines := []string{}
-		scanner := bufio.NewScanner(strings.NewReader(stdout))
+		scanner := bufio.NewScanner(stdout)
 		for scanner.Scan() {
 			lines = append(lines, scanner.Text())
 		}
@@ -124,18 +142,33 @@ func main() {
 
 	worker := func(queue chan work) {
 		for w := range queue {
-			cmd := []string{}
-			cmd = append(cmd, fm)
-			cmd = append(cmd, "-i", *resources)
-			cmd = append(cmd, w.Flags...)
-			cmd = append(cmd, "-s")
-			cmd = append(cmd, w.Sources...)
-
-			if _, err := exec.RunCwd(ctx, ".", cmd...); err != nil {
+			stdout := &bytes.Buffer{}
+			stderr := &bytes.Buffer{}
+			cmd := &exec.Command{Name: fm, Stdout: stdout, Stderr: stderr}
+			cmd.Args = append(cmd.Args, "-i", *resources)
+			cmd.Args = append(cmd.Args, w.Flags...)
+			cmd.Args = append(cmd.Args, "-s")
+			cmd.Args = append(cmd.Args, w.Sources...)
+			if err := exec.Run(ctx, cmd); err != nil {
 				if len(w.Sources) == 1 {
 					// If a source ran alone and failed, that's just a failure.
 					atomic.AddInt32(&failures, 1)
 					td.FailStep(ctx, err)
+					if *local {
+						lines := []string{}
+						scanner := bufio.NewScanner(stderr)
+						for scanner.Scan() {
+							lines = append(lines, scanner.Text())
+						}
+						if err := scanner.Err(); err != nil {
+							td.Fatal(ctx, err)
+						}
+
+						fmt.Fprintf(actualStderr, "%v %v #failed:\n\t%v\n",
+							cmd.Name,
+							strings.Join(cmd.Args, " "),
+							strings.Join(lines, "\n\t"))
+					}
 				} else {
 					// If a batch of sources ran and failed, split them up and try again.
 					for _, source := range w.Sources {
@@ -176,6 +209,12 @@ func main() {
 	wg.Wait()
 
 	if failures > 0 {
-		td.Fatalf(ctx, "%v runs of %v failed after retries.", failures, fm)
+		if *local {
+			// td.Fatalf() would work fine, but barfs up a panic that we don't need to see.
+			fmt.Fprintf(actualStderr, "%v runs of %v failed after retries.\n", failures, fm)
+			os.Exit(1)
+		} else {
+			td.Fatalf(ctx, "%v runs of %v failed after retries.", failures, fm)
+		}
 	}
 }