HTTP GM results viewer: server now returns category summaries along with testData

(SkipBuildbotRuns) R=borenet@google.com Review URL: https://codereview.chromium.org/25045003 git-svn-id: http://skia.googlecode.com/svn/trunk@11520 2bbb7eff-a529-9590-31e7-b0007b416f81
2013-09-30 15:06:25 +00:00 · 2013-09-30 15:06:25 +00:00 · afaad3dd70
commit afaad3dd70
parent 5e49738c99
4 changed files with 129 additions and 61 deletions
--- a/gm/rebaseline_server/results.py
+++ b/gm/rebaseline_server/results.py
@ -31,6 +31,9 @@ if GM_DIRECTORY not in sys.path:
 import gm_json

 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+CATEGORIES_TO_SUMMARIZE = [
+    'builder', 'test', 'config', 'resultType',
+]

 class Results(object):
  """ Loads actual and expected results from all builders, supplying combined
@ -44,24 +47,49 @@ class Results(object):
    """
    self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
    self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
-    self._all_results = self._Combine()
+    self._all_results = Results._Combine(
+        actual_builder_dicts=self._actual_builder_dicts,
+        expected_builder_dicts=self._expected_builder_dicts)

  def GetAll(self):
-    """Return results of all tests, as a list in this form:
+    """Return results of all tests, as a dictionary in this form:

-       [
+       {
+         "categories": # dictionary of categories listed in
+                       # CATEGORIES_TO_SUMMARIZE, with the number of times
+                       # each value appears within its category
         {
-           "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
-           "test": "bigmatrix",
-           "config": "8888",
-           "resultType": "failed",
-           "expectedHashType": "bitmap-64bitMD5",
-           "expectedHashDigest": "10894408024079689926",
-           "actualHashType": "bitmap-64bitMD5",
-           "actualHashDigest": "2409857384569",
-         },
-         ...
-       ]
+           "resultType": # category name
+           {
+             "failed": 29, # category value and total number found of that value
+             "failure-ignored": 948,
+             "no-comparison": 4502,
+             "succeeded": 38609,
+           },
+           "builder":
+           {
+             "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug": 1286,
+             "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release": 1134,
+             ...
+           },
+           ... # other categories from CATEGORIES_TO_SUMMARIZE
+         }, # end of "categories" dictionary
+
+         "testData": # list of test results, with a dictionary for each
+         [
+           {
+             "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
+             "test": "bigmatrix",
+             "config": "8888",
+             "resultType": "failed",
+             "expectedHashType": "bitmap-64bitMD5",
+             "expectedHashDigest": "10894408024079689926",
+             "actualHashType": "bitmap-64bitMD5",
+             "actualHashDigest": "2409857384569",
+           },
+           ...
+         ], # end of "testData" list
+       }
    """
    return self._all_results

@ -84,15 +112,21 @@ class Results(object):
        meta_dict[builder] = gm_json.LoadFromFile(fullpath)
    return meta_dict

-  def _Combine(self):
-    """Returns a list of all tests, across all builders, based on the
-    contents of self._actual_builder_dicts and self._expected_builder_dicts .
-    Returns the list in the same form needed for GetAllResults().
+  @staticmethod
+  def _Combine(actual_builder_dicts, expected_builder_dicts):
+    """Gathers the results of all tests, across all builders (based on the
+    contents of actual_builder_dicts and expected_builder_dicts)
+    and returns it in a list in the same form needed for self.GetAll().
+
+    This is a static method, because once we start refreshing results
+    asynchronously, we need to make sure we are not corrupting the object's
+    member variables.
    """
-    all_tests = []
-    for builder in sorted(self._actual_builder_dicts.keys()):
+    test_data = []
+    category_dict = {}
+    for builder in sorted(actual_builder_dicts.keys()):
      actual_results_for_this_builder = (
-          self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
+          actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
      for result_type in sorted(actual_results_for_this_builder.keys()):
        results_of_this_type = actual_results_for_this_builder[result_type]
        if not results_of_this_type:
@ -102,7 +136,7 @@ class Results(object):
          try:
            # TODO(epoger): assumes a single allowed digest per test
            expected_image = (
-                self._expected_builder_dicts
+                expected_builder_dicts
                    [builder][gm_json.JSONKEY_EXPECTEDRESULTS]
                    [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
                    [0])
@ -159,13 +193,8 @@ class Results(object):
          else:
            updated_result_type = result_type

-          # TODO(epoger): For now, don't include succeeded results.
-          # There are so many of them that they make the client too slow.
-          if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
-            continue
-
          (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
-          all_tests.append({
+          results_for_this_test = {
              "builder": builder,
              "test": test,
              "config": config,
@ -174,5 +203,35 @@ class Results(object):
              "actualHashDigest": str(actual_image[1]),
              "expectedHashType": expected_image[0],
              "expectedHashDigest": str(expected_image[1]),
-          })
-    return all_tests
+          }
+          Results._AddToCategoryDict(category_dict, results_for_this_test)
+
+          # TODO(epoger): For now, don't include succeeded results in the raw
+          # data. There are so many of them that they make the client too slow.
+          if updated_result_type != gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
+            test_data.append(results_for_this_test)
+    return {"categories": category_dict, "testData": test_data}
+
+  @staticmethod
+  def _AddToCategoryDict(category_dict, test_results):
+    """Add test_results to the category dictionary we are building
+    (see documentation of self.GetAll() for the format of this dictionary).
+
+    params:
+      category_dict: category dict-of-dicts to add to; modify this in-place
+      test_results: test data with which to update category_list, in a dict:
+         {
+           "category_name": "category_value",
+           "category_name": "category_value",
+           ...
+         }
+    """
+    for category in CATEGORIES_TO_SUMMARIZE:
+      category_value = test_results.get(category)
+      if not category_value:
+        continue  # test_results did not include this category, keep going
+      if not category_dict.get(category):
+        category_dict[category] = {}
+      if not category_dict[category].get(category_value):
+        category_dict[category][category_value] = 0
+      category_dict[category][category_value] += 1
--- a/gm/rebaseline_server/server.py
+++ b/gm/rebaseline_server/server.py
@ -214,10 +214,10 @@ def main():
                            'to access this server.  WARNING: doing so will '
                            'allow users on other hosts to modify your '
                            'GM expectations!'))
-  parser.add_argument('--port',
-                    help=('Which TCP port to listen on for HTTP requests; '
-                          'defaults to %(default)s'),
-                    default=DEFAULT_PORT)
+  parser.add_argument('--port', type=int,
+                      help=('Which TCP port to listen on for HTTP requests; '
+                            'defaults to %(default)s'),
+                      default=DEFAULT_PORT)
  args = parser.parse_args()
  global _SERVER
  _SERVER = Server(expectations_dir=args.expectations_dir,
--- a/gm/rebaseline_server/static/loader.js
+++ b/gm/rebaseline_server/static/loader.js
@ -1,7 +1,7 @@
 /*
 * Loader:
- * Reads GM result reports written out by results_loader.py, and imports
- * their data into $scope.results .
+ * Reads GM result reports written out by results.py, and imports
+ * them into $scope.categories and $scope.testData .
 */
 var Loader = angular.module(
    'Loader',
@ -12,8 +12,10 @@ Loader.controller(
  function($scope, $http) {
    $http.get("/results/all").then(
      function(response) {
-        $scope.results = response.data;
+        $scope.categories = response.data.categories;
+        $scope.testData = response.data.testData;
        $scope.sortColumn = 'test';
+	$scope.showResultsOfType = 'failed';
      }
    );
  }
--- a/gm/rebaseline_server/static/view.html
+++ b/gm/rebaseline_server/static/view.html
@ -15,6 +15,9 @@
  --export mode
  -->

+  <!-- TODO(epoger): Add some indication of how old the
+  expected/actual data is -->
+
 Settings:
  <ul>
    <!-- TODO(epoger): Now that we get multiple result types in a single
@ -24,45 +27,49 @@
    <li>show results of type
      <select ng-model="showResultsOfType"
              ng-init="showResultsOfType='failed'">
-        <option>failed</option>
-        <option>failure-ignored</option>
-        <!--
-        <option>no-comparison</option>
-
-        TODO(epoger): For now, I have disabled viewing the
-        no-comparison results because there are so many of them, and
-        the browser takes forever to download all the images.  Maybe
-        we should use some sort of lazy-loading technique
-        (e.g. http://www.appelsiini.net/projects/lazyload ), so that
-        the images are only loaded as they become viewable...
-        -->
-        <!--
-        <option>succeeded</option>
-
+        <option ng-repeat="(resultType, count) in categories['resultType']"
+                value="{{resultType}}">
+          {{resultType}} ({{count}})
+        </option>
+      </select>
+      <!--
        TODO(epoger): See results.py: for now, I have disabled
        returning succeeded tests as part of the JSON, because it
        makes the returned JSON too big (and slows down the client).
-        -->
-      </select>
+
+        Also, we should use some sort of lazy-loading technique
+        (e.g. http://www.appelsiini.net/projects/lazyload ), so that
+        the images are only loaded as they become viewable...
+        that will help with long lists like resultType='no-comparison'.
+      -->
+      <br>
+      TODO(epoger): 'no-comparison' will probably take forever;
+      see HTML source for details
+      <br>
+      TODO(epoger): 'succeeded' will not show any results;
+      see HTML source for details
    </li>
    <li>image size
-      <input type="text" ng-model="imageSize" ng-init="imageSize=100"
+      <input type="text" ng-model="imageSizePending"
+             ng-init="imageSizePending=100; imageSize=100"
             maxlength="4"/>
+      <button ng:click="imageSize=imageSizePending">apply</button>
    </li>
  </ul>

    <p>
+      Click on the column header radio buttons to re-sort by that column...<br>
      <!-- TODO(epoger): Show some sort of "loading" message, instead of
           an empty table, while the data is loading.  Otherwise, if there are
           a lot of failures and it takes a long time to load them, the user
           might think there are NO failures and leave the page! -->
      <table border="1">
        <tr>
-          <th ng:click="sortColumn='builder'">Builder</th>
-          <th ng:click="sortColumn='test'">Test</th>
-          <th ng:click="sortColumn='config'">Config</th>
-          <th ng:click="sortColumn='expectedHashDigest'">Expected Image</th>
-          <th ng:click="sortColumn='actualHashDigest'">Actual Image</th>
+          <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="builder">Builder</input></th>
+          <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="test">Test</input></th>
+          <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="config">Config</input></th>
+          <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="expectedHashDigest">Expected Image</input></th>
+          <th><input ng-model="sortColumn" name="sortColumnRadio" type="radio" value="actualHashDigest">Actual Image</input></th>
          <!-- TODO(epoger): Add more columns, such as...
               pixel diff
               notes/bugs
@ -71,7 +78,7 @@
        </tr>
        <!-- TODO(epoger): improve the column sorting, as per
             http://jsfiddle.net/vojtajina/js64b/14/ -->
-        <tr ng-repeat="result in results | filter: { resultType: showResultsOfType } | orderBy: sortColumn">
+        <tr ng-repeat="result in testData | filter: { resultType: showResultsOfType } | orderBy: sortColumn">
          <td>{{result.builder}}</td>
          <td>{{result.test}}</td>
          <td>{{result.config}}</td>