v8/tools/locs.py

#!/usr/bin/env python3

# Copyright 2018 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

""" locs.py - Count lines of code before and after preprocessor expansion
  Consult --help for more information.
"""

# for py2/py3 compatibility
from __future__ import print_function

import argparse
import json
import multiprocessing
import os
import re
import subprocess
import sys
import tempfile
import time
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

# for py2/py3 compatibility
try:
  FileNotFoundError
except NameError:
  FileNotFoundError = IOError

ARGPARSE = argparse.ArgumentParser(
    description=("A script that computes LoC for a build dir"),
    epilog="""Examples:
 Count with default settings for build in out/Default:
   locs.py --build-dir out/Default
 Count only a custom group of files settings for build in out/Default:
   tools/locs.py --build-dir out/Default
                 --group src-compiler '\.\./\.\./src/compiler'
                 --only src-compiler
 Report the 10 files with the worst expansion:
   tools/locs.py --build-dir out/Default --worst 10
 Report the 10 files with the worst expansion in src/compiler:
   tools/locs.py --build-dir out/Default --worst 10
                 --group src-compiler '\.\./\.\./src/compiler'
                 --only src-compiler
 Report the 10 largest files after preprocessing:
   tools/locs.py --build-dir out/Default --largest 10
 Report the 10 smallest input files:
   tools/locs.py --build-dir out/Default --smallest 10""",
    formatter_class=argparse.RawTextHelpFormatter
)

ARGPARSE.add_argument(
    '--json',
    action='store_true',
    default=False,
    help="output json instead of short summary")
ARGPARSE.add_argument(
    '--build-dir',
    type=str,
    help="Use specified build dir and generate necessary files",
    required=True)
ARGPARSE.add_argument(
    '--echocmd',
    action='store_true',
    default=False,
    help="output command used to compute LoC")
ARGPARSE.add_argument(
    '--only',
    action='append',
    default=[],
    help="Restrict counting to report group (can be passed multiple times)")
ARGPARSE.add_argument(
    '--not',
    action='append',
    default=[],
    help="Exclude specific group (can be passed multiple times)")
ARGPARSE.add_argument(
    '--list-groups',
    action='store_true',
    default=False,
    help="List groups and associated regular expressions")
ARGPARSE.add_argument(
    '--group',
    nargs=2,
    action='append',
    default=[],
    help="Add a report group (can be passed multiple times)")
ARGPARSE.add_argument(
    '--largest',
    type=int,
    nargs='?',
    default=0,
    const=3,
    help="Output the n largest files after preprocessing")
ARGPARSE.add_argument(
    '--worst',
    type=int,
    nargs='?',
    default=0,
    const=3,
    help="Output the n files with worst expansion by preprocessing")
ARGPARSE.add_argument(
    '--smallest',
    type=int,
    nargs='?',
    default=0,
    const=3,
    help="Output the n smallest input files")
ARGPARSE.add_argument(
    '--files',
    type=int,
    nargs='?',
    default=0,
    const=3,
    help="Output results for each file separately")
ARGPARSE.add_argument(
    '--jobs',
    type=int,
    default=multiprocessing.cpu_count(),
    help="Process specified number of files concurrently")

ARGS = vars(ARGPARSE.parse_args())


def MaxWidth(strings):
  max_width = 0
  for s in strings:
    max_width = max(max_width, len(s))
  return max_width


def GenerateCompileCommandsAndBuild(build_dir, out):
  if not os.path.isdir(build_dir):
    print("Error: Specified build dir {} is not a directory.".format(
        build_dir), file=sys.stderr)
    exit(1)

  autoninja = "autoninja -C {}".format(build_dir)
  if subprocess.call(autoninja, shell=True, stdout=out) != 0:
    print("Error: Building {} failed.".format(build_dir), file=sys.stderr)
    exit(1)

  compile_commands_file = "{}/compile_commands.json".format(build_dir)
  print("Generating compile commands in {}.".format(
      compile_commands_file), file=out)
  ninja = "ninja -C {} -t compdb cxx cc > {}".format(
      build_dir, compile_commands_file)
  if subprocess.call(ninja, shell=True, stdout=out) != 0:
    print("Error: Cound not generate {} for {}.".format(
        compile_commands_file, build_dir), file=sys.stderr)
    exit(1)

  ninja_deps_file = "{}/ninja-deps.txt".format(build_dir)
  print("Generating ninja dependencies in {}.".format(
      ninja_deps_file), file=out)
  ninja = "ninja -C {} -t deps > {}".format(
      build_dir, ninja_deps_file)
  if subprocess.call(ninja, shell=True, stdout=out) != 0:
    print("Error: Cound not generate {} for {}.".format(
        ninja_deps_file, build_dir), file=sys.stderr)
    exit(1)

  return compile_commands_file, ninja_deps_file


def fmt_bytes(num_bytes):
  if num_bytes > 1024*1024*1024:
    return int(num_bytes / (1024*1024)), "MB"
  elif num_bytes > 1024*1024:
    return int(num_bytes / (1024)), "kB"
  return int(num_bytes), " B"


class CompilationData:
  def __init__(self, loc, in_bytes, expanded, expanded_bytes):
    self.loc = loc
    self.in_bytes = in_bytes
    self.expanded = expanded
    self.expanded_bytes = expanded_bytes

  def ratio(self):
    return self.expanded / (self.loc+1)

  def to_string(self):
    exp_bytes, exp_unit = fmt_bytes(self.expanded_bytes)
    in_bytes, in_unit = fmt_bytes(self.in_bytes)
    return "{:>9,} LoC ({:>7,} {}) to {:>12,} LoC ({:>7,} {}) ({:>5.0f}x)".format(
        self.loc, in_bytes, in_unit, self.expanded, exp_bytes, exp_unit, self.ratio())


class File(CompilationData):
  def __init__(self, file, target, loc, in_bytes, expanded, expanded_bytes):
    super().__init__(loc, in_bytes, expanded, expanded_bytes)
    self.file = file
    self.target = target

  def to_string(self):
    return "{} {} {}".format(super().to_string(), self.file, self.target)


class Group(CompilationData):
  def __init__(self, name, regexp_string):
    super().__init__(0, 0, 0, 0)
    self.name = name
    self.count = 0
    self.regexp = re.compile(regexp_string)

  def account(self, unit):
    if (self.regexp.match(unit.file)):
      self.loc += unit.loc
      self.in_bytes += unit.in_bytes
      self.expanded += unit.expanded
      self.expanded_bytes += unit.expanded_bytes
      self.count += 1

  def to_string(self, name_width):
    return "{:<{}} ({:>5} files): {}".format(
        self.name, name_width, self.count, super().to_string())


def SetupReportGroups():
  default_report_groups = {"total": '.*',
                           "src": '\\.\\./\\.\\./src',
                           "test": '\\.\\./\\.\\./test',
                           "third_party": '\\.\\./\\.\\./third_party',
                           "gen": 'gen'}

  report_groups = default_report_groups.copy()
  report_groups.update(dict(ARGS['group']))

  if ARGS['only']:
    for only_arg in ARGS['only']:
      if not only_arg in report_groups.keys():
        print("Error: specified report group '{}' is not defined.".format(
            ARGS['only']))
        exit(1)
      else:
        report_groups = {
            k: v for (k, v) in report_groups.items() if k in ARGS['only']}

  if ARGS['not']:
    report_groups = {
        k: v for (k, v) in report_groups.items() if k not in ARGS['not']}

  if ARGS['list_groups']:
    print_cat_max_width = MaxWidth(list(report_groups.keys()) + ["Category"])
    print("  {:<{}}  {}".format("Category",
                                print_cat_max_width, "Regular expression"))
    for cat, regexp_string in report_groups.items():
      print("  {:<{}}: {}".format(
          cat, print_cat_max_width, regexp_string))

  report_groups = {k: Group(k, v) for (k, v) in report_groups.items()}

  return report_groups


class Results:
  def __init__(self):
    self.groups = SetupReportGroups()
    self.units = {}
    self.source_dependencies = {}
    self.header_dependents = {}

  def track(self, filename):
    is_tracked = False
    for group in self.groups.values():
      if group.regexp.match(filename):
        is_tracked = True
    return is_tracked

  def recordFile(self, filename, targetname, loc, in_bytes, expanded, expanded_bytes):
    unit = File(filename, targetname, loc, in_bytes, expanded, expanded_bytes)
    self.units[filename] = unit
    for group in self.groups.values():
      group.account(unit)

  def maxGroupWidth(self):
    return MaxWidth([v.name for v in self.groups.values()])

  def printGroupResults(self, file):
    for key in sorted(self.groups.keys()):
      print(self.groups[key].to_string(self.maxGroupWidth()), file=file)

  def printSorted(self, key, count, reverse, out):
    for unit in sorted(list(self.units.values()), key=key, reverse=reverse)[:count]:
      print(unit.to_string(), file=out)

  def addHeaderDeps(self, source_dependencies, header_dependents):
    self.source_dependencies = source_dependencies
    self.header_dependents = header_dependents


class LocsEncoder(json.JSONEncoder):
  def default(self, o):
    if isinstance(o, File):
      return {"file": o.file, "target": o.target, "loc": o.loc, "in_bytes": o.in_bytes,
              "expanded": o.expanded, "expanded_bytes": o.expanded_bytes}
    if isinstance(o, Group):
      return {"name": o.name, "loc": o.loc, "in_bytes": o.in_bytes,
              "expanded": o.expanded, "expanded_bytes": o.expanded_bytes}
    if isinstance(o, Results):
      return {"groups": o.groups, "units": o.units,
              "source_dependencies": o.source_dependencies,
              "header_dependents": o.header_dependents}
    return json.JSONEncoder.default(self, o)


class StatusLine:
  def __init__(self):
    self.max_width = 0

  def print(self, statusline, end="\r", file=sys.stdout):
    self.max_width = max(self.max_width, len(statusline))
    print("{0:<{1}}".format(statusline, self.max_width),
          end=end, file=file, flush=True)


class CommandSplitter:
  def __init__(self):
    self.cmd_pattern = re.compile(
        "([^\\s]*\\s+)?(?P<clangcmd>[^\\s]*clang.*)"
        " -c (?P<infile>.*) -o (?P<outfile>.*)")

  def process(self, compilation_unit):
    cmd = self.cmd_pattern.match(compilation_unit['command'])
    outfilename = cmd.group('outfile')
    infilename = cmd.group('infile')
    infile = Path(compilation_unit['directory']).joinpath(infilename)
    return (cmd.group('clangcmd'), infilename, infile, outfilename)


def parse_ninja_deps(ninja_deps):
  source_dependencies = {}
  header_dependents = defaultdict(int)
  current_target = None
  for line in ninja_deps:
    line = line.rstrip()
    # Ignore empty lines
    if not line:
      current_target = None
      continue
    if line[0] == ' ':
      # New dependency
      if len(line) < 5 or line[0:4] != '    ' or line[5] == ' ':
        sys.exit('Lines must have no indentation or exactly four ' +
                 'spaces.')
      dep = line[4:]
      if not re.search(r"\.(h|hpp)$", dep):
        continue
      header_dependents[dep] += 1
      continue
    # New target
    colon_pos = line.find(':')
    if colon_pos < 0:
      sys.exit('Unindented line must have a colon')
    if current_target is not None:
      sys.exit('Missing empty line before new target')
    current_target = line[0:colon_pos]
    match = re.search(r"#deps (\d+)", line)
    deps_number = match.group(1)
    source_dependencies[current_target] = int(deps_number)

  return (source_dependencies, header_dependents)


def Main():
  out = sys.stdout
  if ARGS['json']:
    out = sys.stderr

  compile_commands_file, ninja_deps_file = GenerateCompileCommandsAndBuild(
      ARGS['build_dir'], out)

  result = Results()
  status = StatusLine()

  try:
    with open(compile_commands_file) as file:
      compile_commands = json.load(file)
    with open(ninja_deps_file) as file:
      source_dependencies, header_dependents = parse_ninja_deps(file)
      result.addHeaderDeps(source_dependencies, header_dependents)
  except FileNotFoundError:
    print("Error: Cannot read '{}'. Consult --help to get started.".format(
        ninja_deps_file))
    exit(1)

  cmd_splitter = CommandSplitter()

  def count_lines_of_unit(ikey):
    i, key = ikey
    if not result.track(key['file']):
      return
    message = "[{}/{}] Counting LoCs of {}".format(
        i, len(compile_commands), key['file'])
    status.print(message, file=out)
    clangcmd, infilename, infile, outfilename = cmd_splitter.process(key)
    if not infile.is_file():
      return

    clangcmd = clangcmd + " -E -P " + \
        str(infile) + " -o /dev/stdout | sed '/^\\s*$/d' | wc -lc"
    loccmd = ("cat {}  | sed '\\;^\\s*//;d' | sed '\\;^/\\*;d'"
              " | sed '/^\\*/d' | sed '/^\\s*$/d' | wc -lc")
    loccmd = loccmd.format(infile)
    runcmd = " {} ; {}".format(clangcmd, loccmd)
    if ARGS['echocmd']:
      print(runcmd)
    process = subprocess.Popen(
        runcmd, shell=True, cwd=key['directory'], stdout=subprocess.PIPE)
    p = {'process': process, 'infile': infilename, 'outfile': outfilename}
    output, _ = p['process'].communicate()
    expanded, expanded_bytes, loc, in_bytes = list(map(int, output.split()))
    result.recordFile(p['infile'], p['outfile'], loc,
                      in_bytes, expanded, expanded_bytes)

  with tempfile.TemporaryDirectory(dir='/tmp/', prefix="locs.") as temp:
    start = time.time()

    with ThreadPoolExecutor(max_workers=ARGS['jobs']) as executor:
      list(executor.map(count_lines_of_unit, enumerate(compile_commands)))

    end = time.time()
    if ARGS['json']:
      print(json.dumps(result, ensure_ascii=False, cls=LocsEncoder))
    status.print("Processed {:,} files in {:,.2f} sec.".format(
        len(compile_commands), end-start), end="\n", file=out)
    result.printGroupResults(file=out)

    if ARGS['largest']:
      print("Largest {} files after expansion:".format(ARGS['largest']))
      result.printSorted(
          lambda v: v.expanded, ARGS['largest'], reverse=True, out=out)

    if ARGS['worst']:
      print("Worst expansion ({} files):".format(ARGS['worst']))
      result.printSorted(
          lambda v: v.ratio(), ARGS['worst'], reverse=True, out=out)

    if ARGS['smallest']:
      print("Smallest {} input files:".format(ARGS['smallest']))
      result.printSorted(
          lambda v: v.loc, ARGS['smallest'], reverse=False, out=out)

    if ARGS['files']:
      print("List of input files:")
      result.printSorted(
          lambda v: v.file, ARGS['files'], reverse=False, out=out)

  return 0


if __name__ == '__main__':
  sys.exit(Main())