v8/tools/locs.py

#!/usr/bin/env python3
# Copyright 2018 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

""" locs.py - Count lines of code before and after preprocessor expansion
  Consult --help for more information.
"""

import json
import sys
import re
import os
import subprocess
import tempfile
import time
from pathlib import Path
import argparse

ARGPARSE = argparse.ArgumentParser(
    description="A script that computes LoC for a build dir or from a compile_commands.json file",
    epilog="""Examples:
 Count with default settings for build in out/Default:
   locs.py --build-dir out/Default
 Count with default settings according to given compile_commands file:
   locs.py --compile-commands compile_commands.json
 Count only a custom group of files settings for build in out/Default:
   tools/locs.py --build-dir out/Default --group src-compiler '\.\./\.\./src/compiler' --only src-compiler
 Report the 10 files with the worst expansion:
   tools/locs.py --build-dir out/Default --worst 10
 Report the 10 files with the worst expansion in src/compiler:
   tools/locs.py --build-dir out/Default --worst 10 --group src-compiler '\.\./\.\./src/compiler' --only src-compiler
 Report the 10 largest files after preprocessing:
   tools/locs.py --build-dir out/Default --largest 10
 Report the 10 smallest input files:
   tools/locs.py --build-dir out/Default --smallest 10""",
    formatter_class=argparse.RawTextHelpFormatter
)

ARGPARSE.add_argument(
    '--json',
    action='store_true',
    default=False,
    help="output json instead of short summary")
ARGPARSE.add_argument(
    '--build-dir',
    type=str,
    default="",
    help="Use specified build dir and generate necessary files")
ARGPARSE.add_argument(
    '--echocmd',
    action='store_true',
    default=False,
    help="output command used to compute LoC")
ARGPARSE.add_argument(
    '--compile-commands',
    type=str,
    default='compile_commands.json',
    help="Use specified compile_commands.json file")
ARGPARSE.add_argument(
    '--only',
    action='append',
    default=[],
    help="Restrict counting to specific report group (can be passed multiple times)")
ARGPARSE.add_argument(
    '--not',
    action='append',
    default=[],
    help="Exclude specific group (can be passed multiple times)")
ARGPARSE.add_argument(
    '--list-groups',
    action='store_true',
    default=False,
    help="List groups and associated regular expressions")
ARGPARSE.add_argument(
    '--group',
    nargs=2,
    action='append',
    default=[],
    help="Add a report group (can be passed multiple times)")
ARGPARSE.add_argument(
    '--largest',
    type=int,
    nargs='?',
    default=0,
    const=3,
    help="Output the n largest files after preprocessing")
ARGPARSE.add_argument(
    '--worst',
    type=int,
    nargs='?',
    default=0,
    const=3,
    help="Output the n files with worst expansion by preprocessing")
ARGPARSE.add_argument(
    '--smallest',
    type=int,
    nargs='?',
    default=0,
    const=3,
    help="Output the n smallest input files")
ARGPARSE.add_argument(
    '--files',
    type=int,
    nargs='?',
    default=0,
    const=3,
    help="Output results for each file separately")

ARGS = vars(ARGPARSE.parse_args())


def MaxWidth(strings):
  max_width = 0
  for s in strings:
    max_width = max(max_width, len(s))
  return max_width


def GenerateCompileCommandsAndBuild(build_dir, compile_commands_file):
  if not os.path.isdir(build_dir):
    print("Error: Specified build dir {} is not a directory.".format(build_dir))
    exit(1)
  compile_commands_file = "{}/compile_commands.json".format(build_dir)
  print("Generating compile commands in {}.".format(compile_commands_file))
  subprocess.call("ninja -C {} -t compdb cxx cc > {}".format(build_dir,
                                                             compile_commands_file), shell=True)
  subprocess.call(
      "autoninja -C {}".format(build_dir, compile_commands_file), shell=True)
  return compile_commands_file


class CompilationData:
  def __init__(self, loc, expanded):
    self.loc = loc
    self.expanded = expanded

  def ratio(self):
    return self.expanded / (self.loc+1)

  def to_string(self):
    return "{:>9,} to {:>12,} ({:>5.0f}x)".format(
        self.loc, self.expanded, self.ratio())


class File(CompilationData):
  def __init__(self, file, loc, expanded):
    super().__init__(loc, expanded)
    self.file = file

  def to_string(self):
    return "{} {}".format(super().to_string(), self.file)


class Group(CompilationData):
  def __init__(self, name, regexp_string):
    super().__init__(0, 0)
    self.name = name
    self.count = 0
    self.regexp = re.compile(regexp_string)

  def account(self, unit):
    if (self.regexp.match(unit.file)):
      self.loc += unit.loc
      self.expanded += unit.expanded
      self.count += 1

  def to_string(self, name_width):
    return "{:<{}} ({:>5} files): {}".format(self.name, name_width, self.count, super().to_string())


class CompilationUnitEncoder(json.JSONEncoder):
  def default(self, obj):
    if isinstance(obj, File):
      return {"file": obj.file, "loc": obj.loc, "expanded": obj.expanded}
    return json.JSONEncoder.default(self, obj)


def SetupReportGroups():
  default_report_groups = {"total": '.*',
                           "src": '\\.\\./\\.\\./src',
                           "test": '\\.\\./\\.\\./test',
                           "third_party": '\\.\\./\\.\\./third_party',
                           "gen": 'gen'}

  report_groups = {**default_report_groups, **dict(ARGS['group'])}

  if ARGS['only']:
    for only_arg in ARGS['only']:
      if not only_arg in report_groups.keys():
        print("Error: specified report group '{}' is not defined.".format(
            ARGS['only']))
        exit(1)
      else:
        report_groups = {
            k: v for (k, v) in report_groups.items() if k in ARGS['only']}

  if ARGS['not']:
    report_groups = {
        k: v for (k, v) in report_groups.items() if k not in ARGS['not']}

  if ARGS['list_groups']:
    print_cat_max_width = MaxWidth(list(report_groups.keys()) + ["Category"])
    print("  {:<{}}  {}".format("Category",
                                print_cat_max_width, "Regular expression"))
    for cat, regexp_string in report_groups.items():
      print("  {:<{}}: {}".format(
          cat, print_cat_max_width, regexp_string))

  report_groups = {k: Group(k, v) for (k, v) in report_groups.items()}

  return report_groups


class Results:
  def __init__(self):
    self.groups = SetupReportGroups()
    self.units = []

  def isTracked(self, filename):
    is_tracked = False
    for group in self.groups.values():
      if group.regexp.match(filename):
        is_tracked = True
    return is_tracked

  def recordFile(self, filename, loc, expanded):
    unit = File(filename, loc, expanded)
    self.units.append(unit)
    for group in self.groups.values():
      group.account(unit)

  def maxGroupWidth(self):
    return MaxWidth([v.name for v in self.groups.values()])

  def printGroupResults(self):
    for key in sorted(self.groups.keys()):
      print(self.groups[key].to_string(self.maxGroupWidth()))


class StatusLine:
  def __init__(self):
    self.max_width = 0

  def print(self, statusline, end="\r"):
    self.max_width = max(self.max_width, len(statusline))
    print("{0:<{1}}".format(statusline, self.max_width), end=end)


class CommandSplitter:
  def __init__(self):
    self.cmd_pattern = re.compile(
        "([^\\s]*\\s+)?(?P<clangcmd>[^\\s]*clang.*) -c (?P<infile>.*) -o (?P<outfile>.*)")

  def process(self, compilation_unit, temp_file_name):
    cmd = self.cmd_pattern.match(compilation_unit['command'])
    outfilename = cmd.group('outfile') + ".cc"
    infilename = cmd.group('infile')
    infile = Path(compilation_unit['directory']).joinpath(infilename)
    outfile = Path(str(temp_file_name)).joinpath(outfilename)
    return [cmd.group('clangcmd'), infilename, infile, outfile]

def Main():
  compile_commands_file = ARGS['compile_commands']

  if ARGS['build_dir']:
    compile_commands_file = GenerateCompileCommandsAndBuild(
        ARGS['build_dir'], compile_commands_file)

  try:
    with open(compile_commands_file) as file:
      data = json.load(file)
  except FileNotFoundError:
    print("Error: Cannot read '{}'. Consult --help to get started.")
    exit(1)

  result = Results()
  status = StatusLine()

  with tempfile.TemporaryDirectory(dir='/tmp/', prefix="locs.") as temp:
    processes = []
    start = time.time()
    cmd_splitter = CommandSplitter()

    for i, key in enumerate(data):
      if not result.isTracked(key['file']):
        continue
      if not ARGS['json']:
        status.print("[{}/{}] Counting LoCs of {}".format(i, len(data), key['file']))
      clangcmd, infilename, infile, outfile = cmd_splitter.process(key, temp)
      outfile.parent.mkdir(parents=True, exist_ok=True)
      if infile.is_file():
        clangcmd = clangcmd + " -E -P " + str(infile) + " -o /dev/stdout | sed '/^\\s*$/d' | wc -l"
        loccmd = "cat {}  | sed '\\;^\\s*//;d' | sed '\\;^/\\*;d' | sed '/^\\*/d' | sed '/^\\s*$/d' | wc -l".format(
            infile)
        runcmd = " {} ; {}".format(clangcmd, loccmd)
        if ARGS['echocmd']:
          print(runcmd)
        p = subprocess.Popen(
            runcmd, shell=True, cwd=key['directory'], stdout=subprocess.PIPE)
        processes.append({'process': p, 'infile': infilename})

    for i, p in enumerate(processes):
      if not ARGS['json']:
        status.print("[{}/{}] Summing up {}".format(i, len(processes), p['infile']))
      output, err = p['process'].communicate()
      expanded, loc = list(map(int, output.split()))
      result.recordFile(p['infile'], loc, expanded)

    end = time.time()
    if ARGS['json']:
      print(json.dumps(result, ensure_ascii=False, cls=CompilationUnitEncoder))
    else:
      status.print("Processed {:,} files in {:,.2f} sec.".format(len(processes), end-start), end="\n")
      result.printGroupResults()

      if ARGS['largest']:
        print("Largest {} files after expansion:".format(ARGS['largest']))
        for u in sorted(result.units, key=lambda v: v.expanded, reverse=True)[:ARGS['largest']]:
          print(u.to_string())

      if ARGS['worst']:
        print("Worst expansion ({} files):".format(ARGS['worst']))
        for u in sorted(result.units, key=lambda v: v.ratio(), reverse=True)[:ARGS['worst']]:
          print(u.to_string())

      if ARGS['smallest']:
        print("Smallest {} input files:".format(ARGS['smallest']))
        for u in sorted(result.units, key=lambda v: v.loc, reverse=False)[:ARGS['smallest']]:
          print(u.to_string())

      if ARGS['files']:
        print("List of input files:")
        for u in sorted(result.units, key=lambda v: v.file, reverse=False):
          print(u.to_string())

  return 0

if __name__ == '__main__':
  sys.exit(Main())