[torque-ls] Add prototype language server implementation for Torque

Design Doc: https://goo.gl/9G9d9k

The initial prototype consists of a few parts:

The VS Code extension is now built using TypeScript. The build artifact
is checked-in along side the extension. The extension now starts up
the language server when it is activated. The path to the LS
executable is configurable via VS Code settings.

The language server is a separate executable. It adds a light-weight
object model on top of a Json Parser for reading/writing LSP requests
and responses. The current server is very much bare-bones featurewise:
    - Tell the client that the server can handle "goto definition"
    - Recompile when Torque files change
    - Goto definition support for Macros/Builtins, local variables
      and arguments.

R=mathias@chromium.org, mvstanton@chromium.org, tebbi@chromium.org

Bug: v8:8880
Change-Id: Ie9b433e64ee63e9aa757b6bf71e5d52beb15b079
Reviewed-on: https://chromium-review.googlesource.com/c/1494354
Reviewed-by: Michael Stanton <mvstanton@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Reviewed-by: Mathias Bynens <mathias@chromium.org>
Commit-Queue: Simon Zünd <szuend@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59960}

BUILD.gn

@@ -3159,10 +3159,16 @@ v8_source_set("torque_base") {
 
 v8_source_set("torque_ls_base") {
   sources = [
+    "src/torque/ls/globals.h",
     "src/torque/ls/json-parser.cc",
     "src/torque/ls/json-parser.h",
     "src/torque/ls/json.cc",
     "src/torque/ls/json.h",
+    "src/torque/ls/message-handler.cc",
+    "src/torque/ls/message-handler.h",
+    "src/torque/ls/message-macros.h",
+    "src/torque/ls/message-pipe.h",
+    "src/torque/ls/message.h",
   ]
 
   deps = [
@@ -3547,6 +3553,25 @@ if (current_toolchain == v8_snapshot_toolchain) {
   }
 }
 
+v8_executable("torque-language-server") {
+  visibility = [ ":*" ]  # Only targets in this file can depend on this.
+
+  sources = [
+    "src/torque/ls/torque-language-server.cc",
+  ]
+
+  deps = [
+    ":torque_base",
+    ":torque_ls_base",
+    "//build/win:default_exe_manifest",
+  ]
+
+  configs = [ ":internal_config" ]
+  if (is_win && is_asan) {
+    remove_configs = [ "//build/config/sanitizers:default_sanitizer_flags" ]
+  }
+}
+
 ###############################################################################
 # Public targets
 #

src/torque/ls/globals.h

@@ -0,0 +1,58 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_TORQUE_LS_GLOBALS_H_
+#define V8_TORQUE_LS_GLOBALS_H_
+
+#include <fstream>
+#include "src/torque/contextual.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+
+// When the language server is run by VS code, stdout can not be seen, as it is
+// used as the communication channel. For debugging purposes a simple
+// Log class is added, that allows writing diagnostics to a file configurable
+// via command line flag.
+class Logger : public ContextualClass<Logger> {
+ public:
+  Logger() : enabled_(false) {}
+  ~Logger() {
+    if (enabled_) logfile_.close();
+  }
+
+  static void Enable(std::string path) {
+    Get().enabled_ = true;
+    Get().logfile_.open(path);
+  }
+
+  template <class... Args>
+  static void Log(Args&&... args) {
+    if (Enabled()) {
+      USE((Stream() << std::forward<Args>(args))...);
+      Flush();
+    }
+  }
+
+ private:
+  static bool Enabled() { return Get().enabled_; }
+  static std::ofstream& Stream() {
+    CHECK(Get().enabled_);
+    return Get().logfile_;
+  }
+  static void Flush() { Get().logfile_.flush(); }
+
+ private:
+  bool enabled_;
+  std::ofstream logfile_;
+};
+
+DECLARE_CONTEXTUAL_VARIABLE(TorqueFileList, std::vector<std::string>);
+
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_TORQUE_LS_GLOBALS_H_

src/torque/ls/json.h

@@ -65,6 +65,12 @@ inline JsonValue From(JsonArray array) {
   return result;
 }
 
+inline JsonValue JsonNull() {
+  JsonValue result;
+  result.tag = JsonValue::IS_NULL;
+  return result;
+}
+
 std::string SerializeToString(const JsonValue& value);
 
 }  // namespace ls

src/torque/ls/message-handler.cc

@@ -0,0 +1,224 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/torque/ls/message-handler.h"
+
+#include "src/torque/ls/globals.h"
+#include "src/torque/ls/json-parser.h"
+#include "src/torque/ls/message-pipe.h"
+#include "src/torque/ls/message.h"
+#include "src/torque/server-data.h"
+#include "src/torque/source-positions.h"
+#include "src/torque/torque-compiler.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+
+DEFINE_CONTEXTUAL_VARIABLE(Logger)
+DEFINE_CONTEXTUAL_VARIABLE(TorqueFileList)
+
+namespace ls {
+
+static const char kContentLength[] = "Content-Length: ";
+static const size_t kContentLengthSize = sizeof(kContentLength) - 1;
+
+static const char kFileUriPrefix[] = "file://";
+static const int kFileUriPrefixLength = sizeof(kFileUriPrefix) - 1;
+
+JsonValue ReadMessage() {
+  std::string line;
+  std::getline(std::cin, line);
+
+  if (line.rfind(kContentLength) != 0) {
+    // Invalid message, we just crash.
+    Logger::Log("[fatal] Did not find Content-Length ...\n");
+    v8::base::OS::Abort();
+  }
+
+  const int content_length = std::atoi(line.substr(kContentLengthSize).c_str());
+  std::getline(std::cin, line);
+  std::string content(content_length, ' ');
+  std::cin.read(&content[0], content_length);
+
+  Logger::Log("[incoming] ", content, "\n\n");
+
+  return ParseJson(content);
+}
+
+void WriteMessage(JsonValue& message) {
+  std::string content = SerializeToString(message);
+
+  Logger::Log("[outgoing] ", content, "\n\n");
+
+  std::cout << kContentLength << content.size() << "\r\n\r\n";
+  std::cout << content;
+}
+
+namespace {
+
+void RecompileTorque() {
+  Logger::Log("[info] Start compilation run ...\n");
+
+  LanguageServerData::Get() = LanguageServerData();
+  SourceFileMap::Get() = SourceFileMap();
+
+  TorqueCompilerOptions options;
+  options.output_directory = "";
+  options.verbose = false;
+  options.collect_language_server_data = true;
+  options.abort_on_lint_errors = false;
+  CompileTorque(TorqueFileList::Get(), options);
+
+  Logger::Log("[info] Finished compilation run ...\n");
+}
+
+void HandleInitializeRequest(InitializeRequest request, MessageWriter writer) {
+  InitializeResponse response;
+  response.set_id(request.id());
+  response.result().capabilities().textDocumentSync();
+  response.result().capabilities().set_definitionProvider(true);
+
+  // TODO(szuend): Register for document synchronisation here,
+  //               so we work with the content that the client
+  //               provides, not directly read from files.
+  // TODO(szuend): Check that the client actually supports dynamic
+  //               "workspace/didChangeWatchedFiles" capability.
+  // TODO(szuend): Check if client supports "LocationLink". This will
+  //               influence the result of "goto definition".
+  writer(response.GetJsonValue());
+}
+
+void HandleInitializedNotification(MessageWriter writer) {
+  RegistrationRequest request;
+  // TODO(szuend): The language server needs a "global" request id counter.
+  request.set_id(2000);
+  request.set_method("client/registerCapability");
+
+  Registration reg = request.params().add_registrations();
+  auto options =
+      reg.registerOptions<DidChangeWatchedFilesRegistrationOptions>();
+  FileSystemWatcher watcher = options.add_watchers();
+  watcher.set_globPattern("**/*.tq");
+  watcher.set_kind(FileSystemWatcher::WatchKind::kAll);
+
+  reg.set_id("did-change-id");
+  reg.set_method("workspace/didChangeWatchedFiles");
+
+  writer(request.GetJsonValue());
+}
+
+void HandleTorqueFileListNotification(TorqueFileListNotification notification) {
+  CHECK_EQ(notification.params().object()["files"].tag, JsonValue::ARRAY);
+
+  std::vector<std::string>& files = TorqueFileList::Get();
+  Logger::Log("[info] Initial file list:\n");
+  for (const auto& fileJson : *notification.params().object()["files"].array) {
+    CHECK_EQ(fileJson.tag, JsonValue::STRING);
+    // We only consider file URIs (there shouldn't be anything else).
+    if (fileJson.string.rfind(kFileUriPrefix) != 0) continue;
+
+    std::string file = fileJson.string.substr(kFileUriPrefixLength);
+    files.push_back(file);
+    Logger::Log("    ", file, "\n");
+  }
+
+  // The Torque compiler expects to see some files first,
+  // we need to order them in the correct way.
+  std::sort(files.begin(), files.end(),
+            [](const std::string& a, const std::string& b) {
+              if (a.find("base.tq") != std::string::npos) return true;
+              if (b.find("base.tq") != std::string::npos) return false;
+
+              if (a.find("array.tq") != std::string::npos) return true;
+              if (b.find("array.tq") != std::string::npos) return false;
+
+              return false;
+            });
+
+  RecompileTorque();
+}
+
+void HandleGotoDefinitionRequest(GotoDefinitionRequest request,
+                                 MessageWriter writer) {
+  GotoDefinitionResponse response;
+  response.set_id(request.id());
+
+  std::string file = request.params().textDocument().uri();
+  CHECK_EQ(file.rfind(kFileUriPrefix), 0);
+  SourceId id = SourceFileMap::GetSourceId(file.substr(kFileUriPrefixLength));
+
+  // If we do not know about the source file, send back an empty response,
+  // i.e. we did not find anything.
+  if (!id.IsValid()) {
+    response.SetNull("result");
+    writer(response.GetJsonValue());
+    return;
+  }
+
+  LineAndColumn pos{request.params().position().line(),
+                    request.params().position().character()};
+
+  if (auto maybe_definition = LanguageServerData::FindDefinition(id, pos)) {
+    SourcePosition definition = *maybe_definition;
+
+    std::string definition_file = SourceFileMap::GetSource(definition.source);
+    response.result().set_uri(kFileUriPrefix + definition_file);
+
+    Range range = response.result().range();
+    range.start().set_line(definition.start.line);
+    range.start().set_character(definition.start.column);
+    range.end().set_line(definition.end.line);
+    range.end().set_character(definition.end.column);
+  } else {
+    response.SetNull("result");
+  }
+
+  writer(response.GetJsonValue());
+}
+
+void HandleChangeWatchedFilesNotification(
+    DidChangeWatchedFilesNotification notification) {
+  // TODO(szuend): Implement updates to the TorqueFile list when create/delete
+  //               notifications are received. Currently we simply re-compile.
+  RecompileTorque();
+}
+
+}  // namespace
+
+void HandleMessage(JsonValue& raw_message, MessageWriter writer) {
+  Request<bool> request(raw_message);
+
+  // We ignore responses for now. They are matched to requests
+  // by id and don't have a method set.
+  // TODO(szuend): Implement proper response handling for requests
+  //               that originate from the server.
+  if (!request.has_method()) {
+    Logger::Log("[info] Unhandled response with id ", request.id(), "\n\n");
+    return;
+  }
+
+  const std::string method = request.method();
+  if (method == "initialize") {
+    HandleInitializeRequest(InitializeRequest(request.GetJsonValue()), writer);
+  } else if (method == "initialized") {
+    HandleInitializedNotification(writer);
+  } else if (method == "torque/fileList") {
+    HandleTorqueFileListNotification(
+        TorqueFileListNotification(request.GetJsonValue()));
+  } else if (method == "textDocument/definition") {
+    HandleGotoDefinitionRequest(GotoDefinitionRequest(request.GetJsonValue()),
+                                writer);
+  } else if (method == "workspace/didChangeWatchedFiles") {
+    HandleChangeWatchedFilesNotification(
+        DidChangeWatchedFilesNotification(request.GetJsonValue()));
+  } else {
+    Logger::Log("[error] Message of type ", method, " is not handled!\n\n");
+  }
+}
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8

src/torque/ls/message-handler.h

@@ -0,0 +1,26 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_TORQUE_LS_MESSAGE_HANDLER_H_
+#define V8_TORQUE_LS_MESSAGE_HANDLER_H_
+
+#include "src/torque/ls/json.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+// The message handler might send responses or follow up requests.
+// To allow unit testing, the "sending" function is configurable.
+using MessageWriter = void (*)(JsonValue& message);
+
+void HandleMessage(JsonValue& raw_message, MessageWriter);
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_TORQUE_LS_MESSAGE_HANDLER_H_

src/torque/ls/message-macros.h

@@ -0,0 +1,57 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_TORQUE_LS_MESSAGE_MACROS_H_
+#define V8_TORQUE_LS_MESSAGE_MACROS_H_
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+#define JSON_STRING_ACCESSORS(name)                                            \
+  inline const std::string& name() const { return object().at(#name).string; } \
+  inline void set_##name(const std::string& str) {                             \
+    object()[#name] = From(str);                                               \
+  }                                                                            \
+  inline bool has_##name() const {                                             \
+    return object().find(#name) != object().end();                             \
+  }
+
+#define JSON_BOOL_ACCESSORS(name)                              \
+  inline bool name() const { return object().at(#name).flag; } \
+  inline void set_##name(bool b) { object()[#name] = From(b); }
+
+#define JSON_INT_ACCESSORS(name)                                \
+  inline int name() const { return object().at(#name).number; } \
+  inline void set_##name(int n) {                               \
+    object()[#name] = From(static_cast<double>(n));             \
+  }
+
+#define JSON_OBJECT_ACCESSORS(type, name) \
+  inline type name() { return GetObject<type>(#name); }
+
+#define JSON_DYNAMIC_OBJECT_ACCESSORS(name) \
+  template <class T>                        \
+  inline T name() {                         \
+    return GetObject<T>(#name);             \
+  }
+
+#define JSON_ARRAY_OBJECT_ACCESSORS(type, name)                               \
+  inline type add_##name() {                                                  \
+    JsonObject& new_element = AddObjectElementToArrayProperty(#name);         \
+    return type(new_element);                                                 \
+  }                                                                           \
+  inline std::size_t name##_size() { return GetArrayProperty(#name).size(); } \
+  inline type name(size_t idx) {                                              \
+    CHECK(idx < name##_size());                                               \
+    return type(*GetArrayProperty(#name)[idx].object);                        \
+  }
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_TORQUE_LS_MESSAGE_MACROS_H_

src/torque/ls/message-pipe.h

@@ -0,0 +1,24 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_TORQUE_LS_MESSAGE_PIPE_H_
+#define V8_TORQUE_LS_MESSAGE_PIPE_H_
+
+#include <memory>
+#include "src/torque/ls/json.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+JsonValue ReadMessage();
+void WriteMessage(JsonValue& message);
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_TORQUE_LS_MESSAGE_PIPE_H_

src/torque/ls/message.h

@@ -0,0 +1,289 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_TORQUE_LS_MESSAGE_H_
+#define V8_TORQUE_LS_MESSAGE_H_
+
+#include "src/base/logging.h"
+#include "src/torque/ls/json.h"
+#include "src/torque/ls/message-macros.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+// Base class for Messages and Objects that are backed by either a
+// JsonValue or a reference to a JsonObject.
+// Helper methods are used by macros to implement typed accessors.
+class BaseJsonAccessor {
+ public:
+  template <class T>
+  T GetObject(const std::string& property) {
+    return T(GetObjectProperty(property));
+  }
+
+  bool HasProperty(const std::string& property) const {
+    return object().count(property) > 0;
+  }
+
+  void SetNull(const std::string& property) { object()[property] = JsonNull(); }
+
+  bool IsNull(const std::string& property) const {
+    return HasProperty(property) &&
+           object().at(property).tag == JsonValue::IS_NULL;
+  }
+
+ protected:
+  virtual const JsonObject& object() const = 0;
+  virtual JsonObject& object() = 0;
+
+  JsonObject& GetObjectProperty(const std::string& property) {
+    if (!object()[property].object) {
+      object()[property] = From(JsonObject{});
+    }
+    return *object()[property].object;
+  }
+
+  JsonArray& GetArrayProperty(const std::string& property) {
+    if (!object()[property].array) {
+      object()[property] = From(JsonArray{});
+    }
+    return *object()[property].array;
+  }
+
+  JsonObject& AddObjectElementToArrayProperty(const std::string& property) {
+    JsonArray& array = GetArrayProperty(property);
+    array.push_back(From(JsonObject{}));
+
+    return *array.back().object;
+  }
+};
+
+// Base class for Requests, Responses and Notifications.
+// In contrast to "BaseObject", a Message owns the backing JsonValue of the
+// whole object tree; i.e. value_ serves as root.
+class Message : public BaseJsonAccessor {
+ public:
+  Message() {
+    value_ = From(JsonObject{});
+    set_jsonrpc("2.0");
+  }
+  explicit Message(JsonValue& value) : value_(std::move(value)) {
+    CHECK(value_.tag == JsonValue::OBJECT);
+  }
+
+  JsonValue& GetJsonValue() { return value_; }
+
+  JSON_STRING_ACCESSORS(jsonrpc)
+
+ protected:
+  const JsonObject& object() const { return *value_.object; }
+  JsonObject& object() { return *value_.object; }
+
+ private:
+  JsonValue value_;
+};
+
+// Base class for complex type that might be part of a Message.
+// Instead of creating theses directly, use the accessors on the
+// root Message or a parent object.
+class NestedJsonAccessor : public BaseJsonAccessor {
+ public:
+  explicit NestedJsonAccessor(JsonObject& object) : object_(object) {}
+
+  const JsonObject& object() const { return object_; }
+  JsonObject& object() { return object_; }
+
+ private:
+  JsonObject& object_;
+};
+
+class ResponseError : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_INT_ACCESSORS(code)
+  JSON_STRING_ACCESSORS(message)
+};
+
+class InitializeParams : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_INT_ACCESSORS(processId)
+  JSON_STRING_ACCESSORS(rootPath)
+  JSON_STRING_ACCESSORS(rootUri)
+  JSON_STRING_ACCESSORS(trace)
+};
+
+class FileListParams : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  // TODO(szuend): Implement read accessor for string
+  //               arrays. "files" is managed directly.
+};
+
+class FileSystemWatcher : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_STRING_ACCESSORS(globPattern)
+  JSON_INT_ACCESSORS(kind)
+
+  enum WatchKind {
+    kCreate = 1,
+    kChange = 2,
+    kDelete = 4,
+
+    kAll = kCreate | kChange | kDelete,
+  };
+};
+
+class DidChangeWatchedFilesRegistrationOptions : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_ARRAY_OBJECT_ACCESSORS(FileSystemWatcher, watchers)
+};
+
+class FileEvent : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_STRING_ACCESSORS(uri)
+  JSON_INT_ACCESSORS(type)
+};
+
+class DidChangeWatchedFilesParams : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_ARRAY_OBJECT_ACCESSORS(FileEvent, changes)
+};
+
+class SaveOptions : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_BOOL_ACCESSORS(includeText)
+};
+
+class TextDocumentSyncOptions : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_BOOL_ACCESSORS(openClose)
+  JSON_INT_ACCESSORS(change)
+  JSON_BOOL_ACCESSORS(willSave)
+  JSON_BOOL_ACCESSORS(willSaveWaitUntil)
+  JSON_OBJECT_ACCESSORS(SaveOptions, save)
+};
+
+class ServerCapabilities : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_OBJECT_ACCESSORS(TextDocumentSyncOptions, textDocumentSync)
+  JSON_BOOL_ACCESSORS(definitionProvider)
+};
+
+class InitializeResult : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_OBJECT_ACCESSORS(ServerCapabilities, capabilities)
+};
+
+class Registration : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_STRING_ACCESSORS(id)
+  JSON_STRING_ACCESSORS(method)
+  JSON_DYNAMIC_OBJECT_ACCESSORS(registerOptions)
+};
+
+class RegistrationParams : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_ARRAY_OBJECT_ACCESSORS(Registration, registrations)
+};
+
+class JsonPosition : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_INT_ACCESSORS(line)
+  JSON_INT_ACCESSORS(character)
+};
+
+class Range : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_OBJECT_ACCESSORS(JsonPosition, start)
+  JSON_OBJECT_ACCESSORS(JsonPosition, end)
+};
+
+class Location : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_STRING_ACCESSORS(uri)
+  JSON_OBJECT_ACCESSORS(Range, range)
+};
+
+class TextDocumentIdentifier : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_STRING_ACCESSORS(uri)
+};
+
+class TextDocumentPositionParams : public NestedJsonAccessor {
+ public:
+  using NestedJsonAccessor::NestedJsonAccessor;
+
+  JSON_OBJECT_ACCESSORS(TextDocumentIdentifier, textDocument)
+  JSON_OBJECT_ACCESSORS(JsonPosition, position)
+};
+
+template <class T>
+class Request : public Message {
+ public:
+  explicit Request(JsonValue& value) : Message(value) {}
+  Request() : Message() {}
+
+  JSON_INT_ACCESSORS(id)
+  JSON_STRING_ACCESSORS(method)
+  JSON_OBJECT_ACCESSORS(T, params)
+};
+using InitializeRequest = Request<InitializeParams>;
+using RegistrationRequest = Request<RegistrationParams>;
+using TorqueFileListNotification = Request<FileListParams>;
+using GotoDefinitionRequest = Request<TextDocumentPositionParams>;
+using DidChangeWatchedFilesNotification = Request<DidChangeWatchedFilesParams>;
+
+template <class T>
+class Response : public Message {
+ public:
+  explicit Response(JsonValue& value) : Message(value) {}
+  Response() : Message() {}
+
+  JSON_INT_ACCESSORS(id)
+  JSON_OBJECT_ACCESSORS(ResponseError, error)
+  JSON_OBJECT_ACCESSORS(T, result)
+};
+using InitializeResponse = Response<InitializeResult>;
+using GotoDefinitionResponse = Response<Location>;
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_TORQUE_LS_MESSAGE_H_

src/torque/ls/torque-language-server.cc

@@ -0,0 +1,52 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+
+#include "src/torque/ls/globals.h"
+#include "src/torque/ls/message-handler.h"
+#include "src/torque/ls/message-pipe.h"
+#include "src/torque/server-data.h"
+#include "src/torque/source-positions.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+int WrappedMain(int argc, const char** argv) {
+  Logger::Scope log_scope;
+  TorqueFileList::Scope files_scope;
+  LanguageServerData::Scope server_data_scope;
+  SourceFileMap::Scope source_file_map_scope;
+
+  for (int i = 1; i < argc; ++i) {
+    if (!strcmp("-l", argv[i])) {
+      Logger::Enable(argv[++i]);
+      break;
+    }
+  }
+
+  while (true) {
+    auto message = ReadMessage();
+
+    // TODO(szuend): We should probably offload the actual message handling
+    //               (even the parsing) to a background thread, so we can
+    //               keep receiving messages. We might also receive
+    //               $/cancelRequests or contet updates, that require restarts.
+    HandleMessage(message, &WriteMessage);
+  }
+  return 0;
+}
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8
+
+int main(int argc, const char** argv) {
+  return v8::internal::torque::ls::WrappedMain(argc, argv);
+}

src/torque/source-positions.h

@@ -16,12 +16,14 @@ namespace torque {
 class SourceId {
  public:
   static SourceId Invalid() { return SourceId(-1); }
+  bool IsValid() const { return id_ != -1; }
   int operator==(const SourceId& s) const { return id_ == s.id_; }
   bool operator<(const SourceId& s) const { return id_ < s.id_; }
 
  private:
   explicit SourceId(int id) : id_(id) {}
   int id_;
+  friend struct SourcePosition;
   friend class SourceFileMap;
 };
 
@@ -71,6 +73,15 @@ class SourceFileMap : public ContextualClass<SourceFileMap> {
     return SourceId(static_cast<int>(Get().sources_.size()) - 1);
   }
 
+  static SourceId GetSourceId(const std::string& path) {
+    for (size_t i = 0; i < Get().sources_.size(); ++i) {
+      if (Get().sources_[i] == path) {
+        return SourceId(static_cast<int>(i));
+      }
+    }
+    return SourceId::Invalid();
+  }
+
  private:
   std::vector<std::string> sources_;
 };

test/cctest/BUILD.gn

@@ -251,7 +251,7 @@ v8_source_set("cctest_sources") {
     "test-weakmaps.cc",
     "test-weaksets.cc",
     "torque/test-torque-ls-json.cc",
-    "torque/test-torque.cc",
+    "torque/test-torque-ls-message.cc",
     "trace-extension.cc",
     "trace-extension.h",
     "unicode-helpers.cc",

test/cctest/torque/test-torque-ls-message.cc

@@ -0,0 +1,116 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/torque/ls/json.h"
+#include "src/torque/ls/message-handler.h"
+#include "src/torque/ls/message.h"
+#include "src/torque/server-data.h"
+#include "src/torque/source-positions.h"
+#include "test/cctest/cctest.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+TEST(InitializeRequest) {
+  InitializeRequest request;
+  request.set_id(5);
+  request.set_method("initialize");
+  request.params();
+
+  HandleMessage(request.GetJsonValue(), [](JsonValue& raw_response) {
+    InitializeResponse response(raw_response);
+
+    // Check that the response id matches up with the request id, and that
+    // the language server signals its support for definitions.
+    CHECK_EQ(response.id(), 5);
+    CHECK_EQ(response.result().capabilities().definitionProvider(), true);
+  });
+}
+
+TEST(RegisterDynamicCapabilitiesAfterInitializedNotification) {
+  Request<bool> notification;
+  notification.set_method("initialized");
+
+  HandleMessage(notification.GetJsonValue(), [](JsonValue& raw_request) {
+    RegistrationRequest request(raw_request);
+
+    CHECK_EQ(request.method(), "client/registerCapability");
+    CHECK_EQ(request.params().registrations_size(), 1);
+
+    Registration registration = request.params().registrations(0);
+    CHECK_EQ(registration.method(), "workspace/didChangeWatchedFiles");
+
+    auto options =
+        registration
+            .registerOptions<DidChangeWatchedFilesRegistrationOptions>();
+    CHECK_EQ(options.watchers_size(), 1);
+  });
+}
+
+TEST(GotoDefinitionUnkownFile) {
+  SourceFileMap::Scope source_file_map_scope;
+
+  GotoDefinitionRequest request;
+  request.set_id(42);
+  request.set_method("textDocument/definition");
+  request.params().textDocument().set_uri("file:///unknown.tq");
+
+  HandleMessage(request.GetJsonValue(), [](JsonValue& raw_response) {
+    GotoDefinitionResponse response(raw_response);
+    CHECK_EQ(response.id(), 42);
+    CHECK(response.IsNull("result"));
+  });
+}
+
+TEST(GotoDefinition) {
+  SourceFileMap::Scope source_file_map_scope;
+  SourceId test_id = SourceFileMap::AddSource("test.tq");
+  SourceId definition_id = SourceFileMap::AddSource("base.tq");
+
+  LanguageServerData::Scope server_data_scope;
+  LanguageServerData::AddDefinition({test_id, {1, 0}, {1, 10}},
+                                    {definition_id, {4, 1}, {4, 5}});
+
+  // First, check a unknown definition. The result must be null.
+  GotoDefinitionRequest request;
+  request.set_id(42);
+  request.set_method("textDocument/definition");
+  request.params().textDocument().set_uri("file://test.tq");
+  request.params().position().set_line(2);
+  request.params().position().set_character(0);
+
+  HandleMessage(request.GetJsonValue(), [](JsonValue& raw_response) {
+    GotoDefinitionResponse response(raw_response);
+    CHECK_EQ(response.id(), 42);
+    CHECK(response.IsNull("result"));
+  });
+
+  // Second, check a known defintion.
+  request = GotoDefinitionRequest();
+  request.set_id(43);
+  request.set_method("textDocument/definition");
+  request.params().textDocument().set_uri("file://test.tq");
+  request.params().position().set_line(1);
+  request.params().position().set_character(5);
+
+  HandleMessage(request.GetJsonValue(), [](JsonValue& raw_response) {
+    GotoDefinitionResponse response(raw_response);
+    CHECK_EQ(response.id(), 43);
+    CHECK(!response.IsNull("result"));
+
+    Location location = response.result();
+    CHECK_EQ(location.uri(), "file://base.tq");
+    CHECK_EQ(location.range().start().line(), 4);
+    CHECK_EQ(location.range().start().character(), 1);
+    CHECK_EQ(location.range().end().line(), 4);
+    CHECK_EQ(location.range().end().character(), 5);
+  });
+}
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8

tools/torque/vscode-torque/.npmrc

@@ -0,0 +1 @@
+package-lock=false

tools/torque/vscode-torque/.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "extensionHost",
+            "request": "launch",
+            "name": "test:client",
+            "runtimeExecutable": "${execPath}",
+            "args": ["--extensionDevelopmentPath=${workspaceRoot}"],
+            "stopOnEntry": false,
+            "sourceMaps": true,
+            "outFiles": ["${workspaceRoot}/lib/test/**/*.js"],
+            "preLaunchTask": "watch:client"
+        }
+    ]
+}

tools/torque/vscode-torque/.vscode/tasks.json

@@ -0,0 +1,19 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "watch:client",
+            "type": "shell",
+            "command": "npm run watch",
+            "group": "build",
+            "presentation": {
+                "panel": "dedicated",
+                "reveal": "never"
+            },
+            "isBackground": true,
+            "problemMatcher": [
+                "$tsc-watch"
+            ]
+        }
+    ]
+}

tools/torque/vscode-torque/README.md

@@ -1,7 +1,6 @@
-# Torque syntax support
+# Torque support
 
-This extensions adds rudimentary syntax highlighting support for the WIP
-Torque language used in V8.
+This extension adds language support for [the Torque language used in V8](https://v8.dev/docs/torque).
 
 ## Installation
 
@@ -11,4 +10,16 @@ directory:
 
 ```
 ln -s $V8/tools/torque/vscode-torque $HOME/.vscode/extensions/vscode-torque
-```
+```
+
+### Language server
+
+The language server is not built by default. To build the language server manually:
+
+```
+autoninja -C <output dir> torque-language-server
+```
+
+The default directory where the extension looks for the executable is "out/x64.release",
+but the absolute path to the executable can be configured with the `torque.ls.executable`
+setting.

tools/torque/vscode-torque/out/extension.js

@@ -0,0 +1,99 @@
+"use strict";
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+// The file out/extension.js gets automatically created from
+// src/extension.ts. out/extension.js should not be modified manually.
+const path = require("path");
+const vscode_1 = require("vscode");
+const vscode_languageclient_1 = require("vscode-languageclient");
+const vscode_languageclient_2 = require("vscode-languageclient");
+let client;
+let outputChannel;
+class TorqueErrorHandler {
+    constructor(config) {
+        this.config = config;
+    }
+    error(error, message, count) {
+        outputChannel.appendLine("TorqueErrorHandler: ");
+        outputChannel.append(error.toString());
+        outputChannel.append(message.toString());
+        return vscode_languageclient_1.ErrorAction.Continue;
+    }
+    closed() {
+        return vscode_languageclient_1.CloseAction.DoNotRestart;
+    }
+}
+function activate(context) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // Create a status bar item that displays the current status of the language server.
+        const statusBarItem = vscode_1.window.createStatusBarItem(vscode_1.StatusBarAlignment.Left, 0);
+        statusBarItem.text = "torque-ls: <unknown>";
+        statusBarItem.show();
+        const torqueConfiguration = vscode_1.workspace.getConfiguration("torque.ls");
+        let serverExecutable = torqueConfiguration.get("executable");
+        if (serverExecutable == null) {
+            serverExecutable = path.join(vscode_1.workspace.rootPath, "out", "x64.release", "torque-language-server");
+        }
+        let serverArguments = [];
+        const loggingEnabled = torqueConfiguration.get("logging");
+        if (loggingEnabled) {
+            const logfile = torqueConfiguration.get("logfile");
+            serverArguments = ["-l", logfile];
+        }
+        const serverOptions = { command: serverExecutable, args: serverArguments };
+        outputChannel = vscode_1.window.createOutputChannel("Torque Language Server");
+        const clientOptions = {
+            diagnosticCollectionName: "torque",
+            documentSelector: [{ scheme: "file", language: "torque" }],
+            errorHandler: new TorqueErrorHandler(vscode_1.workspace.getConfiguration("torque")),
+            initializationFailedHandler: (e) => {
+                outputChannel.appendLine(e);
+                return false;
+            },
+            outputChannel,
+            revealOutputChannelOn: vscode_languageclient_1.RevealOutputChannelOn.Info,
+        };
+        // Create the language client and start the client.
+        client = new vscode_languageclient_2.LanguageClient("torque", "Torque Language Server", serverOptions, clientOptions);
+        client.trace = vscode_languageclient_1.Trace.Verbose;
+        // Update the status bar according to the client state.
+        client.onDidChangeState((event) => {
+            if (event.newState === vscode_languageclient_1.State.Running) {
+                statusBarItem.text = "torque-ls: Running";
+            }
+            else if (event.newState === vscode_languageclient_1.State.Starting) {
+                statusBarItem.text = "torque-ls: Starting";
+            }
+            else {
+                statusBarItem.text = "torque-ls: Stopped";
+            }
+        });
+        // This will start client and server.
+        client.start();
+        yield client.onReady();
+        // The server needs an initial list of all the Torque files
+        // in the workspace, send them over.
+        vscode_1.workspace.findFiles("**/*.tq").then((urls) => {
+            client.sendNotification("torque/fileList", { files: urls.map((url) => url.toString()) });
+        });
+    });
+}
+exports.activate = activate;
+function deactivate() {
+    if (!client) {
+        return undefined;
+    }
+    return client.stop();
+}
+exports.deactivate = deactivate;
+//# sourceMappingURL=extension.js.map

tools/torque/vscode-torque/package.json

@@ -5,22 +5,76 @@
     "version": "0.0.1",
     "publisher": "szuend",
     "engines": {
-        "vscode": "^1.22.0"
+        "vscode": "^1.31.0"
     },
     "categories": [
-        "Languages"
+        "Programming Languages"
     ],
+    "activationEvents": [
+        "onLanguage:torque",
+        "workspaceContains:**/*.tq"
+    ],
+    "main": "./out/extension",
     "contributes": {
-        "languages": [{
-            "id": "torque",
-            "aliases": ["Torque", "torque"],
-            "extensions": [".tq"],
-            "configuration": "./language-configuration.json"
-        }],
-        "grammars": [{
-            "language": "torque",
-            "scopeName": "source.torque",
-            "path": "./syntaxes/torque.tmLanguage.json"
-        }]
+        "configuration": {
+            "type": "object",
+            "title": "Torque",
+            "properties": {
+                "torque.ls.executable": {
+                    "type": [
+                        "string",
+                        null
+                    ],
+                    "default": null,
+                    "description": "Path to the torque language server executable (absolute)"
+                },
+                "torque.ls.logging": {
+                    "type": "boolean",
+                    "default": false,
+                    "description": "Enable language server diagnostics output to log file"
+                },
+                "torque.ls.logfile": {
+                    "type": "string",
+                    "default": "torque-log.txt",
+                    "description": "Target file for language server logging output"
+                }
+            }
+        },
+        "languages": [
+            {
+                "id": "torque",
+                "aliases": [
+                    "Torque",
+                    "torque"
+                ],
+                "extensions": [
+                    ".tq"
+                ],
+                "configuration": "./language-configuration.json"
+            }
+        ],
+        "grammars": [
+            {
+                "language": "torque",
+                "scopeName": "source.torque",
+                "path": "./syntaxes/torque.tmLanguage.json"
+            }
+        ]
+    },
+    "dependencies": {
+        "vscode-languageclient": "^5.2.1"
+    },
+    "devDependencies": {
+        "@types/node": "^8.0.0",
+        "vscode": "^1.1.21",
+        "tslint": "^5.11.0",
+        "typescript": "^3.1.3"
+    },
+    "scripts": {
+        "update-vscode": "vscode-install",
+        "postinstall": "vscode-install",
+        "vscode:prepublish": "npm run update-vscode && npm run compile",
+        "compile": "tsc -b",
+        "watch": "tsc -b -w"
     }
-}
+}

tools/torque/vscode-torque/src/extension.ts

@@ -0,0 +1,104 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// The file out/extension.js gets automatically created from
+// src/extension.ts. out/extension.js should not be modified manually.
+
+import * as path from "path";
+import { ExtensionContext, OutputChannel, StatusBarAlignment,
+    window, workspace, WorkspaceConfiguration } from "vscode";
+import { CloseAction, ErrorAction, ErrorHandler, Message,
+    RevealOutputChannelOn, State, Trace } from "vscode-languageclient";
+
+import {
+  LanguageClient,
+  LanguageClientOptions,
+  ServerOptions,
+} from "vscode-languageclient";
+
+let client: LanguageClient;
+let outputChannel: OutputChannel;
+
+class TorqueErrorHandler implements ErrorHandler {
+  constructor(readonly config: WorkspaceConfiguration) {}
+
+  public error(error: Error, message: Message, count: number): ErrorAction {
+    outputChannel.appendLine("TorqueErrorHandler: ");
+    outputChannel.append(error.toString());
+    outputChannel.append(message.toString());
+    return ErrorAction.Continue;
+  }
+
+  public closed(): CloseAction {
+    return CloseAction.DoNotRestart;
+  }
+}
+
+export async function activate(context: ExtensionContext) {
+  // Create a status bar item that displays the current status of the language server.
+  const statusBarItem = window.createStatusBarItem(StatusBarAlignment.Left, 0);
+  statusBarItem.text = "torque-ls: <unknown>";
+  statusBarItem.show();
+
+  const torqueConfiguration = workspace.getConfiguration("torque.ls");
+  let serverExecutable: string | null = torqueConfiguration.get("executable");
+  if (serverExecutable == null) {
+    serverExecutable = path.join(workspace.rootPath, "out", "x64.release", "torque-language-server");
+  }
+
+  let serverArguments = [];
+  const loggingEnabled: boolean = torqueConfiguration.get("logging");
+  if (loggingEnabled) {
+    const logfile = torqueConfiguration.get("logfile");
+    serverArguments = ["-l", logfile];
+  }
+
+  const serverOptions: ServerOptions = { command: serverExecutable, args: serverArguments };
+
+  outputChannel = window.createOutputChannel("Torque Language Server");
+
+  const clientOptions: LanguageClientOptions = {
+    diagnosticCollectionName: "torque",
+    documentSelector: [{ scheme: "file", language: "torque" }],
+    errorHandler: new TorqueErrorHandler(workspace.getConfiguration("torque")),
+    initializationFailedHandler: (e) => {
+      outputChannel.appendLine(e);
+      return false;
+    },
+    outputChannel,
+    revealOutputChannelOn: RevealOutputChannelOn.Info,
+  };
+
+  // Create the language client and start the client.
+  client = new LanguageClient("torque", "Torque Language Server", serverOptions, clientOptions);
+  client.trace = Trace.Verbose;
+
+  // Update the status bar according to the client state.
+  client.onDidChangeState((event) => {
+    if (event.newState === State.Running) {
+      statusBarItem.text = "torque-ls: Running";
+    } else if (event.newState === State.Starting) {
+      statusBarItem.text = "torque-ls: Starting";
+    } else {
+      statusBarItem.text = "torque-ls: Stopped";
+    }
+  });
+
+  // This will start client and server.
+  client.start();
+
+  await client.onReady();
+
+  // The server needs an initial list of all the Torque files
+  // in the workspace, send them over.
+  workspace.findFiles("**/*.tq").then((urls) => {
+    client.sendNotification("torque/fileList",
+      { files: urls.map((url) => url.toString())});
+  });
+}
+
+export function deactivate(): Thenable<void> | undefined {
+  if (!client) { return undefined; }
+  return client.stop();
+}

tools/torque/vscode-torque/tsconfig.json

@@ -0,0 +1,17 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "target": "es6",
+        "outDir": "out",
+        "rootDir": "src",
+        "lib": [ "es6" ],
+        "sourceMap": true
+    },
+    "include": [
+        "src"
+    ],
+    "exclude": [
+        "node_modules",
+        ".vscode-test"
+    ]
+}

tools/torque/vscode-torque/tslint.json

@@ -0,0 +1,11 @@
+{
+    "defaultSeverity": "error",
+    "extends": [
+        "tslint:recommended"
+    ],
+    "jsRules": {},
+    "rules": {
+        "indent": [true, "spaces", 2]
+    },
+    "rulesDirectory": []
+}