[torque] Better SourcePositions for declarables

This CL improves SourcePosition support inside the Torque compiler.
It starts with the parser, where the SourcePosition of the
MatchedInput now encompasses all tokens, not just the first one.

Second, AST nodes can now be created with an explicit source position.
This can be used to forward the "all encompassing" source position
via MatchedInput -> ParseResultIterator to AST nodes.

Third, declarables are extended to hold two different SourcePositions:
  - One represents the whole declarable. For a macro this would
    inlcude the body as well as the signature.
  - The other is the SourcePosition of the identifying part of a
    declarable. In most cases this is the name. For the rest this
    will stay invalid.

R=sigurds@chromium.org, tebbi@chromium.org

Bug: v8:7793
Change-Id: I509f83aeef7a040d0ea6363b5b7c31ff1b11f47b
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1591600
Commit-Queue: Simon Zünd <szuend@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#61234}

src/torque/declarable.h

@@ -72,7 +72,23 @@ class Declarable {
   }
   virtual const char* type_name() const { return "<<unknown>>"; }
   Scope* ParentScope() const { return parent_scope_; }
-  const SourcePosition& pos() const { return pos_; }
+
+  // The SourcePosition of the whole declarable. For example, for a macro
+  // this will encompass not only the signature, but also the body.
+  SourcePosition Position() const { return position_; }
+  void SetPosition(const SourcePosition& position) { position_ = position; }
+
+  // The SourcePosition of the identifying name of the declarable. For example,
+  // for a macro this will be the SourcePosition of the name.
+  // Note that this SourcePosition might not make sense for all kinds of
+  // declarables, in that case, the default SourcePosition is returned.
+  SourcePosition IdentifierPosition() const {
+    return identifier_position_.source.IsValid() ? identifier_position_
+                                                 : position_;
+  }
+  void SetIdentifierPosition(const SourcePosition& position) {
+    identifier_position_ = position;
+  }
 
  protected:
   explicit Declarable(Kind kind) : kind_(kind) {}
@@ -80,7 +96,8 @@ class Declarable {
  private:
   const Kind kind_;
   Scope* const parent_scope_ = CurrentScope::Get();
-  SourcePosition pos_ = CurrentSourcePosition::Get();
+  SourcePosition position_ = CurrentSourcePosition::Get();
+  SourcePosition identifier_position_ = SourcePosition::Invalid();
 };
 
 #define DECLARE_DECLARABLE_BOILERPLATE(x, y)                  \

src/torque/declaration-visitor.cc

@@ -152,8 +152,12 @@ void DeclarationVisitor::Visit(TorqueBuiltinDeclaration* decl,
 void DeclarationVisitor::Visit(TorqueMacroDeclaration* decl,
                                const Signature& signature,
                                base::Optional<Statement*> body) {
+  Macro* macro =
       Declarations::DeclareMacro(decl->name, base::nullopt, signature,
                                  decl->transitioning, body, decl->op);
+  // TODO(szuend): Set identifier_position to decl->name->pos once all callable
+  // names are changed from std::string to Identifier*.
+  macro->SetPosition(decl->pos);
 }
 
 void DeclarationVisitor::Visit(IntrinsicDeclaration* decl,
@@ -395,7 +399,7 @@ Callable* DeclarationVisitor::SpecializeImplicit(const SpecializationKey& key) {
           nullptr) {
     ReportError("missing specialization of ", key.generic->name(),
                 " with types <", key.specialized_types, "> declared at ",
-                key.generic->pos());
+                key.generic->Position());
   }
   CurrentScope::Scope generic_scope(key.generic->ParentScope());
   Callable* result =

src/torque/earley-parser.h

@@ -303,10 +303,13 @@ class Item {
   void CheckAmbiguity(const Item& other, const LexerResult& tokens) const;
 
   MatchedInput GetMatchedInput(const LexerResult& tokens) const {
-    return {tokens.token_contents[start_].begin,
-            start_ == pos_ ? tokens.token_contents[start_].begin
-                           : tokens.token_contents[pos_ - 1].end,
-            tokens.token_contents[start_].pos};
+    const MatchedInput& start = tokens.token_contents[start_];
+    const MatchedInput& end = start_ == pos_ ? tokens.token_contents[start_]
+                                             : tokens.token_contents[pos_ - 1];
+    CHECK_EQ(start.pos.source, end.pos.source);
+    SourcePosition combined{start.pos.source, start.pos.start, end.pos.end};
+
+    return {start.begin, end.end, combined};
   }
 
   // We exclude {prev_} and {child_} from equality and hash computations,

src/torque/implementation-visitor.cc

@@ -1990,7 +1990,8 @@ LocationReference ImplementationVisitor::GetLocationReference(
       QualifiedName(expr->namespace_qualification, expr->name->value);
   if (base::Optional<Builtin*> builtin = Declarations::TryLookupBuiltin(name)) {
     if (GlobalContext::collect_language_server_data()) {
-      LanguageServerData::AddDefinition(expr->name->pos, (*builtin)->pos());
+      LanguageServerData::AddDefinition(expr->name->pos,
+                                        (*builtin)->Position());
     }
     return LocationReference::Temporary(GetBuiltinCode(*builtin),
                                         "builtin " + expr->name->value);
@@ -2446,7 +2447,7 @@ VisitResult ImplementationVisitor::Visit(CallExpression* expr,
       Callable* callable = LookupCallable(name, Declarations::Lookup(name),
                                           arguments, specialization_types);
       LanguageServerData::AddDefinition(expr->callee->name->pos,
-                                        callable->pos());
+                                        callable->IdentifierPosition());
     }
     return scope.Yield(
         GenerateCall(name, arguments, specialization_types, is_tailcall));
@@ -2688,7 +2689,7 @@ void ImplementationVisitor::VisitAllDeclarables() {
 
 void ImplementationVisitor::Visit(Declarable* declarable) {
   CurrentScope::Scope current_scope(declarable->ParentScope());
-  CurrentSourcePosition::Scope current_source_position(declarable->pos());
+  CurrentSourcePosition::Scope current_source_position(declarable->Position());
   switch (declarable->kind()) {
     case Declarable::kMacro:
       return Visit(Macro::cast(declarable));