[builtins] Remap builtins on Linux

This is a CL similar to
https://chromium-review.googlesource.com/c/v8/v8/+/3553006, but on Linux
rather than macOS. The goal is to allow builtins to use short builtin
calls without paying a memory cost, by remapping rather than copying
them.

However, while macOS has a system call making this easier, on Linux we
don't have one on most kernels. There is the recently-introduced
mremap(MREMAP_DONTUNMMAP), which is available in 5.7, but only works on
anonymous mappings until 5.13, which is too recent for most Android
devices.

Instead, we open() the file containing the builtins, and mmap() it at
the desired location.

Change-Id: I4524f349948b8f48c4536cf392a1cd179662a6cc
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3570426
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Jakob Linke <jgruber@chromium.org>
Commit-Queue: Benoit Lize <lizeb@chromium.org>
Cr-Commit-Position: refs/heads/main@{#80022}

BUILD.bazel

@@ -688,6 +688,7 @@ filegroup(
         "@v8//bazel/config:is_linux": [
             "src/base/debug/stack_trace_posix.cc",
             "src/base/platform/platform-linux.cc",
+            "src/base/platform/platform-linux.h",
         ],
         "@v8//bazel/config:is_android": [
             "src/base/debug/stack_trace_android.cc",

BUILD.gn

@@ -418,7 +418,7 @@ if (v8_enable_zone_compression == "") {
 }
 if (v8_enable_short_builtin_calls == "") {
   v8_enable_short_builtin_calls =
-      v8_current_cpu == "x64" || (!is_android && v8_current_cpu == "arm64")
+      v8_current_cpu == "x64" || v8_current_cpu == "arm64"
 }
 if (v8_enable_external_code_space == "") {
   v8_enable_external_code_space =
@@ -5245,6 +5245,7 @@ v8_component("v8_libbase") {
     sources += [
       "src/base/debug/stack_trace_posix.cc",
       "src/base/platform/platform-linux.cc",
+      "src/base/platform/platform-linux.h",
     ]
 
     libs = [

src/base/platform/platform-linux.cc

@@ -5,6 +5,8 @@
 // Platform-specific code for Linux goes here. For the POSIX-compatible
 // parts, the implementation is in platform-posix.cc.
 
+#include "src/base/platform/platform-linux.h"
+
 #include <pthread.h>
 #include <semaphore.h>
 #include <signal.h>
@@ -21,13 +23,18 @@
 #include <errno.h>
 #include <fcntl.h>  // open
 #include <stdarg.h>
-#include <strings.h>    // index
+#include <strings.h>   // index
-#include <sys/mman.h>   // mmap & munmap & mremap
+#include <sys/mman.h>  // mmap & munmap & mremap
-#include <sys/stat.h>   // open
+#include <sys/stat.h>  // open
+#include <sys/sysmacros.h>
 #include <sys/types.h>  // mmap & munmap
 #include <unistd.h>     // sysconf
 
 #include <cmath>
+#include <memory>
+
+#include "src/base/logging.h"
+#include "src/base/memory.h"
 
 #undef MAP_TYPE
 
@@ -206,5 +213,163 @@ std::vector<OS::MemoryRange> OS::GetFreeMemoryRangesWithin(
   return result;
 }
 
+//  static
+base::Optional<MemoryRegion> MemoryRegion::FromMapsLine(const char* line) {
+  MemoryRegion region;
+  uint8_t dev_major = 0, dev_minor = 0;
+  uintptr_t inode = 0;
+  int path_index = 0;
+  uintptr_t offset = 0;
+  // The format is:
+  // address           perms offset  dev   inode   pathname
+  // 08048000-08056000 r-xp 00000000 03:0c 64593   /usr/sbin/gpm
+  //
+  // The final %n term captures the offset in the input string, which is used
+  // to determine the path name. It *does not* increment the return value.
+  // Refer to man 3 sscanf for details.
+  if (sscanf(line,
+             "%" V8PRIxPTR "-%" V8PRIxPTR " %4c %" V8PRIxPTR
+             " %hhx:%hhx %" V8PRIdPTR " %n",
+             &region.start, &region.end, region.permissions, &offset,
+             &dev_major, &dev_minor, &inode, &path_index) < 7) {
+    return base::nullopt;
+  }
+  region.permissions[4] = '\0';
+  region.inode = inode;
+  region.offset = offset;
+  region.dev = makedev(dev_major, dev_minor);
+  region.pathname.assign(line + path_index);
+
+  return region;
+}
+
+namespace {
+// Parses /proc/self/maps.
+std::unique_ptr<std::vector<MemoryRegion>> ParseProcSelfMaps(
+    std::function<bool(const MemoryRegion&)> predicate, bool early_stopping) {
+  auto result = std::make_unique<std::vector<MemoryRegion>>();
+
+  FILE* fp = fopen("/proc/self/maps", "r");
+  if (!fp) return nullptr;
+
+  // Allocate enough room to be able to store a full file name.
+  // 55ac243aa000-55ac243ac000 r--p 00000000 fe:01 31594735 /usr/bin/head
+  const int kMaxLineLength = 2 * FILENAME_MAX;
+  std::unique_ptr<char[]> line = std::make_unique<char[]>(kMaxLineLength);
+
+  // This loop will terminate once the scanning hits an EOF.
+  bool error = false;
+  while (true) {
+    error = true;
+
+    // Read to the end of the line. Exit if the read fails.
+    if (fgets(line.get(), kMaxLineLength, fp) == nullptr) break;
+    size_t line_length = strlen(line.get());
+    // Line was truncated.
+    if (line.get()[line_length - 1] != '\n') break;
+    line.get()[line_length - 1] = '\0';
+
+    base::Optional<MemoryRegion> region =
+        MemoryRegion::FromMapsLine(line.get());
+    if (!region) break;
+
+    error = false;
+
+    if (predicate(*region)) {
+      result->push_back(std::move(*region));
+      if (early_stopping) break;
+    }
+  }
+
+  fclose(fp);
+  if (!error && result->size()) return result;
+
+  return nullptr;
+}
+
+MemoryRegion FindEnclosingMapping(uintptr_t target_start, size_t size) {
+  auto result = ParseProcSelfMaps(
+      [=](const MemoryRegion& region) {
+        return region.start <= target_start && target_start + size < region.end;
+      },
+      true);
+  if (result)
+    return (*result)[0];
+  else
+    return {};
+}
+}  // namespace
+
+// static
+bool OS::RemapPages(const void* address, size_t size, void* new_address,
+                    MemoryPermission access) {
+  uintptr_t address_addr = reinterpret_cast<uintptr_t>(address);
+
+  DCHECK(IsAligned(address_addr, AllocatePageSize()));
+  DCHECK(
+      IsAligned(reinterpret_cast<uintptr_t>(new_address), AllocatePageSize()));
+  DCHECK(IsAligned(size, AllocatePageSize()));
+
+  MemoryRegion enclosing_region = FindEnclosingMapping(address_addr, size);
+  // Not found.
+  if (!enclosing_region.start) return false;
+
+  // Anonymous mapping?
+  if (enclosing_region.pathname.empty()) return false;
+
+  // Since the file is already in use for executable code, this is most likely
+  // to fail due to sandboxing, e.g. if open() is blocked outright.
+  //
+  // In Chromium on Android, the sandbox allows openat() but prohibits
+  // open(). However, the libc uses openat() in its open() wrapper, and the
+  // SELinux restrictions allow us to read from the path we want to look at,
+  // so we are in the clear.
+  //
+  // Note that this may not be allowed by the sandbox on Linux (and Chrome
+  // OS). On these systems, consider using mremap() with the MREMAP_DONTUNMAP
+  // flag. However, since we need it on non-anonymous mapping, this would only
+  // be available starting with version 5.13.
+  int fd = open(enclosing_region.pathname.c_str(), O_RDONLY);
+  if (fd == -1) return false;
+
+  // Now we have a file descriptor to the same path the data we want to remap
+  // comes from. But... is it the *same* file? This is not guaranteed (e.g. in
+  // case of updates), so to avoid hard-to-track bugs, check that the
+  // underlying file is the same using the device number and the inode. Inodes
+  // are not unique across filesystems, and can be reused. The check works
+  // here though, since we have the problems:
+  // - Inode uniqueness: check device numbers.
+  // - Inode reuse: the initial file is still open, since we are running code
+  //   from it. So its inode cannot have been reused.
+  struct stat stat_buf;
+  if (fstat(fd, &stat_buf)) {
+    close(fd);
+    return false;
+  }
+
+  // Not the same file.
+  if (stat_buf.st_dev != enclosing_region.dev ||
+      stat_buf.st_ino != enclosing_region.inode) {
+    close(fd);
+    return false;
+  }
+
+  size_t offset_in_mapping = address_addr - enclosing_region.start;
+  size_t offset_in_file = enclosing_region.offset + offset_in_mapping;
+  int protection = GetProtectionFromMemoryPermission(access);
+
+  void* mapped_address = mmap(new_address, size, protection,
+                              MAP_FIXED | MAP_PRIVATE, fd, offset_in_file);
+  // mmap() keeps the file open.
+  close(fd);
+
+  if (mapped_address != new_address) {
+    // Should not happen, MAP_FIXED should always map where we want.
+    UNREACHABLE();
+  }
+
+  return true;
+}
+
 }  // namespace base
 }  // namespace v8

src/base/platform/platform-linux.h

@@ -0,0 +1,37 @@
+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_BASE_PLATFORM_PLATFORM_LINUX_H_
+#define V8_BASE_PLATFORM_PLATFORM_LINUX_H_
+
+#include <sys/types.h>
+
+#include <cstdint>
+#include <string>
+
+#include "src/base/base-export.h"
+#include "src/base/optional.h"
+
+namespace v8 {
+namespace base {
+
+// Represents a memory region, as parsed from /proc/PID/maps.
+// Visible for testing.
+struct V8_BASE_EXPORT MemoryRegion {
+  uintptr_t start;
+  uintptr_t end;
+  char permissions[5];
+  off_t offset;
+  dev_t dev;
+  ino_t inode;
+  std::string pathname;
+
+  // |line| must not contains the tail '\n'.
+  static base::Optional<MemoryRegion> FromMapsLine(const char* line);
+};
+
+}  // namespace base
+}  // namespace v8
+
+#endif  // V8_BASE_PLATFORM_PLATFORM_LINUX_H_

src/base/platform/platform-posix.cc

@@ -123,25 +123,6 @@ constexpr int kAppleArmPageSize = 1 << 14;
 
 const int kMmapFdOffset = 0;
 
-// TODO(v8:10026): Add the right permission flag to make executable pages
-// guarded.
-int GetProtectionFromMemoryPermission(OS::MemoryPermission access) {
-  switch (access) {
-    case OS::MemoryPermission::kNoAccess:
-    case OS::MemoryPermission::kNoAccessWillJitLater:
-      return PROT_NONE;
-    case OS::MemoryPermission::kRead:
-      return PROT_READ;
-    case OS::MemoryPermission::kReadWrite:
-      return PROT_READ | PROT_WRITE;
-    case OS::MemoryPermission::kReadWriteExecute:
-      return PROT_READ | PROT_WRITE | PROT_EXEC;
-    case OS::MemoryPermission::kReadExecute:
-      return PROT_READ | PROT_EXEC;
-  }
-  UNREACHABLE();
-}
-
 enum class PageType { kShared, kPrivate };
 
 int GetFlagsForMemoryPermission(OS::MemoryPermission access,
@@ -196,6 +177,25 @@ void* Allocate(void* hint, size_t size, OS::MemoryPermission access,
 
 }  // namespace
 
+// TODO(v8:10026): Add the right permission flag to make executable pages
+// guarded.
+int GetProtectionFromMemoryPermission(OS::MemoryPermission access) {
+  switch (access) {
+    case OS::MemoryPermission::kNoAccess:
+    case OS::MemoryPermission::kNoAccessWillJitLater:
+      return PROT_NONE;
+    case OS::MemoryPermission::kRead:
+      return PROT_READ;
+    case OS::MemoryPermission::kReadWrite:
+      return PROT_READ | PROT_WRITE;
+    case OS::MemoryPermission::kReadWriteExecute:
+      return PROT_READ | PROT_WRITE | PROT_EXEC;
+    case OS::MemoryPermission::kReadExecute:
+      return PROT_READ | PROT_EXEC;
+  }
+  UNREACHABLE();
+}
+
 #if V8_OS_LINUX || V8_OS_FREEBSD
 #ifdef __arm__
 

src/base/platform/platform-posix.h

@@ -23,6 +23,8 @@ class PosixTimezoneCache : public TimezoneCache {
   static const int msPerSecond = 1000;
 };
 
+int GetProtectionFromMemoryPermission(OS::MemoryPermission access);
+
 }  // namespace base
 }  // namespace v8
 

src/base/platform/platform.h

@@ -317,7 +317,7 @@ class V8_BASE_EXPORT OS {
   // Whether the platform supports mapping a given address in another location
   // in the address space.
   V8_WARN_UNUSED_RESULT static constexpr bool IsRemapPageSupported() {
-#ifdef V8_OS_MACOS
+#if defined(V8_OS_MACOS) || defined(V8_OS_LINUX)
     return true;
 #else
     return false;
@@ -330,6 +330,9 @@ class V8_BASE_EXPORT OS {
   // be a multiple of the system page size.  If there is already memory mapped
   // at the target address, it is replaced by the new mapping.
   //
+  // In addition, this is only meant to remap memory which is file-backed, and
+  // mapped from a file which is still accessible.
+  //
   // Must not be called if |IsRemapPagesSupported()| return false.
   // Returns true for success.
   V8_WARN_UNUSED_RESULT static bool RemapPages(const void* address, size_t size,

src/execution/isolate.cc

@@ -3980,13 +3980,18 @@ bool Isolate::Init(SnapshotData* startup_snapshot_data,
   }
 
   if (V8_SHORT_BUILTIN_CALLS_BOOL && FLAG_short_builtin_calls) {
+#if defined(V8_OS_ANDROID)
+    // On Android, the check is not operative to detect memory, and re-embedded
+    // builtins don't have a memory cost.
+    is_short_builtin_calls_enabled_ = true;
+#else
     // Check if the system has more than 4GB of physical memory by comparing the
     // old space size with respective threshold value.
-    //
-    // Additionally, enable if there is already a process-wide CodeRange that
-    // has re-embedded builtins.
     is_short_builtin_calls_enabled_ = (heap_.MaxOldGenerationSize() >=
                                        kShortBuiltinCallsOldSpaceSizeThreshold);
+#endif  // defined(V8_OS_ANDROID)
+    // Additionally, enable if there is already a process-wide CodeRange that
+    // has re-embedded builtins.
     if (COMPRESS_POINTERS_IN_SHARED_CAGE_BOOL) {
       std::shared_ptr<CodeRange> code_range =
           CodeRange::GetProcessWideCodeRange();

src/snapshot/embedded/embedded-file-writer.cc

@@ -164,7 +164,7 @@ void EmbeddedFileWriter::WriteCodeSection(PlatformEmbeddedFileWriterBase* w,
        ++builtin) {
     WriteBuiltin(w, blob, builtin);
   }
-  w->PaddingAfterCode();
+  w->AlignToPageSizeIfNeeded();
   w->Newline();
 }
 

src/snapshot/embedded/platform-embedded-file-writer-base.h

@@ -58,7 +58,7 @@ class PlatformEmbeddedFileWriterBase {
   virtual void SectionRoData() = 0;
 
   virtual void AlignToCodeAlignment() = 0;
-  virtual void PaddingAfterCode() {}
+  virtual void AlignToPageSizeIfNeeded() {}
   virtual void AlignToDataAlignment() = 0;
 
   virtual void DeclareUint32(const char* name, uint32_t value) = 0;

src/snapshot/embedded/platform-embedded-file-writer-generic.cc

@@ -74,7 +74,12 @@ void PlatformEmbeddedFileWriterGeneric::DeclareSymbolGlobal(const char* name) {
 }
 
 void PlatformEmbeddedFileWriterGeneric::AlignToCodeAlignment() {
-#if V8_TARGET_ARCH_X64
+#if (V8_OS_ANDROID || V8_OS_LINUX) && \
+    (V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64)
+  // On these architectures and platforms, we remap the builtins, so need these
+  // to be aligned on a page boundary.
+  fprintf(fp_, ".balign 4096\n");
+#elif V8_TARGET_ARCH_X64
   // On x64 use 64-bytes code alignment to allow 64-bytes loop header alignment.
   STATIC_ASSERT(64 >= kCodeAlignment);
   fprintf(fp_, ".balign 64\n");
@@ -89,6 +94,14 @@ void PlatformEmbeddedFileWriterGeneric::AlignToCodeAlignment() {
 #endif
 }
 
+void PlatformEmbeddedFileWriterGeneric::AlignToPageSizeIfNeeded() {
+#if (V8_OS_ANDROID || V8_OS_LINUX) && \
+    (V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64)
+  // Since the builtins are remapped, need to pad until the next page boundary.
+  fprintf(fp_, ".balign 4096\n");
+#endif
+}
+
 void PlatformEmbeddedFileWriterGeneric::AlignToDataAlignment() {
   // On Windows ARM64, s390, PPC and possibly more platforms, aligned load
   // instructions are used to retrieve v8_Default_embedded_blob_ and/or

src/snapshot/embedded/platform-embedded-file-writer-generic.h

@@ -28,6 +28,7 @@ class PlatformEmbeddedFileWriterGeneric
   void SectionRoData() override;
 
   void AlignToCodeAlignment() override;
+  void AlignToPageSizeIfNeeded() override;
   void AlignToDataAlignment() override;
 
   void DeclareUint32(const char* name, uint32_t value) override;

src/snapshot/embedded/platform-embedded-file-writer-mac.cc

@@ -79,7 +79,7 @@ void PlatformEmbeddedFileWriterMac::AlignToCodeAlignment() {
 #endif
 }
 
-void PlatformEmbeddedFileWriterMac::PaddingAfterCode() {
+void PlatformEmbeddedFileWriterMac::AlignToPageSizeIfNeeded() {
 #if V8_TARGET_ARCH_ARM64
   // ARM64 macOS has a 16kiB page size. Since we want to remap builtins on the
   // heap, make sure that the trailing part of the page doesn't contain anything

src/snapshot/embedded/platform-embedded-file-writer-mac.h

@@ -26,7 +26,7 @@ class PlatformEmbeddedFileWriterMac : public PlatformEmbeddedFileWriterBase {
   void SectionRoData() override;
 
   void AlignToCodeAlignment() override;
-  void PaddingAfterCode() override;
+  void AlignToPageSizeIfNeeded() override;
   void AlignToDataAlignment() override;
 
   void DeclareUint32(const char* name, uint32_t value) override;

test/unittests/base/platform/platform-unittest.cc

@@ -6,8 +6,15 @@
 
 #include <cstring>
 
+#include "src/base/build_config.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
+#if V8_TARGET_OS_LINUX
+#include <sys/sysmacros.h>
+
+#include "src/base/platform/platform-linux.h"
+#endif
+
 #if V8_OS_WIN
 #include <windows.h>
 #endif
@@ -15,6 +22,17 @@
 namespace v8 {
 namespace base {
 
+#if V8_TARGET_OS_WIN
+// Alignemnt is constrained on Windows.
+constexpr size_t kMaxPageSize = 4096;
+#else
+constexpr size_t kMaxPageSize = 16384;
+#endif
+
+alignas(kMaxPageSize) const char kArray[kMaxPageSize] =
+    "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod "
+    "tempor incididunt ut labore et dolore magna aliqua.";
+
 TEST(OS, GetCurrentProcessId) {
 #if V8_OS_POSIX
   EXPECT_EQ(static_cast<int>(getpid()), OS::GetCurrentProcessId());
@@ -28,12 +46,9 @@ TEST(OS, GetCurrentProcessId) {
 
 TEST(OS, RemapPages) {
   if constexpr (OS::IsRemapPageSupported()) {
-    size_t size = base::OS::AllocatePageSize();
+    const size_t size = base::OS::AllocatePageSize();
-    // Data to be remapped, filled with data.
+    ASSERT_TRUE(size <= kMaxPageSize);
-    void* data = OS::Allocate(nullptr, size, base::OS::AllocatePageSize(),
+    const void* data = static_cast<const void*>(kArray);
-                              OS::MemoryPermission::kReadWrite);
-    ASSERT_TRUE(data);
-    memset(data, 0xab, size);
 
     // Target mapping.
     void* remapped_data =
@@ -45,11 +60,65 @@ TEST(OS, RemapPages) {
                                OS::MemoryPermission::kReadExecute));
     EXPECT_EQ(0, memcmp(remapped_data, data, size));
 
-    OS::Free(data, size);
     OS::Free(remapped_data, size);
   }
 }
 
+#if V8_TARGET_OS_LINUX
+TEST(OS, ParseProcMaps) {
+  // Truncated
+  std::string line = "00000000-12345678 r--p";
+  EXPECT_FALSE(MemoryRegion::FromMapsLine(line.c_str()));
+
+  // Constants below are for 64 bit architectures.
+#if V8_TARGET_ARCH_64_BIT
+  // File-backed.
+  line =
+      "7f861d1e3000-7f861d33b000 r-xp 00026000 fe:01 12583839                  "
+      " /lib/x86_64-linux-gnu/libc-2.33.so";
+  auto region = MemoryRegion::FromMapsLine(line.c_str());
+  EXPECT_TRUE(region);
+
+  EXPECT_EQ(region->start, 0x7f861d1e3000u);
+  EXPECT_EQ(region->end, 0x7f861d33b000u);
+  EXPECT_EQ(std::string(region->permissions), std::string("r-xp"));
+  EXPECT_EQ(region->offset, 0x00026000u);
+  EXPECT_EQ(region->dev, makedev(0xfe, 0x01));
+  EXPECT_EQ(region->inode, 12583839u);
+  EXPECT_EQ(region->pathname,
+            std::string("/lib/x86_64-linux-gnu/libc-2.33.so"));
+
+  // Anonymous, but named.
+  line =
+      "5611cc7eb000-5611cc80c000 rw-p 00000000 00:00 0                         "
+      " [heap]";
+  region = MemoryRegion::FromMapsLine(line.c_str());
+  EXPECT_TRUE(region);
+
+  EXPECT_EQ(region->start, 0x5611cc7eb000u);
+  EXPECT_EQ(region->end, 0x5611cc80c000u);
+  EXPECT_EQ(std::string(region->permissions), std::string("rw-p"));
+  EXPECT_EQ(region->offset, 0u);
+  EXPECT_EQ(region->dev, makedev(0x0, 0x0));
+  EXPECT_EQ(region->inode, 0u);
+  EXPECT_EQ(region->pathname, std::string("[heap]"));
+
+  // Anonymous, not named.
+  line = "5611cc7eb000-5611cc80c000 rw-p 00000000 00:00 0";
+  region = MemoryRegion::FromMapsLine(line.c_str());
+  EXPECT_TRUE(region);
+
+  EXPECT_EQ(region->start, 0x5611cc7eb000u);
+  EXPECT_EQ(region->end, 0x5611cc80c000u);
+  EXPECT_EQ(std::string(region->permissions), std::string("rw-p"));
+  EXPECT_EQ(region->offset, 0u);
+  EXPECT_EQ(region->dev, makedev(0x0, 0x0));
+  EXPECT_EQ(region->inode, 0u);
+  EXPECT_EQ(region->pathname, std::string(""));
+#endif  // V8_TARGET_ARCH_64_BIT
+}
+#endif  // V8_TARGET_OS_LINUX
+
 namespace {
 
 class ThreadLocalStorageTest : public Thread, public ::testing::Test {
@@ -105,7 +174,6 @@ class ThreadLocalStorageTest : public Thread, public ::testing::Test {
 
 }  // namespace
 
-
 TEST_F(ThreadLocalStorageTest, DoTest) {
   Run();
   CHECK(Start());