AuroraMiddleware/protobuf
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

[Bazel/C++] Augment cc_dist_library to generate lists of source files (#10026)

Browse Source 
This change largely moves internal logic that creates the the `CcFileList` provider from `build_systems.bzl` to `cc_dist_library.bzl`.

There are also some associated changes to the particular `cc_dist_library` targets, since the output didn't really make sense after previous BUILD.bazel refactoring. There is also a target now for `libprotoc`.
This commit is contained in:
David L. Jones 2022-05-24 12:31:56 -07:00 committed by GitHub
parent bbcc459aa0
commit 586b72c1ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 304 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
pkg/BUILD.bazel
View File

@ -305,9 +305,9 @@ gen_file_lists(
out_stem = "src_file_lists",
src_libs = {
# source rule: name in generated file
"//:protobuf": "libprotobuf",
"//src/google/protobuf/compiler:protoc_lib": "libprotoc",
"//:protobuf_lite": "libprotobuf_lite",
":protobuf": "libprotobuf",
":protoc": "libprotoc",
":protobuf_lite": "libprotobuf_lite",
},
)
@ -343,8 +343,8 @@ cc_dist_library(
}),
tags = ["manual"],
deps = [
"//:protobuf_lite",
"//src/google/protobuf:arena",
"//src/google/protobuf:protobuf_lite",
"//src/google/protobuf/io",
"//src/google/protobuf/io:io_win32",
"//src/google/protobuf/stubs:lite",
@ -362,8 +362,6 @@ cc_dist_library(
}),
tags = ["manual"],
deps = [
"//:protobuf",
"//:protobuf_lite",
"//src/google/protobuf:arena",
"//src/google/protobuf/compiler:importer",
"//src/google/protobuf/io",
@ -371,6 +369,8 @@ cc_dist_library(
"//src/google/protobuf/io:io_win32",
"//src/google/protobuf/io:printer",
"//src/google/protobuf/io:tokenizer",
"//src/google/protobuf:protobuf",
"//src/google/protobuf:protobuf_lite",
"//src/google/protobuf/stubs",
"//src/google/protobuf/stubs:lite",
"//src/google/protobuf/util:delimited_message_util",
@ -382,6 +382,22 @@ cc_dist_library(
],
)
cc_dist_library(
name = "protoc",
tags = ["manual"],
deps = [
"//src/google/protobuf/compiler:code_generator",
"//src/google/protobuf/compiler:command_line_interface",
"//src/google/protobuf/compiler/cpp",
"//src/google/protobuf/compiler/csharp",
"//src/google/protobuf/compiler/java",
"//src/google/protobuf/compiler/objectivec",
"//src/google/protobuf/compiler/php",
"//src/google/protobuf/compiler/python",
"//src/google/protobuf/compiler/ruby",
],
)
################################################################################
# Distribution sources
################################################################################

78
pkg/build_systems.bzl
View File

@ -1,6 +1,7 @@
# Starlark utilities for working with other build systems
load("@rules_pkg//:providers.bzl", "PackageFilegroupInfo", "PackageFilesInfo")
load(":cc_dist_library.bzl", "CcFileList")
################################################################################
# Macro to create CMake and Automake source lists.
@ -31,21 +32,6 @@ def gen_file_lists(name, out_stem, **kwargs):
# Aspect that extracts srcs, hdrs, etc.
################################################################################
CcFileList = provider(
doc = "List of files to be built into a library.",
fields = {
# As a rule of thumb, `hdrs` and `textual_hdrs` are the files that
# would be installed along with a prebuilt library.
"hdrs": "public header files, including those used by generated code",
"textual_hdrs": "files which are included but are not self-contained",
# The `internal_hdrs` are header files which appear in `srcs`.
# These are only used when compiling the library.
"internal_hdrs": "internal header files (only used to build .cc files)",
"srcs": "source files",
},
)
ProtoFileList = provider(
doc = "List of proto files and generated code to be built into a library.",
fields = {
@ -65,56 +51,11 @@ def _flatten_target_files(targets):
files.append(tfile)
return files
def _combine_cc_file_lists(file_lists):
hdrs = {}
textual_hdrs = {}
internal_hdrs = {}
srcs = {}
for file_list in file_lists:
hdrs.update({f: 1 for f in file_list.hdrs})
textual_hdrs.update({f: 1 for f in file_list.textual_hdrs})
internal_hdrs.update({f: 1 for f in file_list.internal_hdrs})
srcs.update({f: 1 for f in file_list.srcs})
return CcFileList(
hdrs = sorted(hdrs.keys()),
textual_hdrs = sorted(textual_hdrs.keys()),
internal_hdrs = sorted(internal_hdrs.keys()),
srcs = sorted(srcs.keys()),
)
def _file_list_aspect_impl(target, ctx):
# We're going to reach directly into the attrs on the traversed rule.
rule_attr = ctx.rule.attr
providers = []
# Extract sources from a `cc_library` (or similar):
if CcInfo in target:
# CcInfo is a proxy for what we expect this rule to look like.
# However, some deps may expose `CcInfo` without having `srcs`,
# `hdrs`, etc., so we use `getattr` to handle that gracefully.
internal_hdrs = []
srcs = []
# Filter `srcs` so it only contains source files. Headers will go
# into `internal_headers`.
for src in _flatten_target_files(getattr(rule_attr, "srcs", [])):
if src.extension.lower() in ["c", "cc", "cpp", "cxx"]:
srcs.append(src)
else:
internal_hdrs.append(src)
providers.append(CcFileList(
hdrs = _flatten_target_files(getattr(rule_attr, "hdrs", [])),
textual_hdrs = _flatten_target_files(getattr(
rule_attr,
"textual_hdrs",
[],
)),
internal_hdrs = internal_hdrs,
srcs = srcs,
))
# Extract sources from a `proto_library`:
if ProtoInfo in target:
proto_srcs = []
@ -197,19 +138,28 @@ def _create_file_list_impl(ctx, fragment_generator):
for srcrule, libname in ctx.attr.src_libs.items():
if CcFileList in srcrule:
cc_file_list = srcrule[CcFileList]
# Turn depsets of files into sorted lists.
srcs = sorted(cc_file_list.srcs.to_list())
hdrs = sorted(
depset(transitive = [
cc_file_list.textual_hdrs,
cc_file_list.hdrs,
]).to_list(),
)
fragments.extend([
fragment_generator(
srcrule.label,
libname + "_srcs",
ctx.attr.source_prefix,
[f.short_path for f in cc_file_list.srcs],
[f.short_path for f in srcs],
),
fragment_generator(
srcrule.label,
libname + "_hdrs",
ctx.attr.source_prefix,
[f.short_path for f in (cc_file_list.hdrs +
cc_file_list.textual_hdrs)],
[f.short_path for f in hdrs],
),
])
@ -247,7 +197,7 @@ def _create_file_list_impl(ctx, fragment_generator):
# keys are the destination:
files.update(srcrule[PackageFilesInfo].dest_src_map)
if files == {} and DefaultInfo in srcrule and CcInfo not in srcrule:
if files == {} and DefaultInfo in srcrule and CcFileList not in srcrule:
# This could be an individual file or filegroup.
# We explicitly ignore rules with CcInfo, since their
# output artifacts are libraries or binaries.

304
pkg/cc_dist_library.bzl
View File

@ -3,12 +3,32 @@
load("@rules_cc//cc:action_names.bzl", cc_action_names = "ACTION_NAMES")
load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cc_toolchain")
################################################################################
# Archive/linking support
################################################################################
def _collect_linker_input_objects(dep_label, cc_info, objs, pic_objs):
"""Accumulate .o and .pic.o files into `objs` and `pic_objs`."""
link_ctx = cc_info.linking_context
if link_ctx == None:
return
linker_inputs = link_ctx.linker_inputs.to_list()
for link_input in linker_inputs:
if link_input.owner != dep_label:
# This is a transitive dep: skip it.
continue
for lib in link_input.libraries:
objs.extend(lib.objects or [])
pic_objs.extend(lib.pic_objects or [])
# Creates an action to build the `output_file` static library (archive)
# using `object_files`.
def _create_archive_action(
ctx,
feature_configuration,
cc_toolchain,
cc_toolchain_info,
output_file,
object_files):
# Based on Bazel's src/main/starlark/builtins_bzl/common/cc/cc_import.bzl:
@ -16,7 +36,7 @@ def _create_archive_action(
# Build the command line and add args for all of the input files:
archiver_variables = cc_common.create_link_variables(
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
cc_toolchain = cc_toolchain_info,
output_file = output_file.path,
is_using_linker = False,
)
@ -48,7 +68,7 @@ def _create_archive_action(
inputs = depset(
direct = object_files,
transitive = [
cc_toolchain.all_files,
cc_toolchain_info.all_files,
],
),
use_default_shell_env = False,
@ -56,72 +76,15 @@ def _create_archive_action(
mnemonic = "CppArchiveDist",
)
# Implementation for cc_dist_library rule.
def _cc_dist_library_impl(ctx):
cc_toolchain_info = find_cc_toolchain(ctx)
if cc_toolchain_info.ar_executable == None:
return []
feature_configuration = cc_common.configure_features(
ctx = ctx,
cc_toolchain = cc_toolchain_info,
)
# Collect the set of object files from the immediate deps.
objs = []
pic_objs = []
for dep in ctx.attr.deps:
if CcInfo not in dep:
continue
link_ctx = dep[CcInfo].linking_context
if link_ctx == None:
continue
linker_inputs = link_ctx.linker_inputs.to_list()
for link_input in linker_inputs:
if link_input.owner != dep.label:
# This is a transitive dep: skip it.
continue
for lib in link_input.libraries:
objs.extend(lib.objects or [])
pic_objs.extend(lib.pic_objects or [])
# For static libraries, build separately with and without pic.
stemname = "lib" + ctx.label.name
outputs = []
if len(objs) > 0:
archive_out = ctx.actions.declare_file(stemname + ".a")
_create_archive_action(
ctx,
feature_configuration,
cc_toolchain_info,
archive_out,
objs,
)
outputs.append(archive_out)
if len(pic_objs) > 0:
pic_archive_out = ctx.actions.declare_file(stemname + ".pic.a")
_create_archive_action(
ctx,
feature_configuration,
cc_toolchain_info,
pic_archive_out,
pic_objs,
)
outputs.append(pic_archive_out)
# For dynamic libraries, use the `cc_common.link` command to ensure
# everything gets built correctly according to toolchain definitions.
def _create_dso_link_action(
ctx,
feature_configuration,
cc_toolchain_info,
object_files,
pic_object_files):
compilation_outputs = cc_common.create_compilation_outputs(
objects = depset(objs),
pic_objects = depset(pic_objs),
objects = depset(object_files),
pic_objects = depset(pic_object_files),
)
link_output = cc_common.link(
actions = ctx.actions,
@ -134,6 +97,8 @@ def _cc_dist_library_impl(ctx):
)
library_to_link = link_output.library_to_link
outputs = []
# Note: library_to_link.dynamic_library and interface_library are often
# symlinks in the solib directory. For DefaultInfo, prefer reporting
# the resolved artifact paths.
@ -147,6 +112,207 @@ def _cc_dist_library_impl(ctx):
elif library_to_link.interface_library != None:
outputs.append(library_to_link.interface_library)
return outputs
################################################################################
# Source file/header support
################################################################################
CcFileList = provider(
doc = "List of files to be built into a library.",
fields = {
# As a rule of thumb, `hdrs` and `textual_hdrs` are the files that
# would be installed along with a prebuilt library.
"hdrs": "public header files, including those used by generated code",
"textual_hdrs": "files which are included but are not self-contained",
# The `internal_hdrs` are header files which appear in `srcs`.
# These are only used when compiling the library.
"internal_hdrs": "internal header files (only used to build .cc files)",
"srcs": "source files",
},
)
def _flatten_target_files(targets):
return depset(transitive = [target.files for target in targets])
files = []
for target in targets:
files.extend(target.files.to_list())
return files
def _cc_file_list_aspect_impl(target, ctx):
# Extract sources from a `cc_library` (or similar):
if CcInfo not in target:
return []
# We're going to reach directly into the attrs on the traversed rule.
rule_attr = ctx.rule.attr
# CcInfo is a proxy for what we expect this rule to look like.
# However, some deps may expose `CcInfo` without having `srcs`,
# `hdrs`, etc., so we use `getattr` to handle that gracefully.
internal_hdrs = []
srcs = []
# Filter `srcs` so it only contains source files. Headers will go
# into `internal_headers`.
for src in _flatten_target_files(getattr(rule_attr, "srcs", [])).to_list():
if src.extension.lower() in ["c", "cc", "cpp", "cxx"]:
srcs.append(src)
else:
internal_hdrs.append(src)
return [CcFileList(
hdrs = _flatten_target_files(getattr(rule_attr, "hdrs", depset())),
textual_hdrs = _flatten_target_files(getattr(
rule_attr,
"textual_hdrs",
depset(),
)),
internal_hdrs = depset(internal_hdrs),
srcs = depset(srcs),
)]
cc_file_list_aspect = aspect(
doc = """
Aspect to provide the list of sources and headers from a rule.
Output is CcFileList. Example:
cc_library(
name = "foo",
srcs = [
"foo.cc",
"foo_internal.h",
],
hdrs = ["foo.h"],
textual_hdrs = ["foo_inl.inc"],
)
# produces:
# CcFileList(
# hdrs = depset([File("foo.h")]),
# textual_hdrs = depset([File("foo_inl.inc")]),
# internal_hdrs = depset([File("foo_internal.h")]),
# srcs = depset([File("foo.cc")]),
# )
""",
implementation = _cc_file_list_aspect_impl,
)
################################################################################
# Rule impl
################################################################################
def _collect_inputs(deps):
"""Collects files from a list of immediate deps.
This rule collects source files and linker inputs for C++ deps. Only
these immediate deps are considered, not transitive deps.
The return value is a struct with object files (linker inputs),
partitioned by PIC and non-pic, and the rules' source and header files:
struct(
objects = ..., # non-PIC object files
pic_objects = ..., # PIC objects
cc_file_list = ..., # a CcFileList
)
Args:
deps: Iterable of immediate deps. These will be treated as the "inputs,"
but not the transitive deps.
Returns:
A struct with linker inputs, source files, and header files.
"""
objs = []
pic_objs = []
# The returned CcFileList will contain depsets of the deps' file lists.
# These lists hold `depset()`s from each of `deps`.
srcs = []
hdrs = []
internal_hdrs = []
textual_hdrs = []
for dep in deps:
if CcInfo in dep:
_collect_linker_input_objects(
dep.label,
dep[CcInfo],
objs,
pic_objs,
)
if CcFileList in dep:
cfl = dep[CcFileList]
srcs.append(cfl.srcs)
hdrs.append(cfl.hdrs)
internal_hdrs.append(cfl.internal_hdrs)
textual_hdrs.append(cfl.textual_hdrs)
return struct(
objects = objs,
pic_objects = pic_objs,
cc_file_list = CcFileList(
srcs = depset(transitive = srcs),
hdrs = depset(transitive = hdrs),
internal_hdrs = depset(transitive = internal_hdrs),
textual_hdrs = depset(transitive = textual_hdrs),
),
)
# Implementation for cc_dist_library rule.
def _cc_dist_library_impl(ctx):
cc_toolchain_info = find_cc_toolchain(ctx)
feature_configuration = cc_common.configure_features(
ctx = ctx,
cc_toolchain = cc_toolchain_info,
)
inputs = _collect_inputs(ctx.attr.deps)
# For static libraries, build separately with and without pic.
stemname = "lib" + ctx.label.name
outputs = []
if len(inputs.objects) > 0:
archive_out = ctx.actions.declare_file(stemname + ".a")
_create_archive_action(
ctx,
feature_configuration,
cc_toolchain_info,
archive_out,
inputs.objects,
)
outputs.append(archive_out)
if len(inputs.pic_objects) > 0:
pic_archive_out = ctx.actions.declare_file(stemname + ".pic.a")
_create_archive_action(
ctx,
feature_configuration,
cc_toolchain_info,
pic_archive_out,
inputs.pic_objects,
)
outputs.append(pic_archive_out)
# For dynamic libraries, use the `cc_common.link` command to ensure
# everything gets built correctly according to toolchain definitions.
outputs.extend(_create_dso_link_action(
ctx,
feature_configuration,
cc_toolchain_info,
inputs.objects,
inputs.pic_objects,
))
# We could expose the libraries for use from cc rules:
#
# linking_context = cc_common.create_linking_context(
@ -169,6 +335,7 @@ def _cc_dist_library_impl(ctx):
return [
DefaultInfo(files = depset(outputs)),
inputs.cc_file_list,
]
cc_dist_library = rule(
@ -214,6 +381,7 @@ Example:
"Only these targets' compilation outputs will be " +
"included (i.e., the transitive dependencies are not " +
"included in the output)."),
aspects = [cc_file_list_aspect],
),
"linkopts": attr.string_list(
doc = ("Add these flags to the C++ linker command when creating " +

2
src/google/protobuf/BUILD.bazel
View File

@ -160,6 +160,7 @@ cc_library(
linkopts = LINK_OPTS,
visibility = [
"//:__pkg__",
"//pkg:__pkg__",
"//src/google/protobuf:__subpackages__",
],
# In Bazel 6.0+, these will be `interface_deps`:
@ -209,6 +210,7 @@ cc_library(
linkopts = LINK_OPTS,
visibility = [
"//:__pkg__",
"//pkg:__pkg__",
"//src/google/protobuf:__subpackages__",
],
deps = [

5
src/google/protobuf/compiler/cpp/BUILD.bazel
View File

@ -48,7 +48,10 @@ cc_library(
],
copts = COPTS,
include_prefix = "google/protobuf/compiler/cpp",
visibility = ["//src/google/protobuf/compiler:__pkg__"],
visibility = [
"//pkg:__pkg__",
"//src/google/protobuf/compiler:__pkg__",
],
deps = [
"//:protobuf",
"//src/google/protobuf/compiler:code_generator",

7
src/google/protobuf/compiler/csharp/BUILD.bazel
View File

@ -51,7 +51,10 @@ cc_library(
"//conditions:default": ["-Wno-overloaded-virtual"],
}),
include_prefix = "google/protobuf/compiler/csharp",
visibility = ["//src/google/protobuf/compiler:__pkg__"],
visibility = [
"//pkg:__pkg__",
"//src/google/protobuf/compiler:__pkg__",
],
deps = [
"//:protobuf",
"//src/google/protobuf/compiler:code_generator",
@ -62,11 +65,11 @@ cc_test(
name = "bootstrap_unittest",
srcs = ["csharp_bootstrap_unittest.cc"],
data = [
"//src/google/protobuf:descriptor_proto_srcs",
"//:well_known_type_protos",
"//conformance:all_files",
"//conformance:conformance_proto",
"//csharp:wkt_cs_srcs",
"//src/google/protobuf:descriptor_proto_srcs",
"//src/google/protobuf:testdata",
],
deps = [

7
src/google/protobuf/compiler/java/BUILD.bazel
View File

@ -74,7 +74,10 @@ cc_library(
],
copts = COPTS,
include_prefix = "google/protobuf/compiler/java",
visibility = ["//src/google/protobuf/compiler:__pkg__"],
visibility = [
"//pkg:__pkg__",
"//src/google/protobuf/compiler:__pkg__",
],
deps = [
"//:protobuf",
"//src/google/protobuf/compiler:code_generator",
@ -85,9 +88,9 @@ cc_test(
name = "doc_comment_unittest",
srcs = ["doc_comment_unittest.cc"],
data = [
"//src/google/protobuf:descriptor_proto_srcs",
"//:well_known_type_protos",
"//conformance:conformance_proto",
"//src/google/protobuf:descriptor_proto_srcs",
],
deps = [
":java",

5
src/google/protobuf/compiler/objectivec/BUILD.bazel
View File

@ -39,7 +39,10 @@ cc_library(
],
copts = COPTS,
include_prefix = "google/protobuf/compiler/objectivec",
visibility = ["//src/google/protobuf/compiler:__pkg__"],
visibility = [
"//pkg:__pkg__",
"//src/google/protobuf/compiler:__pkg__",
],
deps = [
"//:protobuf",
"//src/google/protobuf/compiler:code_generator",

5
src/google/protobuf/compiler/php/BUILD.bazel
View File

@ -12,7 +12,10 @@ cc_library(
hdrs = ["php_generator.h"],
copts = COPTS,
include_prefix = "google/protobuf/compiler/php",
visibility = ["//src/google/protobuf/compiler:__pkg__"],
visibility = [
"//pkg:__pkg__",
"//src/google/protobuf/compiler:__pkg__",
],
deps = [
"//:protobuf",
"//src/google/protobuf/compiler:code_generator",

5
src/google/protobuf/compiler/python/BUILD.bazel
View File

@ -20,7 +20,10 @@ cc_library(
],
copts = COPTS,
include_prefix = "google/protobuf/compiler/python",
visibility = ["//src/google/protobuf/compiler:__pkg__"],
visibility = [
"//pkg:__pkg__",
"//src/google/protobuf/compiler:__pkg__",
],
deps = [
"//:protobuf",
"//src/google/protobuf/compiler:code_generator",

5
src/google/protobuf/compiler/ruby/BUILD.bazel
View File

@ -12,7 +12,10 @@ cc_library(
hdrs = ["ruby_generator.h"],
copts = COPTS,
include_prefix = "google/protobuf/compiler/ruby",
visibility = ["//src/google/protobuf/compiler:__pkg__"],
visibility = [
"//pkg:__pkg__",
"//src/google/protobuf/compiler:__pkg__",
],
deps = [
"//:protobuf",
"//src/google/protobuf/compiler:code_generator",