Added a Bazel dependency for upb (#9812)

* WIP.

* Restore deleted __init__.py files.

* Updated version of upb.

* Moved upb from a submodule to a Bazel dep.

* Updated to new upb which can tolerate missing system_python.

* Updated to a new upb.

BUILD

@@ -531,7 +531,10 @@ cc_binary(
 filegroup(
     name = "testdata",
     srcs = glob(["src/google/protobuf/testdata/**/*"]),
-    visibility = ["//:__subpackages__"],
+    visibility = [
+        "//:__subpackages__",
+        "@upb//:__subpackages__",
+    ],
 )
 
 RELATIVE_LITE_TEST_PROTOS = [
@@ -893,13 +896,23 @@ py_library(
         [
             "python/google/protobuf/**/*.py",
         ],
-        exclude = [
+    ),
+    imports = ["python"],
+    srcs_version = "PY2AND3",
+    visibility = ["@upb//:__subpackages__"],
+)
+
+py_library(
+    name = "python_test_srcs",
+    srcs = glob(
+        [
             "python/google/protobuf/internal/*_test.py",
             "python/google/protobuf/internal/test_util.py",
         ],
     ),
     imports = ["python"],
-    srcs_version = "PY2AND3",
+    srcs_version = "PY3",
+    visibility = ["@upb//:__subpackages__"],
 )
 
 cc_binary(
@@ -996,9 +1009,21 @@ internal_copied_filegroup(
 COPIED_WELL_KNOWN_PROTOS = ["python/" + s[4:] for s in WELL_KNOWN_PROTOS]
 
 py_proto_library(
-    name = "protobuf_python",
+    name = "well_known_types_py_pb2",
     srcs = COPIED_WELL_KNOWN_PROTOS,
     include = "python",
+    default_runtime = "",
+    protoc = ":protoc",
+    srcs_version = "PY2AND3",
+    visibility = ["@upb//:__subpackages__"],
+)
+
+py_library(
+    name = "protobuf_python",
+    deps = [
+        ":well_known_types_py_pb2",
+        ":python_srcs",
+    ],
     data = select({
         "//conditions:default": [],
         ":use_fast_cpp_protos": [
@@ -1006,13 +1031,6 @@ py_proto_library(
             ":python/google/protobuf/pyext/_message.so",
         ],
     }),
-    default_runtime = "",
-    protoc = ":protoc",
-    py_libs = [
-        ":python_srcs",
-    ],
-    srcs_version = "PY2AND3",
-    visibility = ["//visibility:public"],
 )
 
 # Copy the test proto files from src/google/protobuf to
@@ -1040,7 +1058,8 @@ py_proto_library(
     default_runtime = "",
     protoc = ":protoc",
     srcs_version = "PY2AND3",
-    deps = [":protobuf_python"],
+    deps = [":well_known_types_py_pb2"],
+    visibility = ["//visibility:public"],
 )
 
 py_proto_library(
@@ -1054,6 +1073,7 @@ py_proto_library(
     protoc = ":protoc",
     srcs_version = "PY2AND3",
     deps = [":python_common_test_protos"],
+    visibility = ["//visibility:public"],
 )
 
 py_library(

WORKSPACE

@@ -66,3 +66,6 @@ kotlin_repositories()
 
 load("@io_bazel_rules_kotlin//kotlin:core.bzl", "kt_register_toolchains")
 kt_register_toolchains()
+
+load("@upb//bazel:workspace_deps.bzl", "upb_deps")
+upb_deps()

protobuf_deps.bzl

@@ -1,6 +1,7 @@
 """Load dependencies needed to compile the protobuf library as a 3rd-party consumer."""
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 PROTOBUF_MAVEN_ARTIFACTS = [
     "com.google.code.findbugs:jsr305:3.0.2",
@@ -72,8 +73,9 @@ def protobuf_deps():
     if not native.existing_rule("rules_python"):
         http_archive(
             name = "rules_python",
-            sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0",
-            urls = ["https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz"],
+            sha256 = "9fcf91dbcc31fde6d1edb15f117246d912c33c36f44cf681976bd886538deba6",
+            strip_prefix = "rules_python-0.8.0",
+            url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.8.0.tar.gz",
         )
 
     if not native.existing_rule("rules_jvm_external"):
@@ -100,3 +102,10 @@ def protobuf_deps():
             urls = ["https://github.com/bazelbuild/rules_kotlin/releases/download/v1.5.0-beta-4/rules_kotlin_release.tgz"],
             sha256 = "6cbd4e5768bdfae1598662e40272729ec9ece8b7bded8f0d2c81c8ff96dc139d",
         )
+
+    if not native.existing_rule("upb"):
+        git_repository(
+            name = "upb",
+            remote = "https://github.com/protocolbuffers/upb.git",
+            commit = "a0bc6693b069e6d1f083c7727ea08621cea4155e",
+        )

python/google/protobuf/internal/test_util.py

@@ -637,6 +637,13 @@ def GoldenFile(filename):
     # Found it.  Load the golden file from the testdata directory.
     return open(full_path, 'rb')
 
+  # Search for cross-repo path.
+  full_path = os.path.join('external/com_google_protobuf/src/google/protobuf/testdata',
+                           filename)
+  if os.path.exists(full_path):
+    # Found it.  Load the golden file from the testdata directory.
+    return open(full_path, 'rb')
+
   raise RuntimeError(
       'Could not find golden files.  This test must be run from within the '
       'protobuf source package so that it can read test data files from the '

python/google/protobuf/internal/testing_refleaks.py

@@ -67,6 +67,12 @@ class ReferenceLeakCheckerMixin(object):
   NB_RUNS = 3
 
   def run(self, result=None):
+    testMethod = getattr(self, self._testMethodName)
+    expecting_failure_method = getattr(testMethod, "__unittest_expecting_failure__", False)
+    expecting_failure_class = getattr(self, "__unittest_expecting_failure__", False)
+    if expecting_failure_class or expecting_failure_method:
+      return
+
     # python_message.py registers all Message classes to some pickle global
     # registry, which makes the classes immortal.
     # We save a copy of this registry, and reset it before we could references.