From 7628f293961b67275db26607e50e570c873e0a49 Mon Sep 17 00:00:00 2001 From: Chris Eason Date: Sun, 13 Jan 2019 19:22:04 -0500 Subject: [PATCH 1/2] Reimplement java_grpc_library using an aspect Using an aspect allows the implementation to more closely match the user interface and behavior of the 'java_proto_library' native rule. Also noteworthy is this implementation reconfigures itself for protobuf-lite generated messages+runtime and okhttp gRPC transport when consumed by an android_* rule. --- java_grpc_library/BUILD.bazel | 52 ++++++++ java_grpc_library/def.bzl | 149 ++++++++++++++++++++++ java_grpc_library/toolchain.bzl | 213 ++++++++++++++++++++++++++++++++ protobuf-lite/BUILD.bazel | 1 + 4 files changed, 415 insertions(+) create mode 100644 java_grpc_library/BUILD.bazel create mode 100644 java_grpc_library/def.bzl create mode 100644 java_grpc_library/toolchain.bzl diff --git a/java_grpc_library/BUILD.bazel b/java_grpc_library/BUILD.bazel new file mode 100644 index 00000000000..1e3bfab503a --- /dev/null +++ b/java_grpc_library/BUILD.bazel @@ -0,0 +1,52 @@ +load(":toolchain.bzl", "grpc_proto_toolchain") + +package(default_visibility = ["//visibility:public"]) + +grpc_proto_toolchain( + name = "toolchain", + grpc_plugin = "//compiler:grpc_java_plugin", + grpc_plugin_opts = select({ + "//protobuf-lite:android": ["lite"], + "//conditions:default": [], + }), + java_plugin = select({ + "//protobuf-lite:android": "@com_google_protobuf_javalite//:protoc_gen_javalite", + "//conditions:default": None, + }), + output_prefix = select({ + "//protobuf-lite:android": "lib{name}-grpc-lite", + "//conditions:default": "lib{name}-grpc", + }), + protoc = "@com_google_protobuf//:protoc", + single_jar = select({ + "//protobuf-lite:android": True, + "//conditions:default": False, + }), + exports = [ + "@io_grpc_grpc_java//core", + "@io_grpc_grpc_java//stub", + ] + select({ + "//protobuf-lite:android": ["@com_google_protobuf_javalite//:protobuf_java_lite"], + "//conditions:default": ["@com_google_protobuf//:protobuf_java"], + }), + runtime_deps = select({ + "//protobuf-lite:android": ["@io_grpc_grpc_java//okhttp"], + "//conditions:default": ["@io_grpc_grpc_java//netty"], + }), + deps = [ + "@com_google_errorprone_error_prone_annotations//jar", + "@com_google_guava_guava//jar", + "@io_grpc_grpc_java//core", + "@io_grpc_grpc_java//stub", + "@javax_annotation_javax_annotation_api//jar", + ] + select({ + "//protobuf-lite:android": [ + "@com_google_protobuf_javalite//:protobuf_java_lite", + "@io_grpc_grpc_java//protobuf-lite", + ], + "//conditions:default": [ + "@com_google_protobuf//:protobuf_java", + "@io_grpc_grpc_java//protobuf", + ], + }), +) diff --git a/java_grpc_library/def.bzl b/java_grpc_library/def.bzl new file mode 100644 index 00000000000..15c227b349b --- /dev/null +++ b/java_grpc_library/def.bzl @@ -0,0 +1,149 @@ +load(":toolchain.bzl", "GrpcProtoInfo") + +_GRPC_TOOLCHAIN = "//java_grpc_library:toolchain" + +_AspectInfo = provider() + +def _aspect_impl(target, ctx): + # proto_info = target[ProtoInfo] # TODO: update provider when ProtoInfo is a real thing + proto_info = target.proto + tc = ctx.attr._toolchain[platform_common.ToolchainInfo] + transitive_compiled_jars = [t[_AspectInfo].compiled_jars for t in ctx.rule.attr.deps] + + # return (and merge) transitive deps if there are no sources to compile + if len(proto_info.direct_sources) == 0: + java_info = java_common.merge([dep[JavaInfo] for dep in ctx.rule.attr.deps]) + grpc_info = tc.merge([dep[GrpcProtoInfo] for dep in ctx.rule.attr.deps]) + compiled_jars = depset(transitive = transitive_compiled_jars) + return struct( + proto_java = java_info, + providers = [ + java_info, + grpc_info, + _AspectInfo(compiled_jars = depset(transitive = transitive_compiled_jars)), + ], + ) + + # Compile protos to srcs + compiled_jar = tc.declare_file(ctx, ".jar") + java_srcs = tc.declare_file(ctx, "-java-sources.jar") + grpc_srcs = tc.declare_file(ctx, "-grpc-sources.jar") + source_jars = [grpc_srcs, java_srcs] + grpc_info = tc.compile_proto( + ctx, + output_grpc_jar = grpc_srcs, + output_java_jar = java_srcs, + proto_info = proto_info, + deps = [t[GrpcProtoInfo] for t in ctx.rule.attr.deps], + ) + + # Compile srcs to jars + java_info = tc.compile_srcjars( + ctx, + source_jars = source_jars, + output_jar = compiled_jar, + deps = [t[JavaInfo] for t in ctx.rule.attr.deps], + ) + return struct( + proto_java = java_info, + providers = [ + grpc_info, + java_info, + _AspectInfo(compiled_jars = depset( + direct = [compiled_jar], + transitive = transitive_compiled_jars, + )), + ], + ) + +_protocompiler_aspect = aspect( + _aspect_impl, + # provide 'proto_java' legacy provider so IntelliJ plugin is happy :-\ + provides = ["proto_java", JavaInfo, GrpcProtoInfo], + attr_aspects = ["deps"], + fragments = ["java", "android"], + attrs = { + "_toolchain": attr.label( + providers = [platform_common.ToolchainInfo], + default = Label(_GRPC_TOOLCHAIN), + ), + }, +) + +def _rule_impl(ctx): + tc = ctx.attr._toolchain[platform_common.ToolchainInfo] + if ctx.attr.single_jar == "auto": + single_jar = tc.single_jar + elif ctx.attr.single_jar == "yes": + single_jar = True + elif ctx.attr.single_jar == "no": + single_jar = False + else: + fail("Unreachable.") + + if single_jar: + # Aggregate all sources and compile a new JavaInfo. This is really + # not ideal, but is here for compatibility reasons (Android dex'ing + # aspect, I'm looking at you..) + compiled_jar = tc.declare_file(ctx, ".jar") + grpc_info = tc.merge([t[GrpcProtoInfo] for t in ctx.attr.deps]) + java_info = tc.compile_srcjars( + ctx, + source_jars = grpc_info.source_jars.to_list(), + output_jar = compiled_jar, + exports = [dep[JavaInfo] for dep in tc.exports], + ) + runfiles = ctx.runfiles(files = [compiled_jar]) + for dep in tc.deps + tc.runtime_deps: + runfiles = dep.default_runfiles.merge(runfiles) + return [ + java_info, + DefaultInfo( + files = depset(direct = [compiled_jar]), + runfiles = runfiles, + ), + ] + else: + # merge & return the aspect-generated JavaInfos + java_info = java_common.merge([ + dep[JavaInfo] + for dep in ctx.attr.deps + tc.exports + ]) + compiled_jars = depset(transitive = [t[_AspectInfo].compiled_jars for t in ctx.attr.deps]) + runfiles = ctx.runfiles(transitive_files = compiled_jars) + for dep in tc.deps + tc.runtime_deps: + runfiles = dep.default_runfiles.merge(runfiles) + return [ + java_info, + DefaultInfo( + files = compiled_jars, + runfiles = runfiles, + ), + ] + fail("Unreachable.") + +java_grpc_library = rule( + _rule_impl, + provides = [JavaInfo], + fragments = ["android", "java"], + attrs = { + "deps": attr.label_list( + providers = [ + ["proto"], + # TODO: enable when ProtoInfo is real + #[ProtoInfo], + ], + aspects = [_protocompiler_aspect], + mandatory = True, + ), + "single_jar": attr.string( + values = ["yes", "no", "auto"], + default = "auto", + doc = "Combine generated code from all transitive deps into a single compiled jar instead of individual jars per transitive dependency. This is less efficient, but may be necessary for some rules whose aspects don't properly propagate along this rule's transitive dependencies.", + ), + "_toolchain": attr.label( + providers = [platform_common.ToolchainInfo], + default = Label(_GRPC_TOOLCHAIN), + ), + }, +) diff --git a/java_grpc_library/toolchain.bzl b/java_grpc_library/toolchain.bzl new file mode 100644 index 00000000000..f5074c73d18 --- /dev/null +++ b/java_grpc_library/toolchain.bzl @@ -0,0 +1,213 @@ +_toolchain_attrs = { + "deps": attr.label_list(default = [], providers = [JavaInfo]), + "runtime_deps": attr.label_list(default = [], providers = [JavaInfo]), + "exports": attr.label_list(default = [], providers = [JavaInfo]), + "output_prefix": attr.string(default = "lib{name}"), + "single_jar": attr.bool(default = False), + "java_plugin_opts": attr.string_list(default = []), + "java_plugin": attr.label( + executable = True, + cfg = "host", + default = None, + ), + "grpc_plugin_opts": attr.string_list(default = []), + "grpc_plugin": attr.label( + executable = True, + cfg = "host", + default = Label("//compiler:grpc_java_plugin"), + ), + "protoc": attr.label( + executable = True, + cfg = "host", + default = Label("@com_google_protobuf//:protoc"), + ), + "_java_toolchain": attr.label( + default = Label("@bazel_tools//tools/jdk:toolchain"), + cfg = "host", + ), + "_host_javabase": attr.label( + default = Label("@bazel_tools//tools/jdk:current_java_runtime"), + cfg = "host", + ), +} + +GrpcProtoInfo = provider( + fields = { + "source_jars": "Depset[File] of source jars", + "importmap": "(preorder)Depset[Tuple[Importpath,File]] of transitive protos", + "imports": "Depset[File] of transitive protos", + }, +) + +def _get_toolchain(ctx): + return ctx.attr._toolchain[platform_common.ToolchainInfo] + +def _path_ignoring_repository(f): + if (len(f.owner.workspace_root) == 0): + return f.short_path + return f.path[f.path.find(f.owner.workspace_root) + len(f.owner.workspace_root) + 1:] + +def _importmap_args(import_file_tuple): + i, f = import_file_tuple + return "-I%s=%s" % (i, f.path) + +def _compile_proto( + ctx, + output_grpc_jar = None, + output_java_jar = None, + proto_info = None, + deps = []): + """ + Args: + proto_info: (ProtoInfo) + deps: (list[GrpcProtoInfo]) + grpc_output_jar: (File) + java_output_jar: (File) + + Returns: + GrpcProtoInfo + """ + tc = _get_toolchain(ctx) + + # Tuples of [ImportedName => File] + direct_importmap = [] + proto_files = [] + + # TODO: update this logic once '{strip_,}import_prefix' attrs are real + prefix = proto_info.proto_source_root + "/" + for f in proto_info.direct_sources: + imported_name = _path_ignoring_repository(f) + if f.short_path.startswith(prefix): + imported_name = imported_name[len(prefix):] + direct_importmap += [(imported_name, f)] + proto_files += [f] + + # create depsets for use in compilation action & to output via provider + importmap = depset(direct = direct_importmap, transitive = [ + dep.importmap + for dep in deps + ], order = "preorder") + imports = depset(direct = proto_files, transitive = [ + dep.imports + for dep in deps + ]) + + protoc = tc.protoc.files_to_run.executable + grpc_plugin = tc.grpc_plugin.files_to_run.executable + java_plugin = tc.java_plugin.files_to_run.executable if tc.java_plugin else None + + # generate java & grpc srcs + args = ctx.actions.args() + args.add_all(importmap, map_each = _importmap_args) + args.add("--plugin=protoc-gen-grpc-java=%s" % grpc_plugin.path) + args.add("--grpc-java_out={opts}:{file}".format( + opts = ",".join(tc.grpc_plugin_opts), + file = output_grpc_jar.path, + )) + if java_plugin: + args.add("--plugin=protoc-gen-javaplugin=%s" % java_plugin.path) + args.add("--javaplugin_out={opts}:{file}".format( + opts = ",".join(tc.java_plugin_opts), + file = output_java_jar.path, + )) + else: + args.add("--java_out={opts}:{file}".format( + opts = ",".join(tc.java_plugin_opts), + file = output_java_jar.path, + )) + args.add_all(proto_files) + + ctx.actions.run( + inputs = imports, + outputs = [output_grpc_jar, output_java_jar], + executable = protoc, + arguments = [args], + tools = [ + grpc_plugin, + ] + ([java_plugin] if java_plugin else []), + ) + return GrpcProtoInfo( + imports = imports, + importmap = importmap, + source_jars = depset( + direct = [output_java_jar, output_grpc_jar], + transitive = [dep.source_jars for dep in deps], + ), + ) + +def _declare_file(ctx, suffix, **kwargs): + tc = _get_toolchain(ctx) + + # so this can work from an aspect _or_ rule + name = ctx.attr.name if hasattr(ctx.attr, "name") else ctx.rule.attr.name + filename = tc.output_prefix.format(name = name) + suffix + return ctx.actions.declare_file(filename, **kwargs) + +def _merge(grpc_proto_infos): + """Description + Returns: + GrpcProtoInfo + + Args: + grpc_proto_infos: (list[GrpcProtoInfo]) + """ + return GrpcProtoInfo( + source_jars = depset(transitive = [i.source_jars for i in grpc_proto_infos]), + importmap = depset(transitive = [i.importmap for i in grpc_proto_infos]), + imports = depset(transitive = [i.imports for i in grpc_proto_infos]), + ) + +def _compile_srcjars(ctx, source_jars = [], output_jar = None, deps = [], exports = []): + tc = _get_toolchain(ctx) + compiled_jar = output_jar + sources_jar = java_common.pack_sources( + ctx.actions, + output_jar = compiled_jar, + source_jars = source_jars, + java_toolchain = tc._java_toolchain, + host_javabase = tc._host_javabase, + ) + compile_deps = deps + [ + dep[JavaInfo] + for dep in tc.deps + tc.exports + ] + java_common.compile( + ctx, + source_jars = source_jars, + deps = compile_deps, + output = compiled_jar, + java_toolchain = tc._java_toolchain, + host_javabase = tc._host_javabase, + ) + ijar = java_common.run_ijar( + ctx.actions, + jar = compiled_jar, + java_toolchain = tc._java_toolchain, + ) + java_info = JavaInfo( + output_jar = compiled_jar, + compile_jar = ijar, + source_jar = sources_jar, + deps = compile_deps, + exports = exports, + runtime_deps = [t[JavaInfo] for t in tc.runtime_deps], + ) + return java_info + +def _toolchain_impl(ctx): + # Pass configured attrs through via provider + props = {k: getattr(ctx.attr, k) for k in _toolchain_attrs.keys()} + props["declare_file"] = _declare_file + props["compile_srcjars"] = _compile_srcjars + props["merge"] = _merge + props["compile_proto"] = _compile_proto + + # I think the android dexer requires this? + props["runtime"] = props["deps"] + props["exports"] + return [platform_common.ToolchainInfo(**props)] + +grpc_proto_toolchain = rule( + _toolchain_impl, + attrs = _toolchain_attrs, + provides = [platform_common.ToolchainInfo], +) diff --git a/protobuf-lite/BUILD.bazel b/protobuf-lite/BUILD.bazel index 8abb3027034..501593d7914 100644 --- a/protobuf-lite/BUILD.bazel +++ b/protobuf-lite/BUILD.bazel @@ -24,4 +24,5 @@ config_setting( values = { "crosstool_top": "//external:android/crosstool", }, + visibility = ["//java_grpc_library:__pkg__"], ) From 8425b9af42bdaee2400cc342928ebcb98fa5806a Mon Sep 17 00:00:00 2001 From: Chris Eason Date: Sun, 13 Jan 2019 20:49:56 -0500 Subject: [PATCH 2/2] Migrate java_grpc_library to new impl, add deprecation message --- BUILD.bazel | 10 ++-------- alts/BUILD.bazel | 11 ++--------- grpclb/BUILD.bazel | 11 ++--------- java_grpc_library.bzl | 7 +++++++ 4 files changed, 13 insertions(+), 26 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index e4175b0ecee..1e41991c863 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -12,15 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":java_grpc_library.bzl", "java_grpc_library") - -java_proto_library( - name = "api_proto_java", - deps = ["@com_google_protobuf//:api_proto"], -) +load("//java_grpc_library:def.bzl", "java_grpc_library") java_grpc_library( name = "java_grpc_library__external_repo_test", - srcs = ["@com_google_protobuf//:api_proto"], - deps = [":api_proto_java"], + deps = ["@com_google_protobuf//:api_proto"], ) diff --git a/alts/BUILD.bazel b/alts/BUILD.bazel index 8924d7f1cc9..460191e23d6 100644 --- a/alts/BUILD.bazel +++ b/alts/BUILD.bazel @@ -1,4 +1,4 @@ -load("//:java_grpc_library.bzl", "java_grpc_library") +load("//java_grpc_library:def.bzl", "java_grpc_library") java_library( name = "alts_internal", @@ -7,7 +7,6 @@ java_library( ]), deps = [ ":handshaker_java_grpc", - ":handshaker_java_proto", "//core", "//core:internal", "//netty", @@ -73,13 +72,7 @@ proto_library( ], ) -java_proto_library( - name = "handshaker_java_proto", - deps = [":handshaker_proto"], -) - java_grpc_library( name = "handshaker_java_grpc", - srcs = [":handshaker_proto"], - deps = [":handshaker_java_proto"], + deps = [":handshaker_proto"], ) diff --git a/grpclb/BUILD.bazel b/grpclb/BUILD.bazel index c5578662749..3755870a092 100644 --- a/grpclb/BUILD.bazel +++ b/grpclb/BUILD.bazel @@ -1,4 +1,4 @@ -load("//:java_grpc_library.bzl", "java_grpc_library") +load("//java_grpc_library:def.bzl", "java_grpc_library") java_library( name = "grpclb", @@ -8,7 +8,6 @@ java_library( visibility = ["//visibility:public"], deps = [ ":load_balancer_java_grpc", - ":load_balancer_java_proto", "//core", "//core:internal", "//core:util", @@ -29,13 +28,7 @@ proto_library( ], ) -java_proto_library( - name = "load_balancer_java_proto", - deps = [":load_balancer_proto"], -) - java_grpc_library( name = "load_balancer_java_grpc", - srcs = [":load_balancer_proto"], - deps = [":load_balancer_java_proto"], + deps = [":load_balancer_proto"], ) diff --git a/java_grpc_library.bzl b/java_grpc_library.bzl index 9c271a83ee5..50f38143536 100644 --- a/java_grpc_library.bzl +++ b/java_grpc_library.bzl @@ -90,6 +90,13 @@ def java_grpc_library(name, srcs, deps, flavor=None, visibility: (list) the visibility list **kwargs: Passed through to generated targets """ + + print("Deprecated. Please update your load path to @io_grpc_grpc_java//java_grpc_library:def.bzl " + + "for the new implementation which mirrors the native rule interface " + + "({repo}//{package})".format( + repo = native.repository_name(), + package = native.package_name(), + )) if flavor == None: flavor = "normal"