diff --git a/.bazelrc b/.bazelrc
index 9f33b600dcdd3e..1dd928acdb4455 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -496,7 +496,7 @@ build:rbe_win --shell_executable=C:\\tools\\msys64\\usr\\bin\\bash.exe
# TODO(gunan): Remove once we use MSVC 2019 with latest patches.
build:rbe_win --define=override_eigen_strong_inline=true
-build:rbe_win --jobs=500
+build:rbe_win --jobs=100
build:rbe_win_py37 --config=rbe
build:rbe_win_py37 --repo_env=TF_PYTHON_CONFIG_REPO="@windows_py37_config_python"
diff --git a/README.md b/README.md
index 1392da3f60bf58..9cf595bbf6173d 100644
--- a/README.md
+++ b/README.md
@@ -133,6 +133,7 @@ Build Type
**Linux ppc64le CPU** Stable Release | [](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Release_Build/) | Release [1.15](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Release_Build/) / [2.x](https://powerci.osuosl.org/job/TensorFlow2_PPC64LE_CPU_Release_Build/)
**Linux ppc64le GPU** Nightly | [](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Build/) | [Nightly](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/)
**Linux ppc64le GPU** Stable Release | [](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/) | Release [1.15](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/) / [2.x](https://powerci.osuosl.org/job/TensorFlow2_PPC64LE_GPU_Release_Build/)
+**Linux aarch64 CPU** Nightly
Python 3.6 | [](https://status.openlabtesting.org/builds/builds?project=tensorflow%2Ftensorflow) | [Nightly](https://status.openlabtesting.org/builds/builds?project=tensorflow%2Ftensorflow&job_name=tensorflow-arm64-build-daily-master)
**Linux CPU with Intel oneAPI Deep Neural Network Library (oneDNN)** Nightly | [](https://tensorflow-ci.intel.com/job/tensorflow-mkl-build-whl-nightly/) | [Nightly](https://tensorflow-ci.intel.com/job/tensorflow-mkl-build-whl-nightly/)
**Linux CPU with Intel oneAPI Deep Neural Network Library (oneDNN)** Stable Release |  | Release [1.15](https://pypi.org/project/intel-tensorflow/1.15.0/) / [2.x](https://pypi.org/project/intel-tensorflow/)
**Red Hat® Enterprise Linux® 7.6 CPU & GPU**
Python 2.7, 3.6 | [](https://jenkins-tensorflow.apps.ci.centos.org/job/tensorflow-rhel7-3.6/2/) | [1.13.1 PyPI](https://tensorflow.pypi.thoth-station.ninja/index/)
diff --git a/RELEASE.md b/RELEASE.md
index 5c05f2a4285ed2..69eca82c5f21a2 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -11,6 +11,10 @@
* C-API functions `TF_StringDecode`, `TF_StringEncode`, and
`TF_StringEncodedSize` are no longer relevant and have been removed; see
core/platform/ctstring.h for string access/modification in C.
+* In batching library, rename parameter
+ SharedBatchScheduler::QueueOptions::max_batch_size to a more accurate name
+ (input_batch_size_limit) for a recent feature to enable split of large batch
+ sizes.
## Known Caveats
@@ -27,7 +31,11 @@
*
*
* TF Core:
- *
+ *
+ * `tf.Tensor` is now a subclass of `typing.Generic`, allowing type annotations
+ to be parameterized by dtype: `tf.Tensor[tf.Int32]`. This requires Python 3,
+ and will become fully compatible with static type checkers in the future.
+
* `tf.data`:
* Added optional `exclude_cols` parameter to CsvDataset. This parameter is
the complement of `select_cols`; at most one of these should be specified.
@@ -50,6 +58,9 @@
* Tracing and Debugging:
*
* Other:
+ * We have replaced uses of "whitelist" with "allowlist" where possible.
+ Please see https://developers.google.com/style/word-list#blacklist for more
+ context.
*
## Thanks to our Contributors
diff --git a/SECURITY.md b/SECURITY.md
index f3a6c148b2eaf5..6c722766b3a214 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -44,7 +44,7 @@ Even if the untrusted party only supplies the serialized computation
graph (in form of a `GraphDef`, `SavedModel`, or equivalent on-disk format), the
set of computation primitives available to TensorFlow is powerful enough that
you should assume that the TensorFlow process effectively executes arbitrary
-code. One common solution is to whitelist only a few safe Ops. While this is
+code. One common solution is to allow only a few safe Ops. While this is
possible in theory, we still recommend you sandbox the execution.
It depends on the computation graph whether a user provided checkpoint is safe.
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index d00608ccc98114..8a0918b416fb74 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -467,6 +467,13 @@ config_setting(
visibility = ["//visibility:public"],
)
+# This flag enables experimental MLIR bridge support.
+config_setting(
+ name = "enable_mlir_bridge",
+ values = {"define": "enable_mlir_bridge=true"},
+ visibility = ["//visibility:public"],
+)
+
# This flag enables experimental TPU support
config_setting(
name = "with_tpu_support",
diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 9696a3415bf3ff..5c101bef85fc54 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -177,7 +177,9 @@ cc_library(
visibility = [
"//tensorflow:internal",
],
- deps = [],
+ deps = [
+ "//tensorflow/core:protos_all_cc",
+ ],
)
cc_library(
@@ -480,6 +482,7 @@ tf_cuda_cc_test(
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core/common_runtime:function_optimization_registry",
+ "//tensorflow/core/common_runtime:optimization_registry",
"//tensorflow/core/common_runtime/eager:eager_operation",
"//tensorflow/core/distributed_runtime/rpc:grpc_server_lib",
"@com_google_absl//absl/strings",
diff --git a/tensorflow/c/eager/abstract_operation.h b/tensorflow/c/eager/abstract_operation.h
index ff17fcf3cea5a0..b332679cc7cc70 100644
--- a/tensorflow/c/eager/abstract_operation.h
+++ b/tensorflow/c/eager/abstract_operation.h
@@ -73,7 +73,8 @@ class AbstractOperation {
virtual Status SetDeviceName(const char* name) = 0;
virtual Status AddInput(AbstractTensorHandle* input) = 0;
- virtual Status AddInputList(absl::Span inputs) = 0;
+ virtual Status AddInputList(
+ absl::Span inputs) = 0;
virtual Status Execute(absl::Span retvals,
int* num_retvals) = 0;
diff --git a/tensorflow/c/eager/abstract_tensor_handle.h b/tensorflow/c/eager/abstract_tensor_handle.h
index d50bd4530db8b0..de041690420552 100644
--- a/tensorflow/c/eager/abstract_tensor_handle.h
+++ b/tensorflow/c/eager/abstract_tensor_handle.h
@@ -16,6 +16,8 @@ limitations under the License.
#define TENSORFLOW_C_EAGER_ABSTRACT_TENSOR_HANDLE_H_
#include
+
+#include "tensorflow/core/framework/types.pb.h"
namespace tensorflow {
// Abstract interface to a Tensor handle in either tracing or immediate
@@ -27,6 +29,9 @@ class AbstractTensorHandle {
virtual ~AbstractTensorHandle() {}
public:
+ // Returns tensor dtype.
+ virtual tensorflow::DataType DataType() const = 0;
+
AbstractTensorHandleKind getKind() const { return kind_; }
// Release any underlying resources, including the interface object.
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 4be3cdd7c2db34..70acd710166fd5 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -337,10 +337,13 @@ tensorflow::Status CreateRemoteContexts(
});
}
counter.Wait();
+ tensorflow::StatusGroup sg;
for (int i = 0; i < num_remote_workers; i++) {
- TF_RETURN_IF_ERROR(statuses[i]);
+ if (TF_PREDICT_FALSE(!statuses[i].ok())) {
+ sg.Update(statuses[i]);
+ }
}
- return tensorflow::Status::OK();
+ return sg.as_summary_status();
}
tensorflow::Status UpdateRemoteContexts(
@@ -611,10 +614,21 @@ tensorflow::Status UpdateTFE_ContextWithServerDef(
// Initialize remote eager workers.
if (reset_context) {
- LOG_AND_RETURN_IF_ERROR(CreateRemoteContexts(
+ const tensorflow::Status s = CreateRemoteContexts(
ctx, remote_workers, context_id, context_view_id, keep_alive_secs,
server_def, remote_eager_workers.get(), context->Executor().Async(),
- context->LazyCopyFunctionRemoteInputs(), base_request));
+ context->LazyCopyFunctionRemoteInputs(), base_request);
+ // NOTE: the remote tasks could fail after `GetAllRemoteDevices` and cause
+ // the CreateRemoteContexts to fail. We currently only log instead of
+ // directly returning the error, since returning here will cause the server
+ // object to be destroyed (which currently CHECK-fails). The client will
+ // see additional errors if ops are subsequently sent to the failed workers.
+ if (TF_PREDICT_FALSE(!s.ok())) {
+ LOG(ERROR) << "Error when creating contexts on remote targets: "
+ << s.error_message()
+ << "\nExecuting remote ops or functions on these remote "
+ "targets will fail.";
+ }
} else {
// The master's context_view_id will be incremented by one
// the UpdateRemoteMaster call later. We want all new workers and
@@ -644,15 +658,16 @@ tensorflow::Status UpdateTFE_ContextWithServerDef(
grpc_server->worker_env()->rendezvous_mgr->Find(context_id);
auto* device_mgr = grpc_server->worker_env()->device_mgr;
std::shared_ptr worker_session;
- TF_RETURN_IF_ERROR(grpc_server->worker_env()->session_mgr->CreateSession(
- session_name, server_def, base_request.cluster_device_attributes(),
- true));
- TF_RETURN_IF_ERROR(
+ LOG_AND_RETURN_IF_ERROR(
+ grpc_server->worker_env()->session_mgr->CreateSession(
+ session_name, server_def, base_request.cluster_device_attributes(),
+ true));
+ LOG_AND_RETURN_IF_ERROR(
grpc_server->worker_env()->session_mgr->WorkerSessionForSession(
session_name, &worker_session));
// Initialize remote tensor communication based on worker session.
- TF_RETURN_IF_ERROR(r->Initialize(worker_session.get()));
+ LOG_AND_RETURN_IF_ERROR(r->Initialize(worker_session.get()));
tensorflow::DistributedFunctionLibraryRuntime* cluster_flr =
tensorflow::eager::CreateClusterFLR(context_id, context,
diff --git a/tensorflow/c/eager/c_api_distributed_test.cc b/tensorflow/c/eager/c_api_distributed_test.cc
index 65f8d3cc646328..a6547e23454817 100644
--- a/tensorflow/c/eager/c_api_distributed_test.cc
+++ b/tensorflow/c/eager/c_api_distributed_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
#include "tensorflow/c/eager/tfe_tensorhandle_internal.h"
#include "tensorflow/core/common_runtime/eager/eager_operation.h"
#include "tensorflow/core/common_runtime/function_optimization_registry.h"
+#include "tensorflow/core/common_runtime/optimization_registry.h"
#include "tensorflow/core/distributed_runtime/rpc/grpc_server_lib.h"
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/graph/graph.h"
@@ -316,6 +317,114 @@ string VariableAddFunction() {
return def.SerializeAsString();
}
+// A graph optimization pass that would fail when triggered for more than once.
+class GraphErrorInjectionPass : public tensorflow::GraphOptimizationPass {
+ public:
+ static bool enabled_;
+ GraphErrorInjectionPass() {}
+
+ tensorflow::Status Run(
+ const tensorflow::GraphOptimizationPassOptions& options) override {
+ if (!enabled_) {
+ return tensorflow::Status::OK();
+ }
+ if (first_call_) {
+ first_call_ = false;
+ return tensorflow::Status::OK();
+ }
+ return tensorflow::errors::Internal("Graph pass runs for more than once!");
+ }
+
+ private:
+ bool first_call_ = true;
+};
+
+// After the graph pass is registered, it takes effect globally and can affect
+// other test cases. Define a static variable to switch it on and off.
+bool GraphErrorInjectionPass::enabled_ = false;
+
+// Test to ensure that a registered graph optimization pass is only executed
+// once (i.e., on the main function side) in running distributed functions.
+// This test creates a cluster with two workers, create a variable on the
+// second worker, and run a distributed function (VariableAddFunction) whose ops
+// span the local and remote workers. If the graph optimization pass is executed
+// on both the main function side and the component function side, an error will
+// be thrown in the registered graph optimization pass.
+TEST(CAPI, DistributedFunctionGraphPassOnlyOnce) {
+ // Register graph pass that will raise error if called more than once.
+ tensorflow::optimization_registration::OptimizationPassRegistration
+ register_test_pass(tensorflow::OptimizationPassRegistry::PRE_PLACEMENT, 0,
+ std::make_unique(),
+ "error_injector");
+ GraphErrorInjectionPass::enabled_ = true;
+
+ tensorflow::ServerDef server_def = GetServerDef(3);
+ // This server def has the task index set to 0.
+ string serialized = server_def.SerializeAsString();
+
+ server_def.set_task_index(1);
+ std::unique_ptr worker_server1;
+ ASSERT_TRUE(tensorflow::GrpcServer::Create(
+ server_def, tensorflow::Env::Default(), &worker_server1)
+ .ok());
+ ASSERT_TRUE(worker_server1->Start().ok());
+ server_def.set_task_index(2);
+ std::unique_ptr worker_server2;
+ ASSERT_TRUE(tensorflow::GrpcServer::Create(
+ server_def, tensorflow::Env::Default(), &worker_server2)
+ .ok());
+ ASSERT_TRUE(worker_server2->Start().ok());
+ const char dev2_name[] = "/job:localhost/replica:0/task:2/device:CPU:0";
+
+ TF_Status* status = TF_NewStatus();
+ TFE_ContextOptions* opts = TFE_NewContextOptions();
+ TFE_ContextOptionsSetDevicePlacementPolicy(opts, TFE_DEVICE_PLACEMENT_SILENT);
+ TFE_Context* ctx = TFE_NewContext(opts, status);
+ EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+ TFE_DeleteContextOptions(opts);
+
+ TFE_ContextSetServerDef(ctx, 0, serialized.data(), serialized.size(), status);
+ EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+
+ TFE_TensorHandle* var_handle = TestVariable(ctx, 2.0, dev2_name);
+ EXPECT_NE(var_handle, nullptr);
+
+ const string function_def = VariableAddFunction();
+ TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(),
+ status);
+ ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status);
+
+ TFE_Op* func = TFE_NewOp(ctx, "VariableAddFunction", status);
+ ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status);
+ TFE_OpAddInput(func, var_handle, status);
+ ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status);
+ TFE_TensorHandle* retvals[1] = {nullptr};
+ int num_retvals = 1;
+ TFE_Execute(func, &retvals[0], &num_retvals, status);
+ ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+ ASSERT_EQ(1, num_retvals);
+ TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
+ ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+ TFE_DeleteTensorHandle(retvals[0]);
+ float sum = 0;
+ ASSERT_EQ(sizeof(sum), TF_TensorByteSize(t));
+ memcpy(&sum, TF_TensorData(t), TF_TensorByteSize(t));
+ TF_DeleteTensor(t);
+ ASSERT_EQ(sum, 4.0);
+
+ TFE_DeleteOp(func);
+ TFE_DeleteTensorHandle(var_handle);
+ TFE_DeleteContext(ctx);
+ TF_DeleteStatus(status);
+
+ // TODO(b/136478427): Figure out how to correctly shut the server down.
+ worker_server1.release();
+ worker_server2.release();
+
+ // Disable the test graph pass so it does not affect other test cases.
+ GraphErrorInjectionPass::enabled_ = false;
+}
+
class FunctionErrorInjectionPass : public tensorflow::FunctionOptimizationPass {
public:
FunctionErrorInjectionPass(string error_node, string error_device)
diff --git a/tensorflow/c/eager/c_api_unified_experimental_graph.cc b/tensorflow/c/eager/c_api_unified_experimental_graph.cc
index e7e1ef8486821e..6165a7d14a3637 100644
--- a/tensorflow/c/eager/c_api_unified_experimental_graph.cc
+++ b/tensorflow/c/eager/c_api_unified_experimental_graph.cc
@@ -49,6 +49,10 @@ class GraphTensor : public TracingTensorHandle {
explicit GraphTensor(TF_Output output)
: TracingTensorHandle(kGraph), output_(output) {}
void Release() override { delete this; }
+
+ tensorflow::DataType DataType() const override {
+ return static_cast(TF_OperationOutputType(output_));
+ }
TF_Output output_;
// For LLVM style RTTI.
@@ -102,9 +106,18 @@ class GraphOperation : public TracingOperation {
TF_AddInput(op_.get(), t->output_);
return Status::OK();
}
- Status AddInputList(absl::Span inputs) override {
- return tensorflow::errors::Unimplemented(
- "AddInputList has not been implemented yet.");
+ Status AddInputList(absl::Span inputs) override {
+ std::vector tf_outputs(inputs.size());
+ for (int i = 0; i < inputs.size(); i++) {
+ GraphTensor* t = dyn_cast(inputs[i]);
+ if (!t) {
+ return tensorflow::errors::InvalidArgument(
+ "Unable to cast input to GraphTensor");
+ }
+ tf_outputs[i] = t->output_;
+ }
+ TF_AddInputList(op_.get(), tf_outputs.data(), tf_outputs.size());
+ return Status::OK();
}
Status Execute(absl::Span retvals,
int* num_retvals) override {
diff --git a/tensorflow/c/eager/immediate_execution_tensor_handle.h b/tensorflow/c/eager/immediate_execution_tensor_handle.h
index c9e39a80663077..f7c77aa06db38f 100644
--- a/tensorflow/c/eager/immediate_execution_tensor_handle.h
+++ b/tensorflow/c/eager/immediate_execution_tensor_handle.h
@@ -33,8 +33,6 @@ namespace tensorflow {
// is needed a static_cast can be applied.
class ImmediateExecutionTensorHandle : public AbstractTensorHandle {
public:
- // Returns tensor dtype.
- virtual tensorflow::DataType DataType() const = 0;
// Returns number of dimensions.
virtual Status NumDims(int* num_dims) const = 0;
// Returns number of elements across all dimensions.
diff --git a/tensorflow/c/eager/parallel_device/parallel_device.cc b/tensorflow/c/eager/parallel_device/parallel_device.cc
index 5740fc4631e257..d0e9f351478817 100644
--- a/tensorflow/c/eager/parallel_device/parallel_device.cc
+++ b/tensorflow/c/eager/parallel_device/parallel_device.cc
@@ -243,8 +243,10 @@ TFE_TensorHandle* CopyTensorFromParallelDevice(TFE_Context* context,
const char* target_device_name,
TF_Status* status,
void* device_info) {
- TF_SetStatus(status, TF_INTERNAL,
- "Trying to copy a tensor out of a parallel device.");
+ TF_SetStatus(status, TF_UNIMPLEMENTED,
+ "Trying to copy a tensor out of a parallel device. Since there "
+ "are multiple components to parallel tensors, they must be "
+ "unpacked explicitly.");
return nullptr;
}
diff --git a/tensorflow/c/eager/parallel_device/parallel_device_test.cc b/tensorflow/c/eager/parallel_device/parallel_device_test.cc
index 2fa183d50f6829..06a26ab2710092 100644
--- a/tensorflow/c/eager/parallel_device/parallel_device_test.cc
+++ b/tensorflow/c/eager/parallel_device/parallel_device_test.cc
@@ -157,7 +157,7 @@ TEST(PARALLEL_DEVICE, TestExplicitCopies) {
// Copies off of parallel devices must be explicit.
TensorHandlePtr copy_back(TFE_TensorHandleCopyToDevice(
device_value.get(), context.get(), first_device_name, status.get()));
- ASSERT_EQ(TF_GetCode(status.get()), TF_INTERNAL);
+ ASSERT_EQ(TF_GetCode(status.get()), TF_UNIMPLEMENTED);
}
TEST(PARALLEL_DEVICE, TestDifferentShapes) {
diff --git a/tensorflow/c/experimental/filesystem/plugins/gcs/BUILD b/tensorflow/c/experimental/filesystem/plugins/gcs/BUILD
index 2a886dee4cb24e..a0c137017664c8 100644
--- a/tensorflow/c/experimental/filesystem/plugins/gcs/BUILD
+++ b/tensorflow/c/experimental/filesystem/plugins/gcs/BUILD
@@ -52,6 +52,40 @@ cc_library(
],
)
+cc_library(
+ name = "cleanup",
+ hdrs = ["cleanup.h"],
+)
+
+cc_library(
+ name = "ram_file_block_cache",
+ srcs = ["ram_file_block_cache.cc"],
+ hdrs = ["ram_file_block_cache.h"],
+ deps = [
+ ":cleanup",
+ ":file_block_cache",
+ "//tensorflow/c:env",
+ "//tensorflow/c:tf_status",
+ "@com_google_absl//absl/base:core_headers",
+ "@com_google_absl//absl/synchronization",
+ ],
+)
+
+tf_cc_test(
+ name = "ram_file_block_cache_test",
+ size = "small",
+ srcs = ["ram_file_block_cache_test.cc"],
+ deps = [
+ ":ram_file_block_cache",
+ "//tensorflow/c:tf_status_internal",
+ "//tensorflow/core:lib",
+ "//tensorflow/core:test",
+ "//tensorflow/core:test_main",
+ "//tensorflow/core/platform:blocking_counter",
+ "//tensorflow/core/platform/cloud:now_seconds_env",
+ ],
+)
+
tf_cc_test(
name = "gcs_filesystem_test",
srcs = [
@@ -69,3 +103,29 @@ tf_cc_test(
"@com_google_absl//absl/strings",
],
)
+
+cc_library(
+ name = "expiring_lru_cache",
+ hdrs = ["expiring_lru_cache.h"],
+ deps = [
+ "//tensorflow/c:env",
+ "//tensorflow/c:tf_status",
+ "@com_google_absl//absl/base:core_headers",
+ "@com_google_absl//absl/synchronization",
+ ],
+)
+
+tf_cc_test(
+ name = "expiring_lru_cache_test",
+ size = "small",
+ srcs = ["expiring_lru_cache_test.cc"],
+ deps = [
+ ":expiring_lru_cache",
+ "//tensorflow/c:tf_status_helper",
+ "//tensorflow/c:tf_status_internal",
+ "//tensorflow/core:lib",
+ "//tensorflow/core:test",
+ "//tensorflow/core:test_main",
+ "//tensorflow/core/platform/cloud:now_seconds_env",
+ ],
+)
diff --git a/tensorflow/c/experimental/filesystem/plugins/gcs/cleanup.h b/tensorflow/c/experimental/filesystem/plugins/gcs/cleanup.h
new file mode 100644
index 00000000000000..cc7a7451bb8fa8
--- /dev/null
+++ b/tensorflow/c/experimental/filesystem/plugins/gcs/cleanup.h
@@ -0,0 +1,109 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// MakeCleanup(f) returns an RAII cleanup object that calls 'f' in its
+// destructor. The easiest way to use MakeCleanup is with a lambda argument,
+// capturing the return value in an 'auto' local variable. Most users will not
+// need more sophisticated syntax than that.
+//
+// Example:
+// void func() {
+// FILE* fp = fopen("data.txt", "r");
+// if (fp == nullptr) return;
+// auto fp_cleaner = gtl::MakeCleanup([fp] { fclose(fp); });
+// // No matter what, fclose(fp) will happen.
+// DataObject d;
+// while (ReadDataObject(fp, &d)) {
+// if (d.IsBad()) {
+// LOG(ERROR) << "Bad Data";
+// return;
+// }
+// PushGoodData(d);
+// }
+// }
+//
+// You can use Cleanup directly, instead of using MakeCleanup and auto,
+// but there's rarely a reason to do that.
+//
+// You can call 'release()' on a Cleanup object to cancel the cleanup.
+
+#ifndef TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_GCS_CLEANUP_H_
+#define TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_GCS_CLEANUP_H_
+
+#include
+#include
+
+namespace tf_gcs_filesystem {
+
+// A move-only RAII object that calls a stored cleanup functor when
+// destroyed. Cleanup is the return type of gtl::MakeCleanup(F).
+template
+class Cleanup {
+ public:
+ Cleanup() : released_(true), f_() {}
+
+ template
+ explicit Cleanup(G&& f) // NOLINT
+ : f_(std::forward(f)) {} // NOLINT(build/c++11)
+
+ Cleanup(Cleanup&& src) // NOLINT
+ : released_(src.is_released()), f_(src.release()) {}
+
+ // Implicitly move-constructible from any compatible Cleanup.
+ // The source will be released as if src.release() were called.
+ // A moved-from Cleanup can be safely destroyed or reassigned.
+ template
+ Cleanup(Cleanup&& src) // NOLINT
+ : released_(src.is_released()), f_(src.release()) {}
+
+ // Assignment to a Cleanup object behaves like destroying it
+ // and making a new one in its place, analogous to unique_ptr
+ // semantics.
+ Cleanup& operator=(Cleanup&& src) { // NOLINT
+ if (!released_) f_();
+ released_ = src.released_;
+ f_ = src.release();
+ return *this;
+ }
+
+ ~Cleanup() {
+ if (!released_) f_();
+ }
+
+ // Releases the cleanup function instead of running it.
+ // Hint: use c.release()() to run early.
+ F release() {
+ released_ = true;
+ return std::move(f_);
+ }
+
+ bool is_released() const { return released_; }
+
+ private:
+ static_assert(!std::is_reference::value, "F must not be a reference");
+
+ bool released_ = false;
+ F f_;
+};
+
+template ::type>
+Cleanup MakeCleanup(F&& f) {
+ return Cleanup(std::forward(f));
+}
+
+} // namespace tf_gcs_filesystem
+
+#endif // TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_GCS_CLEANUP_H_
diff --git a/tensorflow/c/experimental/filesystem/plugins/gcs/expiring_lru_cache.h b/tensorflow/c/experimental/filesystem/plugins/gcs/expiring_lru_cache.h
new file mode 100644
index 00000000000000..c0347faa16dff4
--- /dev/null
+++ b/tensorflow/c/experimental/filesystem/plugins/gcs/expiring_lru_cache.h
@@ -0,0 +1,191 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_GCS_EXPIRING_LRU_CACHE_H_
+#define TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_PLUGINS_GCS_EXPIRING_LRU_CACHE_H_
+
+#include
+#include
+#include