Uses absl::log for logging. deperecates --minloglevel flag, but introduced --quiet.

Author: taku910

CMakeLists.txt

@@ -36,6 +36,10 @@ set_property(CACHE SPM_PROTOBUF_PROVIDER PROPERTY STRINGS "internal" "package")
 set(SPM_ABSL_PROVIDER "internal" CACHE STRING "Provider of absl library")
 set_property(CACHE SPM_ABSL_PROVIDER PROPERTY STRINGS "internal" "module" "package")
 
+if (SPM_ENABLE_SHARED)
+ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+endif() 
+
 # Includes processor name to avoid conflicts.
 set(CPACK_PACKAGE_FILE_NAME
     "sentencepiece-${SPM_VERSION}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}")

src/CMakeLists.txt

@@ -22,7 +22,9 @@ if (SPM_ABSL_PROVIDER STREQUAL "module" OR SPM_ABSL_PROVIDER STREQUAL "package")
   list(APPEND SPM_LIBS absl::check)
   add_compile_definitions(_USE_EXTERNAL_ABSL)
 elseif (SPM_ABSL_PROVIDER STREQUAL "internal")
-  set(ABSL_FLAGS_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/absl/flags/flag.cc)
+ set(ABSL_FLAGS_SRCS
+    ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/absl/flags/flag.cc
+    ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/absl/log/log.cc)
 endif()
 
 if (SPM_PROTOBUF_PROVIDER STREQUAL "internal")
@@ -102,6 +104,7 @@ set(SPM_SRCS
   char_model.cc
   error.cc
   filesystem.cc
+  init.cc
   model_factory.cc
   model_interface.cc
   normalizer.cc

src/common.h

@@ -15,17 +15,12 @@
 #ifndef COMMON_H_
 #define COMMON_H_
 
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
-
+#include <cstdint>
 #include <iostream>
-#include <memory>
-#include <string>
-#include <utility>
-#include <vector>
 
-#include "config.h"
+#include "third_party/absl/log/check.h"
+#include "third_party/absl/log/globals.h"
+#include "third_party/absl/log/log.h"
 #include "third_party/absl/strings/string_view.h"
 
 #if defined(_WIN32) && !defined(__CYGWIN__)
@@ -41,134 +36,33 @@
 #include <windows.h>
 #endif
 
-typedef uint32_t char32;
+using char32 = uint32_t;
 
 static constexpr uint32_t kUnicodeError = 0xFFFD;
 
-#if defined(_FREEBSD)
-#include <sys/endian.h>
-#endif
-#if !defined(__APPLE__) && !defined(_WIN32) && !defined(_FREEBSD) && \
-    !defined(_AIX)
-#include <endian.h>
-#if BYTE_ORDER == __BIG_ENDIAN
-#define IS_BIG_ENDIAN
-#endif
-#endif
-
-#if defined(_AIX) && BYTE_ORDER == BIG_ENDIAN
-#define IS_BIG_ENDIAN
-#endif
-
-namespace sentencepiece {
-namespace util {
-
-inline uint32_t Swap32(uint32_t x) {
-#ifdef OS_WIN
-  return _byteswap_ulong(x);
-#else   // OS_WIN
-  return __builtin_bswap32(x);
-#endif  // OS_WIN
-}
-
-}  // namespace util
-
-constexpr bool is_bigendian() {
-#ifdef IS_BIG_ENDIAN
-  return true;
-#else   // IS_BIG_ENDIAN
-  return false;
-#endif  // IS_BIG_ENDIAN
-}
-
-namespace error {
-
-void Abort();
-void Exit(int code);
-
-class Die {
- public:
-  explicit Die(bool die) : die_(die) {}
-  ~Die() {
-    std::cerr << std::endl;
-    if (die_) {
-      Abort();
-    }
-  }
-  int operator&(std::ostream &) { return 0; }
-
- private:
-  bool die_;
-};
-}  // namespace error
-
-namespace logging {
-enum LogSeverity {
-  LOG_INFO = 0,
-  LOG_WARNING = 1,
-  LOG_ERROR = 2,
-  LOG_FATAL = 3,
-  LOG_SEVERITY_SIZE = 4,
-};
-
-int GetMinLogLevel();
-void SetMinLogLevel(int v);
-
-inline absl::string_view BaseName(absl::string_view path) {
-#ifdef OS_WIN
-  const size_t pos = path.find_last_of('\\');
-#else
-  const size_t pos = path.find_last_of('/');
-#endif
-  return pos == absl::string_view::npos ? path : path.substr(pos + 1);
-}
-
-}  // namespace logging
-}  // namespace sentencepiece
-
-#define LOG(severity)                                                        \
-  (::sentencepiece::logging::GetMinLogLevel() >                              \
-   ::sentencepiece::logging::LOG_##severity)                                 \
-      ? 0                                                                    \
-      : ::sentencepiece::error::Die(                                         \
-            ::sentencepiece::logging::LOG_##severity >=                      \
-            ::sentencepiece::logging::LOG_FATAL) &                           \
-            std::cerr << ::sentencepiece::logging::BaseName(__FILE__) << "(" \
-                      << __LINE__ << ") "                                    \
-                      << "LOG(" << #severity << ") "
+#define FRIEND_TEST(a, b) friend class a##_Test_##b;
 
-#define CHECK(condition)                                                      \
-  (condition) ? 0                                                             \
-              : ::sentencepiece::error::Die(true) &                           \
-                    std::cerr << ::sentencepiece::logging::BaseName(__FILE__) \
-                              << "(" << __LINE__ << ") [" << #condition       \
-                              << "] "
+#define RETURN_IF_ERROR(expr)          \
+  do {                                 \
+    const auto _status = expr;         \
+    if (!_status.ok()) return _status; \
+  } while (0)
 
-#define CHECK_EQ(a, b) CHECK((a) == (b))
-#define CHECK_NE(a, b) CHECK((a) != (b))
-#define CHECK_GE(a, b) CHECK((a) >= (b))
-#define CHECK_LE(a, b) CHECK((a) <= (b))
-#define CHECK_GT(a, b) CHECK((a) > (b))
-#define CHECK_LT(a, b) CHECK((a) < (b))
+// CHECK_OK must work on util::Status, not absl::Status.
+#if defined CHECK_OK
+#undef CHECK_OK
+#endif  // CHECK_OK
 
-#define FRIEND_TEST(a, b) friend class a##_Test_##b;
+#if defined QCHECK_OK
+#undef QCHECK_OK
+#endif  // QCHECK_OK
 
 #define CHECK_OK(expr)                         \
   do {                                         \
     const auto _status = expr;                 \
     CHECK(_status.ok()) << _status.ToString(); \
   } while (0)
 
-#define CHECK_NOT_OK(expr)                      \
-  do {                                          \
-    const auto _status = expr;                  \
-    CHECK(!_status.ok()) << _status.ToString(); \
-  } while (0)
-
-#define RETURN_IF_ERROR(expr)          \
-  do {                                 \
-    const auto _status = expr;         \
-    if (!_status.ok()) return _status; \
-  } while (0)
+#define QCHECK_OK CHECK_OK
 
 #endif  // COMMON_H_

src/error.cc

@@ -15,34 +15,9 @@
 #include <cstring>
 
 #include "common.h"
-#include "init.h"
 #include "sentencepiece_processor.h"
 
-#ifdef _USE_EXTERNAL_ABSL
-// Naive workaround to define minloglevel on external absl package.
-// We want to define them in other cc file.
-#include "third_party/absl/flags/flag.h"
-#include "third_party/absl/flags/parse.h"
-ABSL_FLAG(int32_t, minloglevel, 0,
-          "Messages logged at a lower level than this don't actually.");
-#endif
-
 namespace sentencepiece {
-namespace error {
-
-void Abort() {
-  std::cerr << "Program terminated with an unrecoverable error." << std::endl;
-  ShutdownLibrary();
-  std::exit(-1);
-}
-
-void Exit(int code) {
-  ShutdownLibrary();
-  std::exit(code);
-}
-
-}  // namespace error
-
 namespace util {
 
 Status::Status() {}

src/init.cc

@@ -0,0 +1,60 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.!
+
+#include "init.h"
+
+#include "common.h"
+#include "config.h"
+#include "third_party/absl/flags/flag.h"
+#include "third_party/absl/flags/parse.h"
+#include "third_party/absl/flags/usage.h"
+#include "third_party/absl/flags/usage_config.h"
+#include "third_party/absl/strings/str_cat.h"
+#include "util.h"
+
+#ifdef _USE_EXTERNAL_PROTOBUF
+#include "google/protobuf/message_lite.h"
+#else
+#include "third_party/protobuf-lite/google/protobuf/message_lite.h"
+#endif
+
+ABSL_FLAG(bool, quiet, false, "Suppress logging message.");
+
+namespace sentencepiece {
+void ParseCommandLineFlags(const char *usage, int *argc, char ***argv,
+                           bool remove_arg) {
+  absl::SetProgramUsageMessage(absl::StrCat(PACKAGE_STRING, " ", VERSION,
+                                            "\n\n", "Usage: ", *argv[0],
+                                            " [options] files"));
+  absl::SetFlagsUsageConfig({.version_string = [&]() {
+    return absl::StrCat(PACKAGE_STRING, " ", VERSION, "\n");
+  }});
+
+  const auto unused_args = absl::ParseCommandLine(*argc, *argv);
+
+  if (remove_arg) {
+    char **argv_val = *argv;
+    *argv = argv_val = argv_val + *argc - unused_args.size();
+    std::copy(unused_args.begin(), unused_args.end(), argv_val);
+    *argc = static_cast<int>(unused_args.size());
+  }
+
+  if (absl::GetFlag(FLAGS_quiet)) {
+    absl::SetMinLogLevel(static_cast<absl::LogSeverityAtLeast>(100));
+  }
+}
+
+void ShutdownLibrary() { google::protobuf::ShutdownProtobufLibrary(); }
+
+}  // namespace sentencepiece

src/init.h

@@ -16,40 +16,13 @@
 #define INIT_H_
 
 #include "common.h"
-#include "third_party/absl/flags/flag.h"
-#include "third_party/absl/flags/parse.h"
-#include "third_party/absl/flags/usage.h"
 
-#ifdef _USE_EXTERNAL_PROTOBUF
-#include "google/protobuf/message_lite.h"
-#else
-#include "third_party/protobuf-lite/google/protobuf/message_lite.h"
-#endif
+namespace sentencepiece {
 
-ABSL_DECLARE_FLAG(int32_t, minloglevel);
+void ParseCommandLineFlags(const char *usage, int *argc, char ***argv,
+                           bool remove_arg = true);
 
-namespace sentencepiece {
-inline void ParseCommandLineFlags(const char *usage, int *argc, char ***argv,
-                                  bool remove_arg = true) {
-  absl::SetProgramUsageMessage(*argv[0]);
-  const auto unused_args = absl::ParseCommandLine(*argc, *argv);
-
-  if (remove_arg) {
-    char **argv_val = *argv;
-    *argv = argv_val = argv_val + *argc - unused_args.size();
-    std::copy(unused_args.begin(), unused_args.end(), argv_val);
-    *argc = static_cast<int>(unused_args.size());
-  }
-
-  logging::SetMinLogLevel(absl::GetFlag(FLAGS_minloglevel));
-}
-
-inline void ShutdownLibrary() {
-  google::protobuf::ShutdownProtobufLibrary();
-#ifdef HAS_ABSL_CLEANUP_FLAGS
-  absl::CleanupFlags();
-#endif
-}
+void ShutdownLibrary();
 
 class ScopedResourceDestructor {
  public:

src/normalizer.cc

@@ -258,7 +258,7 @@ std::string Normalizer::EncodePrecompiledCharsMap(
   blob.append(string_util::EncodePOD<uint32_t>(trie_blob.size()));
   blob.append(trie_blob.data(), trie_blob.size());
 
-  if constexpr (is_bigendian()) {
+  if constexpr (util::is_bigendian()) {
     uint32_t *data = reinterpret_cast<uint32_t *>(blob.data());
     for (int i = 0; i < blob.size() / 4; ++i) data[i] = util::Swap32(data[i]);
   }
@@ -280,7 +280,7 @@ util::Status Normalizer::DecodePrecompiledCharsMap(
     return util::InternalError("Blob for normalization rule is broken.");
   }
 
-  if constexpr (is_bigendian()) {
+  if constexpr (util::is_bigendian()) {
     trie_blob_size = util::Swap32(trie_blob_size);
   }
 
@@ -295,7 +295,7 @@ util::Status Normalizer::DecodePrecompiledCharsMap(
 
   blob.remove_prefix(sizeof(trie_blob_size));
 
-  if constexpr (is_bigendian()) {
+  if constexpr (util::is_bigendian()) {
     CHECK_OR_RETURN(buffer);
     buffer->assign(blob.data(), trie_blob_size);
     uint32_t *data =

src/sentencepiece_trainer.cc

@@ -150,7 +150,7 @@ util::Status SentencePieceTrainer::MergeSpecsFromArgs(
     } else if (key == "minloglevel") {
       int v = 0;
       CHECK_OR_RETURN(absl::SimpleAtoi(value, &v));
-      logging::SetMinLogLevel(v);
+      SetMinLogLevel(v);
       continue;
     }