support direct local address

cherry-pick: envoyproxy@090e73d

use directLocalAddress instead of localAddress in LocalPortValueExtractorImpl of srds
Link: https://code.alibaba-inc.com/Ingress/envoy/codereview/21602142
* support direct local address

* add metrics stats

Author: johnlanni

docs/root/configuration/observability/access_log/usage.rst

@@ -624,14 +624,54 @@ The following command operators are supported:
   If the original connection was redirected by iptables TPROXY, and the listener's transparent
   option was set to true, this represents the original destination address and port.
 
+  .. note::
+
+    This may not be the physical remote address of the peer if the address has been inferred from
+    :ref:`Proxy Protocol filter <config_listener_filters_proxy_protocol>`.
+
+%DOWNSTREAM_DIRECT_LOCAL_ADDRESS%
+  Direct local address of the downstream connection.
+
+  .. note::
+
+    This is always the physical local address even if the downstream remote address has been inferred from
+    :ref:`Proxy Protocol filter <config_listener_filters_proxy_protocol>`.
+
 %DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT%
   Local address of the downstream connection, without any port component.
   IP addresses are the only address type with a port component.
 
+  .. note::
+
+    This may not be the physical local address if the downstream local address has been inferred from
+    :ref:`Proxy Protocol filter <config_listener_filters_proxy_protocol>`.
+
+%DOWNSTREAM_DIRECT_LOCAL_ADDRESS_WITHOUT_PORT%
+  Direct local address of the downstream connection, without any port component.
+
+  .. note::
+
+    This is always the physical local address even if the downstream local address has been inferred from
+    :ref:`Proxy Protocol filter <config_listener_filters_proxy_protocol>`.
+
 %DOWNSTREAM_LOCAL_PORT%
   Local port of the downstream connection.
   IP addresses are the only address type with a port component.
 
+  .. note::
+
+    This may not be the physical port if the downstream local address has been inferred from
+    :ref:`Proxy Protocol filter <config_listener_filters_proxy_protocol>`.
+
+%DOWNSTREAM_DIRECT_LOCAL_PORT%
+  Direct local port of the downstream connection.
+  IP addresses are the only address type with a port component.
+
+  .. note::
+
+    This is always the listener port even if the downstream local address has been inferred from
+    :ref:`Proxy Protocol filter <config_listener_filters_proxy_protocol>`.
+
 .. _config_access_log_format_connection_id:
 
 %CONNECTION_ID%

envoy/network/socket.h

@@ -176,6 +176,12 @@ class ConnectionInfoProvider {
    */
   virtual const Address::InstanceConstSharedPtr& localAddress() const PURE;
 
+  /**
+   * @return the direct local address of the socket. This is the listener address and it can not be
+   * modified by listener filters.
+   */
+  virtual const Address::InstanceConstSharedPtr& directLocalAddress() const PURE;
+
   /**
    * @return true if the local address has been restored to a value that is different from the
    *         address the socket was initially accepted at.

source/common/http/conn_manager_config.h

@@ -22,6 +22,12 @@
 namespace Envoy {
 namespace Http {
 
+#if defined(ALIMESH)
+#define HIGRESS_EXT_HTTP_CONN_MAN_STATS(COUNTER, GAUGE, HISTOGRAM)                                 \
+  COUNTER(downstream_rq_retry_scope_found_total)                                                   \
+  COUNTER(downstream_rq_retry_scope_not_found_total)
+#endif
+
 /**
  * All stats for the connection manager. @see stats_macros.h
  */
@@ -92,6 +98,10 @@ namespace Http {
  */
 struct ConnectionManagerNamedStats {
   ALL_HTTP_CONN_MAN_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, GENERATE_HISTOGRAM_STRUCT)
+#if defined(ALIMESH)
+  HIGRESS_EXT_HTTP_CONN_MAN_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT,
+                                  GENERATE_HISTOGRAM_STRUCT)
+#endif
 };
 
 struct ConnectionManagerStats {

source/common/http/conn_manager_impl.cc

@@ -80,8 +80,16 @@ bool requestWasConnect(const RequestHeaderMapSharedPtr& headers, Protocol protoc
 ConnectionManagerStats ConnectionManagerImpl::generateStats(const std::string& prefix,
                                                             Stats::Scope& scope) {
   return ConnectionManagerStats(
+#if defined(ALIMESH)
+      {ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER_PREFIX(scope, prefix), POOL_GAUGE_PREFIX(scope, prefix),
+                               POOL_HISTOGRAM_PREFIX(scope, prefix))
+           HIGRESS_EXT_HTTP_CONN_MAN_STATS(POOL_COUNTER_PREFIX(scope, prefix),
+                                           POOL_GAUGE_PREFIX(scope, prefix),
+                                           POOL_HISTOGRAM_PREFIX(scope, prefix))},
+#else
       {ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER_PREFIX(scope, prefix), POOL_GAUGE_PREFIX(scope, prefix),
                                POOL_HISTOGRAM_PREFIX(scope, prefix))},
+#endif
       prefix, scope);
 }
 
@@ -1673,9 +1681,15 @@ void ConnectionManagerImpl::ActiveStream::refreshCachedRoute(const Router::Route
       }
       route = snapped_route_config_->route(cb, *request_headers_, filter_manager_.streamInfo(),
                                            stream_id_);
+      bool retry_found = route != nullptr;
       ENVOY_STREAM_LOG(debug,
                        "after the route was not found, search again in other scopes and found:{}",
-                       *this, route != nullptr);
+                       *this, retry_found);
+      if (retry_found) {
+        connection_manager_.stats_.named_.downstream_rq_retry_scope_found_total_.inc();
+      } else {
+        connection_manager_.stats_.named_.downstream_rq_retry_scope_not_found_total_.inc();
+      }
     }
   }
 #endif

source/common/http/filter_manager.h

@@ -569,6 +569,9 @@ class OverridableRemoteConnectionInfoSetterStreamInfo : public StreamInfo::Strea
   const Network::Address::InstanceConstSharedPtr& localAddress() const override {
     return StreamInfoImpl::downstreamAddressProvider().localAddress();
   }
+  const Network::Address::InstanceConstSharedPtr& directLocalAddress() const override {
+    return StreamInfoImpl::downstreamAddressProvider().directLocalAddress();
+  }
   bool localAddressRestored() const override {
     return StreamInfoImpl::downstreamAddressProvider().localAddressRestored();
   }

source/common/network/socket_impl.h

@@ -13,8 +13,8 @@ class ConnectionInfoSetterImpl : public ConnectionInfoSetter {
 public:
   ConnectionInfoSetterImpl(const Address::InstanceConstSharedPtr& local_address,
                            const Address::InstanceConstSharedPtr& remote_address)
-      : local_address_(local_address), remote_address_(remote_address),
-        direct_remote_address_(remote_address) {}
+      : local_address_(local_address), direct_local_address_(local_address),
+        remote_address_(remote_address), direct_remote_address_(remote_address) {}
 
   void setDirectRemoteAddressForTest(const Address::InstanceConstSharedPtr& direct_remote_address) {
     direct_remote_address_ = direct_remote_address;
@@ -31,6 +31,9 @@ class ConnectionInfoSetterImpl : public ConnectionInfoSetter {
 
   // ConnectionInfoSetter
   const Address::InstanceConstSharedPtr& localAddress() const override { return local_address_; }
+  const Address::InstanceConstSharedPtr& directLocalAddress() const override {
+    return direct_local_address_;
+  }
   void setLocalAddress(const Address::InstanceConstSharedPtr& local_address) override {
     local_address_ = local_address;
   }
@@ -76,6 +79,7 @@ class ConnectionInfoSetterImpl : public ConnectionInfoSetter {
 
 private:
   Address::InstanceConstSharedPtr local_address_;
+  Address::InstanceConstSharedPtr direct_local_address_;
   bool local_address_restored_{false};
   Address::InstanceConstSharedPtr remote_address_;
   Address::InstanceConstSharedPtr direct_remote_address_;

source/common/router/scoped_config_impl.cc

@@ -43,7 +43,7 @@ LocalPortValueExtractorImpl::LocalPortValueExtractorImpl(
 std::unique_ptr<ScopeKeyFragmentBase> LocalPortValueExtractorImpl::computeFragment(
     const Http::HeaderMap&, const StreamInfo::StreamInfo* info, ReComputeCbPtr&) const {
   ASSERT(info != nullptr, "streamInfo is nullptr.");
-  auto port = info->downstreamAddressProvider().localAddress()->ip()->port();
+  auto port = info->downstreamAddressProvider().directLocalAddress()->ip()->port();
   return std::make_unique<StringKeyFragment>(std::to_string(long(port)));
 }
 

source/extensions/filters/http/lua/wrappers.cc

@@ -192,6 +192,13 @@ int StreamInfoWrapper::luaDownstreamLocalAddress(lua_State* state) {
   return 1;
 }
 
+int StreamInfoWrapper::luaDownstreamDirectLocalAddress(lua_State* state) {
+  const std::string& local_address =
+      stream_info_.downstreamAddressProvider().directLocalAddress()->asString();
+  lua_pushlstring(state, local_address.data(), local_address.size());
+  return 1;
+}
+
 int StreamInfoWrapper::luaDownstreamDirectRemoteAddress(lua_State* state) {
   const std::string& direct_remote_address =
       stream_info_.downstreamAddressProvider().directRemoteAddress()->asString();

source/extensions/filters/http/lua/wrappers.h

@@ -207,6 +207,7 @@ class StreamInfoWrapper : public Filters::Common::Lua::BaseLuaObject<StreamInfoW
   static ExportedFunctions exportedFunctions() {
     return {{"protocol", static_luaProtocol},
             {"dynamicMetadata", static_luaDynamicMetadata},
+            {"downstreamDirectLocalAddress", static_luaDownstreamDirectLocalAddress},
             {"downstreamLocalAddress", static_luaDownstreamLocalAddress},
             {"downstreamDirectRemoteAddress", static_luaDownstreamDirectRemoteAddress},
             {"downstreamSslConnection", static_luaDownstreamSslConnection},
@@ -239,7 +240,14 @@ class StreamInfoWrapper : public Filters::Common::Lua::BaseLuaObject<StreamInfoW
   DECLARE_LUA_FUNCTION(StreamInfoWrapper, luaDownstreamLocalAddress);
 
   /**
-   * Get current downstream local address
+   * Get current direct downstream local address
+   * @return string representation of downstream directly connected local address.
+   * This is equivalent to the local address of the physical connection.
+   */
+  DECLARE_LUA_FUNCTION(StreamInfoWrapper, luaDownstreamDirectLocalAddress);
+
+  /**
+   * Get current direct downstream remote address
    * @return string representation of downstream directly connected address.
    * This is equivalent to the address of the physical connection.
    */

test/common/formatter/substitution_formatter_test.cc

@@ -880,6 +880,18 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) {
                 ProtoEq(ValueUtil::stringValue("127.0.0.2:0")));
   }
 
+  {
+    StreamInfoFormatter format("DOWNSTREAM_DIRECT_LOCAL_ADDRESS");
+    auto address = Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance(
+             "127.1.2.3", 6745)},
+         original_address = stream_info.downstream_connection_info_provider_->localAddress();
+    stream_info.downstream_connection_info_provider_->setLocalAddress(address);
+    EXPECT_EQ("127.0.0.2:0", format.formatWithContext({}, stream_info));
+    EXPECT_THAT(format.formatValueWithContext({}, stream_info),
+                ProtoEq(ValueUtil::stringValue("127.0.0.2:0")));
+    stream_info.downstream_connection_info_provider_->setLocalAddress(original_address);
+  }
+
   {
     StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT");
     EXPECT_EQ("127.0.0.2",
@@ -891,8 +903,33 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) {
   }
 
   {
-    StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_PORT");
+    StreamInfoFormatter format("DOWNSTREAM_DIRECT_LOCAL_ADDRESS_WITHOUT_PORT");
+    EXPECT_EQ("127.0.0.2", format.formatWithContext({}, stream_info));
+    EXPECT_THAT(format.formatValueWithContext({}, stream_info),
+                ProtoEq(ValueUtil::stringValue("127.0.0.2")));
+  }
 
+  {
+    StreamInfoFormatter format("DOWNSTREAM_DIRECT_LOCAL_ADDRESS_WITHOUT_PORT");
+    auto address = Network::Address::InstanceConstSharedPtr{
+        new Network::Address::Ipv4Instance("127.1.2.3", 8900)};
+    stream_info.downstream_connection_info_provider_->setLocalAddress(address);
+    EXPECT_EQ("127.0.0.2", format.formatWithContext({}, stream_info));
+    EXPECT_THAT(format.formatValueWithContext({}, stream_info),
+                ProtoEq(ValueUtil::stringValue("127.0.0.2")));
+  }
+
+  {
+    StreamInfoFormatter format("DOWNSTREAM_DIRECT_LOCAL_PORT");
+    EXPECT_EQ("0", format.formatWithContext({}, stream_info));
+    EXPECT_THAT(format.formatValueWithContext({}, stream_info), ProtoEq(ValueUtil::numberValue(0)));
+  }
+
+  {
+    StreamInfoFormatter downstream_local_port_format("DOWNSTREAM_LOCAL_PORT"),
+        downstream_direct_downstream_local_port_format("DOWNSTREAM_DIRECT_LOCAL_PORT");
+
+    StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_PORT");
     // Validate for IPv4 address
     auto address = Network::Address::InstanceConstSharedPtr{
         new Network::Address::Ipv4Instance("127.1.2.3", 8443)};
@@ -942,6 +979,11 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) {
     EXPECT_THAT(upstream_format.formatValue(request_headers, response_headers, response_trailers,
                                             stream_info, body, AccessLog::AccessLogType::NotSet),
                 ProtoEq(ValueUtil::nullValue()));
+    EXPECT_EQ("0",
+              downstream_direct_downstream_local_port_format.formatWithContext({}, stream_info));
+    EXPECT_THAT(
+        downstream_direct_downstream_local_port_format.formatValueWithContext({}, stream_info),
+        ProtoEq(ValueUtil::numberValue(0)));
   }
 
   {