Enable statsbeat

eng/versioning/version_client.txt

@@ -441,8 +441,8 @@ com.azure.tools:azure-sdk-build-tool;1.0.0;1.1.0-beta.1
 # note: The unreleased dependencies will not be manipulated with the automatic PR creation code.
 # In the pom, the version update tag after the version should name the unreleased package and the dependency version:
 # <!-- {x-version-update;unreleased_com.azure:azure-core;dependency} -->
-unreleased_com.azure:azure-core-test;1.22.0-beta.1
 unreleased_com.azure:azure-messaging-eventgrid;4.19.0-beta.1
+unreleased_com.azure:azure-core-test;1.22.0-beta.1
 
 # Released Beta dependencies: Copy the entry from above, prepend "beta_", remove the current
 # version and set the version to the released beta. Released beta dependencies are only valid

sdk/monitor/azure-monitor-opentelemetry-exporter/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "java",
   "TagPrefix": "java/monitor/azure-monitor-opentelemetry-exporter",
-  "Tag": "java/monitor/azure-monitor-opentelemetry-exporter_4051462e7c"
+  "Tag": "java/monitor/azure-monitor-opentelemetry-exporter_28d6ecbf1a"
 }

sdk/monitor/azure-monitor-opentelemetry-exporter/pom.xml

@@ -151,7 +151,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core-test</artifactId>
-      <version>1.21.0</version>  <!-- {x-version-update;com.azure:azure-core-test;dependency} -->
+      <version>1.22.0-beta.1</version>  <!-- {x-version-update;unreleased_com.azure:azure-core-test;dependency} -->
       <scope>test</scope>
     </dependency>
     <dependency>

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/AzureMonitorExporterBuilder.java

@@ -27,13 +27,17 @@
 import com.azure.monitor.opentelemetry.exporter.implementation.SpanDataMapper;
 import com.azure.monitor.opentelemetry.exporter.implementation.builders.AbstractTelemetryBuilder;
 import com.azure.monitor.opentelemetry.exporter.implementation.configuration.ConnectionString;
+import com.azure.monitor.opentelemetry.exporter.implementation.configuration.StatsbeatConnectionString;
 import com.azure.monitor.opentelemetry.exporter.implementation.heartbeat.HeartbeatExporter;
 import com.azure.monitor.opentelemetry.exporter.implementation.localstorage.LocalStorageStats;
 import com.azure.monitor.opentelemetry.exporter.implementation.localstorage.LocalStorageTelemetryPipelineListener;
 import com.azure.monitor.opentelemetry.exporter.implementation.models.ContextTagKeys;
 import com.azure.monitor.opentelemetry.exporter.implementation.pipeline.TelemetryItemExporter;
 import com.azure.monitor.opentelemetry.exporter.implementation.pipeline.TelemetryPipeline;
 import com.azure.monitor.opentelemetry.exporter.implementation.pipeline.TelemetryPipelineListener;
+import com.azure.monitor.opentelemetry.exporter.implementation.statsbeat.Feature;
+import com.azure.monitor.opentelemetry.exporter.implementation.statsbeat.StatsbeatModule;
+import com.azure.monitor.opentelemetry.exporter.implementation.utils.PropertyHelper;
 import com.azure.monitor.opentelemetry.exporter.implementation.utils.ResourceParser;
 import com.azure.monitor.opentelemetry.exporter.implementation.utils.TempDirs;
 import com.azure.monitor.opentelemetry.exporter.implementation.utils.VersionGenerator;
@@ -55,9 +59,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.BiConsumer;
 
+import static java.util.concurrent.TimeUnit.DAYS;
 import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
 
 /**
  * This class provides a fluent builder API to configure the OpenTelemetry SDK with Azure Monitor Exporters.
@@ -71,6 +78,9 @@ public final class AzureMonitorExporterBuilder {
     private static final String APPLICATIONINSIGHTS_AUTHENTICATION_SCOPE =
         "https://monitor.azure.com//.default";
 
+    private static final String STATSBEAT_LONG_INTERVAL_SECONDS = "STATSBEAT_LONG_INTERVAL_SECONDS";
+    private static final String STATSBEAT_SHORT_INTERVAL_SECONDS = "STATSBEAT_SHORT_INTERVAL_SECONDS";
+
     private static final Map<String, String> PROPERTIES =
         CoreUtils.getProperties("azure-monitor-opentelemetry-exporter.properties");
 
@@ -257,7 +267,7 @@ public SpanExporter buildTraceExporter() {
     public MetricExporter buildMetricExporter() {
         internalBuildAndFreeze();
         // TODO (trask) how to pass along configuration properties?
-        return buildMetricExporter(DefaultConfigProperties.createForTest(Collections.emptyMap()));
+        return buildMetricExporter(DefaultConfigProperties.create(Collections.emptyMap()));
     }
 
     /**
@@ -271,7 +281,7 @@ public MetricExporter buildMetricExporter() {
     public LogRecordExporter buildLogRecordExporter() {
         internalBuildAndFreeze();
         // TODO (trask) how to pass along configuration properties?
-        return buildLogRecordExporter(DefaultConfigProperties.createForTest(Collections.emptyMap()));
+        return buildLogRecordExporter(DefaultConfigProperties.create(Collections.emptyMap()));
     }
 
     /**
@@ -351,10 +361,45 @@ private SpanExporter buildTraceExporter(ConfigProperties configProperties) {
     private MetricExporter buildMetricExporter(ConfigProperties configProperties) {
         HeartbeatExporter.start(
             MINUTES.toSeconds(15), createDefaultsPopulator(configProperties), builtTelemetryItemExporter::send);
+        if (connectionString != null) {
+            StatsbeatModule statsbeatModule = new StatsbeatModule(PropertyHelper::lazyUpdateVmRpIntegration);
+            statsbeatModule.start(
+                builtTelemetryItemExporter,
+                this::getStatsbeatConnectionString,
+                connectionString::getInstrumentationKey,
+                false,
+                getStatsbeaLongInterval(configProperties.getInt(STATSBEAT_LONG_INTERVAL_SECONDS)),
+                getStatsbeatShortInterval(configProperties.getInt(STATSBEAT_SHORT_INTERVAL_SECONDS)),
+                false,
+                initStatsbeatFeatures());
+        }
         return new AzureMonitorMetricExporter(
             new MetricDataMapper(createDefaultsPopulator(configProperties), true), builtTelemetryItemExporter);
     }
 
+    private long getStatsbeaLongInterval(Integer intervalFromConfig) {
+        if (intervalFromConfig == null) {
+            return DAYS.toSeconds(1); // default to every 24 hours
+        }
+        return SECONDS.toSeconds(intervalFromConfig);
+    }
+
+    private long getStatsbeatShortInterval(Integer intervalFromConfig) {
+        if (intervalFromConfig == null) {
+            return MINUTES.toSeconds(15); // default to every 15 mins
+        }
+        return SECONDS.toSeconds(intervalFromConfig);
+    }
+
+    private Set<Feature> initStatsbeatFeatures() {
+        // TODO (jean): start tracking native image usage based on a system property or env var to indicate it's from the native image path
+        return Collections.emptySet();
+    }
+
+    private StatsbeatConnectionString getStatsbeatConnectionString() {
+        return StatsbeatConnectionString.create(connectionString, null, null);
+    }
+
     private LogRecordExporter buildLogRecordExporter(ConfigProperties configProperties) {
         return new AzureMonitorLogRecordExporter(
             new LogDataMapper(true, false, createDefaultsPopulator(configProperties)), builtTelemetryItemExporter);

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/configuration/StatsbeatConnectionString.java

@@ -91,7 +91,7 @@ public final class StatsbeatConnectionString {
     public static StatsbeatConnectionString create(
         ConnectionString connectionString,
         @Nullable String instrumentationKey,
-        String ingestionEndpoint) {
+        @Nullable String ingestionEndpoint) {
 
         if (instrumentationKey == null || instrumentationKey.isEmpty()) {
             InstrumentationKeyEndpointPair pair =

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/pipeline/TelemetryItemExporter.java

@@ -137,7 +137,6 @@ public CompletableResultCode shutdown() {
     CompletableResultCode internalSendByBatch(TelemetryItemBatchKey telemetryItemBatchKey,
                                               List<TelemetryItem> telemetryItems) {
         List<ByteBuffer> byteBuffers;
-
         // Don't send _OTELRESOURCE_ custom metric when OTEL_RESOURCE_ATTRIBUTES env var is empty
         // Don't send _OTELRESOURCE_ custom metric to Statsbeat yet
         // insert _OTELRESOURCE_ at the beginning of each batch

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/statsbeat/AzureMetadataService.java

@@ -8,9 +8,6 @@
 import com.azure.core.http.HttpPipelineBuilder;
 import com.azure.core.http.HttpRequest;
 import com.azure.core.http.HttpResponse;
-import com.azure.core.http.policy.CookiePolicy;
-import com.azure.core.http.policy.RetryPolicy;
-import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.monitor.opentelemetry.exporter.implementation.NoopTracer;
 import com.azure.monitor.opentelemetry.exporter.implementation.utils.ThreadPoolUtils;
 import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -81,7 +78,7 @@ void updateMetadata(String response) throws IOException {
     private void updateMetadata(MetadataInstanceResponse metadataInstanceResponse) {
         vmMetadataServiceCallback.accept(metadataInstanceResponse);
         attachStatsbeat.updateMetadataInstance(metadataInstanceResponse);
-        customDimensions.setResourceProvider(ResourceProvider.RP_VM);
+        customDimensions.setResourceProviderVm();
 
         // osType from the Azure Metadata Service has a higher precedence over the running app’s
         // operating system.

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/statsbeat/CustomDimensions.java

@@ -6,6 +6,7 @@
 import com.azure.monitor.opentelemetry.exporter.implementation.builders.StatsbeatTelemetryBuilder;
 import com.azure.monitor.opentelemetry.exporter.implementation.utils.PropertyHelper;
 import com.azure.monitor.opentelemetry.exporter.implementation.utils.SystemInformation;
+import com.azure.monitor.opentelemetry.exporter.implementation.utils.VersionGenerator;
 
 public class CustomDimensions {
 
@@ -19,51 +20,28 @@ public class CustomDimensions {
 
     // visible for testing
     CustomDimensions() {
-        String qualifiedSdkVersion = PropertyHelper.getQualifiedSdkVersionString();
-
-        if (qualifiedSdkVersion.startsWith("aw")) {
-            resourceProvider = ResourceProvider.RP_APPSVC;
-            operatingSystem = OperatingSystem.OS_WINDOWS;
-        } else if (qualifiedSdkVersion.startsWith("al")) {
-            resourceProvider = ResourceProvider.RP_APPSVC;
-            operatingSystem = OperatingSystem.OS_LINUX;
-        } else if (qualifiedSdkVersion.startsWith("kw")) {
-            resourceProvider = ResourceProvider.RP_AKS;
-            operatingSystem = OperatingSystem.OS_WINDOWS;
-        } else if (qualifiedSdkVersion.startsWith("kl")) {
-            resourceProvider = ResourceProvider.RP_AKS;
-            operatingSystem = OperatingSystem.OS_LINUX;
-        } else if (qualifiedSdkVersion.startsWith("fw")) {
-            resourceProvider = ResourceProvider.RP_FUNCTIONS;
-            operatingSystem = OperatingSystem.OS_WINDOWS;
-        } else if (qualifiedSdkVersion.startsWith("fl")) {
-            resourceProvider = ResourceProvider.RP_FUNCTIONS;
-            operatingSystem = OperatingSystem.OS_LINUX;
-        } else {
-            resourceProvider = ResourceProvider.UNKNOWN;
-            operatingSystem = initOperatingSystem();
-        }
-
-        sdkVersion = qualifiedSdkVersion.substring(qualifiedSdkVersion.lastIndexOf(':') + 1);
+        resourceProvider = ResourceProvider.initResourceProvider();
+        operatingSystem = initOperatingSystem();
+        sdkVersion = initSdkVersion();
         runtimeVersion = System.getProperty("java.version");
 
-        attachType = RpAttachType.getRpAttachType();
+        attachType = RpAttachType.getRpAttachTypeString();
         language = "java";
     }
 
-    public ResourceProvider getResourceProvider() {
+    ResourceProvider getResourceProvider() {
         return resourceProvider;
     }
 
-    public OperatingSystem getOperatingSystem() {
+    OperatingSystem getOperatingSystem() {
         return operatingSystem;
     }
 
-    public void setResourceProvider(ResourceProvider resourceProvider) {
-        this.resourceProvider = resourceProvider;
+    void setResourceProviderVm() {
+        this.resourceProvider = ResourceProvider.RP_VM;
     }
 
-    public void setOperatingSystem(OperatingSystem operatingSystem) {
+    void setOperatingSystem(OperatingSystem operatingSystem) {
         this.operatingSystem = operatingSystem;
     }
 
@@ -86,4 +64,12 @@ private static OperatingSystem initOperatingSystem() {
             return OperatingSystem.OS_UNKNOWN;
         }
     }
+
+    private static String initSdkVersion() {
+        if (RpAttachType.getRpAttachType() == RpAttachType.MANUAL) {
+            return VersionGenerator.getSdkVersion();
+        }
+        String qualifiedSdkVersionString = PropertyHelper.getQualifiedSdkVersionString();
+        return qualifiedSdkVersionString.substring(qualifiedSdkVersionString.lastIndexOf(':')+1);
+    }
 }

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/statsbeat/FeatureStatsbeat.java

@@ -69,11 +69,14 @@ protected void send(TelemetryItemExporter telemetryItemExporter) {
             featureType = "1";
         }
 
-        StatsbeatTelemetryBuilder telemetryBuilder = createStatsbeatTelemetry(FEATURE_METRIC_NAME, 0);
-        telemetryBuilder.addProperty("feature", featureValue);
-        telemetryBuilder.addProperty("type", featureType);
+        // don't send feature/instrumentation statsbeat when it's empty
+        if (!featureValue.isEmpty()) {
+            StatsbeatTelemetryBuilder telemetryBuilder = createStatsbeatTelemetry(FEATURE_METRIC_NAME, 0);
+            telemetryBuilder.addProperty("feature", featureValue);
+            telemetryBuilder.addProperty("type", featureType);
 
-        telemetryItemExporter.send(Collections.singletonList(telemetryBuilder.build()));
+            telemetryItemExporter.send(Collections.singletonList(telemetryBuilder.build()));
+        }
     }
 
     void trackConfigurationOptions(Set<Feature> featureSet) {

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/statsbeat/ResourceProvider.java

@@ -3,11 +3,14 @@
 
 package com.azure.monitor.opentelemetry.exporter.implementation.statsbeat;
 
+import com.azure.monitor.opentelemetry.exporter.implementation.utils.Strings;
+
 enum ResourceProvider {
     RP_FUNCTIONS("functions"),
     RP_APPSVC("appsvc"),
     RP_VM("vm"),
     RP_AKS("aks"),
+    RP_SPRING_CLOUD("springcloud"),
     UNKNOWN("unknown");
 
     private final String value;
@@ -19,4 +22,18 @@ enum ResourceProvider {
     String getValue() {
         return value;
     }
+
+    static ResourceProvider initResourceProvider() {
+        if ("java".equals(System.getenv("FUNCTIONS_WORKER_RUNTIME"))) {
+            return ResourceProvider.RP_FUNCTIONS;
+        } else if (!Strings.isNullOrEmpty(System.getenv("WEBSITE_SITE_NAME"))) {
+            return ResourceProvider.RP_APPSVC;
+        } else if (!Strings.isNullOrEmpty(System.getenv("KUBERNETES_SERVICE_HOST"))) {
+            return ResourceProvider.RP_AKS;
+        } else if (!Strings.isNullOrEmpty(System.getenv("APPLICATIONINSIGHTS_SPRINGCLOUD_SERVICE_ID"))) {
+            return ResourceProvider.RP_SPRING_CLOUD;
+        } else {
+            return ResourceProvider.UNKNOWN;
+        }
+    }
 }

sdk/monitor/azure-monitor-opentelemetry-exporter/src/main/java/com/azure/monitor/opentelemetry/exporter/implementation/statsbeat/RpAttachType.java

@@ -9,7 +9,7 @@ public enum RpAttachType {
     STANDALONE_AUTO("StandaloneAuto"), // RP attach is enabled via a custom JAVA_OPTS or on premise resources
     INTEGRATED_AUTO("IntegratedAuto"); // RP attach is on by default
 
-    private static volatile RpAttachType attachType;
+    private static volatile RpAttachType attachType = MANUAL;
     private final String label;
 
     private RpAttachType(String label) {
@@ -20,7 +20,11 @@ public static void setRpAttachType(RpAttachType type) {
         attachType = type;
     }
 
-    public static String getRpAttachType() {
+    public static RpAttachType getRpAttachType() {
+        return attachType;
+    }
+
+    public static String getRpAttachTypeString() {
         return attachType != null ? attachType.label : null;
     }
 }