Adding new Dell EMC ECS Object Storage Plugin for CloudStack

client/pom.xml

@@ -632,6 +632,11 @@
             <artifactId>cloud-plugin-storage-object-minio</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-plugin-storage-object-ecs</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.cloudstack</groupId>
             <artifactId>cloud-plugin-storage-object-ceph</artifactId>

plugins/pom.xml

@@ -140,6 +140,7 @@
         <module>storage/object/ceph</module>
         <module>storage/object/cloudian</module>
         <module>storage/object/simulator</module>
+        <module>storage/object/ECS</module>
 
 
         <module>storage-allocators/random</module>

plugins/storage/object/ECS/pom.xml

@@ -0,0 +1,47 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>cloud-plugin-storage-object-ecs</artifactId>
+    <name>Apache CloudStack Plugin - ECS object storage provider</name>
+    <parent>
+        <groupId>org.apache.cloudstack</groupId>
+        <artifactId>cloudstack-plugins</artifactId>
+        <version>4.23.0.0-SNAPSHOT</version>
+        <relativePath>../../../pom.xml</relativePath>
+    </parent>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-engine-storage</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-engine-storage-object</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-engine-schema</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+</project>

plugins/storage/object/ECS/src/main/java/org/apache/cloudstack/storage/datastore/driver/EcsConstants.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.storage.datastore.driver;
+
+public final class EcsConstants {
+    private EcsConstants() {}
+
+    // Object store details keys
+    public static final String MGMT_URL  = "mgmt_url";
+    public static final String SA_USER   = "sa_user";
+    public static final String SA_PASS   = "sa_password";
+    public static final String NAMESPACE = "namespace";
+    public static final String INSECURE  = "insecure";
+    public static final String S3_HOST   = "s3_host";
+    public static final String USER_PREFIX = "user_prefix";
+    public static final String DEFAULT_USER_PREFIX = "cs-";
+
+    // Per-account keys
+    public static final String AD_KEY_ACCESS = "ecs.accesskey";
+    public static final String AD_KEY_SECRET = "ecs.secretkey";
+}

plugins/storage/object/ECS/src/main/java/org/apache/cloudstack/storage/datastore/driver/EcsMgmtTokenManager.java

@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.storage.datastore.driver;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class EcsMgmtTokenManager {
+    private static final long DEFAULT_TOKEN_MAX_AGE_SEC = 300;
+    private static final long EXPIRY_SKEW_SEC = 30;
+
+    private static final ConcurrentHashMap<TokenKey, TokenEntry> TOKEN_CACHE = new ConcurrentHashMap<>();
+    private static final ConcurrentHashMap<TokenKey, Object> TOKEN_LOCKS = new ConcurrentHashMap<>();
+
+    static final class EcsUnauthorizedException extends RuntimeException {
+        EcsUnauthorizedException(final String msg) { super(msg); }
+    }
+
+    @FunctionalInterface
+    public interface WithToken<T> { T run(String token) throws Exception; }
+
+    private static final class TokenKey {
+        final String mgmtUrl;
+        final String user;
+        TokenKey(final String mgmtUrl, final String user) {
+            this.mgmtUrl = mgmtUrl;
+            this.user = user;
+        }
+        @Override public boolean equals(final Object o) {
+            if (this == o) return true;
+            if (!(o instanceof TokenKey)) return false;
+            final TokenKey k = (TokenKey) o;
+            return Objects.equals(mgmtUrl, k.mgmtUrl) && Objects.equals(user, k.user);
+        }
+        @Override public int hashCode() { return Objects.hash(mgmtUrl, user); }
+    }
+
+    private static final class TokenEntry {
+        final String token;
+        final long expiresAtMs;
+        TokenEntry(final String token, final long expiresAtMs) {
+            this.token = token;
+            this.expiresAtMs = expiresAtMs;
+        }
+        boolean validNow() {
+            return token != null && !token.isBlank() && System.currentTimeMillis() < expiresAtMs;
+        }
+    }
+
+    public <T> T callWithRetry401(final EcsObjectStoreDriverImpl.EcsCfg cfg,
+                                  final WithToken<T> op,
+                                  final HttpClientFactory httpFactory) throws Exception {
+        try {
+            return op.run(getAuthToken(cfg, httpFactory));
+        } catch (EcsUnauthorizedException u) {
+            invalidate(cfg);
+            return op.run(getAuthToken(cfg, httpFactory));
+        }
+    }
+
+    public void invalidate(final EcsObjectStoreDriverImpl.EcsCfg cfg) {
+        TOKEN_CACHE.remove(new TokenKey(trimTail(cfg.mgmtUrl), cfg.saUser));
+    }
+
+    public String getAuthToken(final EcsObjectStoreDriverImpl.EcsCfg cfg,
+                               final HttpClientFactory httpFactory) {
+        final String mu = trimTail(cfg.mgmtUrl);
+        final TokenKey key = new TokenKey(mu, cfg.saUser);
+
+        final TokenEntry cached = TOKEN_CACHE.get(key);
+        if (cached != null && cached.validNow()) return cached.token;
+
+        final Object lock = TOKEN_LOCKS.computeIfAbsent(key, k -> new Object());
+        synchronized (lock) {
+            final TokenEntry cached2 = TOKEN_CACHE.get(key);
+            if (cached2 != null && cached2.validNow()) return cached2.token;
+
+            final TokenEntry fresh = loginAndGetTokenFresh(mu, cfg.saUser, cfg.saPass, cfg.insecure, httpFactory);
+            TOKEN_CACHE.put(key, fresh);
+            return fresh.token;
+        }
+    }
+
+    private TokenEntry loginAndGetTokenFresh(final String mgmtUrl,
+                                             final String user,
+                                             final String pass,
+                                             final boolean insecure,
+                                             final HttpClientFactory httpFactory) {
+        try (CloseableHttpClient http = httpFactory.build(insecure)) {
+            final HttpGet get = new HttpGet(mgmtUrl + "/login");
+            UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pass);
+            get.addHeader(new BasicScheme().authenticate(creds, get, null));
+
+            try (CloseableHttpResponse resp = http.execute(get)) {
+                final int status = resp.getStatusLine().getStatusCode();
+                if (status != 200 && status != 201) {
+                    final String body = resp.getEntity() != null
+                            ? EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8)
+                            : "";
+                    throw new CloudRuntimeException("ECS /login failed: HTTP " + status + " body=" + body);
+                }
+                if (resp.getFirstHeader("X-SDS-AUTH-TOKEN") == null) {
+                    throw new CloudRuntimeException("ECS /login did not return X-SDS-AUTH-TOKEN header");
+                }
+
+                final String token = resp.getFirstHeader("X-SDS-AUTH-TOKEN").getValue();
+
+                long maxAgeSec = DEFAULT_TOKEN_MAX_AGE_SEC;
+                try {
+                    if (resp.getFirstHeader("X-SDS-AUTH-MAX-AGE") != null) {
+                        maxAgeSec = Long.parseLong(resp.getFirstHeader("X-SDS-AUTH-MAX-AGE").getValue().trim());
+                    }
+                } catch (Exception ignore) { }
+
+                final long effectiveSec = Math.max(5, maxAgeSec - EXPIRY_SKEW_SEC);
+                final long expiresAtMs = System.currentTimeMillis() + (effectiveSec * 1000L);
+                return new TokenEntry(token, expiresAtMs);
+            }
+        } catch (Exception e) {
+            throw new CloudRuntimeException("Failed to obtain ECS auth token: " + e.getMessage(), e);
+        }
+    }
+
+    private static String trimTail(final String s) {
+        if (s == null) return null;
+        return s.endsWith("/") ? s.substring(0, s.length() - 1) : s;
+    }
+
+    /** Simple seam for testability; implemented by the driver using its existing buildHttpClient(). */
+    @FunctionalInterface
+    public interface HttpClientFactory {
+        CloseableHttpClient build(boolean insecure);
+    }
+}