From 0665a0400d9419e9510cfe118516ca7351805b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Sexenian?= <99925035+tomas-sexenian@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:22:45 -0300 Subject: [PATCH 1/5] STORAGE_PROVIDER_PRIVACY property rename Issue:108183 --- .../main/java/com/genexus/db/driver/ExternalProviderBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java index 9d6fc7ed1..3fa99f40a 100644 --- a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java +++ b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java @@ -16,7 +16,7 @@ public abstract class ExternalProviderBase { static final String FOLDER = "FOLDER_NAME"; @Deprecated - static final String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_ACL"; + static final String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_PRIVACY"; @Deprecated static final String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; From 66c9aa37e401e8a8aa12b364fc14f1d1b9b0f582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Sexenian?= <99925035+tomas-sexenian@users.noreply.github.com> Date: Thu, 25 Apr 2024 12:25:35 -0300 Subject: [PATCH 2/5] Dynamically choose between ACL and no ACL method --- .../db/driver/ExternalProviderS3V2.java | 457 +++++++++++------- 1 file changed, 290 insertions(+), 167 deletions(-) diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index a250db53b..c57cbf775 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -13,8 +13,6 @@ import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.presigner.S3Presigner; -import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; -import software.amazon.awssdk.utils.IoUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -23,9 +21,12 @@ import com.genexus.util.GXService; import com.genexus.util.StorageUtils; import com.genexus.StructSdtMessages_Message; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.utils.IoUtils; import java.io.*; import java.net.URI; +import java.net.URL; import java.net.URLEncoder; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -75,6 +76,8 @@ public class ExternalProviderS3V2 extends ExternalProviderBase implements Extern private String endpointUrl = ".s3.amazonaws.com/"; private int defaultExpirationMinutes = DEFAULT_EXPIRATION_MINUTES; private Boolean pathStyleUrls = false; + private Boolean objectOwnershipEnabled; + public String getName() { return NAME; @@ -100,6 +103,9 @@ private void initialize() throws Exception { String bucket = getEncryptedPropertyValue(BUCKET, BUCKET_DEPRECATED); String folder = getPropertyValue(FOLDER, FOLDER_DEPRECATED, ""); clientRegion = getPropertyValue(REGION, REGION_DEPRECATED, DEFAULT_REGION); + + objectOwnershipEnabled = !getPropertyValue(DEFAULT_ACL, DEFAULT_ACL_DEPRECATED, "").equals("Bucket owner enforced"); + String endpointValue = getPropertyValue(STORAGE_ENDPOINT, STORAGE_ENDPOINT_DEPRECATED, ""); if (endpointValue.equals("custom")) { endpointValue = getPropertyValue(STORAGE_CUSTOM_ENDPOINT, STORAGE_CUSTOM_ENDPOINT_DEPRECATED); @@ -112,7 +118,7 @@ private void initialize() throws Exception { } if (this.client == null) { - if (clientRegion.length() == 0) { + if (clientRegion.isEmpty()) { clientRegion = DEFAULT_REGION; } @@ -142,7 +148,7 @@ private S3Client buildS3Client(String accessKey, String secretKey, String endpoi logger.debug("Using IAM Credentials"); } - if (endpoint.length() > 0 && !endpoint.contains(".amazonaws.com")) { + if (!endpoint.isEmpty() && !endpoint.contains(".amazonaws.com")) { pathStyleUrls = true; s3Client = builder @@ -222,7 +228,7 @@ public void download(String externalFileName, String localFile, ResourceAccessCo .build(), ResponseTransformer.toBytes()); try (InputStream objectData = objectBytes.asInputStream()) { - try (OutputStream outputStream = new FileOutputStream(localFile)) { + try (OutputStream outputStream = Files.newOutputStream(Paths.get(localFile))) { int read; byte[] bytes = new byte[1024]; while ((read = objectData.read(bytes)) != -1) { @@ -230,67 +236,21 @@ public void download(String externalFileName, String localFile, ResourceAccessCo } } } - } catch (FileNotFoundException ex) { - logger.error("Error while downloading file to the external provider", ex); } catch (IOException ex) { logger.error("Error while downloading file to the external provider", ex); } } public String upload(String localFile, String externalFileName, ResourceAccessControlList acl) { - client.putObject(PutObjectRequest.builder() - .bucket(bucket) - .key(externalFileName) - .build(), - RequestBody.fromFile(Paths.get(localFile))); - // As of December 2023, some S3-compatible storages do not - // implement every AWS S3 feature such as setting an object ACL - if (endpointUrl.contains(".amazonaws.com")) - client.putObjectAcl(PutObjectAclRequest.builder() - .bucket(bucket) - .key(externalFileName) - .acl(internalToAWSACL(acl)) - .build()); - return getResourceUrl(externalFileName, acl, defaultExpirationMinutes); - } - - private ObjectCannedACL internalToAWSACL(ResourceAccessControlList acl) { - if (acl == ResourceAccessControlList.Default) { - acl = this.defaultAcl; - } - - ObjectCannedACL accessControl = ObjectCannedACL.PRIVATE; - if (acl == ResourceAccessControlList.Private) { - accessControl = ObjectCannedACL.PRIVATE; - } else if (acl == ResourceAccessControlList.PublicRead) { - accessControl = ObjectCannedACL.PUBLIC_READ; - } else if (acl == ResourceAccessControlList.PublicReadWrite) { - accessControl = ObjectCannedACL.PUBLIC_READ_WRITE; - } - return accessControl; + return objectOwnershipEnabled ? + uploadWithACL(localFile, externalFileName, acl) : + uploadWithoutACL(localFile, externalFileName); } public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) { - try { - ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input)); - PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() - .bucket(bucket) - .key(externalFileName) - .contentType(externalFileName.endsWith(".tmp") ? "image/jpeg" : null); - if (endpointUrl.contains(".amazonaws.com")) - putObjectRequestBuilder = putObjectRequestBuilder.acl(internalToAWSACL(acl)); - PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); - - PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer)); - if (!response.sdkHttpResponse().isSuccessful()) { - logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error")); - } - - return getResourceUrl(externalFileName, acl, defaultExpirationMinutes); - } catch (IOException ex) { - logger.error("Error while uploading file to the external provider.", ex); - return ""; - } + return objectOwnershipEnabled ? + uploadWithACL(externalFileName, input, acl) : + uploadWithoutACL(externalFileName, input); } public String get(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) { @@ -306,41 +266,9 @@ public String get(String externalFileName, ResourceAccessControlList acl, int ex } private String getResourceUrl(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) { - if (internalToAWSACL(acl) == ObjectCannedACL.PRIVATE) { - expirationMinutes = expirationMinutes > 0 ? expirationMinutes : defaultExpirationMinutes; - Instant expiration = Instant.now().plus(Duration.ofMinutes(expirationMinutes)); - - GetObjectRequest getObjectRequest = GetObjectRequest.builder() - .bucket(bucket) - .key(externalFileName) - .build(); - - PresignedGetObjectRequest presignedGetObjectRequest = - presigner.presignGetObject(r -> r.signatureDuration(Duration.between(Instant.now(), expiration)) - .getObjectRequest(getObjectRequest)); - - return presignedGetObjectRequest.url().toString(); - } else { - try { - int lastIndex = Math.max(externalFileName.lastIndexOf('/'), externalFileName.lastIndexOf('\\')); - String path = externalFileName.substring(0, lastIndex + 1); - String fileName = externalFileName.substring(lastIndex + 1); - String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8.toString()); - - String url = String.format( - "https://%s.s3.%s.amazonaws.com/%s%s", - bucket, - clientRegion, - path, - encodedFileName - ); - - return url; - } catch (UnsupportedEncodingException uee) { - logger.error("Failed to encode resource URL for " + externalFileName, uee); - return ""; - } - } + return objectOwnershipEnabled ? + getResourceUrlWithACL(externalFileName, acl, expirationMinutes) : + getResourceUrlWithoutACL(externalFileName); } public void delete(String objectName, ResourceAccessControlList acl) { @@ -358,50 +286,15 @@ public String rename(String objectName, String newName, ResourceAccessControlLis } public String copy(String objectName, String newName, ResourceAccessControlList acl) { - CopyObjectRequest.Builder requestBuilder = CopyObjectRequest.builder() - .sourceBucket(bucket) - .sourceKey(objectName) - .destinationBucket(bucket) - .destinationKey(newName); - if (endpointUrl.contains(".amazonaws.com")) - requestBuilder = requestBuilder.acl(internalToAWSACL(acl)); - CopyObjectRequest request = requestBuilder.build(); - client.copyObject(request); - return getResourceUrl(newName, acl, defaultExpirationMinutes); + return objectOwnershipEnabled ? + copyWithACL(objectName, newName, acl) : + copyWithoutACL(objectName, newName); } public String copy(String objectUrl, String newName, String tableName, String fieldName, ResourceAccessControlList acl) { - String resourceFolderName = buildPath(folder, tableName, fieldName); - String resourceKey = resourceFolderName + StorageUtils.DELIMITER + newName; - - try { - objectUrl = new URI(objectUrl).getPath(); - } catch (Exception e) { - logger.error("Failed to Parse Storage Object URI for Copy operation", e); - } - - Map metadata = new HashMap<>(); - metadata.put("Table", tableName); - metadata.put("Field", fieldName); - metadata.put("KeyValue", StorageUtils.encodeNonAsciiCharacters(resourceKey)); - - GetObjectRequest getObjectRequest = GetObjectRequest.builder() - .bucket(bucket) - .key(objectUrl) - .build(); - ResponseBytes objectBytes = client.getObjectAsBytes(getObjectRequest); - - PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() - .bucket(bucket) - .key(resourceKey) - .metadata(metadata) - .contentType(getContentType(newName)); - if (endpointUrl.contains(".amazonaws.com")) - putObjectRequestBuilder = putObjectRequestBuilder.acl(internalToAWSACL(acl)); - PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); - client.putObject(putObjectRequest, RequestBody.fromBytes(objectBytes.asByteArray())); - - return getResourceUrl(resourceKey, acl, defaultExpirationMinutes); + return objectOwnershipEnabled ? + copyWithACL(objectUrl, newName, tableName, fieldName, acl) : + copyWithoutACL(objectUrl, newName, tableName, fieldName); } private String getContentType(String fileName) { @@ -425,45 +318,45 @@ private String findContentTypeByExtension(String fileName) { } private static Map contentTypes = new HashMap() {{ - put("txt" , "text/plain"); - put("rtx" , "text/richtext"); - put("htm" , "text/html"); - put("html" , "text/html"); - put("xml" , "text/xml"); - put("aif" , "audio/x-aiff"); - put("au" , "audio/basic"); - put("wav" , "audio/wav"); - put("bmp" , "image/bmp"); - put("gif" , "image/gif"); - put("jpe" , "image/jpeg"); - put("jpeg" , "image/jpeg"); - put("jpg" , "image/jpeg"); - put("jfif" , "image/pjpeg"); - put("tif" , "image/tiff"); - put("tiff" , "image/tiff"); - put("png" , "image/x-png"); - put("3gp" , "video/3gpp"); - put("3g2" , "video/3gpp2"); - put("mpg" , "video/mpeg"); - put("mpeg" , "video/mpeg"); - put("mov" , "video/quicktime"); - put("qt" , "video/quicktime"); - put("avi" , "video/x-msvideo"); - put("exe" , "application/octet-stream"); - put("dll" , "application/x-msdownload"); - put("ps" , "application/postscript"); - put("pdf" , "application/pdf"); - put("svg" , "image/svg+xml"); - put("tgz" , "application/x-compressed"); - put("zip" , "application/x-zip-compressed"); - put("gz" , "application/x-gzip"); - put("json" , "application/json"); + put("txt" , "text/plain"); + put("rtx" , "text/richtext"); + put("htm" , "text/html"); + put("html" , "text/html"); + put("xml" , "text/xml"); + put("aif" , "audio/x-aiff"); + put("au" , "audio/basic"); + put("wav" , "audio/wav"); + put("bmp" , "image/bmp"); + put("gif" , "image/gif"); + put("jpe" , "image/jpeg"); + put("jpeg" , "image/jpeg"); + put("jpg" , "image/jpeg"); + put("jfif" , "image/pjpeg"); + put("tif" , "image/tiff"); + put("tiff" , "image/tiff"); + put("png" , "image/x-png"); + put("3gp" , "video/3gpp"); + put("3g2" , "video/3gpp2"); + put("mpg" , "video/mpeg"); + put("mpeg" , "video/mpeg"); + put("mov" , "video/quicktime"); + put("qt" , "video/quicktime"); + put("avi" , "video/x-msvideo"); + put("exe" , "application/octet-stream"); + put("dll" , "application/x-msdownload"); + put("ps" , "application/postscript"); + put("pdf" , "application/pdf"); + put("svg" , "image/svg+xml"); + put("tgz" , "application/x-compressed"); + put("zip" , "application/x-zip-compressed"); + put("gz" , "application/x-gzip"); + put("json" , "application/json"); }}; private String buildPath(String... pathPart) { ArrayList pathParts = new ArrayList<>(); for (String part : pathPart) { - if (part.length() > 0) { + if (!part.isEmpty()) { pathParts.add(part); } } @@ -682,6 +575,236 @@ private String getStorageUriWithoutRegion() { "https://" + bucket + ".s3.amazonaws.com/" : ".s3.amazonaws.com//" + bucket + "/"; } + + // With ACL implementation + + private String uploadWithACL(String localFile, String externalFileName, ResourceAccessControlList acl) { + client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(externalFileName) + .build(), + RequestBody.fromFile(Paths.get(localFile))); + if (endpointUrl.contains(".amazonaws.com")) + client.putObjectAcl(PutObjectAclRequest.builder() + .bucket(bucket) + .key(externalFileName) + .acl(internalToAWSACLWithACL(acl)) + .build()); + return getResourceUrl(externalFileName, acl, defaultExpirationMinutes); + } + + private ObjectCannedACL internalToAWSACLWithACL(ResourceAccessControlList acl) { + if (acl == ResourceAccessControlList.Default) + acl = this.defaultAcl; + + ObjectCannedACL accessControl = ObjectCannedACL.PRIVATE; + if (acl == ResourceAccessControlList.Private) + accessControl = ObjectCannedACL.PRIVATE; + else if (acl == ResourceAccessControlList.PublicRead) + accessControl = ObjectCannedACL.PUBLIC_READ; + else if (acl == ResourceAccessControlList.PublicReadWrite) + accessControl = ObjectCannedACL.PUBLIC_READ_WRITE; + return accessControl; + } + + private String uploadWithACL(String externalFileName, InputStream input, ResourceAccessControlList acl) { + try { + ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input)); + PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() + .bucket(bucket) + .key(externalFileName) + .contentType(externalFileName.endsWith(".tmp") ? "image/jpeg" : null); + if (endpointUrl.contains(".amazonaws.com")) + putObjectRequestBuilder = putObjectRequestBuilder.acl(internalToAWSACLWithACL(acl)); + PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); + + PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer)); + if (!response.sdkHttpResponse().isSuccessful()) { + logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error")); + } + + return getResourceUrl(externalFileName, acl, defaultExpirationMinutes); + } catch (IOException ex) { + logger.error("Error while uploading file to the external provider.", ex); + return ""; + } + } + + private String getResourceUrlWithACL(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) { + if (internalToAWSACLWithACL(acl) == ObjectCannedACL.PRIVATE) { + expirationMinutes = expirationMinutes > 0 ? expirationMinutes : defaultExpirationMinutes; + Instant expiration = Instant.now().plus(Duration.ofMinutes(expirationMinutes)); + + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(externalFileName) + .build(); + + PresignedGetObjectRequest presignedGetObjectRequest = + presigner.presignGetObject(r -> r.signatureDuration(Duration.between(Instant.now(), expiration)) + .getObjectRequest(getObjectRequest)); + + return presignedGetObjectRequest.url().toString(); + } else { + try { + int lastIndex = Math.max(externalFileName.lastIndexOf('/'), externalFileName.lastIndexOf('\\')); + String path = externalFileName.substring(0, lastIndex + 1); + String fileName = externalFileName.substring(lastIndex + 1); + String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8.toString()); + + String url = String.format( + "https://%s.s3.%s.amazonaws.com/%s%s", + bucket, + clientRegion, + path, + encodedFileName + ); + + return url; + } catch (UnsupportedEncodingException uee) { + logger.error("Failed to encode resource URL for " + externalFileName, uee); + return ""; + } + } + } + + private String copyWithACL(String objectName, String newName, ResourceAccessControlList acl) { + CopyObjectRequest.Builder requestBuilder = CopyObjectRequest.builder() + .sourceBucket(bucket) + .sourceKey(objectName) + .destinationBucket(bucket) + .destinationKey(newName); + if (endpointUrl.contains(".amazonaws.com")) + requestBuilder = requestBuilder.acl(internalToAWSACLWithACL(acl)); + CopyObjectRequest request = requestBuilder.build(); + client.copyObject(request); + return getResourceUrl(newName, acl, defaultExpirationMinutes); + } + + private String copyWithACL(String objectUrl, String newName, String tableName, String fieldName, ResourceAccessControlList acl) { + String resourceFolderName = buildPath(folder, tableName, fieldName); + String resourceKey = resourceFolderName + StorageUtils.DELIMITER + newName; + + try { + objectUrl = new URI(objectUrl).getPath(); + } catch (Exception e) { + logger.error("Failed to Parse Storage Object URI for Copy operation", e); + } + + Map metadata = new HashMap<>(); + metadata.put("Table", tableName); + metadata.put("Field", fieldName); + metadata.put("KeyValue", StorageUtils.encodeNonAsciiCharacters(resourceKey)); + + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(objectUrl) + .build(); + ResponseBytes objectBytes = client.getObjectAsBytes(getObjectRequest); + + PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() + .bucket(bucket) + .key(resourceKey) + .metadata(metadata) + .contentType(getContentType(newName)); + if (endpointUrl.contains(".amazonaws.com")) + putObjectRequestBuilder = putObjectRequestBuilder.acl(internalToAWSACLWithACL(acl)); + PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); + client.putObject(putObjectRequest, RequestBody.fromBytes(objectBytes.asByteArray())); + + return getResourceUrl(resourceKey, acl, defaultExpirationMinutes); + } + + // Without ACL implementation + + private String uploadWithoutACL(String localFile, String externalFileName) { + client.putObject(PutObjectRequest.builder() + .bucket(bucket) + .key(externalFileName) + .build(), + RequestBody.fromFile(Paths.get(localFile))); + return getResourceUrlWithoutACL(externalFileName); + } + + private String uploadWithoutACL(String externalFileName, InputStream input) { + try { + ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input)); + PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() + .bucket(bucket) + .key(externalFileName) + .contentType(externalFileName.endsWith(".tmp") ? "image/jpeg" : null); + PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); + + PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer)); + if (!response.sdkHttpResponse().isSuccessful()) { + logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error")); + } + + return getResourceUrlWithoutACL(externalFileName); + } catch (IOException ex) { + logger.error("Error while uploading file to the external provider.", ex); + return ""; + } + } + + private String getResourceUrlWithoutACL(String externalFileName) { + try { + GetUrlRequest request = GetUrlRequest.builder() + .bucket(bucket) + .key(externalFileName) + .build(); + + URL url = client.utilities().getUrl(request); + return url.toString(); + + } catch (S3Exception e) { + logger.error("Failed to get the URL for the given resource because " + e.awsErrorDetails().errorMessage(), e); + return ""; + } + } + + private String copyWithoutACL(String objectName, String newName) { + CopyObjectRequest.Builder requestBuilder = CopyObjectRequest.builder() + .sourceBucket(bucket) + .sourceKey(objectName) + .destinationBucket(bucket) + .destinationKey(newName); + CopyObjectRequest request = requestBuilder.build(); + client.copyObject(request); + return getResourceUrlWithoutACL(newName); + } + + private String copyWithoutACL(String objectUrl, String newName, String tableName, String fieldName) { + String resourceFolderName = buildPath(folder, tableName, fieldName); + String resourceKey = resourceFolderName + StorageUtils.DELIMITER + newName; + + try { + objectUrl = new URI(objectUrl).getPath(); + } catch (Exception e) { + logger.error("Failed to Parse Storage Object URI for Copy operation", e); + } + + Map metadata = new HashMap<>(); + metadata.put("Table", tableName); + metadata.put("Field", fieldName); + metadata.put("KeyValue", StorageUtils.encodeNonAsciiCharacters(resourceKey)); + + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(objectUrl) + .build(); + ResponseBytes objectBytes = client.getObjectAsBytes(getObjectRequest); + + PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() + .bucket(bucket) + .key(resourceKey) + .metadata(metadata) + .contentType(getContentType(newName)); + PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); + client.putObject(putObjectRequest, RequestBody.fromBytes(objectBytes.asByteArray())); + + return getResourceUrlWithoutACL(resourceKey); + } } //http://192.168.254.78:9000/java-classes-unittests/text.txt \ No newline at end of file From c327c1f99cf7d8e9d47e19d7b17abd50bc23ec14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Sexenian?= <99925035+tomas-sexenian@users.noreply.github.com> Date: Tue, 7 May 2024 10:10:10 -0300 Subject: [PATCH 3/5] Presign every link to a bucket resource if the bucket is private --- .../db/driver/ExternalProviderS3V2.java | 57 ++++++++++++++----- .../db/driver/ExternalProviderBase.java | 6 +- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index c57cbf775..5a4e75c56 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -104,7 +104,7 @@ private void initialize() throws Exception { String folder = getPropertyValue(FOLDER, FOLDER_DEPRECATED, ""); clientRegion = getPropertyValue(REGION, REGION_DEPRECATED, DEFAULT_REGION); - objectOwnershipEnabled = !getPropertyValue(DEFAULT_ACL, DEFAULT_ACL_DEPRECATED, "").equals("Bucket owner enforced"); + objectOwnershipEnabled = !getPropertyValue(DEFAULT_ACL, DEFAULT_STORAGE_PRIVACY, "").contains("Bucket owner enforced"); String endpointValue = getPropertyValue(STORAGE_ENDPOINT, STORAGE_ENDPOINT_DEPRECATED, ""); if (endpointValue.equals("custom")) { @@ -268,7 +268,7 @@ public String get(String externalFileName, ResourceAccessControlList acl, int ex private String getResourceUrl(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) { return objectOwnershipEnabled ? getResourceUrlWithACL(externalFileName, acl, expirationMinutes) : - getResourceUrlWithoutACL(externalFileName); + getResourceUrlWithoutACL(externalFileName, expirationMinutes); } public void delete(String objectName, ResourceAccessControlList acl) { @@ -717,13 +717,18 @@ private String copyWithACL(String objectUrl, String newName, String tableName, S // Without ACL implementation + private enum BucketPrivacy {PRIVATE, PUBLIC}; + private final BucketPrivacy ownerEnforcedBucketPrivacy = getPropertyValue(DEFAULT_ACL, DEFAULT_STORAGE_PRIVACY, "").contains("Bucket owner enforced") ? + (getPropertyValue(DEFAULT_ACL, DEFAULT_STORAGE_PRIVACY, "").contains("private") ? BucketPrivacy.PRIVATE : BucketPrivacy.PUBLIC) + : null; + private String uploadWithoutACL(String localFile, String externalFileName) { client.putObject(PutObjectRequest.builder() .bucket(bucket) .key(externalFileName) .build(), RequestBody.fromFile(Paths.get(localFile))); - return getResourceUrlWithoutACL(externalFileName); + return getResourceUrlWithoutACL(externalFileName, defaultExpirationMinutes); } private String uploadWithoutACL(String externalFileName, InputStream input) { @@ -740,27 +745,49 @@ private String uploadWithoutACL(String externalFileName, InputStream input) { logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error")); } - return getResourceUrlWithoutACL(externalFileName); + return getResourceUrlWithoutACL(externalFileName, defaultExpirationMinutes); } catch (IOException ex) { logger.error("Error while uploading file to the external provider.", ex); return ""; } } - private String getResourceUrlWithoutACL(String externalFileName) { - try { - GetUrlRequest request = GetUrlRequest.builder() + private String getResourceUrlWithoutACL(String externalFileName, int expirationMinutes) { + if (ownerEnforcedBucketPrivacy == BucketPrivacy.PRIVATE) { + expirationMinutes = expirationMinutes > 0 ? expirationMinutes : defaultExpirationMinutes; + Instant expiration = Instant.now().plus(Duration.ofMinutes(expirationMinutes)); + + GetObjectRequest getObjectRequest = GetObjectRequest.builder() .bucket(bucket) .key(externalFileName) .build(); - URL url = client.utilities().getUrl(request); - return url.toString(); + PresignedGetObjectRequest presignedGetObjectRequest = + presigner.presignGetObject(r -> r.signatureDuration(Duration.between(Instant.now(), expiration)) + .getObjectRequest(getObjectRequest)); - } catch (S3Exception e) { - logger.error("Failed to get the URL for the given resource because " + e.awsErrorDetails().errorMessage(), e); - return ""; - } + return presignedGetObjectRequest.url().toString(); + } else if (ownerEnforcedBucketPrivacy == BucketPrivacy.PUBLIC){ + try { + int lastIndex = Math.max(externalFileName.lastIndexOf('/'), externalFileName.lastIndexOf('\\')); + String path = externalFileName.substring(0, lastIndex + 1); + String fileName = externalFileName.substring(lastIndex + 1); + String encodedFileName = URLEncoder.encode(fileName, StandardCharsets.UTF_8.toString()); + + String url = String.format( + "https://%s.s3.%s.amazonaws.com/%s%s", + bucket, + clientRegion, + path, + encodedFileName + ); + + return url; + } catch (UnsupportedEncodingException uee) { + logger.error("Failed to encode resource URL for {}", externalFileName, uee); + return ""; + } + } else return ""; } private String copyWithoutACL(String objectName, String newName) { @@ -771,7 +798,7 @@ private String copyWithoutACL(String objectName, String newName) { .destinationKey(newName); CopyObjectRequest request = requestBuilder.build(); client.copyObject(request); - return getResourceUrlWithoutACL(newName); + return getResourceUrlWithoutACL(newName, defaultExpirationMinutes); } private String copyWithoutACL(String objectUrl, String newName, String tableName, String fieldName) { @@ -803,7 +830,7 @@ private String copyWithoutACL(String objectUrl, String newName, String tableName PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); client.putObject(putObjectRequest, RequestBody.fromBytes(objectBytes.asByteArray())); - return getResourceUrlWithoutACL(resourceKey); + return getResourceUrlWithoutACL(resourceKey, defaultExpirationMinutes); } } diff --git a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java index 3fa99f40a..cfe61a019 100644 --- a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java +++ b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java @@ -14,9 +14,7 @@ public abstract class ExternalProviderBase { static final String DEFAULT_ACL = "DEFAULT_ACL"; static final String DEFAULT_EXPIRATION = "DEFAULT_EXPIRATION"; static final String FOLDER = "FOLDER_NAME"; - - @Deprecated - static final String DEFAULT_ACL_DEPRECATED = "STORAGE_PROVIDER_PRIVACY"; + static final String DEFAULT_STORAGE_PRIVACY = "STORAGE_PROVIDER_PRIVACY"; @Deprecated static final String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; @@ -33,7 +31,7 @@ public ExternalProviderBase(GXService s) { } private void init() { - String aclS = getPropertyValue(DEFAULT_ACL, DEFAULT_ACL_DEPRECATED, ""); + String aclS = getPropertyValue(DEFAULT_ACL, DEFAULT_STORAGE_PRIVACY, ""); if (aclS.length() > 0) { this.defaultAcl = ResourceAccessControlList.parse(aclS); } From cca20a14a6dc1d9bc301b53011c49b031b8bee4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Sexenian?= <99925035+tomas-sexenian@users.noreply.github.com> Date: Thu, 9 May 2024 10:33:48 -0300 Subject: [PATCH 4/5] Revert property name for compatibility reasons --- .../main/java/com/genexus/db/driver/ExternalProviderBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java index cfe61a019..b51cf48a3 100644 --- a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java +++ b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderBase.java @@ -14,7 +14,7 @@ public abstract class ExternalProviderBase { static final String DEFAULT_ACL = "DEFAULT_ACL"; static final String DEFAULT_EXPIRATION = "DEFAULT_EXPIRATION"; static final String FOLDER = "FOLDER_NAME"; - static final String DEFAULT_STORAGE_PRIVACY = "STORAGE_PROVIDER_PRIVACY"; + static final String DEFAULT_STORAGE_PRIVACY = "STORAGE_PROVIDER_DEFAULT_ACL"; @Deprecated static final String DEFAULT_EXPIRATION_DEPRECATED = "STORAGE_PROVIDER_DEFAULT_EXPIRATION"; From 733aa0a51e59e97e58bdc6bffae5fbdd5f727bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Sexenian?= <99925035+tomas-sexenian@users.noreply.github.com> Date: Tue, 2 Jul 2024 11:19:55 -0300 Subject: [PATCH 5/5] bucketOwnerEnforced constant --- .../java/com/genexus/db/driver/ExternalProviderS3V2.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index 5a4e75c56..799e461eb 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -77,7 +77,7 @@ public class ExternalProviderS3V2 extends ExternalProviderBase implements Extern private int defaultExpirationMinutes = DEFAULT_EXPIRATION_MINUTES; private Boolean pathStyleUrls = false; private Boolean objectOwnershipEnabled; - + private final String bucketOwnerEnforced = "Bucket owner enforced"; public String getName() { return NAME; @@ -104,7 +104,7 @@ private void initialize() throws Exception { String folder = getPropertyValue(FOLDER, FOLDER_DEPRECATED, ""); clientRegion = getPropertyValue(REGION, REGION_DEPRECATED, DEFAULT_REGION); - objectOwnershipEnabled = !getPropertyValue(DEFAULT_ACL, DEFAULT_STORAGE_PRIVACY, "").contains("Bucket owner enforced"); + objectOwnershipEnabled = !getPropertyValue(DEFAULT_ACL, DEFAULT_STORAGE_PRIVACY, "").contains(bucketOwnerEnforced); String endpointValue = getPropertyValue(STORAGE_ENDPOINT, STORAGE_ENDPOINT_DEPRECATED, ""); if (endpointValue.equals("custom")) { @@ -718,7 +718,7 @@ private String copyWithACL(String objectUrl, String newName, String tableName, S // Without ACL implementation private enum BucketPrivacy {PRIVATE, PUBLIC}; - private final BucketPrivacy ownerEnforcedBucketPrivacy = getPropertyValue(DEFAULT_ACL, DEFAULT_STORAGE_PRIVACY, "").contains("Bucket owner enforced") ? + private final BucketPrivacy ownerEnforcedBucketPrivacy = getPropertyValue(DEFAULT_ACL, DEFAULT_STORAGE_PRIVACY, "").contains(bucketOwnerEnforced) ? (getPropertyValue(DEFAULT_ACL, DEFAULT_STORAGE_PRIVACY, "").contains("private") ? BucketPrivacy.PRIVATE : BucketPrivacy.PUBLIC) : null;