From 4c656db089bf507fb3cd7a7c74509a5b57be9e55 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 4 Jul 2025 16:33:26 -0300 Subject: [PATCH 1/6] The method createNewFile converts all the inputstream to memory. This could cause out of memory errors. Now the process use a buffer to avoid this. Issue: 205339 --- .../src/main/java/com/genexus/util/GXFileInfo.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/com/genexus/util/GXFileInfo.java b/common/src/main/java/com/genexus/util/GXFileInfo.java index 7b5e2e2a1..bc2706525 100644 --- a/common/src/main/java/com/genexus/util/GXFileInfo.java +++ b/common/src/main/java/com/genexus/util/GXFileInfo.java @@ -68,7 +68,7 @@ public boolean createNewFile() throws IOException{ return fileSource.createNewFile(); } public boolean createNewFile(InputStream input) throws IOException{ - fromBytes(SpecificImplementation.GXutil.toByteArray(input)); + fromInputStream(input); return true; } public boolean delete(){ @@ -180,6 +180,16 @@ public void fromBytes(byte[] data) throws IOException destination.write(data, 0, data.length); } } + private void fromInputStream(InputStream input) throws IOException + { + try (OutputStream output = new BufferedOutputStream(new FileOutputStream(fileSource))) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + output.write(buffer, 0, bytesRead); + } + } + } public String readAllText(String encoding)throws IOException{ return SpecificImplementation.FileUtils.readFileToString(fileSource, CommonUtil.normalizeEncodingName(encoding)); } From 860e6bfa6b1d4c51e012c5af47f359b33990f866 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 4 Jul 2025 19:19:17 -0300 Subject: [PATCH 2/6] The method createNewFile converts all the inputstream to memory. This could cause out of memory errors. Now the process use a buffer to avoid this. Issue: 205339 --- .../db/driver/ExternalProviderS3V1.java | 23 ++++++++----- .../db/driver/ExternalProviderS3V2.java | 32 +++++++++++++++---- .../driver/ExternalProviderAzureStorage.java | 8 ++--- .../db/driver/ExternalProviderHelper.java | 28 ++++++++++++++++ .../db/driver/ExternalProviderGoogle.java | 17 ++++++++-- .../db/driver/ExternalProviderIBM.java | 25 +++++++++------ 6 files changed, 102 insertions(+), 31 deletions(-) diff --git a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java index bebdc4c84..a15dc2381 100644 --- a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java +++ b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java @@ -17,7 +17,6 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.util.IOUtils; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -220,23 +219,31 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) { } public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) { - byte[] bytes; + ExternalProviderHelper.InputStreamWithLength streamInfo = null; try { - bytes = IOUtils.toByteArray(input); + streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); + ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(bytes.length); + metadata.setContentLength(streamInfo.contentLength); + if (externalFileName.endsWith(".tmp")) { metadata.setContentType("image/jpeg"); } String upload = ""; - try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes)) { - client.putObject(new PutObjectRequest(bucket, externalFileName, byteArrayInputStream, metadata).withCannedAcl(internalToAWSACL(acl))); - upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes); - } + client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl))); + upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes); return upload; } catch (IOException ex) { logger.error("Error while uploading file to the external provider.", ex); return ""; + } finally { + if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { + try { + streamInfo.tempFile.delete(); + } catch (Exception e) { + logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); + } + } } } diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index 878f4947a..25c55d38c 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -21,13 +21,11 @@ import com.genexus.util.StorageUtils; import com.genexus.StructSdtMessages_Message; import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; -import software.amazon.awssdk.utils.IoUtils; import java.io.*; import java.net.URI; import java.net.URL; import java.net.URLEncoder; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -603,8 +601,10 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) } private String uploadWithACL(String externalFileName, InputStream input, ResourceAccessControlList acl) { + ExternalProviderHelper.InputStreamWithLength streamInfo = null; try { - ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input)); + streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); + PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() .bucket(bucket) .key(externalFileName) @@ -613,7 +613,7 @@ private String uploadWithACL(String externalFileName, InputStream input, Resourc putObjectRequestBuilder = putObjectRequestBuilder.acl(internalToAWSACLWithACL(acl)); PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); - PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer)); + PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromInputStream(streamInfo.inputStream, streamInfo.contentLength)); if (!response.sdkHttpResponse().isSuccessful()) { logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error")); } @@ -622,6 +622,15 @@ private String uploadWithACL(String externalFileName, InputStream input, Resourc } catch (IOException ex) { logger.error("Error while uploading file to the external provider.", ex); return ""; + } finally { + // Clean up the temporary file if it was created + if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { + try { + streamInfo.tempFile.delete(); + } catch (Exception e) { + logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); + } + } } } @@ -727,15 +736,17 @@ private String uploadWithoutACL(String localFile, String externalFileName) { } private String uploadWithoutACL(String externalFileName, InputStream input) { + ExternalProviderHelper.InputStreamWithLength streamInfo = null; try { - ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input)); + streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); + PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() .bucket(bucket) .key(externalFileName) .contentType(externalFileName.endsWith(".tmp") ? "image/jpeg" : null); PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); - PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer)); + PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromInputStream(streamInfo.inputStream, streamInfo.contentLength)); if (!response.sdkHttpResponse().isSuccessful()) { logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error")); } @@ -744,6 +755,15 @@ private String uploadWithoutACL(String externalFileName, InputStream input) { } catch (IOException ex) { logger.error("Error while uploading file to the external provider.", ex); return ""; + } finally { + // Clean up the temporary file if it was created + if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { + try { + streamInfo.tempFile.delete(); + } catch (Exception e) { + logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); + } + } } } diff --git a/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java b/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java index ed25d917b..6e97e428f 100644 --- a/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java +++ b/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java @@ -146,10 +146,10 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC blob.getProperties().setContentType("image/jpeg"); } try (BlobOutputStream blobOutputStream = blob.openOutputStream()) { - int next = input.read(); - while (next != -1) { - blobOutputStream.write(next); - next = input.read(); + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + blobOutputStream.write(buffer, 0, bytesRead); } } return getResourceUrl(externalFileName, acl, DEFAULT_EXPIRATION_MINUTES); diff --git a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java index fb8710752..416039472 100644 --- a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java +++ b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java @@ -3,6 +3,8 @@ import com.genexus.util.Encryption; import com.genexus.util.GXService; +import java.io.*; + public class ExternalProviderHelper { public static String getServicePropertyValue(GXService s, String propName, boolean isSecure){ @@ -23,4 +25,30 @@ public static String getEnvironmentVariable(String name, boolean required) throw } return value; } + + public static InputStreamWithLength getInputStreamContentLength(InputStream input) throws IOException { + File tempFile = File.createTempFile("upload-", ".tmp"); + try (OutputStream out = new FileOutputStream(tempFile)) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + long size = tempFile.length(); + InputStream newInput = new FileInputStream(tempFile); + return new InputStreamWithLength(newInput, size, tempFile); + } + + public static class InputStreamWithLength { + public final InputStream inputStream; + public final long contentLength; + public final File tempFile; // nullable + + public InputStreamWithLength(InputStream inputStream, long contentLength, File tempFile) { + this.inputStream = inputStream; + this.contentLength = contentLength; + this.tempFile = tempFile; + } + } } diff --git a/gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java b/gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java index 7c39c45a0..ccf320c86 100644 --- a/gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java +++ b/gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java @@ -16,7 +16,6 @@ import com.google.api.services.storage.model.StorageObject; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.storage.*; -import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -170,16 +169,28 @@ private void setBlobAcl(BlobId blobId, ResourceAccessControlList acl) { } public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) { + ExternalProviderHelper.InputStreamWithLength streamInfo = null; try { + streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); + BlobId blobId = BlobId.of(bucket, externalFileName); BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); - byte[] targetArray = IOUtils.toByteArray(input); - storageClient.create(blobInfo, targetArray); + + storageClient.createFrom(blobInfo, streamInfo.tempFile.toPath()); + setBlobAcl(blobId, acl); return getResourceUrl(blobInfo, acl); } catch (IOException ex) { handleIOException(ex); return ""; + } finally { + if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { + try { + streamInfo.tempFile.delete(); + } catch (Exception e) { + logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); + } + } } } diff --git a/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java b/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java index b38c8eb4b..ae10a6fb1 100644 --- a/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java +++ b/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java @@ -14,7 +14,6 @@ import com.ibm.cloud.objectstorage.services.s3.AmazonS3Client; import com.ibm.cloud.objectstorage.services.s3.AmazonS3ClientBuilder; import com.ibm.cloud.objectstorage.services.s3.model.*; -import com.ibm.cloud.objectstorage.util.IOUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -150,24 +149,30 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) { } public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) { - byte[] bytes; - try { - bytes = IOUtils.toByteArray(input); + ExternalProviderHelper.InputStreamWithLength streamInfo = null; + try { + streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(bytes.length); + metadata.setContentLength(streamInfo.contentLength); if (externalFileName.endsWith(".tmp")) { metadata.setContentType("image/jpeg"); } String upload = ""; - try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes)) { - client.putObject(new PutObjectRequest(bucket, externalFileName, byteArrayInputStream, metadata).withCannedAcl(internalToAWSACL(acl))); - upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes); - } + client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl))); + upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes); return upload; } catch (IOException ex) { logger.error("Error while uploading file to the external provider.", ex); return ""; - } + } finally { + if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { + try { + streamInfo.tempFile.delete(); + } catch (Exception e) { + logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); + } + } + } } public String get(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) { From e131aea2f280fca5e6df0bbf6c2c4dbb7b168aba Mon Sep 17 00:00:00 2001 From: iroqueta Date: Fri, 11 Jul 2025 12:33:24 -0300 Subject: [PATCH 3/6] =?UTF-8?q?Tries=20to=20determine=20the=20content-type?= =?UTF-8?q?=20of=20files=20uploaded=20to=20external=20storage.=20Previousl?= =?UTF-8?q?y,=20if=20the=20file=20extension=20was=20.tmp,=20it=20was=20alw?= =?UTF-8?q?ays=20set=20to=20image/jpeg,=20which=20caused=20issues=E2=80=94?= =?UTF-8?q?such=20as=20when=20uploading=20a=20PDF:=20upon=20downloading=20?= =?UTF-8?q?it,=20the=20file=20couldn't=20be=20opened=20properly.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/pom.xml | 2 +- .../db/driver/ExternalProviderS3V1.java | 4 +--- .../db/driver/ExternalProviderS3V2.java | 2 +- .../driver/ExternalProviderAzureStorage.java | 19 ++++++++++++++----- gxcloudstorage-common/pom.xml | 5 +++++ .../db/driver/ExternalProviderHelper.java | 13 ++++++++++--- .../db/driver/ExternalProviderIBM.java | 4 +--- java/pom.xml | 5 ----- wrappercommon/pom.xml | 5 ----- 9 files changed, 33 insertions(+), 26 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 920640fd2..313903e28 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -38,7 +38,7 @@ commons-io commons-io - 2.11.0 + 2.12.0 org.bouncycastle diff --git a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java index a15dc2381..edaa99cbe 100644 --- a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java +++ b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java @@ -225,10 +225,8 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(streamInfo.contentLength); + metadata.setContentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType); - if (externalFileName.endsWith(".tmp")) { - metadata.setContentType("image/jpeg"); - } String upload = ""; client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl))); upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes); diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index 25c55d38c..23a0eadcf 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -743,7 +743,7 @@ private String uploadWithoutACL(String externalFileName, InputStream input) { PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() .bucket(bucket) .key(externalFileName) - .contentType(externalFileName.endsWith(".tmp") ? "image/jpeg" : null); + .contentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType); PutObjectRequest putObjectRequest = putObjectRequestBuilder.build(); PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromInputStream(streamInfo.inputStream, streamInfo.contentLength)); diff --git a/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java b/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java index 6e97e428f..38e564914 100644 --- a/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java +++ b/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java @@ -139,16 +139,16 @@ public String upload(String localFile, String externalFileName, ResourceAccessCo } public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) { - + ExternalProviderHelper.InputStreamWithLength streamInfo = null; try { + streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); + CloudBlockBlob blob = getCloudBlockBlob(externalFileName, acl); - if (externalFileName.endsWith(".tmp")) { - blob.getProperties().setContentType("image/jpeg"); - } + blob.getProperties().setContentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType); try (BlobOutputStream blobOutputStream = blob.openOutputStream()) { byte[] buffer = new byte[8192]; int bytesRead; - while ((bytesRead = input.read(buffer)) != -1) { + while ((bytesRead = streamInfo.inputStream.read(buffer)) != -1) { blobOutputStream.write(buffer, 0, bytesRead); } } @@ -163,6 +163,15 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC logger.error("Error uploading file", ex); return ""; } + finally { + if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { + try { + streamInfo.tempFile.delete(); + } catch (Exception e) { + logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); + } + } + } } public String get(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) { diff --git a/gxcloudstorage-common/pom.xml b/gxcloudstorage-common/pom.xml index 8f8724f1c..f3a749793 100644 --- a/gxcloudstorage-common/pom.xml +++ b/gxcloudstorage-common/pom.xml @@ -34,6 +34,11 @@ log4j-api ${log4j.version} + + org.apache.tika + tika-core + 3.2.0 + diff --git a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java index 416039472..e980993ab 100644 --- a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java +++ b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java @@ -4,6 +4,7 @@ import com.genexus.util.GXService; import java.io.*; +import org.apache.tika.Tika; public class ExternalProviderHelper { @@ -36,19 +37,25 @@ public static InputStreamWithLength getInputStreamContentLength(InputStream inpu } } long size = tempFile.length(); - InputStream newInput = new FileInputStream(tempFile); - return new InputStreamWithLength(newInput, size, tempFile); + InputStream bufferedInput = new BufferedInputStream(new FileInputStream(tempFile)); + bufferedInput.mark(128 * 1024); + Tika tika = new Tika(); + String detectedContentType = tika.detect(bufferedInput); + bufferedInput.reset(); + return new InputStreamWithLength(bufferedInput, size, tempFile, detectedContentType); } public static class InputStreamWithLength { public final InputStream inputStream; public final long contentLength; public final File tempFile; // nullable + public final String detectedContentType; - public InputStreamWithLength(InputStream inputStream, long contentLength, File tempFile) { + public InputStreamWithLength(InputStream inputStream, long contentLength, File tempFile, String detectedContentType) { this.inputStream = inputStream; this.contentLength = contentLength; this.tempFile = tempFile; + this.detectedContentType = detectedContentType; } } } diff --git a/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java b/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java index ae10a6fb1..9e9779055 100644 --- a/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java +++ b/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java @@ -154,9 +154,7 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(streamInfo.contentLength); - if (externalFileName.endsWith(".tmp")) { - metadata.setContentType("image/jpeg"); - } + metadata.setContentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType); String upload = ""; client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl))); upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes); diff --git a/java/pom.xml b/java/pom.xml index 6c03bd23a..6f69f9e61 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -59,11 +59,6 @@ javax.jms 3.1.2.2 - - commons-io - commons-io - 2.11.0 - commons-net commons-net diff --git a/wrappercommon/pom.xml b/wrappercommon/pom.xml index b52622955..c94606440 100644 --- a/wrappercommon/pom.xml +++ b/wrappercommon/pom.xml @@ -29,11 +29,6 @@ log4j-core ${log4j.version} - - commons-io - commons-io - 2.11.0 - org.apache.ws.security wss4j From 7d9ba6e76b29eb51e8c3bc964f9af5dea5986c26 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Tue, 15 Jul 2025 14:37:33 -0300 Subject: [PATCH 4/6] Update Tika version to be java 8 compliant. --- gxcloudstorage-common/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gxcloudstorage-common/pom.xml b/gxcloudstorage-common/pom.xml index f3a749793..9944cc52d 100644 --- a/gxcloudstorage-common/pom.xml +++ b/gxcloudstorage-common/pom.xml @@ -37,7 +37,7 @@ org.apache.tika tika-core - 3.2.0 + 2.9.4 From 4b034f7254ebdc5dff626fef0af0564b164b9621 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Mon, 21 Jul 2025 16:10:00 -0300 Subject: [PATCH 5/6] Add log4j impl for slf4j to avoid "No SLF4J providers were found" in Tika logs --- gxcloudstorage-common/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gxcloudstorage-common/pom.xml b/gxcloudstorage-common/pom.xml index 9944cc52d..c09876e36 100644 --- a/gxcloudstorage-common/pom.xml +++ b/gxcloudstorage-common/pom.xml @@ -34,6 +34,11 @@ log4j-api ${log4j.version} + + org.apache.logging.log4j + log4j-slf4j2-impl + ${log4j.version} + org.apache.tika tika-core From c61bbd95c2910eb121e381217bdfa27249643f68 Mon Sep 17 00:00:00 2001 From: iroqueta Date: Wed, 23 Jul 2025 16:59:57 -0300 Subject: [PATCH 6/6] Implemented better way to delete temporary files --- .../db/driver/ExternalProviderS3V1.java | 13 +-------- .../db/driver/ExternalProviderS3V2.java | 28 ++----------------- .../driver/ExternalProviderAzureStorage.java | 14 +--------- .../db/driver/ExternalProviderHelper.java | 19 +++++++++++-- .../db/driver/ExternalProviderGoogle.java | 13 +-------- .../db/driver/ExternalProviderIBM.java | 14 ++-------- 6 files changed, 24 insertions(+), 77 deletions(-) diff --git a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java index edaa99cbe..6ee1dd79e 100644 --- a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java +++ b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java @@ -219,10 +219,7 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) { } public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) { - ExternalProviderHelper.InputStreamWithLength streamInfo = null; - try { - streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); - + try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) { ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(streamInfo.contentLength); metadata.setContentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType); @@ -234,14 +231,6 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC } catch (IOException ex) { logger.error("Error while uploading file to the external provider.", ex); return ""; - } finally { - if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { - try { - streamInfo.tempFile.delete(); - } catch (Exception e) { - logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); - } - } } } diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index 23a0eadcf..0da262412 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -601,10 +601,7 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) } private String uploadWithACL(String externalFileName, InputStream input, ResourceAccessControlList acl) { - ExternalProviderHelper.InputStreamWithLength streamInfo = null; - try { - streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); - + try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) { PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() .bucket(bucket) .key(externalFileName) @@ -622,15 +619,6 @@ private String uploadWithACL(String externalFileName, InputStream input, Resourc } catch (IOException ex) { logger.error("Error while uploading file to the external provider.", ex); return ""; - } finally { - // Clean up the temporary file if it was created - if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { - try { - streamInfo.tempFile.delete(); - } catch (Exception e) { - logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); - } - } } } @@ -736,10 +724,7 @@ private String uploadWithoutACL(String localFile, String externalFileName) { } private String uploadWithoutACL(String externalFileName, InputStream input) { - ExternalProviderHelper.InputStreamWithLength streamInfo = null; - try { - streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); - + try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) { PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder() .bucket(bucket) .key(externalFileName) @@ -755,15 +740,6 @@ private String uploadWithoutACL(String externalFileName, InputStream input) { } catch (IOException ex) { logger.error("Error while uploading file to the external provider.", ex); return ""; - } finally { - // Clean up the temporary file if it was created - if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { - try { - streamInfo.tempFile.delete(); - } catch (Exception e) { - logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); - } - } } } diff --git a/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java b/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java index 38e564914..723ea9eca 100644 --- a/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java +++ b/gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java @@ -139,10 +139,7 @@ public String upload(String localFile, String externalFileName, ResourceAccessCo } public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) { - ExternalProviderHelper.InputStreamWithLength streamInfo = null; - try { - streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); - + try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) { CloudBlockBlob blob = getCloudBlockBlob(externalFileName, acl); blob.getProperties().setContentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType); try (BlobOutputStream blobOutputStream = blob.openOutputStream()) { @@ -163,15 +160,6 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC logger.error("Error uploading file", ex); return ""; } - finally { - if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { - try { - streamInfo.tempFile.delete(); - } catch (Exception e) { - logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); - } - } - } } public String get(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) { diff --git a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java index e980993ab..c514b639b 100644 --- a/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java +++ b/gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java @@ -8,6 +8,8 @@ public class ExternalProviderHelper { + public static final int BUFFER_MARK_LIMIT = 128 * 1024; + public static String getServicePropertyValue(GXService s, String propName, boolean isSecure){ String value = s.getProperties().get(propName); if (value != null){ @@ -38,14 +40,14 @@ public static InputStreamWithLength getInputStreamContentLength(InputStream inpu } long size = tempFile.length(); InputStream bufferedInput = new BufferedInputStream(new FileInputStream(tempFile)); - bufferedInput.mark(128 * 1024); + bufferedInput.mark(BUFFER_MARK_LIMIT); Tika tika = new Tika(); String detectedContentType = tika.detect(bufferedInput); bufferedInput.reset(); return new InputStreamWithLength(bufferedInput, size, tempFile, detectedContentType); } - public static class InputStreamWithLength { + public static class InputStreamWithLength implements AutoCloseable { public final InputStream inputStream; public final long contentLength; public final File tempFile; // nullable @@ -57,5 +59,18 @@ public InputStreamWithLength(InputStream inputStream, long contentLength, File t this.tempFile = tempFile; this.detectedContentType = detectedContentType; } + + @Override + public void close() throws IOException { + try { + if (inputStream != null) { + inputStream.close(); + } + } finally { + if (tempFile != null && tempFile.exists()) { + tempFile.delete(); + } + } + } } } diff --git a/gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java b/gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java index ccf320c86..f1220b19d 100644 --- a/gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java +++ b/gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java @@ -169,10 +169,7 @@ private void setBlobAcl(BlobId blobId, ResourceAccessControlList acl) { } public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) { - ExternalProviderHelper.InputStreamWithLength streamInfo = null; - try { - streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); - + try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) { BlobId blobId = BlobId.of(bucket, externalFileName); BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); @@ -183,14 +180,6 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC } catch (IOException ex) { handleIOException(ex); return ""; - } finally { - if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { - try { - streamInfo.tempFile.delete(); - } catch (Exception e) { - logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); - } - } } } diff --git a/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java b/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java index 9e9779055..18dbf8ec2 100644 --- a/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java +++ b/gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java @@ -149,9 +149,7 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) { } public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) { - ExternalProviderHelper.InputStreamWithLength streamInfo = null; - try { - streamInfo = ExternalProviderHelper.getInputStreamContentLength(input); + try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) { ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(streamInfo.contentLength); metadata.setContentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType); @@ -162,15 +160,7 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC } catch (IOException ex) { logger.error("Error while uploading file to the external provider.", ex); return ""; - } finally { - if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) { - try { - streamInfo.tempFile.delete(); - } catch (Exception e) { - logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e); - } - } - } + } } public String get(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) {