From 38b89d01af9434e19c49eccb5155becae1c9e4d3 Mon Sep 17 00:00:00 2001 From: tomas-sexenian Date: Mon, 19 Feb 2024 12:50:06 -0300 Subject: [PATCH 1/5] Drain entire download stream Issue: 107019 --- .../db/driver/ExternalProviderS3V1.java | 32 ++++++++++++++----- .../db/driver/ExternalProviderS3V2.java | 8 +++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java index 6666a8888..8b43a8881 100644 --- a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java +++ b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java @@ -29,7 +29,6 @@ import java.util.Date; import java.util.List; - public class ExternalProviderS3V1 extends ExternalProviderBase implements ExternalProvider { private static Logger logger = LogManager.getLogger(ExternalProviderS3V1.class); @@ -173,21 +172,38 @@ private String ensureFolder(String... pathPart) { } public void download(String externalFileName, String localFile, ResourceAccessControlList acl) { + S3ObjectInputStream objectData = null; try { S3Object object = client.getObject(new GetObjectRequest(bucket, externalFileName)); - try (InputStream objectData = object.getObjectContent()) { - try (OutputStream outputStream = new FileOutputStream(new File(localFile))){ - int read; - byte[] bytes = new byte[1024]; - while ((read = objectData.read(bytes)) != -1) { - outputStream.write(bytes, 0, read); - } + objectData = object.getObjectContent(); + File file = new File(localFile); + File parentDir = file.getParentFile(); + if (parentDir != null && !parentDir.exists()) + parentDir.mkdirs(); + + try (OutputStream outputStream = new FileOutputStream(file)) { + int read; + byte[] bytes = new byte[1024]; + while ((read = objectData.read(bytes)) != -1) { + outputStream.write(bytes, 0, read); } } } catch (FileNotFoundException ex) { logger.error("Error while downloading file to the external provider", ex); } catch (IOException ex) { logger.error("Error while downloading file to the external provider", ex); + } finally { + if (objectData != null) { + try { + // Drain the remainder of the stream to avoid leaving the HTTP connection in an inconsistent state + byte[] drainBuffer = new byte[1024]; + while (objectData.read(drainBuffer) != -1) { + // Continue reading until the end of the stream + } + } catch (IOException e) { + logger.error("Error while draining the S3ObjectInputStream", e); + } + } } } diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index 310be4501..8b59eabc6 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -228,6 +228,14 @@ public void download(String externalFileName, String localFile, ResourceAccessCo while ((read = objectData.read(bytes)) != -1) { outputStream.write(bytes, 0, read); } + } catch (IOException e) { + logger.error("Error while processing the input stream", e); + } finally { + // Drain the remainder of the stream to avoid leaving the HTTP connection in an inconsistent state + byte[] buffer = new byte[1024]; + while (objectData.read(buffer) != -1) { + // Consume the rest of the stream without processing it + } } } } catch (FileNotFoundException ex) { From e858aef4ec5d56281c8675cb2fed8639d3f65fa5 Mon Sep 17 00:00:00 2001 From: tomas-sexenian Date: Wed, 6 Mar 2024 15:31:34 -0300 Subject: [PATCH 2/5] Close objectData after downloading resource --- .../com/genexus/db/driver/ExternalProviderS3V1.java | 12 +----------- .../com/genexus/db/driver/ExternalProviderS3V2.java | 6 +----- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java index 8b43a8881..c0eaa988c 100644 --- a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java +++ b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java @@ -193,17 +193,7 @@ public void download(String externalFileName, String localFile, ResourceAccessCo } catch (IOException ex) { logger.error("Error while downloading file to the external provider", ex); } finally { - if (objectData != null) { - try { - // Drain the remainder of the stream to avoid leaving the HTTP connection in an inconsistent state - byte[] drainBuffer = new byte[1024]; - while (objectData.read(drainBuffer) != -1) { - // Continue reading until the end of the stream - } - } catch (IOException e) { - logger.error("Error while draining the S3ObjectInputStream", e); - } - } + try {objectData.close();} catch (IOException ioe) {logger.error("Error while draining the S3ObjectInputStream", ioe);} } } diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index 8b59eabc6..8efcd4be7 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -231,11 +231,7 @@ public void download(String externalFileName, String localFile, ResourceAccessCo } catch (IOException e) { logger.error("Error while processing the input stream", e); } finally { - // Drain the remainder of the stream to avoid leaving the HTTP connection in an inconsistent state - byte[] buffer = new byte[1024]; - while (objectData.read(buffer) != -1) { - // Consume the rest of the stream without processing it - } + objectData.close(); } } } catch (FileNotFoundException ex) { From b67eb9a3eda6c50d6786119544a70bdeb5267458 Mon Sep 17 00:00:00 2001 From: tomas-sexenian Date: Mon, 18 Mar 2024 12:49:08 -0300 Subject: [PATCH 3/5] Correctly close objectData --- .../java/com/genexus/db/driver/ExternalProviderS3V1.java | 7 +++++-- .../java/com/genexus/db/driver/ExternalProviderS3V2.java | 2 -- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java index 26e1bb7ce..aeed6a2f0 100644 --- a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java +++ b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java @@ -187,13 +187,16 @@ public void download(String externalFileName, String localFile, ResourceAccessCo while ((read = objectData.read(bytes)) != -1) { outputStream.write(bytes, 0, read); } + } finally { + try { + if (objectData != null) + objectData.close(); + } catch (IOException ioe) {logger.error("Error while draining the S3ObjectInputStream", ioe);} } } catch (FileNotFoundException ex) { logger.error("Error while downloading file to the external provider", ex); } catch (IOException ex) { logger.error("Error while downloading file to the external provider", ex); - } finally { - try {objectData.close();} catch (IOException ioe) {logger.error("Error while draining the S3ObjectInputStream", ioe);} } } diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index 9152880c1..a01f7d45f 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -230,8 +230,6 @@ public void download(String externalFileName, String localFile, ResourceAccessCo } } catch (IOException e) { logger.error("Error while processing the input stream", e); - } finally { - objectData.close(); } } } catch (FileNotFoundException ex) { From 4b43426e409ff99d7ac3e8fe9be31b269d3e903f Mon Sep 17 00:00:00 2001 From: tomas-sexenian Date: Mon, 18 Mar 2024 12:52:37 -0300 Subject: [PATCH 4/5] Remove unncesary catch --- .../main/java/com/genexus/db/driver/ExternalProviderS3V2.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java index a01f7d45f..a250db53b 100644 --- a/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java +++ b/gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java @@ -228,8 +228,6 @@ public void download(String externalFileName, String localFile, ResourceAccessCo while ((read = objectData.read(bytes)) != -1) { outputStream.write(bytes, 0, read); } - } catch (IOException e) { - logger.error("Error while processing the input stream", e); } } } catch (FileNotFoundException ex) { From b8e58f8076230991c92c24212b608d1c346b018a Mon Sep 17 00:00:00 2001 From: tomas-sexenian Date: Mon, 18 Mar 2024 14:04:34 -0300 Subject: [PATCH 5/5] Change the position of the finally block --- .../com/genexus/db/driver/ExternalProviderS3V1.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java index aeed6a2f0..830e92e11 100644 --- a/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java +++ b/gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java @@ -187,16 +187,16 @@ public void download(String externalFileName, String localFile, ResourceAccessCo while ((read = objectData.read(bytes)) != -1) { outputStream.write(bytes, 0, read); } - } finally { - try { - if (objectData != null) - objectData.close(); - } catch (IOException ioe) {logger.error("Error while draining the S3ObjectInputStream", ioe);} } } catch (FileNotFoundException ex) { logger.error("Error while downloading file to the external provider", ex); } catch (IOException ex) { logger.error("Error while downloading file to the external provider", ex); + } finally { + try { + if (objectData != null) + objectData.close(); + } catch (IOException ioe) {logger.error("Error while draining the S3ObjectInputStream", ioe);} } }