Merge pull request #12081 from GlobalDataverseCommunityConsortium/DANS-2139-reExportAll

Merging. I'm hacking this part of the code heavily in my never-ending exports pr, but a conflict is minimal. 

reExportAll Improvements

Author: landreev

doc/sphinx-guides/source/admin/metadataexport.rst

@@ -22,12 +22,15 @@ In addition to the automated exports, a Dataverse installation admin can start a
 
 ``curl http://localhost:8080/api/admin/metadata/reExportAll``
 
+``curl http://localhost:8080/api/admin/metadata/reExportAll?olderThan=<YYYY-MM-DD>``
+
 ``curl http://localhost:8080/api/admin/metadata/clearExportTimestamps``
 
 ``curl http://localhost:8080/api/admin/metadata/:persistentId/reExportDataset?persistentId=doi:10.5072/FK2/AAA000``
 
 The first will attempt to export all the published, local (non-harvested) datasets that haven't been exported yet. 
-The second will *force* a re-export of every published, local dataset, regardless of whether it has already been exported or not. 
+The second will *force* a re-export of every published, local dataset, regardless of whether it has already been exported or not.
+With the optional olderThan query parameter, the second will *force* re-export of all published, local datasets that were last exported before the olderThan date.
 
 The first two calls return a status message informing the administrator that the process has been launched (``{"status":"WORKFLOW_IN_PROGRESS"}``). The administrator can check the progress of the process via log files: ``[Payara directory]/glassfish/domains/domain1/logs/export_[time stamp].log``.
 

src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java

@@ -698,6 +698,13 @@ public void reExportAllAsync() {
         exportAllDatasets(true);
     }
 
+    // reExportAll with a date *forces* a reexport on all published datasets that were not exported or were exported before the date;
+    @Asynchronous
+    public void reExportAllAsync(Date reExportDate) {
+        exportAllDatasets(true, reExportDate);
+        
+    }
+
     public void reExportAll() {
         exportAllDatasets(true);
     }
@@ -715,30 +722,27 @@ public void exportAll() {
         exportAllDatasets(false);
     }
 
-    public void exportAllDatasets(boolean forceReExport) {
+    private void exportAllDatasets(boolean b) {
+     exportAllDatasets(b, null);
+    }
+    
+    @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
+    private void exportAllDatasets(boolean forceReExport, Date reExportDate) {
         Integer countAll = 0;
         Integer countSuccess = 0;
         Integer countError = 0;
         String logTimestamp = logFormatter.format(new Date());
         Logger exportLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.DatasetServiceBean." + "ExportAll" + logTimestamp);
         String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "export_" + logTimestamp + ".log";
         FileHandler fileHandler;
-        boolean fileHandlerSuceeded;
         try {
             fileHandler = new FileHandler(logFileName);
             exportLogger.setUseParentHandlers(false);
-            fileHandlerSuceeded = true;
+            exportLogger.addHandler(fileHandler);
         } catch (IOException | SecurityException ex) {
             Logger.getLogger(DatasetServiceBean.class.getName()).log(Level.SEVERE, null, ex);
             return;
         }
-
-        if (fileHandlerSuceeded) {
-            exportLogger.addHandler(fileHandler);
-        } else {
-            exportLogger = logger;
-        }
-
         exportLogger.info("Starting an export all job");
 
         for (Long datasetId : findAllLocalDatasetIds()) {
@@ -757,9 +761,17 @@ public void exportAllDatasets(boolean forceReExport) {
 
                     // can't trust dataset.getPublicationDate(), no.
                     Date publicationDate = dataset.getReleasedVersion().getReleaseTime(); // we know this dataset has a non-null released version! Maybe not - SEK 8/19 (We do now! :)
-                    if (forceReExport || (publicationDate != null
-                            && (dataset.getLastExportTime() == null
-                            || dataset.getLastExportTime().before(publicationDate)))) {
+                    /**
+                     * Three cases: force is true and no date given - reexport every dataset force
+                     * is true and reExport date given - reexport datasets last exported before that
+                     * date force is false, reExportDate ignored - reexport datasets last exported
+                     * before they were last published
+                     */
+                    if ((forceReExport && reExportDate == null)
+                            || (forceReExport && (dataset.getLastExportTime() == null || dataset.getLastExportTime().before(reExportDate)))
+                            || (forceReExport == false
+                                    && (publicationDate != null && (dataset.getLastExportTime() == null
+                                            || dataset.getLastExportTime().before(publicationDate))))) {
                         countAll++;
                         try {
                             recordService.exportAllFormatsInNewTransaction(dataset);
@@ -768,6 +780,13 @@ public void exportAllDatasets(boolean forceReExport) {
                         } catch (Exception ex) {
                             exportLogger.log(Level.INFO, "Error exporting dataset: " + dataset.getDisplayName() + " " + dataset.getGlobalId().asString() + "; " + ex.getMessage(), ex);
                             countError++;
+                        } catch (Throwable t) {
+                            exportLogger.log(Level.SEVERE, "Fatal error exporting dataset: " + dataset.getDisplayName() + " " + dataset.getGlobalId().asString() + "; " + t.getClass().getName() + ": " + t.getMessage(), t);
+                            exportLogger.info("Datasets processed before fatal error: " + countAll.toString());
+                            exportLogger.info("Datasets exported successfully: " + countSuccess.toString());
+                            exportLogger.info("Datasets failures: " + countError.toString());
+                            fileHandler.close();
+                            throw t;
                         }
                     }
                 }
@@ -778,10 +797,7 @@ public void exportAllDatasets(boolean forceReExport) {
         exportLogger.info("Datasets failures: " + countError.toString());
         exportLogger.info("Finished export-all job.");
 
-        if (fileHandlerSuceeded) {
-            fileHandler.close();
-        }
-
+        fileHandler.close();
     }
 
     @Asynchronous
@@ -1140,4 +1156,5 @@ public void saveStorageQuota(Dataset target, Long allocation) {
         }
         em.flush();
     }
+
 }

src/main/java/edu/harvard/iq/dataverse/api/Metadata.java

@@ -8,11 +8,9 @@
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
 
+import java.util.Date;
 import java.util.logging.Logger;
 import jakarta.ejb.EJB;
-import jakarta.json.Json;
-import jakarta.json.JsonArrayBuilder;
-import jakarta.json.JsonObjectBuilder;
 import jakarta.ws.rs.*;
 
 import jakarta.ws.rs.core.Response;
@@ -57,8 +55,18 @@ public Response exportAll() {
     @GET
     @Path("/reExportAll")
     @Produces("application/json")
-    public Response reExportAll() {
-        datasetService.reExportAllAsync();
+    public Response reExportAll(@QueryParam(value = "olderThan") String olderThan) {
+        Date reExportDate = null;
+        if (olderThan != null && !olderThan.isEmpty()) {
+            try {
+                java.text.SimpleDateFormat dateFormat = new java.text.SimpleDateFormat("yyyy-MM-dd");
+                dateFormat.setLenient(false);
+                reExportDate = dateFormat.parse(olderThan);
+            } catch (java.text.ParseException e) {
+                return error(Response.Status.BAD_REQUEST, "Invalid date format for olderThan parameter. Expected format: YYYY-MM-DD");
+            }
+        }
+        datasetService.reExportAllAsync(reExportDate);
         return this.accepted();
     }