From 240e851c7ba43b5037097aa2f0e0e827f2564f12 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 13 Feb 2024 16:38:03 -0500 Subject: [PATCH 01/21] Ingest/Uningest from file page --- .../edu/harvard/iq/dataverse/FilePage.java | 112 ++++++++++++++++++ .../webapp/file-edit-button-fragment.xhtml | 16 +++ 2 files changed, 128 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 479c8a429c6..b6706acd4ff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -475,6 +475,112 @@ public String restrictFile(boolean restricted) throws CommandException{ return returnToDraftVersion(); } + public String ingestFile() throws CommandException{ + + User u = session.getUser(); + if(!u.isAuthenticated() || !(permissionService.permissionsFor(u, file).contains(Permission.PublishDataset))) { + //Shouldn't happen (choice not displayed for users who don't have the right permission), but check anyway + logger.warning("User: " + u.getIdentifier() + " tried to ingest a file"); + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.cantIngestFileWarning")); + return null; + } + + DataFile dataFile = fileMetadata.getDataFile(); + editDataset = dataFile.getOwner(); + + if (dataFile.isTabularData()) { + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.alreadyIngestedWarning")); + return null; + } + + boolean ingestLock = dataset.isLockedFor(DatasetLock.Reason.Ingest); + + if (ingestLock) { + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.ingestInProgressWarning")); + return null; + } + + if (!FileUtil.canIngestAsTabular(dataFile)) { + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.cantIngestFileWarning")); + return null; + + } + + dataFile.SetIngestScheduled(); + + if (dataFile.getIngestRequest() == null) { + dataFile.setIngestRequest(new IngestRequest(dataFile)); + } + + dataFile.getIngestRequest().setForceTypeCheck(true); + + // update the datafile, to save the newIngest request in the database: + save(); + + // queue the data ingest job for asynchronous execution: + String status = ingestService.startIngestJobs(editDataset.getId(), new ArrayList<>(Arrays.asList(dataFile)), (AuthenticatedUser) session.getUser()); + + if (!StringUtil.isEmpty(status)) { + // This most likely indicates some sort of a problem (for example, + // the ingest job was not put on the JMS queue because of the size + // of the file). But we are still returning the OK status - because + // from the point of view of the API, it's a success - we have + // successfully gone through the process of trying to schedule the + // ingest job... + + logger.warning("Ingest Status for file: " + dataFile.getId() + " : " + status); + } + logger.info("File: " + dataFile.getId() + " ingest queued"); + + init(); + JsfHelper.addInfoMessage(BundleUtil.getStringFromBundle("file.ingest.ingestQueued")); + return returnToDraftVersion(); + } + + public String uningestFile() throws CommandException { + + if (!file.isTabularData()) { + if(file.isIngestProblem()) { + User u = session.getUser(); + if(!u.isAuthenticated() || !(permissionService.permissionsFor(u, file).contains(Permission.PublishDataset))) { + logger.warning("User: " + u.getIdentifier() + " tried to uningest a file"); + //Shouldn't happen (choice not displayed for users who don't have the right permission), but check anyway + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); + return null; + } + file.setIngestDone(); + file.setIngestReport(null); + } else { + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); + return null; + } + } else { + commandEngine.submit(new UningestFileCommand(dvRequestService.getDataverseRequest(), file)); + Long dataFileId = file.getId(); + file = datafileService.find(dataFileId); + } + editDataset = file.getOwner(); + if (editDataset.isReleased()) { + try { + ExportService instance = ExportService.getInstance(); + instance.exportAllFormats(editDataset); + + } catch (ExportException ex) { + // Something went wrong! + // Just like with indexing, a failure to export is not a fatal + // condition. We'll just log the error as a warning and keep + // going: + logger.log(Level.WARNING, "Uningest: Exception while exporting:{0}", ex.getMessage()); + } + } + save(); + //Refresh filemetadata with file title, etc. + init(); + JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("file.uningest.complete")); + return returnToDraftVersion(); + } + + private List filesToBeDeleted = new ArrayList<>(); public String deleteFile() { @@ -948,6 +1054,12 @@ public boolean isPubliclyDownloadable() { return FileUtil.isPubliclyDownloadable(fileMetadata); } + public boolean isIngestable() { + DataFile f = fileMetadata.getDataFile(); + //Datafile is an ingestable type and hasn't been ingested yet or had an ingest fail + return (FileUtil.canIngestAsTabular(f)&&!(f.isTabularData() || f.isIngestProblem())); + } + private Boolean lockedFromEditsVar; private Boolean lockedFromDownloadVar; diff --git a/src/main/webapp/file-edit-button-fragment.xhtml b/src/main/webapp/file-edit-button-fragment.xhtml index 4dac1613266..e08de716cda 100644 --- a/src/main/webapp/file-edit-button-fragment.xhtml +++ b/src/main/webapp/file-edit-button-fragment.xhtml @@ -77,6 +77,22 @@ + + + +
  • + + + +
    • +     + +
  • + + + +
    • +     From fcdc24611d26889ba32fda351490c0ae657aef7e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Feb 2024 17:00:07 -0500 Subject: [PATCH 02/21] missing imports/@EJB --- src/main/java/edu/harvard/iq/dataverse/FilePage.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index b6706acd4ff..4e5843964e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -21,6 +21,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand; import edu.harvard.iq.dataverse.engine.command.impl.PersistProvFreeFormCommand; import edu.harvard.iq.dataverse.engine.command.impl.RestrictFileCommand; +import edu.harvard.iq.dataverse.engine.command.impl.UningestFileCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.export.ExportService; import io.gdcc.spi.export.ExportException; @@ -28,6 +29,8 @@ import edu.harvard.iq.dataverse.externaltools.ExternalTool; import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; +import edu.harvard.iq.dataverse.ingest.IngestRequest; +import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; @@ -35,6 +38,8 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.JsfHelper; +import edu.harvard.iq.dataverse.util.StringUtil; + import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -45,6 +50,7 @@ import java.util.Comparator; import java.util.List; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import jakarta.ejb.EJB; import jakarta.ejb.EJBException; @@ -112,10 +118,10 @@ public class FilePage implements java.io.Serializable { GuestbookResponseServiceBean guestbookResponseService; @EJB AuthenticationServiceBean authService; - @EJB DatasetServiceBean datasetService; - + @EJB + IngestServiceBean ingestService; @EJB SystemConfig systemConfig; @@ -209,7 +215,7 @@ public String init() { // If this DatasetVersion is unpublished and permission is doesn't have permissions: // > Go to the Login page // - // Check permisisons + // Check permissions Boolean authorized = (fileMetadata.getDatasetVersion().isReleased()) || (!fileMetadata.getDatasetVersion().isReleased() && this.canViewUnpublishedDataset()); From 15ae19e36250e3a467452cfd41287df1cfe8bd3a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Feb 2024 17:00:28 -0500 Subject: [PATCH 03/21] Change command to publish perm --- .../dataverse/engine/command/impl/UningestFileCommand.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java index 3e85630dd59..e9791809cb2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java @@ -33,7 +33,7 @@ * @author skraffmi * @author Leonid Andreev */ -@RequiredPermissions({}) +@RequiredPermissions(Permission.PublishDataset) public class UningestFileCommand extends AbstractVoidCommand { private static final Logger logger = Logger.getLogger(UningestFileCommand.class.getCanonicalName()); @@ -48,8 +48,8 @@ public UningestFileCommand(DataverseRequest aRequest, DataFile uningest) { protected void executeImpl(CommandContext ctxt) throws CommandException { // first check if user is a superuser - if ( (!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser() ) ) { - throw new PermissionException("Uningest File can only be called by Superusers.", + if (!(getUser() instanceof AuthenticatedUser)) { + throw new PermissionException("Uningest File can only be called by User with the PublishDataset permission.", this, Collections.singleton(Permission.EditDataset), uningest); } From 262fb267a2025872d8f537e937ad31dc0a25a156 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Feb 2024 17:49:09 -0500 Subject: [PATCH 04/21] superuser only in command, add docs --- .../user/tabulardataingest/ingestprocess.rst | 20 ++++++++++++++++++- .../command/impl/UningestFileCommand.java | 10 +++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst index 33ae9b555e6..9e82ff12b9b 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst @@ -32,7 +32,7 @@ format. (more info below) Tabular Data and Metadata -========================== +========================= Data vs. Metadata ----------------- @@ -56,3 +56,21 @@ the Dataverse Software was originally based on the `DDI Codebook `_ format. You can see an example of DDI output under the :ref:`data-variable-metadata-access` section of the :doc:`/api/dataaccess` section of the API Guide. + +Uningest and Reingest +===================== + +Ingest will only work for files whose content can be interpreted as a table. +Multi-sheets spreadsheets and CSV files with different number of entries per row are two examples where ingest will fail. +This is non-fatal. The Dataverse software will not produce a .tab version of the file and will show a warning to users +who can see the draft version of the dataset containing the file that will indicate why ingest failed. When the file is published as +part of the dataset, there will be no indication that ingest was attempted and failed. + +If the warning message is a concern, the Dataverse software includes both an API call (see the Files section of the :doc:`/api/native-api` guide) +and an Edit/Uningest menu option displayed on the file page, that allow a file to be Uningested. These are only available to superusers. +Uningest will remove the warning. Uningest can also be done for a file that was successfully ingested. +This will remove the .tab version of the file that was generated. + +If a file is a tabular format but was never ingested, .e.g. due to the ingest file size limit being lower in the past, or if ingest had failed, +e.g. in a prior Dataverse version, an reingest API (see the Files section of the :doc:`/api/native-api` guide) and a file page Edit/Reingest option +in the user interface allow ingest to be tried again. As with Uningest, this fucntionality is only available to superusers. diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java index e9791809cb2..ba04c4d7931 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java @@ -33,7 +33,7 @@ * @author skraffmi * @author Leonid Andreev */ -@RequiredPermissions(Permission.PublishDataset) +@RequiredPermissions({}) public class UningestFileCommand extends AbstractVoidCommand { private static final Logger logger = Logger.getLogger(UningestFileCommand.class.getCanonicalName()); @@ -47,10 +47,10 @@ public UningestFileCommand(DataverseRequest aRequest, DataFile uningest) { @Override protected void executeImpl(CommandContext ctxt) throws CommandException { - // first check if user is a superuser - if (!(getUser() instanceof AuthenticatedUser)) { - throw new PermissionException("Uningest File can only be called by User with the PublishDataset permission.", - this, Collections.singleton(Permission.EditDataset), uningest); + // first check if user is a superuser + if ((!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser())) { + throw new PermissionException("Uningest File can only be called by Superusers.", this, + Collections.singleton(Permission.EditDataset), uningest); } // is this actually a tabular data file? From 130cfba92e9f3ced2e9497ba74b2f17b20bfec77 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 13 Feb 2024 17:51:37 -0500 Subject: [PATCH 05/21] release note --- doc/release-notes/10318-uningest-and-reingest.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/release-notes/10318-uningest-and-reingest.md diff --git a/doc/release-notes/10318-uningest-and-reingest.md b/doc/release-notes/10318-uningest-and-reingest.md new file mode 100644 index 00000000000..7465f934330 --- /dev/null +++ b/doc/release-notes/10318-uningest-and-reingest.md @@ -0,0 +1,2 @@ +New Uningest/Reingest options are available in the File Page Edit menu for superusers, allowing ingest errors to be cleared and for +ingest to be retried (e.g. after a Dataverse version update or if ingest size limits are changed). From 1dc4825cb1aa2f204958782e238ad77ac4e231b6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 13:19:13 -0500 Subject: [PATCH 06/21] update perms --- .../edu/harvard/iq/dataverse/FilePage.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 4e5843964e7..37798f1cd3c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -484,7 +484,7 @@ public String restrictFile(boolean restricted) throws CommandException{ public String ingestFile() throws CommandException{ User u = session.getUser(); - if(!u.isAuthenticated() || !(permissionService.permissionsFor(u, file).contains(Permission.PublishDataset))) { + if(!u.isAuthenticated() || !u.isSuperuser()) { //Shouldn't happen (choice not displayed for users who don't have the right permission), but check anyway logger.warning("User: " + u.getIdentifier() + " tried to ingest a file"); JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.cantIngestFileWarning")); @@ -544,23 +544,29 @@ public String ingestFile() throws CommandException{ } public String uningestFile() throws CommandException { - + if (!file.isTabularData()) { - if(file.isIngestProblem()) { - User u = session.getUser(); - if(!u.isAuthenticated() || !(permissionService.permissionsFor(u, file).contains(Permission.PublishDataset))) { - logger.warning("User: " + u.getIdentifier() + " tried to uningest a file"); - //Shouldn't happen (choice not displayed for users who don't have the right permission), but check anyway - JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); - return null; - } - file.setIngestDone(); - file.setIngestReport(null); + //Ingest never succeeded, either there was a failure or this is not a tabular data file + User u = session.getUser(); + if (!u.isAuthenticated() || !u.isSuperuser()) { + logger.warning("User: " + u.getIdentifier() + " tried to uningest a file"); + // Shouldn't happen (choice not displayed for users who don't have the right + // permission), but check anyway + JH.addMessage(FacesMessage.SEVERITY_WARN, + BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); + return null; + } + if (file.isIngestProblem()) { + file.setIngestDone(); + file.setIngestReport(null); } else { - JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); - return null; + //Shouldn't happen - got called when there is no tabular data or an ingest problem + JH.addMessage(FacesMessage.SEVERITY_WARN, + BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); + return null; } } else { + //Uningest command does it's own check for isSuperuser commandEngine.submit(new UningestFileCommand(dvRequestService.getDataverseRequest(), file)); Long dataFileId = file.getId(); file = datafileService.find(dataFileId); @@ -580,12 +586,11 @@ public String uningestFile() throws CommandException { } } save(); - //Refresh filemetadata with file title, etc. + // Refresh filemetadata with file title, etc. init(); JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("file.uningest.complete")); return returnToDraftVersion(); - } - + } private List filesToBeDeleted = new ArrayList<>(); From f15122615aea9109ed90b2ff5c4e6a4965f8efcf Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 13:19:23 -0500 Subject: [PATCH 07/21] add bundle strings --- src/main/java/propertyFiles/Bundle.properties | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index f1c8381816c..42c844e532e 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2203,6 +2203,15 @@ ingest.csv.lineMismatch=Mismatch between line counts in first and final passes!, ingest.csv.recordMismatch=Reading mismatch, line {0} of the Data file: {1} delimited values expected, {2} found. ingest.csv.nullStream=Stream can't be null. +file.ingest=Ingest +file.uningest=Uningest +file.ingest.alreadyIngestedWarning=This file has already been ingested +file.ingest.ingestInProgressWarning=Ingestion of this file is already in progress +file.ingest.cantIngestFileWarning=Ingest not supported for this file type +file.ingest.ingestQueued=Ingestion has been requested +file.ingest.cantUningestFileWarning=This file cannot be uningested +file.uningest.complete=Uningestion of this file has been completed + # editdatafile.xhtml # editFilesFragment.xhtml From 14b280cf39dee856dfa9a1a6e97f1a7392418a02 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 13:19:33 -0500 Subject: [PATCH 08/21] doc updates --- doc/sphinx-guides/source/api/native-api.rst | 4 ++++ .../source/user/tabulardataingest/ingestprocess.rst | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index dbe769e2fd1..8cfa5deb96c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2854,6 +2854,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -d true "https://demo.dataverse.org/api/files/:persistentId/restrict?persistentId=doi:10.5072/FK2/AAA000" +.. _file-uningest: + Uningest a File ~~~~~~~~~~~~~~~ @@ -2891,6 +2893,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/uningest?persistentId=doi:10.5072/FK2/AAA000" +.. _file-reingest: + Reingest a File ~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst index 9e82ff12b9b..ac5fb5af4ec 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst @@ -66,11 +66,11 @@ This is non-fatal. The Dataverse software will not produce a .tab version of the who can see the draft version of the dataset containing the file that will indicate why ingest failed. When the file is published as part of the dataset, there will be no indication that ingest was attempted and failed. -If the warning message is a concern, the Dataverse software includes both an API call (see the Files section of the :doc:`/api/native-api` guide) +If the warning message is a concern, the Dataverse software includes both an API call (see :ref:`file-uningest` in the :doc:`/api/native-api` guide) and an Edit/Uningest menu option displayed on the file page, that allow a file to be Uningested. These are only available to superusers. Uningest will remove the warning. Uningest can also be done for a file that was successfully ingested. This will remove the .tab version of the file that was generated. If a file is a tabular format but was never ingested, .e.g. due to the ingest file size limit being lower in the past, or if ingest had failed, -e.g. in a prior Dataverse version, an reingest API (see the Files section of the :doc:`/api/native-api` guide) and a file page Edit/Reingest option +e.g. in a prior Dataverse version, an reingest API (see :ref:`file-reingest` in the :doc:`/api/native-api` guide) and a file page Edit/Reingest option in the user interface allow ingest to be tried again. As with Uningest, this fucntionality is only available to superusers. From 5dffe36c793fa25b9ee8199fda2104fb26f92b9a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 13:21:09 -0500 Subject: [PATCH 09/21] Apply suggestions from code review Co-authored-by: Philip Durbin --- .../source/user/tabulardataingest/ingestprocess.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst index ac5fb5af4ec..4dce441de4a 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst @@ -61,13 +61,13 @@ Uningest and Reingest ===================== Ingest will only work for files whose content can be interpreted as a table. -Multi-sheets spreadsheets and CSV files with different number of entries per row are two examples where ingest will fail. +Multi-sheet spreadsheets and CSV files with a different number of entries per row are two examples where ingest will fail. This is non-fatal. The Dataverse software will not produce a .tab version of the file and will show a warning to users who can see the draft version of the dataset containing the file that will indicate why ingest failed. When the file is published as part of the dataset, there will be no indication that ingest was attempted and failed. If the warning message is a concern, the Dataverse software includes both an API call (see :ref:`file-uningest` in the :doc:`/api/native-api` guide) -and an Edit/Uningest menu option displayed on the file page, that allow a file to be Uningested. These are only available to superusers. +and an Edit/Uningest menu option displayed on the file page, that allow a file to be uningested. These are only available to superusers. Uningest will remove the warning. Uningest can also be done for a file that was successfully ingested. This will remove the .tab version of the file that was generated. From 70db48f7b51c1f789674fed74ba39d6c6bed80c4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 16:21:20 -0500 Subject: [PATCH 10/21] change to require publish to uningest for a problem --- .../edu/harvard/iq/dataverse/FilePage.java | 20 ++++++++++--------- .../webapp/file-edit-button-fragment.xhtml | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 37798f1cd3c..909a616a4a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -547,16 +547,17 @@ public String uningestFile() throws CommandException { if (!file.isTabularData()) { //Ingest never succeeded, either there was a failure or this is not a tabular data file - User u = session.getUser(); - if (!u.isAuthenticated() || !u.isSuperuser()) { - logger.warning("User: " + u.getIdentifier() + " tried to uningest a file"); - // Shouldn't happen (choice not displayed for users who don't have the right - // permission), but check anyway - JH.addMessage(FacesMessage.SEVERITY_WARN, - BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); - return null; - } if (file.isIngestProblem()) { + //We allow anyone who can publish to uningest in order to clear a problem + User u = session.getUser(); + if (!u.isAuthenticated() || !(permissionService.permissionsFor(u, file).contains(Permission.PublishDataset))) { + logger.warning("User: " + u.getIdentifier() + " tried to uningest a file"); + // Shouldn't happen (choice not displayed for users who don't have the right + // permission), but check anyway + JH.addMessage(FacesMessage.SEVERITY_WARN, + BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); + return null; + } file.setIngestDone(); file.setIngestReport(null); } else { @@ -566,6 +567,7 @@ public String uningestFile() throws CommandException { return null; } } else { + //Superuser required to uningest after a success //Uningest command does it's own check for isSuperuser commandEngine.submit(new UningestFileCommand(dvRequestService.getDataverseRequest(), file)); Long dataFileId = file.getId(); diff --git a/src/main/webapp/file-edit-button-fragment.xhtml b/src/main/webapp/file-edit-button-fragment.xhtml index e08de716cda..fd455521c98 100644 --- a/src/main/webapp/file-edit-button-fragment.xhtml +++ b/src/main/webapp/file-edit-button-fragment.xhtml @@ -79,7 +79,7 @@ - +
  • From 3d7f72a532841a1ce0a9635158b04a60ea080bf1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 16:21:39 -0500 Subject: [PATCH 11/21] add uningest for a problem logic in api --- .../edu/harvard/iq/dataverse/api/Files.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 5d400ee1438..1f0e0801c68 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -51,6 +51,7 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.URLTokenUtil; +import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; @@ -65,6 +66,7 @@ import java.util.logging.Logger; import jakarta.ejb.EJB; import jakarta.ejb.EJBException; +import jakarta.faces.application.FacesMessage; import jakarta.inject.Inject; import jakarta.json.Json; import jakarta.json.JsonArray; @@ -637,7 +639,27 @@ public Response uningestDatafile(@Context ContainerRequestContext crc, @PathPara if (dataFile == null) { return error(Response.Status.NOT_FOUND, "File not found for given id."); } - + if (!dataFile.isTabularData()) { + // Ingest never succeeded, either there was a failure or this is not a tabular + // data file + // We allow anyone who can publish to uningest in order to clear a problem + if (dataFile.isIngestProblem()) { + try { + AuthenticatedUser au = getRequestAuthenticatedUserOrDie(crc); + if (!(permissionSvc.permissionsFor(au, dataFile).contains(Permission.PublishDataset))) { + return forbidden( + "Uningesting to remove an ingest problem can only be done by those who can publish the dataset"); + } + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + dataFile.setIngestDone(); + dataFile.setIngestReport(null); + } else { + return error(Response.Status.BAD_REQUEST, + BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); + } + } if (!dataFile.isTabularData()) { return error(Response.Status.BAD_REQUEST, "Cannot uningest non-tabular file."); } From 7de7f43c99d7f79a7a4a255e75bfa08506361a41 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 16:21:56 -0500 Subject: [PATCH 12/21] update docs --- .../source/user/tabulardataingest/ingestprocess.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst index 4dce441de4a..418eb2206c8 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst @@ -67,9 +67,10 @@ who can see the draft version of the dataset containing the file that will indic part of the dataset, there will be no indication that ingest was attempted and failed. If the warning message is a concern, the Dataverse software includes both an API call (see :ref:`file-uningest` in the :doc:`/api/native-api` guide) -and an Edit/Uningest menu option displayed on the file page, that allow a file to be uningested. These are only available to superusers. -Uningest will remove the warning. Uningest can also be done for a file that was successfully ingested. -This will remove the .tab version of the file that was generated. +and an Edit/Uningest menu option displayed on the file page, that allow a file to be uningested by anone who can publish the dataset. + +Uningest will remove the warning. Uningest can also be done for a file that was successfully ingested. This is only available to superusers. +This will remove the variable-level metadata and the .tab version of the file that was generated. If a file is a tabular format but was never ingested, .e.g. due to the ingest file size limit being lower in the past, or if ingest had failed, e.g. in a prior Dataverse version, an reingest API (see :ref:`file-reingest` in the :doc:`/api/native-api` guide) and a file page Edit/Reingest option From 51fe60c095f52e26a6f1be7587c5323de7107993 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 16:26:50 -0500 Subject: [PATCH 13/21] fix logic --- .../edu/harvard/iq/dataverse/api/Files.java | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 1f0e0801c68..d48ae3247b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -655,27 +655,24 @@ public Response uningestDatafile(@Context ContainerRequestContext crc, @PathPara } dataFile.setIngestDone(); dataFile.setIngestReport(null); + return ok("Datafile " + dataFile.getId() + " uningested."); } else { return error(Response.Status.BAD_REQUEST, - BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); + BundleUtil.getStringFromBundle("Cannot uningest non-tabular file.")); + } + } else { + try { + DataverseRequest req = createDataverseRequest(getRequestUser(crc)); + execCommand(new UningestFileCommand(req, dataFile)); + Long dataFileId = dataFile.getId(); + dataFile = fileService.find(dataFileId); + Dataset theDataset = dataFile.getOwner(); + exportDatasetMetadata(settingsService, theDataset); + return ok("Datafile " + dataFileId + " uningested."); + } catch (WrappedResponse wr) { + return wr.getResponse(); } } - if (!dataFile.isTabularData()) { - return error(Response.Status.BAD_REQUEST, "Cannot uningest non-tabular file."); - } - - try { - DataverseRequest req = createDataverseRequest(getRequestUser(crc)); - execCommand(new UningestFileCommand(req, dataFile)); - Long dataFileId = dataFile.getId(); - dataFile = fileService.find(dataFileId); - Dataset theDataset = dataFile.getOwner(); - exportDatasetMetadata(settingsService, theDataset); - return ok("Datafile " + dataFileId + " uningested."); - } catch (WrappedResponse wr) { - return wr.getResponse(); - } - } // reingest attempts to queue an *existing* DataFile From 31d7cbcea224d253325f9baa3b4f4f1d8e802882 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 17:38:19 -0500 Subject: [PATCH 14/21] typo/merge issues --- src/main/webapp/file-edit-button-fragment.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/file-edit-button-fragment.xhtml b/src/main/webapp/file-edit-button-fragment.xhtml index fd455521c98..30c3f6e7938 100644 --- a/src/main/webapp/file-edit-button-fragment.xhtml +++ b/src/main/webapp/file-edit-button-fragment.xhtml @@ -79,9 +79,9 @@ - +
  • - +
    • From 057d2c3c5d9a00b38354416dbaa70ee6637bbe43 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 17:49:28 -0500 Subject: [PATCH 15/21] missing save --- src/main/java/edu/harvard/iq/dataverse/api/Files.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index d48ae3247b5..f735ecfdec8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -655,6 +655,7 @@ public Response uningestDatafile(@Context ContainerRequestContext crc, @PathPara } dataFile.setIngestDone(); dataFile.setIngestReport(null); + fileService.save(dataFile); return ok("Datafile " + dataFile.getId() + " uningested."); } else { return error(Response.Status.BAD_REQUEST, From beb5bf6847469ab9b41b3128837d5c9d4daddf24 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 17:55:59 -0500 Subject: [PATCH 16/21] tweak api doc --- doc/sphinx-guides/source/api/native-api.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8cfa5deb96c..1b04d7c9e12 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2859,7 +2859,10 @@ The fully expanded example above (without environment variables) looks like this Uningest a File ~~~~~~~~~~~~~~~ -Reverse the tabular data ingest process performed on a file where ``ID`` is the database id or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file to process. Note that this requires "superuser" credentials. +Reverse the tabular data ingest process performed on a file where ``ID`` is the database id or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file to process. + +Note that this requires "superuser" credentials to undo a successful ingest and remove the variable-level metadata and .tab version of the file. +It can also be used by a user who can publish the dataset to clear the error from an unsuccessful ingest. A curl example using an ``ID``: From 00d418912d88e202a390a6c2d70d80efb0ec5bfc Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Feb 2024 18:02:53 -0500 Subject: [PATCH 17/21] changelog, release note tweaks --- doc/release-notes/10318-uningest-and-reingest.md | 5 +++-- doc/sphinx-guides/source/api/changelog.rst | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/10318-uningest-and-reingest.md b/doc/release-notes/10318-uningest-and-reingest.md index 7465f934330..9f6f81b4818 100644 --- a/doc/release-notes/10318-uningest-and-reingest.md +++ b/doc/release-notes/10318-uningest-and-reingest.md @@ -1,2 +1,3 @@ -New Uningest/Reingest options are available in the File Page Edit menu for superusers, allowing ingest errors to be cleared and for -ingest to be retried (e.g. after a Dataverse version update or if ingest size limits are changed). +New Uningest/Reingest options are available in the File Page Edit menu, allowing ingest errors to be cleared (by users who can published the associated dataset) +and (by suerpsuers) for a successful ingest to be undone or retried (e.g. after a Dataverse version update or if ingest size limits are changed). +The /api/files//uningest api also now allows users who can publish the dataset to undo an ingest failure. diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index d272086fa2e..99414550c4b 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -12,6 +12,7 @@ v6.2 - **/api/datasets/{id}/versions/{versionId}**: The includeFiles parameter has been renamed to excludeFiles. The default behavior remains the same, which is to include files. However, when excludeFiles is set to true, the files will be excluded. A bug that caused the API to only return a deaccessioned dataset if the user had edit privileges has been fixed. - **/api/datasets/{id}/versions**: The includeFiles parameter has been renamed to excludeFiles. The default behavior remains the same, which is to include files. However, when excludeFiles is set to true, the files will be excluded. +- **/api/files/$ID/uningest**: Can now be used by users with the ability to publish the dataset to undo a failed ingest. (Removing a successful ingest still requires being superuser) v6.1 ---- From 87b5a38bd5511a169f1ccae9d3bb966f2e3cb6b6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 15 Feb 2024 17:01:59 -0500 Subject: [PATCH 18/21] typo #10318 --- .../source/user/tabulardataingest/ingestprocess.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst index 418eb2206c8..1e481a54da6 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst @@ -67,7 +67,7 @@ who can see the draft version of the dataset containing the file that will indic part of the dataset, there will be no indication that ingest was attempted and failed. If the warning message is a concern, the Dataverse software includes both an API call (see :ref:`file-uningest` in the :doc:`/api/native-api` guide) -and an Edit/Uningest menu option displayed on the file page, that allow a file to be uningested by anone who can publish the dataset. +and an Edit/Uningest menu option displayed on the file page, that allow a file to be uningested by anyone who can publish the dataset. Uningest will remove the warning. Uningest can also be done for a file that was successfully ingested. This is only available to superusers. This will remove the variable-level metadata and the .tab version of the file that was generated. From 5760149b5527ebd351acca0b71757d8a4c8540bb Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 28 Feb 2024 16:26:55 -0500 Subject: [PATCH 19/21] typo --- doc/release-notes/10318-uningest-and-reingest.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/10318-uningest-and-reingest.md b/doc/release-notes/10318-uningest-and-reingest.md index 9f6f81b4818..80ca6be57ea 100644 --- a/doc/release-notes/10318-uningest-and-reingest.md +++ b/doc/release-notes/10318-uningest-and-reingest.md @@ -1,3 +1,3 @@ New Uningest/Reingest options are available in the File Page Edit menu, allowing ingest errors to be cleared (by users who can published the associated dataset) -and (by suerpsuers) for a successful ingest to be undone or retried (e.g. after a Dataverse version update or if ingest size limits are changed). +and (by superusers) for a successful ingest to be undone or retried (e.g. after a Dataverse version update or if ingest size limits are changed). The /api/files//uningest api also now allows users who can publish the dataset to undo an ingest failure. From 9f34826e758c5c40ed4ab7b3bd17a06929eab6c0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 29 Feb 2024 14:30:07 -0500 Subject: [PATCH 20/21] just save the file, w/o calling FilePage.save() which creates a draft --- src/main/java/edu/harvard/iq/dataverse/FilePage.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 909a616a4a3..080828f996f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -521,8 +521,8 @@ public String ingestFile() throws CommandException{ dataFile.getIngestRequest().setForceTypeCheck(true); // update the datafile, to save the newIngest request in the database: - save(); - + datafileService.save(file); + // queue the data ingest job for asynchronous execution: String status = ingestService.startIngestJobs(editDataset.getId(), new ArrayList<>(Arrays.asList(dataFile)), (AuthenticatedUser) session.getUser()); @@ -587,7 +587,8 @@ public String uningestFile() throws CommandException { logger.log(Level.WARNING, "Uningest: Exception while exporting:{0}", ex.getMessage()); } } - save(); + datafileService.save(file); + // Refresh filemetadata with file title, etc. init(); JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("file.uningest.complete")); From 6480896a9cf6b2b4c9c858169b32f58a03d722b9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 29 Feb 2024 14:30:48 -0500 Subject: [PATCH 21/21] info to fine --- src/main/java/edu/harvard/iq/dataverse/FilePage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 080828f996f..dcb27b7c31b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -536,7 +536,7 @@ public String ingestFile() throws CommandException{ logger.warning("Ingest Status for file: " + dataFile.getId() + " : " + status); } - logger.info("File: " + dataFile.getId() + " ingest queued"); + logger.fine("File: " + dataFile.getId() + " ingest queued"); init(); JsfHelper.addInfoMessage(BundleUtil.getStringFromBundle("file.ingest.ingestQueued"));