Merge pull request #9096 from GlobalDataverseCommunityConsortium/DVWe…

…bloader_integration

IQSS/9095-dvwebloader integration

doc/release-notes/9096-folder-upload.md

@@ -0,0 +1 @@
+Dataverse can now support upload of an entire folder tree of files and retain the relative paths of files as directory path metadata for the uploaded files, if the installation is configured with S3 direct upload.

doc/sphinx-guides/source/developers/big-data-support.rst

@@ -36,10 +36,20 @@ At present, one potential drawback for direct-upload is that files are only part
 
 ``./asadmin create-jvm-options "-Ddataverse.files.<id>.ingestsizelimit=<size in bytes>"``
 
+.. _cors-s3-bucket:
 
-**IMPORTANT:** One additional step that is required to enable direct uploads via a Dataverse installation and for direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. 
+Allow CORS for S3 Buckets
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**IMPORTANT:** One additional step that is required to enable direct uploads via a Dataverse installation and for direct download to work with previewers and direct upload to work with dvwebloader (:ref:`folder-upload`) is to allow cross site (CORS) requests on your S3 store.
 The example below shows how to enable CORS rules (to support upload and download) on a bucket using the AWS CLI command line tool. Note that you may want to limit the AllowedOrigins and/or AllowedHeaders further.  https://github.com/gdcc/dataverse-previewers/wiki/Using-Previewers-with-download-redirects-from-S3 has some additional information about doing this.
 
+If you'd like to check the CORS configuration on your bucket before making changes:
+
+``aws s3api get-bucket-cors --bucket <BUCKET_NAME>``
+
+To proceed with making changes:
+
 ``aws s3api put-bucket-cors --bucket <BUCKET_NAME> --cors-configuration file://cors.json``
 
 with the contents of the file cors.json as follows:

doc/sphinx-guides/source/installation/config.rst

@@ -2829,6 +2829,7 @@ The URL for your Repository Storage Abstraction Layer (RSAL) installation. This
 This setting controls which upload methods are available to users of your Dataverse installation. The following upload methods are available:
 
 - ``native/http``: Corresponds to "Upload with HTTP via your browser" and APIs that use HTTP (SWORD and native).
+- ``dvwebloader``: Corresponds to :ref:`folder-upload`. Note that ``dataverse.files.<id>.upload-redirect`` must be set to "true" on an S3 store for this method to show up in the UI. In addition, :ref:`:WebloaderUrl` must be set. CORS allowed on the S3 bucket. See :ref:`cors-s3-bucket`.
 - ``dcm/rsync+ssh``: Corresponds to "Upload with rsync+ssh via Data Capture Module (DCM)". A lot of setup is required, as explained in the :doc:`/developers/big-data-support` section of the Developer Guide.
 
 Out of the box only ``native/http`` is enabled and will work without further configuration. To add multiple upload method, separate them using a comma like this:
@@ -3330,6 +3331,11 @@ The interval in seconds between Dataverse calls to Globus to check on upload pro
 
 A true/false option to add a Globus transfer option to the file download menu which is not yet fully supported in the dataverse-globus app. See :ref:`globus-support` for details.
 
+.. _:WebloaderUrl:
 
+:WebloaderUrl
++++++++++++++
+
+The URL for main HTML file in https://github.com/gdcc/dvwebloader when that app is deployed. See also :ref:`:UploadMethods` for another required settings.
 
 .. _supported MicroProfile Config API source: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Overview.html

doc/sphinx-guides/source/user/dataset-management.rst

@@ -93,6 +93,13 @@ Dropbox Upload
 
 Some Dataverse installations support the ability to upload files directly from Dropbox. To do so, click the "Upload from Dropbox" button, log in to Dropbox in the pop-up window, and select the files you'd like to transfer over.
 
+.. _folder-upload:
+
+Folder Upload
+-------------
+
+Some Dataverse installations support the ability to upload files from a local folder and subfolders. To do this, click the "Upload from Folder" button, select the folder you wish to upload, select/unselect specific files, and click "Start Uploads". More detailed instructions are available in the `DVWebloader wiki <https://github.com/gdcc/dvwebloader/wiki#use>`_.
+
 .. _rsync_upload:
 
 rsync + SSH Upload
@@ -268,7 +275,7 @@ After you :ref:`upload your files <dataset-file-upload>`, you can apply a "Workf
 |cw-image4|
 
 How to Describe Your Computational Workflow
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The Dataverse installation you are using may have enabled Computational Workflow metadata fields for your use. If so, when :ref:`editing your dataset metadata <adding-new-dataset>`, you will see the fields described below.
 

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -56,6 +56,8 @@
 
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.URLTokenUtil;
+import edu.harvard.iq.dataverse.util.WebloaderUtil;
 import edu.harvard.iq.dataverse.validation.URLValidator;
 import edu.harvard.iq.dataverse.workflows.WorkflowComment;
 
@@ -1845,7 +1847,9 @@ public boolean globusUploadSupported() {
         return settingsWrapper.isGlobusUpload() && settingsWrapper.isGlobusEnabledStorageDriver(dataset.getEffectiveStorageDriverId());
     }
 
-
+    public boolean webloaderUploadSupported() {
+        return settingsWrapper.isWebloaderUpload() && StorageIO.isDirectUploadEnabled(dataset.getEffectiveStorageDriverId());
+    }
 
     private String init(boolean initFull) {
 
@@ -6062,4 +6066,19 @@ public void startGlobusTransfer() {
         }
         PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken));
     }
+
+    public String getWebloaderUrlForDataset(Dataset d) {
+        String localeCode = session.getLocaleCode();
+        User user = session.getUser();
+        if (user instanceof AuthenticatedUser) {
+            ApiToken apiToken = authService.getValidApiTokenForUser((AuthenticatedUser) user);
+            return WebloaderUtil.getWebloaderUrl(d, apiToken, localeCode,
+                    settingsService.getValueForKey(SettingsServiceBean.Key.WebloaderUrl));
+        } else {
+            // Shouldn't normally happen (seesion timeout? bug?)
+            logger.warning("getWebloaderUrlForDataset called for non-Authenticated user");
+            return null;
+        }
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java

@@ -5,7 +5,9 @@
 import edu.harvard.iq.dataverse.api.AbstractApiBean;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
 import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper;
 import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;
@@ -36,6 +38,8 @@
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.URLTokenUtil;
+import edu.harvard.iq.dataverse.util.WebloaderUtil;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.EjbUtil;
 import edu.harvard.iq.dataverse.util.FileMetadataUtil;
@@ -3067,6 +3071,10 @@ public boolean globusUploadSupported() {
         return settingsWrapper.isGlobusUpload()
                 && settingsWrapper.isGlobusEnabledStorageDriver(dataset.getEffectiveStorageDriverId());
     }
+
+    public boolean webloaderUploadSupported() {
+        return settingsWrapper.isWebloaderUpload() && StorageIO.isDirectUploadEnabled(dataset.getEffectiveStorageDriverId());
+    }
 
     private void populateFileMetadatas() {
         fileMetadatas = new ArrayList<>();
@@ -3106,4 +3114,18 @@ public void setFileAccessRequest(boolean fileAccessRequest) {
     public boolean isHasPublicStore() {
         return settingsWrapper.isTrueForKey(SettingsServiceBean.Key.PublicInstall, StorageIO.isPublicStore(dataset.getEffectiveStorageDriverId()));
     }
+
+    public String getWebloaderUrlForDataset(Dataset d) {
+        String localeCode = session.getLocaleCode();
+        User user = session.getUser();
+        if (user instanceof AuthenticatedUser) {
+            ApiToken apiToken = authService.getValidApiTokenForUser((AuthenticatedUser) user);
+            return WebloaderUtil.getWebloaderUrl(d, apiToken, localeCode,
+                    settingsService.getValueForKey(SettingsServiceBean.Key.WebloaderUrl));
+        } else {
+            // Shouldn't normally happen (seesion timeout? bug?)
+            logger.warning("getWebloaderUrlForDataset called for non-Authenticated user");
+            return null;
+        }
+    }
 }

src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java

@@ -107,6 +107,8 @@ public class SettingsWrapper implements java.io.Serializable {
 
     private Boolean rsyncOnly = null;
 
+    private Boolean webloaderUpload = null;
+
     private String metricsUrl = null; 
 
     private Boolean dataFilePIDSequentialDependent = null;
@@ -338,6 +340,13 @@ public String getGlobusAppUrl() {
 
     }
 
+    public boolean isWebloaderUpload() {
+        if (webloaderUpload == null) {
+            webloaderUpload = systemConfig.isWebloaderUpload();
+        }
+        return webloaderUpload;
+    }
+
     public boolean isRsyncOnly() {
         if (rsyncOnly == null) {
             String downloadMethods = getValueForKey(SettingsServiceBean.Key.DownloadMethods);

src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java

@@ -940,4 +940,14 @@ public List <WorkflowComment> getWorkflowCommentsByAuthenticatedUser(Authenticat
         return query.getResultList();
     }
 
+    public ApiToken getValidApiTokenForUser(AuthenticatedUser user) {
+        ApiToken apiToken = null;
+        apiToken = findApiTokenByUser(user);
+        if ((apiToken == null) || (apiToken.getExpireTime().before(new Date()))) {
+            logger.fine("Created apiToken for user: " + user.getIdentifier());
+            apiToken = generateApiTokenForUser(user);
+        }
+        return apiToken;
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java

@@ -568,11 +568,16 @@ Whether Harvesting (OAI) service is enabled
         /*
          * Allow a custom JavaScript to control values of specific fields.
          */
-        ControlledVocabularyCustomJavaScript,
+        ControlledVocabularyCustomJavaScript, 
         /**
          * A compound setting for disabling signup for remote Auth providers:
          */
-        AllowRemoteAuthSignUp
+        AllowRemoteAuthSignUp,
+        /**
+         * The URL for the DvWebLoader tool (see github.com/gdcc/dvwebloader for details)
+         */
+        WebloaderUrl
+
         ;
 
         @Override

src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java

@@ -767,7 +767,13 @@ public enum FileUploadMethods {
          * Upload through Globus of large files
          */
 
-        GLOBUS("globus")
+        GLOBUS("globus"), 
+
+        /**
+         * Upload folders of files through dvwebloader app
+         */
+
+        WEBLOADER("dvwebloader");
         ;
 
 
@@ -904,6 +910,10 @@ public boolean isRsyncUpload(){
     public boolean isGlobusUpload(){
         return getMethodAvailable(FileUploadMethods.GLOBUS.toString(), true);
     }
+
+    public boolean isWebloaderUpload(){
+        return getMethodAvailable(FileUploadMethods.WEBLOADER.toString(), true);
+    }
 
     // Controls if HTTP upload is enabled for both GUI and API.
     public boolean isHTTPUpload(){       

src/main/java/edu/harvard/iq/dataverse/util/WebloaderUtil.java

@@ -0,0 +1,36 @@
+package edu.harvard.iq.dataverse.util;
+
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map.Entry;
+import java.util.logging.Logger;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpSession;
+
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetPage;
+import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
+import edu.harvard.iq.dataverse.authorization.users.ApiToken;
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.authorization.users.User;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+
+public class WebloaderUtil {
+
+    private static final Logger logger = Logger.getLogger(WebloaderUtil.class.getCanonicalName());
+
+    /**
+     * Create the URL required to launch https://github.com/gdcc/dvwebloader
+     */
+    public static String getWebloaderUrl(Dataset d, ApiToken apiToken, String localeCode, String baseUrl) {
+        // Use URLTokenUtil for params currently in common with external tools.
+        URLTokenUtil tokenUtil = new URLTokenUtil(d, apiToken, localeCode);
+        String appUrl;
+        appUrl = baseUrl
+                + "?datasetPid={datasetPid}&siteUrl={siteUrl}&key={apiToken}&datasetId={datasetId}&datasetVersion={datasetVersion}&dvLocale={localeCode}";
+        return tokenUtil.replaceTokensWithValues(appUrl);
+    }
+}

src/main/java/propertyFiles/Bundle.properties

@@ -1673,6 +1673,10 @@ file.finishGlobus=Globus Transfer has finished
 file.downloadFromGlobus=Download through Globus
 file.globus.transfer=Globus Transfer
 file.globus.of=of:
+file.fromWebloader.tip=Upload a folder of files. This method retains the relative path structure from your local machine. (Using it will cancel any other types of uploads in progress on this page.)
+file.fromWebloaderAfterCreate.tip=An option to upload a folder of files will be enabled after this dataset is created.
+file.fromWebloader=Upload a Folder
+
 file.api.httpDisabled=File upload via HTTP is not available for this installation of Dataverse.
 file.api.alreadyHasPackageFile=File upload via HTTP disabled since this dataset already contains a package file.
 file.replace.original=Original File

src/main/webapp/dataset.xhtml

@@ -846,6 +846,7 @@
                                 <ui:param name="editDatafilesPage" value="false"/>
                                 <ui:param name="rsyncSupported" value="#{DatasetPage.rsyncUploadSupported()}"/>
                                 <ui:param name="globusUploadSupported" value="#{DatasetPage.globusUploadSupported()}"/>
+                                <ui:param name="webloaderUploadSupported" value="#{DatasetPage.webloaderUploadSupported()}"/>
                                 <ui:param name="dataverse" value="#{DatasetPage.dataset.owner}"/>
                                 <ui:param name="dataset" value="#{DatasetPage.dataset}"/>
                                 <ui:param name="version" value="#{DatasetPage.workingVersion}"/>

src/main/webapp/editFilesFragment.xhtml

@@ -158,21 +158,26 @@
                                   widgetVar="fileUploadWidget">
                         <f:passThroughAttribute name="aria-label" value="#{bundle['file.uploadFiles']}"/>
                     </p:fileUpload>
-
-                                <div jsf:id="dropboxBlock" jsf:rendered="#{settingsWrapper.isHasDropBoxKey() and !lockedFromEdits }"  class="margin-top">
+                              <ui:fragment rendered="#{!lockedFromEdits}">
+                                <div jsf:id="webloaderBlock" jsf:rendered="#{webloaderUploadSupported}"  class="ui-fileupload-buttonbar ui-widget-header ui-corner-top">
+                                    <p class="help-block" jsf:rendered="#{!createDataset}">#{bundle['file.fromWebloader.tip']}</p>
+                                    <p class="help-block" jsf:rendered="#{createDataset}">#{bundle['file.fromWebloaderAfterCreate.tip']}</p>
+                                    <p:button target="_blank" rendered="#{!createDataset}" onclick="cancelDatasetEdit();window.open('#{(datasetPage==true) ? DatasetPage.getWebloaderUrlForDataset(dataset):EditDatafilesPage.getWebloaderUrlForDataset(dataset)}');return false;" value="#{bundle['file.fromWebloader']}"  icon="ui-icon-plusthick" />
+                                </div>
+                                <div jsf:id="dropboxBlock" jsf:rendered="#{settingsWrapper.isHasDropBoxKey()}"  class="margin-top">
                                     <!-- Dropbox upload widget -->
                                     <p class="help-block">#{bundle['file.fromDropbox.tip']}</p>
                                     <h:inputText id="dropBoxSelectionInput" style="display:none" value="#{EditDatafilesPage.dropBoxSelection}"/>
                                     <p:commandButton id="dropBoxButton" actionListener="#{EditDatafilesPage.handleDropBoxUpload}" oncomplete="javascript:dropBoxUploadFinished();" update="@none" style="display:none;" />
                                     <p:commandButton id="dropBoxUserButton" disabled="#{!(datasetPage || EditDatafilesPage.showFileUploadComponent())}" value="#{bundle['file.fromDropbox']}" onclick="openDropboxChooser();" icon="dropin-btn-status" />
                                     <p:message for="dropBoxButton" id="dropBoxUploadMessage" display="text" redisplay="false" />
                                 </div>
-                                <div jsf:id="globusBlock" jsf:rendered="#{globusUploadSupported and !lockedFromEdits }"  class="margin-top">
+                                <div jsf:id="globusBlock" jsf:rendered="#{globusUploadSupported}"  class="margin-top">
                                     <p class="help-block" jsf:rendered="#{!createDataset}">#{bundle['file.fromGlobus.tip']}</p>
                                     <p class="help-block" jsf:rendered="#{createDataset}">#{bundle['file.fromGlobusAfterCreate.tip']}</p>
                                     <p:button target="_blank" rendered="#{!createDataset}" onclick="cancelDatasetEdit();window.open('#{GlobusServiceBean.getGlobusAppUrlForDataset(dataset)}');return false;" value="#{bundle['file.fromGlobus']}"  icon="globus-btn" />
                                 </div>
-
+                              </ui:fragment>
                             </div>
                         </div>
 

src/main/webapp/editdatafiles.xhtml

@@ -63,6 +63,7 @@
                                 <ui:param name="showFileButtonUpdate" value="true"/>
                                 <ui:param name="rsyncSupported" value="#{EditDatafilesPage.rsyncUploadSupported()}"/>
                                 <ui:param name="globusUploadSupported" value="#{EditDatafilesPage.globusUploadSupported()}"/>
+                                <ui:param name="webloaderUploadSupported" value="#{EditDatafilesPage.webloaderUploadSupported()}"/>
                                 <ui:param name="lockedFromEdits" value="#{EditDatafilesPage.lockedFromEdits}"/>
                             </ui:include>
                         </div>

src/main/webapp/resources/css/structure.css

@@ -884,7 +884,6 @@ div.panel-body.read-terms{max-height:220px; overflow-y:scroll; width:100%; backg
 .dropin-btn-status.ui-icon {background: url("https://www.dropbox.com/static/images/widgets/dbx-saver-status.png") no-repeat;}
 .globus-btn.ui-icon {background: url("https://docs.globus.org/images/home/transfer.png") no-repeat;background-size:contain;display:inline-block;}
 
-
 /* VERSIONS */
 div[id$="versionsTable"] th.col-select-width * {display:none;}
 #version-details-block > div {width:35%; padding:4px 10px; border: 1px solid #DDDDDD;}