diff --git a/src/js/collections/DataPackage.js b/src/js/collections/DataPackage.js index a51ffa671..c3f07d765 100644 --- a/src/js/collections/DataPackage.js +++ b/src/js/collections/DataPackage.js @@ -1,4 +1,4 @@ -"use strict"; +"use strict"; define([ "jquery", @@ -433,6 +433,145 @@ define([ } }, + /** + * Fetches member models in batches to avoid fetching all members simultaneously. + * @param {Array} models - The array of member models to fetch. + * @param {number} [index] - The current index of the model being fetched. + * @param {number} [batchSize] - The number of models to fetch in each batch. + * @param {number} [timeout] - The timeout for each fetch request in milliseconds. + * @param {number} [maxRetries] - The maximum number of retries for each fetch request. + * @since 0.0.0 + */ + fetchMemberModels( + models, + index = 0, + batchSize = 10, + timeout = 5000, + maxRetries = 3, + ) { + // Update the number of file metadata items being loaded + this.packageModel.set("numLoadingFileMetadata", models.length - index); + + // If the index is greater than or equal to the length of the models array, stop fetching + if (index >= models.length) { + this.triggerComplete(); + return; + } + + // If batchSize is 0, set it to the total number of models + let batchSizeAdjust = batchSize; + if (batchSizeAdjust === 0 && index === 0) + batchSizeAdjust = models.length; + + const collection = this; + // Slice the models array to get the current batch + const batch = models.slice(index, index + batchSizeAdjust); + + // Create an array of promises for fetching each model in the batch + const fetchPromises = batch.map( + (memberModel) => + new Promise((resolve, reject) => { + const attemptFetch = (retriesLeft) => { + // Create a promise for the fetch request + const fetchPromise = new Promise( + (fetchResolve, fetchReject) => { + memberModel.fetch({ + success: () => { + // Once the model is synced, handle the response + memberModel.once("sync", (oldModel) => { + const newModel = collection.getMember(oldModel); + + // If the type of the old model is different from the new model + if (oldModel.type !== newModel.type) { + if (newModel.type === "DataPackage") { + // If the new model is a DataPackage, replace the old model with the new one + oldModel.trigger("replace", newModel); + fetchResolve(); + } else { + // Otherwise, fetch the new model and replace the old model with the new one + newModel.set("synced", false); + newModel.fetch(); + newModel.once("sync", (fetchedModel) => { + fetchedModel.set("synced", true); + collection.remove(oldModel); + collection.add(fetchedModel); + oldModel.trigger("replace", newModel); + if (newModel.type === "EML") + collection.trigger("add:EML"); + fetchResolve(); + }); + } + } else { + // If the type of the old model is the same as the new model, merge the new model into the collection + newModel.set("synced", true); + collection.add(newModel, { merge: true }); + if (newModel.type === "EML") + collection.trigger("add:EML"); + fetchResolve(); + } + }); + }, + error: (model, response) => + fetchReject(new Error(response.statusText)), + }); + }, + ); + + // Create a promise for the timeout + const timeoutPromise = new Promise((__, timeoutReject) => { + setTimeout( + () => timeoutReject(new Error("Fetch timed out")), + timeout, + ); + }); + + // Race the fetch promise against the timeout promise + Promise.race([fetchPromise, timeoutPromise]) + .then(resolve) + .catch((error) => { + if (retriesLeft > 0) { + // Retry the fetch if there are retries left + attemptFetch(retriesLeft - 1); + } else { + // Reject the promise if all retries are exhausted + reject(error); + } + }); + }; + + // Start the fetch attempt with the maximum number of retries + attemptFetch(maxRetries); + }), + ); + + // Once all fetch promises are resolved, fetch the next batch + Promise.allSettled(fetchPromises) + .then((results) => { + const errors = results.filter( + (result) => result.status === "rejected", + ); + if (errors.length > 0) { + /* eslint-disable */ + console.error("Error fetching member models:", errors); + /* eslint-enable */ + } + // Fetch the next batch of models + this.fetchMemberModels.call( + collection, + models, + index + batchSizeAdjust, + batchSizeAdjust, + timeout, + maxRetries, + ); + }) + .catch((error) => { + /* eslint-disable */ + console.error("Error fetching member models:", error); + /* eslint-enable */ + }); + }, + /** * Overload fetch calls for a DataPackage * @param {object} [options] - Optional options for this fetch that get @@ -711,56 +850,13 @@ define([ // Don't fetch each member model if the fetchModels property on this // Collection is set to false if (this.fetchModels !== false) { - // Add the models to the collection now, silently this.add(models, - // {silent: true}); - - // Retrieve the model for each member - const collection = this; - models.forEach((model) => { - model.fetch(); - model.once("sync", (oldModel) => { - // Get the right model type based on the model values - const newModel = collection.getMember(oldModel); - - // If the model type has changed, then mark the model as - // unsynced, since there may be custom fetch() options for the - // new model - if (oldModel.type !== newModel.type) { - // DataPackages shouldn't be fetched until we support nested - // packages better in the UI - if (newModel.type === "DataPackage") { - // Trigger a replace event so other parts of the app know - // when a model has been replaced with a different type - oldModel.trigger("replace", newModel); - } else { - newModel.set("synced", false); - - newModel.fetch(); - newModel.once("sync", (fetchedModel) => { - fetchedModel.set("synced", true); - - // Remove the model from the collection and add it back - collection.remove(oldModel); - collection.add(fetchedModel); - - // Trigger a replace event so other parts of the app know - // when a model has been replaced with a different type - oldModel.trigger("replace", newModel); - - if (newModel.type === "EML") - collection.trigger("add:EML"); - }); - } - } else { - newModel.set("synced", true); - collection.add(newModel, { - merge: true, - }); - - if (newModel.type === "EML") collection.trigger("add:EML"); - } - }); - }); + // Start fetching member models + this.fetchMemberModels.call( + this, + models, + 0, + MetacatUI.appModel.get("batchSizeFetch"), + ); } } catch (error) { // TODO: Handle the error diff --git a/src/js/models/AppModel.js b/src/js/models/AppModel.js index 72ee1825b..2be813509 100644 --- a/src/js/models/AppModel.js +++ b/src/js/models/AppModel.js @@ -2433,6 +2433,40 @@ define(["jquery", "underscore", "backbone"], function ($, _, Backbone) { * @example application%2Fbagit-097 */ packageFormat: "application%2Fbagit-1.0", + /** + * Whether to batch fetch requests to the DataONE API. This is an experimental feature + * and should be used with caution. If set to a number greater than 0, MetacatUI will + * batch requests to the DataONE API and send them in groups of this size. This can + * improve performance when making many requests to the DataONE API, but can also + * cause issues if the requests are too large or if the DataONE API is not able to + * handle the batched requests. + * + * Currently, this feature is only used in the DataPackageModel when fetching the + * list of DataONE member models. + * + * @type {number} + * @default 0 + * @example 20 + * @since 0.0.0 + */ + batchSizeFetch: 0, + /** + * Whether to batch uploads to the DataONE API. This is an experimental feature + * and should be used with caution. If set to a number greater than 0, MetacatUI will + * batch uploads to the DataONE API and send them in groups of this size. This can + * improve performance when uploading many files to the DataONE API, but can also + * cause issues if the requests are too large or if the DataONE API is not able to + * handle the batched requests. + * + * Currently, this feature is only used in the DataPackageModel when uploading files + * to the DataONE API. + * + * @type {number} + * @default 0 + * @example 20 + * @since 0.0.0 + */ + batchSizeUpload: 0, }, MetacatUI.AppConfig, ), diff --git a/src/js/views/DataItemView.js b/src/js/views/DataItemView.js index 4b700e821..13841ca0a 100644 --- a/src/js/views/DataItemView.js +++ b/src/js/views/DataItemView.js @@ -8,7 +8,7 @@ define([ "views/DownloadButtonView", "text!templates/dataItem.html", "text!templates/dataItemHierarchy.html", -], function ( +], ( _, $, Backbone, @@ -18,19 +18,19 @@ define([ DownloadButtonView, DataItemTemplate, DataItemHierarchy, -) { +) => { /** - * @class DataItemView - * @classdesc A DataItemView represents a single data item in a data package as a single row of + * @class DataItemView + * @classdesc A DataItemView represents a single data item in a data package as a single row of a nested table. An item may represent a metadata object (as a folder), or a data object described by the metadata (as a file). Every metadata DataItemView has a resource map associated with it that describes the relationships between the aggregated metadata and data objects. - * @classcategory Views - * @constructor - * @screenshot views/DataItemView.png - */ - var DataItemView = Backbone.View.extend( + * @classcategory Views + * @class + * @screenshot views/DataItemView.png + */ + const DataItemView = Backbone.View.extend( /** @lends DataItemView.prototype */ { tagName: "tr", @@ -44,7 +44,7 @@ define([ /** The HTML template for a data item */ dataItemHierarchyTemplate: _.template(DataItemHierarchy), - //Templates + // Templates metricTemplate: _.template( "" + "" + @@ -88,9 +88,12 @@ define([ "click .downloadAction": "downloadFile", }, - /** Initialize the object - post constructor */ - initialize: function (options) { - if (typeof options == "undefined") var options = {}; + /** + * Initialize the object - post constructor + * @param options + */ + initialize(options) { + if (typeof options === "undefined") var options = {}; this.model = options.model || new DataONEObject(); this.currentlyViewing = options.currentlyViewing || null; @@ -105,26 +108,27 @@ define([ this.parentEditorView = options.parentEditorView || null; this.dataPackageId = options.dataPackageId || null; - if (!(typeof options.metricsModel == "undefined")) { + if (!(typeof options.metricsModel === "undefined")) { this.metricsModel = options.metricsModel; } }, - /** Renders a DataItemView for the given DataONEObject + /** + * Renders a DataItemView for the given DataONEObject * @param {DataONEObject} model */ - render: function (model) { - //Prevent duplicate listeners + render(model) { + // Prevent duplicate listeners this.stopListening(); if (this.itemType === "folder") { // Set the data-id for identifying events to model ids this.$el.attr( "data-id", - (this.itemPath ? this.itemPath : "") + "/" + this.itemName, + `${this.itemPath ? this.itemPath : ""}/${this.itemName}`, ); this.$el.attr("data-parent", this.itemPath ? this.itemPath : ""); - this.$el.attr("data-category", "entities-" + this.itemName); + this.$el.attr("data-category", `entities-${this.itemName}`); var attributes = new Object(); attributes.fileType = undefined; @@ -163,9 +167,9 @@ define([ } else { // Set the data-id for identifying events to model ids this.$el.attr("data-id", this.model.get("id")); - this.$el.attr("data-category", "entities-" + this.model.get("id")); + this.$el.attr("data-category", `entities-${this.model.get("id")}`); - //Destroy the old tooltip + // Destroy the old tooltip this.$(".status .icon, .status .progress") .tooltip("hide") .tooltip("destroy"); @@ -181,12 +185,12 @@ define([ attributes.isMetadata = true; } - //Format the title + // Format the title if (Array.isArray(attributes.title)) { attributes.title = attributes.title[0]; } - //Set some defaults + // Set some defaults attributes.numAttributes = 0; attributes.entityIsValid = true; attributes.hasInvalidAttribute = false; @@ -198,7 +202,7 @@ define([ // Note: .canWrite is set here (at render) instead of at init // because render will get called a few times during page load // as the app updates what it knows about the object - let accessPolicy = this.model.get("accessPolicy"); + const accessPolicy = this.model.get("accessPolicy"); if (accessPolicy) { attributes.canWrite = accessPolicy.isAuthorized("write"); this.canWrite = attributes.canWrite; @@ -213,11 +217,11 @@ define([ this.canShare = this.canShareItem(); attributes.canShare = this.canShare; - //Get the number of attributes for this item + // Get the number of attributes for this item if (this.model.type != "EML") { - //Get the parent EML model + // Get the parent EML model if (this.parentEML) { - var parentEML = this.parentEML; + var { parentEML } = this; } else { var parentEML = MetacatUI.rootDataPackage.where({ id: Array.isArray(this.model.get("isDocumentedBy")) @@ -228,22 +232,22 @@ define([ if (Array.isArray(parentEML)) parentEML = parentEML[0]; - //If we found a parent EML model + // If we found a parent EML model if (parentEML && parentEML.type == "EML") { this.parentEML = parentEML; - //Find the EMLEntity model for this data item - var entity = + // Find the EMLEntity model for this data item + const entity = this.model.get("metadataEntity") || parentEML.getEntity(this.model); - //If we found an EMLEntity model + // If we found an EMLEntity model if (entity) { this.entity = entity; - //Get the file name from the metadata if it is not in the model + // Get the file name from the metadata if it is not in the model if (!this.model.get("fileName")) { - var fileName = ""; + let fileName = ""; if (entity.get("physicalObjectName")) fileName = entity.get("physicalObjectName"); @@ -254,12 +258,12 @@ define([ this.model.set("fileName", fileName); } - //Get the number of attributes for this entity + // Get the number of attributes for this entity attributes.numAttributes = entity.get("attributeList").length; - //Determine if the entity model is valid + // Determine if the entity model is valid attributes.entityIsValid = entity.isValid(); - //Listen to changes to certain attributes of this EMLEntity model + // Listen to changes to certain attributes of this EMLEntity model // to re-render this view this.stopListening(entity); this.listenTo( @@ -268,34 +272,34 @@ define([ this.render, ); - //Check if there are any invalid attribute models - //Also listen to each attribute model + // Check if there are any invalid attribute models + // Also listen to each attribute model _.each( entity.get("attributeList"), function (attr) { - var isValid = attr.isValid(); + const isValid = attr.isValid(); - //Mark that this entity has at least one invalid attribute + // Mark that this entity has at least one invalid attribute if (!attributes.hasInvalidAttribute && !isValid) attributes.hasInvalidAttribute = true; this.stopListening(attr); - //Listen to when the validation status changes and rerender + // Listen to when the validation status changes and rerender if (isValid) this.listenTo(attr, "invalid", this.render); else this.listenTo(attr, "valid", this.render); }, this, ); - //If there are no attributes now, rerender when one is added + // If there are no attributes now, rerender when one is added this.listenTo(entity, "change:attributeList", this.render); } else { - //Rerender when an entity is added + // Rerender when an entity is added this.listenTo(this.model, "change:entities", this.render); } } else { - //When the package is complete, rerender + // When the package is complete, rerender this.listenTo( MetacatUI.rootDataPackage, "add:EML", @@ -306,14 +310,14 @@ define([ this.$el.html(this.template(attributes)); - //Initialize dropdowns + // Initialize dropdowns this.$el.find(".dropdown-toggle").dropdown(); - //Render the Share button + // Render the Share button this.renderShareControl(); if (this.model.get("type") == "Metadata") { - //Add the title data-attribute attribute to the name cell + // Add the title data-attribute attribute to the name cell this.$el.find(".name").attr("data-attribute", "title"); this.$el.addClass("folder"); } else { @@ -332,45 +336,38 @@ define([ container: "body", }); - //Check if the data package is in progress of being uploaded + // Check if the data package is in progress of being uploaded this.toggleSaving(); - //Create tooltips based on the upload status - var uploadStatus = this.model.get("uploadStatus"), - errorMessage = this.model.get("errorMessage"); + // Create tooltips based on the upload status + const uploadStatus = this.model.get("uploadStatus"); + let errorMessage = this.model.get("errorMessage"); // Use a friendlier message for 401 errors (the one returned is a little hard to understand) if (this.model.get("sysMetaErrorCode") == 401) { // If the user at least has write permission, they cannot update the system metadata only, so show this message /** @todo Do an object update when someone has write permission but not changePermission and is trying to change the system metadata (but not the access policy) */ if (accessPolicy && accessPolicy.isAuthorized("write")) { - errorMessage = - "The owner of this data file has not given you permission to rename it or change the " + - MetacatUI.appModel.get("accessPolicyName") + - "."; + errorMessage = `The owner of this data file has not given you permission to rename it or change the ${MetacatUI.appModel.get( + "accessPolicyName", + )}.`; // Otherwise, assume they only have read access } else { - errorMessage = - "The owner of this data file has not given you permission to edit this data file or change the " + - MetacatUI.appModel.get("accessPolicyName") + - "."; + errorMessage = `The owner of this data file has not given you permission to edit this data file or change the ${MetacatUI.appModel.get( + "accessPolicyName", + )}.`; } } // When there's an error or a warninig if (uploadStatus == "e" && errorMessage) { - var tooltipClass = uploadStatus == "e" ? "error" : ""; + const tooltipClass = uploadStatus == "e" ? "error" : ""; this.$(".status .icon").tooltip({ placement: "top", trigger: "hover", html: true, - title: - "
Issue saving:
" + - errorMessage + - "
", + title: `
Issue saving:
${errorMessage}
`, container: "body", }); @@ -424,20 +421,19 @@ define([ this.$el.addClass("loading"); } else if (uploadStatus == "p") { - var model = this.model; + var { model } = this; this.$(".status .progress").tooltip({ placement: "top", trigger: "hover", html: true, - title: function () { + title() { if (model.get("numSaveAttempts") > 0) { - return ( - "
Something went wrong during upload.
Trying again... (attempt " + - (model.get("numSaveAttempts") + 1) + - " of 3)
" - ); - } else if (model.get("uploadProgress")) { + return `
Something went wrong during upload.
Trying again... (attempt ${ + model.get("numSaveAttempts") + 1 + } of 3)
`; + } + if (model.get("uploadProgress")) { var percentDone = model.get("uploadProgress").toString(); if (percentDone.indexOf(".") > -1) percentDone = percentDone.substring( @@ -446,11 +442,7 @@ define([ ); } else var percentDone = "0"; - return ( - "
Uploading: " + - percentDone + - "%
" - ); + return `
Uploading: ${percentDone}%
`; }, container: "body", }); @@ -460,21 +452,21 @@ define([ this.$el.removeClass("loading"); } - //Listen to changes to the upload progress of this object + // Listen to changes to the upload progress of this object this.listenTo( this.model, "change:uploadProgress", this.showUploadProgress, ); - //Listen to changes to the upload status of the entire package + // Listen to changes to the upload status of the entire package this.listenTo( MetacatUI.rootDataPackage.packageModel, "change:uploadStatus", this.toggleSaving, ); - //listen for changes to rerender the view + // listen for changes to rerender the view this.listenTo( this.model, "change:fileName change:title change:id change:formatType " + @@ -484,7 +476,7 @@ define([ ); // render changes to the item var view = this; - this.listenTo(this.model, "replace", function (newModel) { + this.listenTo(this.model, "replace", (newModel) => { view.model = newModel; view.render(); }); @@ -496,28 +488,29 @@ define([ this.model.getFormat() == "metadata" || this.model.get("id") == this.currentlyViewing ) { - attributes.title = "Metadata: " + this.model.get("fileName"); + attributes.title = `Metadata: ${this.model.get("fileName")}`; attributes.icon = "icon-file-text"; attributes.metricIcon = "icon-eye-open"; this.isMetadata = true; this.$el.attr("data-packageId", this.dataPackageId); } - var objectTitleTooltip = + const objectTitleTooltip = attributes.title || attributes.fileName || attributes.id; attributes.objectTitle = objectTitleTooltip.length > 150 - ? objectTitleTooltip.slice(0, 75) + - "..." + - objectTitleTooltip.slice( + ? `${objectTitleTooltip.slice( + 0, + 75, + )}...${objectTitleTooltip.slice( objectTitleTooltip.length - 75, objectTitleTooltip.length, - ) + )}` : objectTitleTooltip; attributes.fileType = this.model.getFormat(); attributes.isFolder = false; - //Determine the icon type based on format type + // Determine the icon type based on format type if (this.model.getFormat() == "program") attributes.icon = "icon-code"; else if (this.model.getFormat() == "data") @@ -530,12 +523,15 @@ define([ attributes.id = this.model.get("id"); attributes.memberRowMetrics = null; - var metricToolTip = null, - view = this; + let metricToolTip = null; + var view = this; // Insert metrics for this item, // if the model has already been fethced. - if (this.metricsModel != null && this.metricsModel.get("views") !== null) { + if ( + this.metricsModel != null && + this.metricsModel.get("views") !== null + ) { metricToolTip = this.getMemberRowMetrics(view.id); attributes.memberRowMetrics = metricToolTip.split(" ")[0]; } else { @@ -543,8 +539,8 @@ define([ // If the fetch() is still in progress. this.listenTo(this.metricsModel, "sync", function () { metricToolTip = this.getMemberRowMetrics(view.id); - let readsCell = this.$( - '.metrics-count.downloads[data-id="' + view.id + '"]', + const readsCell = this.$( + `.metrics-count.downloads[data-id="${view.id}"]`, ); metricToolTip = view.getMemberRowMetrics(view.id); if (typeof metricToolTip !== "undefined" && metricToolTip) @@ -586,7 +582,7 @@ define([ } // var parent = itemPathParts[itemPathParts.length - 2]; - var parentPath = itemPathParts.slice(0, -1).join("/"); + const parentPath = itemPathParts.slice(0, -1).join("/"); if (parentPath !== undefined) { this.$el.attr("data-parent", parentPath); @@ -609,12 +605,9 @@ define([ this.downloadButtonView.render(); const id = this.model.get("id"); - const infoLink = - MetacatUI.root + - "/view/" + - encodeURIComponent(this.currentlyViewing) + - "#" + - encodeURIComponent(id); + const infoLink = `${MetacatUI.root}/view/${encodeURIComponent( + this.currentlyViewing, + )}#${encodeURIComponent(id)}`; attributes.moreInfoLink = infoLink; attributes.insertInfoIcon = this.insertInfoIcon; @@ -652,16 +645,16 @@ define([ * Renders a button that opens the AccessPolicyView for editing permissions on this data item * @since 2.15.0 */ - renderShareControl: function () { - //Get the Share button element - var shareButton = this.$(".sharing button"); + renderShareControl() { + // Get the Share button element + const shareButton = this.$(".sharing button"); if ( this.parentEditorView && this.parentEditorView.isAccessPolicyEditEnabled() ) { - //Start a title for the button tooltip - var sharebuttonTitle; + // Start a title for the button tooltip + let sharebuttonTitle; // If the user is not authorized to change the permissions of // this object, then disable the share button @@ -687,7 +680,7 @@ define([ }, /** Close the view and remove it from the DOM */ - onClose: function () { + onClose() { this.remove(); // remove for the DOM, stop listening this.off(); // remove callbacks, prevent zombies }, @@ -695,16 +688,16 @@ define([ /** Generate a unique id for each data item in the table TODO: This could be replaced with the DataONE identifier - */ - generateId: function () { - var idStr = ""; // the id to return - var length = 30; // the length of the generated string - var chars = + */ + generateId() { + let idStr = ""; // the id to return + const length = 30; // the length of the generated string + const chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXTZabcdefghiklmnopqrstuvwxyz".split( "", ); - for (var i = 0; i < length; i++) { + for (let i = 0; i < length; i++) { idStr += chars[Math.floor(Math.random() * chars.length)]; } @@ -713,21 +706,20 @@ define([ /** * Update the folder name based on the scimeta title - * * @param e The event triggering this method */ - updateName: function (e) { - var enteredText = this.cleanInput($(e.target).text().trim()); + updateName(e) { + const enteredText = this.cleanInput($(e.target).text().trim()); // Set the title if this item is metadata or set the file name // if its not if (this.model.get("type") == "Metadata") { - var title = this.model.get("title"); + const title = this.model.get("title"); // Get the current title which is either an array of titles // or a single string. When it's an array of strings, we // use the first as the canonical title - var currentTitle = Array.isArray(title) ? title[0] : title; + const currentTitle = Array.isArray(title) ? title[0] : title; // Don't set the title if it hasn't changed or is empty if ( @@ -739,7 +731,7 @@ define([ // that aren't Arrays into Arrays if ( (Array.isArray(title) && title.length < 2) || - typeof title == "string" + typeof title === "string" ) { this.model.set("title", [enteredText]); } else { @@ -763,10 +755,10 @@ define([ Handle the add file event, showing the file picker dialog Multiple files are allowed using the shift and or option/alt key @param {Event} event - */ - handleAddFiles: function (event) { + */ + handleAddFiles(event) { event.stopPropagation(); - var fileUploadElement = this.$(".file-upload"); + const fileUploadElement = this.$(".file-upload"); fileUploadElement.val(""); @@ -776,15 +768,110 @@ define([ event.preventDefault(); }, + /** + * Method to handle batch file uploads. + * This method processes files independently to avoid being slowed down by large files. + * @param {FileList} fileList - The list of files to be uploaded. + * @param {number} [batchSize] - The number of files to upload concurrently. + * @since 0.0.0 + */ + uploadFilesInBatch(fileList, batchSize = 10) { + const view = this; + let currentIndex = 0; // Index of the current file being processed + let activeUploads = 0; // Counter for the number of active uploads + + // If batchSize is 0, set it to the total number of files + if (batchSize === 0) batchSize = fileList.length; + + /** + * Function to upload the next file in the list. + * This function is called recursively to ensure that the number of concurrent uploads + * does not exceed the batch size. + */ + function uploadNextFile() { + // If all files have been processed, return + if (currentIndex >= fileList.length) { + return; + } + + // If the number of active uploads is less than the batch size, start a new upload + if (activeUploads < batchSize) { + const dataONEObject = fileList[currentIndex]; + currentIndex++; // Move to the next file + activeUploads++; // Increment the active uploads counter + + // Create a new Promise to handle the file upload + new Promise((resolve, reject) => { + // If the file needs to be uploaded and its checksum is not calculated + if ( + dataONEObject.get("uploadFile") && + !dataONEObject.get("checksum") + ) { + // Stop listening to previous checksumCalculated events + dataONEObject.stopListening( + dataONEObject, + "checksumCalculated", + ); + // Listen to the checksumCalculated event to start the upload + dataONEObject.listenToOnce( + dataONEObject, + "checksumCalculated", + () => { + dataONEObject.save(); // Save the file + // Listen to changes in the uploadStatus to resolve the Promise + dataONEObject.listenTo( + dataONEObject, + "change:uploadStatus", + () => { + if ( + dataONEObject.get("uploadStatus") !== "p" && + dataONEObject.get("uploadStatus") !== "q" && + dataONEObject.get("uploadStatus") !== "l" + ) { + resolve(); // Resolve the Promise when the upload is complete + } + }, + ); + }, + ); + try { + dataONEObject.calculateChecksum(); // Calculate the checksum + } catch (exception) { + reject(exception); // Reject the Promise if an error occurs + } + } else { + resolve(); // Resolve the Promise if the file does not need to be uploaded + } + }) + .then(() => { + activeUploads--; // Decrement the active uploads counter + uploadNextFile(); // Start the next file upload + }) + .catch((error) => { + console.error("Error uploading file:", error); + activeUploads--; // Decrement the active uploads counter + uploadNextFile(); // Start the next file upload + }); + + uploadNextFile(); // Start the next file upload + } + } + + // Start the initial batch of uploads + for (let i = 0; i < batchSize; i++) { + uploadNextFile(); + } + }, + /** With a file list from the file picker or drag and drop, add the files to the collection @param {Event} event - */ - addFiles: function (event) { - var fileList, // The list of chosen files - parentDataPackage, // The id of the first resource of this row's scimeta - self = this; // A reference to this view + */ + addFiles(event) { + let fileList; // The list of chosen files + let parentDataPackage; // The id of the first resource of this row's scimeta + const self = this; // A reference to this view event.stopPropagation(); event.preventDefault(); @@ -793,10 +880,8 @@ define([ fileList = event.originalEvent.dataTransfer.files; // handle file picker files - } else { - if (event.target) { - fileList = event.target.files; - } + } else if (event.target) { + fileList = event.target.files; } this.$el.removeClass("droppable"); @@ -805,71 +890,63 @@ define([ if (typeof event.delegateTarget.dataset.id !== "undefined") { this.parentSciMeta = this.getParentScienceMetadata(event); this.collection = this.getParentDataPackage(event); + // Queue the files for upload + const queueFilesPromise = new Promise((resolve) => { + _.each( + fileList, + function (file) { + let uploadStatus = "l"; + let errorMessage = ""; + + if (file.size == 0) { + uploadStatus = "e"; + errorMessage = + "This is an empty file. It won't be included in the dataset."; + } - // Read each file, and make a DataONEObject - _.each( - fileList, - function (file) { - var uploadStatus = "l", - errorMessage = ""; - - if (file.size == 0) { - uploadStatus = "e"; - errorMessage = - "This is an empty file. It won't be included in the dataset."; - } - - var dataONEObject = new DataONEObject({ - synced: true, - type: "Data", - fileName: file.name, - size: file.size, - mediaType: file.type, - uploadFile: file, - uploadStatus: uploadStatus, - errorMessage: errorMessage, - isDocumentedBy: [this.parentSciMeta.id], - isDocumentedByModels: [this.parentSciMeta], - resourceMap: [this.collection.packageModel.id], - }); + const dataONEObject = new DataONEObject({ + synced: true, + type: "Data", + fileName: file.name, + size: file.size, + mediaType: file.type, + uploadFile: file, + uploadStatus, + errorMessage, + isDocumentedBy: [this.parentSciMeta.id], + isDocumentedByModels: [this.parentSciMeta], + resourceMap: [this.collection.packageModel.id], + }); - // Add it to the parent collection - this.collection.add(dataONEObject); + // Add it to the parent collection + this.collection.add(dataONEObject); + }, + this, + ); + resolve(); + }); - // Asychronously calculate the checksum - if ( - dataONEObject.get("uploadFile") && - !dataONEObject.get("checksum") - ) { - dataONEObject.stopListening( - dataONEObject, - "checksumCalculated", - ); - dataONEObject.listenToOnce( - dataONEObject, - "checksumCalculated", - dataONEObject.save, - ); - try { - dataONEObject.calculateChecksum(); - } catch (exception) { - // TODO: Fail gracefully here for the user - } - } - }, - this, - ); + queueFilesPromise.then(() => { + // Call the batch upload method + this.uploadFilesInBatch( + this.collection.models, + MetacatUI.appModel.get("batchSizeUpload"), + ); + }); } }, /** Show the drop zone for this row in the table */ - showDropzone: function () { + showDropzone() { if (this.model.get("type") !== "Metadata") return; this.$el.addClass("droppable"); }, - /** Hide the drop zone for this row in the table */ - hideDropzone: function (event) { + /** + * Hide the drop zone for this row in the table + * @param event + */ + hideDropzone(event) { if (this.model.get("type") !== "Metadata") return; this.$el.removeClass("droppable"); }, @@ -880,10 +957,9 @@ define([ * * Called indirectly via the "click" event on elements with the * class .replaceFile. See this View's events map. - * * @param {MouseEvent} event Browser Click event */ - handleReplace: function (event) { + handleReplace(event) { event.stopPropagation(); // Stop immediately if we know the user doesn't have privs @@ -892,7 +968,7 @@ define([ return; } - var fileReplaceElement = $(event.target) + const fileReplaceElement = $(event.target) .parents(".dropdown-menu") .children(".file-replace"); @@ -921,10 +997,9 @@ define([ * mistakes that would cause the editor to get into a broken state. * On error, we attempt to return the editor back to its pre-replace * state. - * * @param {Event} event */ - replaceFile: function (event) { + replaceFile(event) { event.stopPropagation(); event.preventDefault(); @@ -932,7 +1007,7 @@ define([ return; } - var fileList = event.target.files; + const fileList = event.target.files; // Pre-check fileList value to make sure we can work with it if (fileList.length != 1) { @@ -946,11 +1021,11 @@ define([ } // Save uploadStatus for reverting if need to - var oldUploadStatus = this.model.get("uploadStatus"); + const oldUploadStatus = this.model.get("uploadStatus"); - var file = fileList[0], - uploadStatus = "q", - errorMessage = ""; + const file = fileList[0]; + let uploadStatus = "q"; + let errorMessage = ""; if (file.size == 0) { uploadStatus = "e"; @@ -966,7 +1041,7 @@ define([ } // Copy model attributes aside for reverting on error - var newAttributes = { + const newAttributes = { synced: false, fileName: file.name, size: file.size, @@ -974,15 +1049,15 @@ define([ uploadFile: file, hasContentChanges: true, checksum: null, - uploadStatus: uploadStatus, + uploadStatus, sysMetaUploadStatus: "c", // I set this so DataPackage::save // wouldn't try to update the sysmeta after the update - errorMessage: errorMessage, + errorMessage, }; // Save a copy of the attributes we're changing so we can revert // later if we encounter an exception - var oldAttributes = {}; + const oldAttributes = {}; _.each( Object.keys(newAttributes), function (k) { @@ -991,7 +1066,7 @@ define([ this, ); - oldAttributes["uploadStatus"] = oldUploadStatus; + oldAttributes.uploadStatus = oldUploadStatus; try { this.model.set(newAttributes); @@ -1002,7 +1077,7 @@ define([ // Grab a reference to the entity in the EML for the object // we're replacing this.parentSciMeta = this.getParentScienceMetadata(event); - var entity = null; + let entity = null; if (this.parentSciMeta) { entity = this.parentSciMeta.getEntity(this.model); @@ -1030,7 +1105,7 @@ define([ MetacatUI.rootDataPackage.packageModel.set("changed", true); // Last, provided a visual indication the replace was completed - var describeButton = this.$el + const describeButton = this.$el .children(".controls") .children(".btn-group") .children("button.edit") @@ -1040,14 +1115,14 @@ define([ return; } - var oldText = describeButton.html(); + const oldText = describeButton.html(); describeButton.html(' Replaced'); - var previousBtnClasses = describeButton.attr("class"); + const previousBtnClasses = describeButton.attr("class"); describeButton.removeClass("warning error").addClass("message"); - window.setTimeout(function () { + window.setTimeout(() => { describeButton.html(oldText); describeButton.addClass(previousBtnClasses).removeClass("message"); }, 3000); @@ -1062,19 +1137,17 @@ define([ this.render(); } - - return; }, /** Handle remove events for this row in the data package table @param {Event} event - */ - handleRemove: function (event) { - var eventId, // The id of the row of this event - removalIds = [], // The list of target ids to remove - dataONEObject, // The model represented by this row - documents; // The list of ids documented by this row (if meta) + */ + handleRemove(event) { + let eventId; // The id of the row of this event + const removalIds = []; // The list of target ids to remove + let dataONEObject; // The model represented by this row + let documents; // The list of ids documented by this row (if meta) event.stopPropagation(); event.preventDefault(); @@ -1113,9 +1186,9 @@ define([ _.each(documents, removalIds.push()); } } - //Data objects may need to be removed from the EML model entities list + // Data objects may need to be removed from the EML model entities list else if (dataONEObject && this.parentSciMeta.type == "EML") { - var matchingEntity = this.parentSciMeta.getEntity(dataONEObject); + const matchingEntity = this.parentSciMeta.getEntity(dataONEObject); if (matchingEntity) this.parentSciMeta.removeEntity(matchingEntity); } @@ -1124,8 +1197,8 @@ define([ _.each( removalIds, function (id) { - var documents = this.parentSciMeta.get("documents"); - var index = documents.indexOf(id); + const documents = this.parentSciMeta.get("documents"); + const index = documents.indexOf(id); if (index > -1) { this.parentSciMeta.get("documents").splice(index, 1); } @@ -1151,11 +1224,11 @@ define([ * data or metadata row of the UI event * @param {Event} event */ - getParentScienceMetadata: function (event) { - var parentMetadata, // The parent metadata array in the collection - eventModels, // The models associated with the event's table row - eventModel, // The model associated with the event's table row - parentSciMeta; // The parent science metadata for the event model + getParentScienceMetadata(event) { + let parentMetadata; // The parent metadata array in the collection + let eventModels; // The models associated with the event's table row + let eventModel; // The model associated with the event's table row + let parentSciMeta; // The parent science metadata for the event model if (typeof event.delegateTarget.dataset.id !== "undefined") { eventModels = MetacatUI.rootDataPackage.where({ @@ -1171,26 +1244,24 @@ define([ // Is this a Data or Metadata model? if (eventModel.get && eventModel.get("type") === "Metadata") { return eventModel; - } else { - // It's data, get the parent scimeta - parentMetadata = MetacatUI.rootDataPackage.where({ - id: Array.isArray(eventModel.get("isDocumentedBy")) - ? eventModel.get("isDocumentedBy")[0] - : null, - }); - - if (parentMetadata.length > 0) { - parentSciMeta = parentMetadata[0]; - return parentSciMeta; - } else { - //If there is only one metadata model in the root data package, then use that metadata model - var metadataModels = MetacatUI.rootDataPackage.where({ - type: "Metadata", - }); + } + // It's data, get the parent scimeta + parentMetadata = MetacatUI.rootDataPackage.where({ + id: Array.isArray(eventModel.get("isDocumentedBy")) + ? eventModel.get("isDocumentedBy")[0] + : null, + }); - if (metadataModels.length == 1) return metadataModels[0]; - } + if (parentMetadata.length > 0) { + parentSciMeta = parentMetadata[0]; + return parentSciMeta; } + // If there is only one metadata model in the root data package, then use that metadata model + const metadataModels = MetacatUI.rootDataPackage.where({ + type: "Metadata", + }); + + if (metadataModels.length == 1) return metadataModels[0]; } }, @@ -1199,8 +1270,10 @@ define([ * data or metadata row of the UI event * @param {Event} event */ - getParentDataPackage: function (event) { - var parentSciMeta, parenResourceMaps, parentResourceMapId; + getParentDataPackage(event) { + let parentSciMeta; + let parenResourceMaps; + let parentResourceMapId; if (typeof event.delegateTarget.dataset.id !== "undefined") { parentSciMeta = this.getParentScienceMetadata(event); @@ -1231,28 +1304,27 @@ define([ return MetacatUI.rootDataPackage; // A nested package - } else { - return MetacatUI.rootDataPackage.where({ - id: parentResourceMapId, - })[0]; } + return MetacatUI.rootDataPackage.where({ + id: parentResourceMapId, + })[0]; } }, /** * Removes invalid characters and formatting from the given input string * @param {string} input The string to clean - * @return {string} + * @returns {string} */ - cleanInput: function (input) { + cleanInput(input) { // 1. remove line breaks / Mso classes - var stringStripper = /(\n|\r| class=(")?Mso[a-zA-Z]+(")?)/g; - var output = input.replace(stringStripper, " "); + const stringStripper = /(\n|\r| class=(")?Mso[a-zA-Z]+(")?)/g; + let output = input.replace(stringStripper, " "); // 2. strip Word generated HTML comments - var commentSripper = new RegExp("", "g"); + const commentSripper = new RegExp("", "g"); output = output.replace(commentSripper, ""); - var tagStripper = new RegExp( + let tagStripper = new RegExp( "<(/)*(meta|link|span|\\?xml:|st1:|o:|font)(.*?)>", "gi", ); @@ -1261,7 +1333,7 @@ define([ output = output.replace(tagStripper, ""); // 4. Remove everything in between and including tags '' - var badTags = [ + const badTags = [ "style", "script", "applet", @@ -1272,17 +1344,17 @@ define([ for (var i = 0; i < badTags.length; i++) { tagStripper = new RegExp( - "<" + badTags[i] + ".*?" + badTags[i] + "(.*?)>", + `<${badTags[i]}.*?${badTags[i]}(.*?)>`, "gi", ); output = output.replace(tagStripper, ""); } // 5. remove attributes ' style="..."' - var badAttributes = ["style", "start"]; + const badAttributes = ["style", "start"]; for (var i = 0; i < badAttributes.length; i++) { - var attributeStripper = new RegExp( - " " + badAttributes[i] + '="(.*?)"', + const attributeStripper = new RegExp( + ` ${badAttributes[i]}="(.*?)"`, "gi", ); output = output.replace(attributeStripper, ""); @@ -1296,7 +1368,7 @@ define([ /** * Style this table row to indicate it will be removed */ - previewRemove: function () { + previewRemove() { this.$el.toggleClass("remove-preview"); }, @@ -1305,8 +1377,8 @@ define([ * an 'empty' class, and remove it when the user focuses back out. * @param {Event} e */ - emptyName: function (e) { - var editableCell = this.$(".canRename [contenteditable]"); + emptyName(e) { + const editableCell = this.$(".canRename [contenteditable]"); editableCell.tooltip("hide"); @@ -1315,7 +1387,7 @@ define([ .attr("data-original-text", editableCell.text().trim()) .text("") .addClass("empty") - .on("focusout", function () { + .on("focusout", () => { if (!editableCell.text()) editableCell .text(editableCell.attr("data-original-text")) @@ -1326,37 +1398,34 @@ define([ /** * Changes the access policy of a data object based on user input. - * * @param {Event} e - The event that triggered this function as a callback */ - changeAccessPolicy: function (e) { + changeAccessPolicy(e) { if (typeof e === "undefined" || !e) return; - var accessPolicy = this.model.get("accessPolicy"); + const accessPolicy = this.model.get("accessPolicy"); - var makePublic = $(e.target).prop("checked"); + const makePublic = $(e.target).prop("checked"); - //If the user has chosen to make this object private + // If the user has chosen to make this object private if (!makePublic) { if (accessPolicy) { - //Make the access policy private + // Make the access policy private accessPolicy.makePrivate(); } else { - //Create an access policy from the default settings + // Create an access policy from the default settings this.model.createAccessPolicy(); - //Make the access policy private + // Make the access policy private this.model.get("accessPolicy").makePrivate(); } + } else if (accessPolicy) { + // Make the access policy public + accessPolicy.makePublic(); } else { - if (accessPolicy) { - //Make the access policy public - accessPolicy.makePublic(); - } else { - //Create an access policy from the default settings - this.model.createAccessPolicy(); - //Make the access policy public - this.model.get("accessPolicy").makePublic(); - } + // Create an access policy from the default settings + this.model.createAccessPolicy(); + // Make the access policy public + this.model.get("accessPolicy").makePublic(); } }, @@ -1365,28 +1434,28 @@ define([ * @param {string} attr The modal attribute that has been validated * @param {string} errorMsg The validation error message to display */ - showValidation: function (attr, errorMsg) { - //Find the element that is required - var requiredEl = this.$("[data-category='" + attr + "']").addClass( + showValidation(attr, errorMsg) { + // Find the element that is required + const requiredEl = this.$(`[data-category='${attr}']`).addClass( "error", ); - //When it is updated, remove the error styling - this.listenToOnce(this.model, "change:" + attr, this.hideRequired); + // When it is updated, remove the error styling + this.listenToOnce(this.model, `change:${attr}`, this.hideRequired); }, /** * Hides the 'required' styling from this view */ - hideRequired: function () { - //Remove the error styling + hideRequired() { + // Remove the error styling this.$("[contenteditable].error").removeClass("error"); }, /** * Show the data item as saving */ - showSaving: function () { + showSaving() { this.$(".controls button").prop("disabled", true); if (this.model.get("type") != "Metadata") @@ -1400,17 +1469,17 @@ define([ /** * Hides the styles applied in {@link DataItemView#showSaving} */ - hideSaving: function () { + hideSaving() { this.$(".controls button").prop("disabled", false); this.$(".disable-layer").remove(); - //Make the name cell editable again + // Make the name cell editable again this.$(".canRename > div").prop("contenteditable", true); this.$el.removeClass("error-saving"); }, - toggleSaving: function () { + toggleSaving() { if ( this.model.get("uploadStatus") == "p" || this.model.get("uploadStatus") == "l" || @@ -1428,13 +1497,13 @@ define([ /** * Shows the current progress of the file upload */ - showUploadProgress: function () { + showUploadProgress() { if (this.model.get("numSaveAttempts") > 0) { this.$(".progress .bar").css("width", "100%"); } else { this.$(".progress .bar").css( "width", - this.model.get("uploadProgress") + "%", + `${this.model.get("uploadProgress")}%`, ); } }, @@ -1449,65 +1518,63 @@ define([ * not. If metadata (ie type is EML), also checks that the resource * map can be shared. Otherwise, only checks if the data item can be * shared. - * - * @return {boolean} Whether the item can be shared + * @returns {boolean} Whether the item can be shared * @since 2.15.0 */ - canShareItem: function () { + canShareItem() { if (this.parentEditorView) { if (this.parentEditorView.isAccessPolicyEditEnabled()) { if (this.model.type === "EML") { // Check whether we can share the resource map - var pkgModel = MetacatUI.rootDataPackage.packageModel, - pkgAccessPolicy = pkgModel.get("accessPolicy"); + const pkgModel = MetacatUI.rootDataPackage.packageModel; + const pkgAccessPolicy = pkgModel.get("accessPolicy"); - var canShareResourceMap = + const canShareResourceMap = pkgModel.isNew() || (pkgAccessPolicy && pkgAccessPolicy.isAuthorized("changePermission")); // Check whether we can share the metadata - var modelAccessPolicy = this.model.get("accessPolicy"); - var canShareMetadata = + const modelAccessPolicy = this.model.get("accessPolicy"); + const canShareMetadata = this.model.isNew() || (modelAccessPolicy && modelAccessPolicy.isAuthorized("changePermission")); // Only return true if we can share both return canShareMetadata && canShareResourceMap; - } else { - return ( - this.model.get("accessPolicy") && - this.model.get("accessPolicy").isAuthorized("changePermission") - ); } + return ( + this.model.get("accessPolicy") && + this.model.get("accessPolicy").isAuthorized("changePermission") + ); } } }, - downloadFile: function (e) { + downloadFile(e) { this.downloadButtonView.download(e); }, // Member row metrics for the package table // Retrieving information from the Metrics Model's result details - getMemberRowMetrics: function (id) { + getMemberRowMetrics(id) { if (typeof this.metricsModel !== "undefined") { - var metricsResultDetails = this.metricsModel.get("resultDetails"); + const metricsResultDetails = this.metricsModel.get("resultDetails"); if ( typeof metricsResultDetails !== "undefined" && metricsResultDetails ) { - var metricsPackageDetails = - metricsResultDetails["metrics_package_counts"]; + const metricsPackageDetails = + metricsResultDetails.metrics_package_counts; - var objectLevelMetrics = metricsPackageDetails[id]; + const objectLevelMetrics = metricsPackageDetails[id]; if (typeof objectLevelMetrics !== "undefined") { if (this.isMetadata) { - var reads = objectLevelMetrics["viewCount"]; + var reads = objectLevelMetrics.viewCount; } else { - var reads = objectLevelMetrics["downloadCount"]; + var reads = objectLevelMetrics.downloadCount; } } else { var reads = 0; diff --git a/src/js/views/metadata/EML211EditorView.js b/src/js/views/metadata/EML211EditorView.js index ada94708d..7e8eca478 100644 --- a/src/js/views/metadata/EML211EditorView.js +++ b/src/js/views/metadata/EML211EditorView.js @@ -552,6 +552,15 @@ define([ "change:numLoadingFiles", this.toggleEnableControls, ); + this.stopListening( + MetacatUI.rootDataPackage.packageModel, + "change:numLoadingFileMetadata", + ); + this.listenTo( + MetacatUI.rootDataPackage.packageModel, + "change:numLoadingFileMetadata", + this.toggleEnableControls, + ); }, /** Render the Data Package View and insert it into this view */ @@ -1200,6 +1209,15 @@ define([ "numLoadingFiles", )}${noun} to upload...`, ); + } else if ( + MetacatUI.rootDataPackage.packageModel.get("numLoadingFileMetadata") > + 0 + ) { + this.disableControls( + `Waiting for ${MetacatUI.rootDataPackage.packageModel.get( + "numLoadingFileMetadata", + )} file metadata to load...`, + ); } else { this.enableControls(); } diff --git a/test/config/tests.json b/test/config/tests.json index 2f79cb313..e626f2c7f 100644 --- a/test/config/tests.json +++ b/test/config/tests.json @@ -91,7 +91,8 @@ "./js/specs/unit/collections/BioontologyResults.spec.js", "./js/specs/unit/models/ontologies/BioontologyOntology.spec.js", "./js/specs/unit/models/accordion/Accordion.spec.js", - "./js/specs/unit/models/accordion/AccordionItem.spec.js" + "./js/specs/unit/models/accordion/AccordionItem.spec.js", + "./js/specs/unit/views/DataItemView.spec.js" ], "integration": [ "./js/specs/integration/collections/SolrResults.spec.js", diff --git a/test/js/specs/unit/collections/DataPackage.spec.js b/test/js/specs/unit/collections/DataPackage.spec.js index 54fba03bb..9ee585c98 100644 --- a/test/js/specs/unit/collections/DataPackage.spec.js +++ b/test/js/specs/unit/collections/DataPackage.spec.js @@ -1,6 +1,7 @@ -define(["../../../../../../../../src/js/collections/DataPackage"], function ( - DataPackage, -) { +define([ + "../../../../../../../../src/js/collections/DataPackage", + "../../../../../../../../src/js/models/DataONEObject", +], function (DataPackage, DataONEObject) { var expect = chai.expect; describe("DataPackage Test Suite", function () { @@ -39,5 +40,98 @@ define(["../../../../../../../../src/js/collections/DataPackage"], function ( expect(result).to.equal("folder1/folder2/file.txt"); }); }); + + describe("fetchMemberModels", function () { + this.timeout(30000); // Increase timeout to 30 seconds + + it("should fetch member models successfully", function (done) { + const models = [new DataONEObject(), new DataONEObject()]; + const originalFetch = DataONEObject.prototype.fetch; + let fetchCallCount = 0; + + DataONEObject.prototype.fetch = function (options) { + fetchCallCount++; + options.success(); + }; + + dataPackage.fetchMemberModels.call(dataPackage, models, 0, 2, 5000, 3); + + setTimeout(function () { + expect(fetchCallCount).to.equal(2); + DataONEObject.prototype.fetch = originalFetch; + done(); + }, 100); + }); + + it("should retry fetching member models on failure", function (done) { + const models = [new DataONEObject(), new DataONEObject()]; + const originalFetch = DataONEObject.prototype.fetch; + let fetchCallCount = 0; + let maxRetries = 3; + + DataONEObject.prototype.fetch = function (options) { + fetchCallCount++; + options.error({ statusText: "Internal Server Error" }); + }; + + dataPackage.fetchMemberModels(models, 0, 2, 5000, maxRetries); + + setTimeout(function () { + console.log( + "[should retry fetching member models on failure] " + + fetchCallCount + + " fetch calls", + ); + expect(fetchCallCount).to.equal(models.length * (maxRetries + 1)); // 2 models * 3 retries + DataONEObject.prototype.fetch = originalFetch; + done(); + }, 100); + }); + + it("should trigger complete event after fetching all models", function (done) { + const models = [ + new DataONEObject({ identifier: "1" }), + new DataONEObject({ identifier: "2" }), + ]; + const originalFetch = DataONEObject.prototype.fetch; + let fetchCallCount = 0; + let completeEventTriggered = false; + let maxRetries = 3; + + DataONEObject.prototype.fetch = function (options) { + console.log( + "[should trigger complete event after fetching all models] fetching model: " + + this.get("identifier"), + ); + fetchCallCount++; + options.success(); + }; + + dataPackage.triggerComplete = function () { + completeEventTriggered = true; + console.log( + "[should trigger complete event after fetching all models] complete event triggered", + ); + }; + + dataPackage.fetchMemberModels(models, 0, 2, 100, maxRetries); + + setTimeout(function () { + console.log( + "[should trigger complete event after fetching all models] " + + fetchCallCount + + " fetch calls", + ); + console.log( + "[should trigger complete event after fetching all models] " + + completeEventTriggered, + ); + expect(fetchCallCount).to.equal(models.length * (maxRetries + 1)); + expect(completeEventTriggered).to.be.true; + DataONEObject.prototype.fetch = originalFetch; + done(); + }, 1000); + }); + }); }); }); diff --git a/test/js/specs/unit/views/DataItemView.spec.js b/test/js/specs/unit/views/DataItemView.spec.js new file mode 100644 index 000000000..67bc6dfc0 --- /dev/null +++ b/test/js/specs/unit/views/DataItemView.spec.js @@ -0,0 +1,171 @@ +define([ + "jquery", + "underscore", + "backbone", + "models/DataONEObject", + "views/DataItemView", +], function ($, _, Backbone, DataONEObject, DataItemView) { + var expect = chai.expect; + + describe("DataItemView Test Suite", function () { + let dataItemView, model, collection; + + // Set up the test environment before each test + beforeEach(function () { + // Create a new DataONEObject model with a test identifier + model = new DataONEObject({ identifier: "test-id" }); + // Create a new Backbone collection + collection = new Backbone.Collection(); + // Initialize the DataItemView with the model and collection + dataItemView = new DataItemView({ + model: model, + collection: collection, + }); + + // Stub the getParentScienceMetadata function to return a mock object + sinon.stub(dataItemView, "getParentScienceMetadata").returns({ + id: "mock-sci-meta-id", + }); + + // Stub the getParentDataPackage function to return a mock object with a spy on the add method + sinon.stub(dataItemView, "getParentDataPackage").returns({ + packageModel: { id: "mock-package-id" }, + add: sinon.spy(), + }); + }); + + // Clean up the test environment after each test + afterEach(function () { + // Restore the stubbed methods to their original implementations + dataItemView.getParentScienceMetadata.restore(); + dataItemView.getParentDataPackage.restore(); + dataItemView.remove(); + }); + + describe("uploadFilesInBatch", function () { + it("should upload files in batches", function (done) { + // Create a list of DataONEObject models with initial upload status + const fileList = [ + new DataONEObject({ uploadFile: true, uploadStatus: "l" }), + new DataONEObject({ uploadFile: true, uploadStatus: "l" }), + new DataONEObject({ uploadFile: true, uploadStatus: "l" }), + ]; + + // Define the batch size for the upload + const batchSize = 2; + // Spy on the uploadFilesInBatch method to verify its call + const uploadSpy = sinon.spy(dataItemView, "uploadFilesInBatch"); + // Stub the save method to simulate setting the upload status to "p" + const saveStub = sinon + .stub(DataONEObject.prototype, "save") + .callsFake(function () { + this.set("uploadStatus", "p"); + }); + // Stub the calculateChecksum method to simulate setting checksum attributes + const checksumStub = sinon + .stub(DataONEObject.prototype, "calculateChecksum") + .callsFake(function () { + this.set("checksum", "fakeChecksum"); + this.set("checksumAlgorithm", "fakeAlgorithm"); + this.trigger("checksumCalculated", this.attributes); + }); + + // Call the method to be tested + dataItemView.uploadFilesInBatch(fileList, batchSize); + + // Simulate the completion of the upload by setting the upload status to "c" + fileList.forEach(function (file) { + file.set("uploadStatus", "c"); + }); + + // Use setTimeout to allow asynchronous operations to complete + setTimeout(function () { + // Log the call counts for debugging purposes + console.log( + "[should upload files in batches] uploadSpy.callCount: ", + uploadSpy.callCount, + ); + console.log( + "[should upload files in batches] checksumSpy.callCount: ", + checksumStub.callCount, + ); + + // Verify that the method was called once with the correct arguments + expect(uploadSpy.calledOnce).to.be.true; + expect(uploadSpy.calledWith(fileList, batchSize)).to.be.true; + // Verify that the calculateChecksum method was called the expected number of times + console.log( + "[should upload files in batches] fileList.length: ", + fileList.length, + ); + console.log( + "[should upload files in batches] saveSpy.callCount: ", + saveStub.callCount, + ); + expect(checksumStub.callCount).to.equal(fileList.length); + expect(saveStub.callCount).to.equal(fileList.length); + // Restore the spies and stubs + uploadSpy.restore(); + checksumStub.restore(); + saveStub.restore(); + // Indicate that the test is complete + done(); + }, 0); + }); + }); + + describe("addFiles", function () { + it("should add files to the collection", function (done) { + // Create a fake file object to simulate a file upload + const fakeFile = new Blob(["fake file content"], { + type: "text/plain", + }); + fakeFile.name = "fakeFile.txt"; + + // Create a mock event object with the necessary properties + const event = { + stopPropagation: sinon.spy(), + preventDefault: sinon.spy(), + target: { files: [fakeFile] }, + originalEvent: { dataTransfer: { files: [fakeFile] } }, + delegateTarget: { dataset: { id: "test-id" } }, + }; + + // Stub the methods to simulate their behavior + const uploadStub = sinon + .stub(dataItemView, "uploadFilesInBatch") + .returns(true); + const d1ObjectStub = sinon + .stub(DataONEObject.prototype, "initialize") + .returns(true); + + // Call the method to be tested + dataItemView.addFiles.call(dataItemView, event); + + // Use setTimeout to allow asynchronous operations to complete + setTimeout(function () { + // Verify that the event methods were called + expect(event.stopPropagation.calledOnce).to.be.true; + expect(event.preventDefault.calledOnce).to.be.true; + // Verify that the DataONEObject initialize method was called + console.log( + "[should add files to the collection] d1ObjectStub.callCount: ", + d1ObjectStub.callCount, + ); + expect(d1ObjectStub.calledOnce).to.be.true; + // Verify that the uploadFilesInBatch method was called + console.log( + "[should add files to the collection] uploadStub.callCount: ", + uploadStub.callCount, + ); + expect(uploadStub.calledOnce).to.be.true; + // Restore the stubs + uploadStub.restore(); + d1ObjectStub.restore(); + // Indicate that the test is complete + done(); + }, 0); + }); + }); + }); +}); diff --git a/test/server.js b/test/server.js index 082def5bc..04e036d82 100644 --- a/test/server.js +++ b/test/server.js @@ -43,7 +43,10 @@ const server = app.listen(port); url = "http://localhost:" + port + url; async function runTests(url) { - const browser = await puppeteer.launch({ headless: true }); + const browser = await puppeteer.launch({ + headless: true, + args: ["--no-sandbox", "--disable-setuid-sandbox"], + }); const page = await browser.newPage(); await page.goto(url, { waitUntil: "networkidle0" }); const html = await page.content(); // serialized HTML of page DOM.