implement a writeToString method for CSV formatters

src/FileExportManager.js

@@ -2,7 +2,6 @@
 const { merge, isEmpty, groupBy } = require('lodash');
 const sanitizeFilename = require('sanitize-filename');
 const { EventEmitter } = require('eventemitter3');
-const queue = require('async/queue');
 const path = require('path');
 const {
   protocolProperty,
@@ -31,6 +30,7 @@
 const UserCancelledExport = require('./errors/UserCancelledExport');
 const { isElectron, isCordova } = require('./utils/Environment');
 
+
 const defaultCSVOptions = {
   adjacencyMatrix: false,
   attributeList: true,
@@ -94,6 +94,7 @@
   constructor(exportOptions = {}) {
     this.exportOptions = getOptions(exportOptions);
     this.events = new EventEmitter();
+    this.logs = [];
   }
 
   on = (...args) => {
@@ -106,7 +107,6 @@
       console.warn('Malformed emit.');
       return;
     }
-
     this.events.emit(event, payload);
   }
 
@@ -134,18 +134,9 @@
       return Promise.reject(new ExportError('Couldn\'t create temp directory.'));
     }
 
-    // This queue instance accepts one or more promises and limits their
-    // concurrency for better usability in consuming apps
-    // https://caolan.github.io/async/v3/docs.html#queue
-
-    // Set concurrency to conservative values for now, based on platform
-    const QUEUE_CONCURRENCY = isElectron() ? 50 : 1;
-
-    const q = queue((task, callback) => {
-      task()
-        .then((result) => callback(null, result))
-        .catch((error) => callback(error));
-    }, QUEUE_CONCURRENCY);
+    const logger = (message) => {
+      this.logs.push(message);
+    };
 
     const exportFormats = [
       ...(this.exportOptions.exportGraphML ? ['graphml'] : []),
@@ -158,7 +149,7 @@
     // Cleanup function called by abort method, after fatal errors, and after
     // the export promise resolves.
     const cleanUp = () => {
-      q.kill();
+      // q.kill();
     };
 
     this.emit('begin', ProgressMessages.Begin);
@@ -187,6 +178,10 @@
         // Short delay to give consumer UI time to render
         sleep(1000)(shouldContinue)
           .then(() => {
+            this.logs.push(`Starting export process...`);
+            this.logs.push(this.exportOptions);
+            // this.logs.push(sessions);
+            this.logs.push(`Temporary directory: ${tmpDir}`);
             this.emit('update', ProgressMessages.Formatting);
             return insertEgoIntoSessionNetworks(sessions);
           })
@@ -207,12 +202,12 @@
           })
           // Resequence IDs for this export
           .then((sessionsWithEgo) => resequenceIds(sessionsWithEgo))
-          .then((unifiedSessions) => {
+          .then(async (unifiedSessions) => {
             if (!shouldContinue()) {
               throw new UserCancelledExport();
             }
 
-            const promisedExports = [];
+            const exportPromises = [];
 
             // Create an array of promises representing each session in each export format
             const finishedSessions = [];
@@ -263,56 +258,56 @@
 
                   partitionedNetworks.forEach((partitionedNetwork) => {
                     const partitionedEntity = partitionedNetwork.partitionEntity;
-                    promisedExports.push(() => new Promise((resolve, reject) => {
+
+                    exportPromises.push(async () => {
                       try {
-                        exportFile(
+                        const result = await exportFile(
                           prefix,
                           partitionedEntity,
                           format,
                           tmpDir,
                           partitionedNetwork,
                           protocol.codebook,
                           this.exportOptions,
-                        ).then((result) => {
-                          if (!finishedSessions.includes(prefix)) {
-                            // If we unified the networks, we need to iterate sessionVariables and
-                            // emit a 'session-exported' event for each sessionID
-                            if (this.exportOptions.globalOptions.unifyNetworks) {
-                              Object.values(session.sessionVariables)
-                                .forEach((sessionVariables) => {
-                                  this.emit('session-exported', sessionVariables.sessionId);
-                                });
-                            } else {
-                              this.emit('session-exported', session.sessionVariables.sessionId);
-                            }
-
-                            this.emit('update', ProgressMessages.ExportSession(finishedSessions.length + 1, sessionExportTotal));
-                            finishedSessions.push(prefix);
+                          logger
+                        );
+
+                        if (!finishedSessions.includes(prefix)) {
+                          // If we unified the networks, we need to iterate sessionVariables and
+                          // emit a 'session-exported' event for each sessionID
+                          if (this.exportOptions.globalOptions.unifyNetworks) {
+                            Object.values(session.sessionVariables)
+                              .forEach((sessionVariables) => {
+                                this.emit('session-exported', sessionVariables.sessionId);
+                              });
+                          } else {
+                            this.emit('session-exported', session.sessionVariables.sessionId);
                           }
-                          resolve(result);
-                        }).catch((e) => reject(e));
+
+                          this.emit('update', ProgressMessages.ExportSession(finishedSessions.length + 1, sessionExportTotal));
+                          finishedSessions.push(prefix);
+                        }
+
+                        succeeded.push(result);
+                        return result;
+
                       } catch (error) {
                         this.emit('error', `Encoding ${prefix} failed: ${error.message}`);
                         this.emit('update', ProgressMessages.ExportSession(finishedSessions.length + 1, sessionExportTotal));
-                        reject(error);
+                        failed.push(error);
                       }
-                    }));
+                    });
                   });
                 });
               });
             });
 
-            q.push(promisedExports, (err, result) => {
-              if (err) {
-                failed.push(err);
-                return;
-              }
-              succeeded.push(result);
-            });
+            // Process one at a time
+            for (const fn of exportPromises) {
+              await fn();
+            }
 
-            return new Promise((resolve, reject) => q.drain()
-              .then(() => resolve({ exportedPaths: succeeded, failedExports: failed }))
-              .catch(reject));
+            return { exportedPaths: succeeded, failedExports: failed };
           })
           // Then, Zip the result.
           .then(({ exportedPaths, failedExports }) => {
@@ -328,7 +323,7 @@
             // If we have no files to encode (but we do have errors), finish
             // the task here so the user can see the errors
             if (exportedPaths.length === 0) {
-              this.emit('finished', ProgressMessages.Finished);
+              this.emit('finished', this.logs);
               cleanUp();
               resolveRun();
               cancelled = true;
@@ -386,7 +381,7 @@
             }
 
             if (!saveCancelled) {
-              this.emit('finished', ProgressMessages.Finished);
+              this.emit('finished', this.logs);
             }
 
             cleanUp();

src/exportFile.js

@@ -1,13 +1,12 @@
 /* eslint-disable global-require */
-const { createWriteStream } = require('./utils/filesystem');
+const { writeFile } = require('./utils/filesystem');
 const {
   getFileExtension,
   makeFilename,
 } = require('./utils/general');
 const { isCordova, isElectron } = require('./utils/Environment');
 const getFormatterClass = require('./utils/getFormatterClass');
 const { ExportError } = require('./errors/ExportError');
-const UserCancelledExport = require('./errors/UserCancelledExport');
 
 /**
  * Export a single (CSV or graphml) file
@@ -23,74 +22,65 @@
  *                    If aborted, the returned promise will never settle.
  * @private
  */
-const exportFile = (
+const exportFile = async (
   namePrefix,
   partitonedEntityName,
   exportFormat,
   outDir,
   network,
   codebook,
   exportOptions,
+  logger,
 ) => {
   const Formatter = getFormatterClass(exportFormat);
   const extension = getFileExtension(exportFormat);
 
   if (!Formatter || !extension) {
-    return Promise.reject(new ExportError(`Invalid export format ${exportFormat}`));
+    throw new ExportError(`Invalid export format ${exportFormat}`);
   }
 
-  // Establish variables to hold the stream controller (needed to handle abort method)
-  // and the stream itself.
-  let streamController;
-  let writeStream;
-  let promiseResolve;
-  let promiseReject;
+  logger(`Exporting session ${namePrefix} (${partitonedEntityName}) in ${exportFormat} format...`);
+  logger(network);
 
-  // Create a promise
-  const pathPromise = new Promise((resolve, reject) => {
-    promiseResolve = resolve;
-    promiseReject = reject;
-    let filePath;
+  let filePath;
 
-    const formatter = new Formatter(network, codebook, exportOptions);
-    const outputName = makeFilename(namePrefix, partitonedEntityName, exportFormat, extension);
-    if (isElectron()) {
-      const path = require('path');
-      filePath = path.join(outDir, outputName);
-    }
-
-    if (isCordova()) {
-      filePath = `${outDir}${outputName}`;
-    }
+  const formatter = new Formatter(network, codebook, exportOptions);
+  const outputName = makeFilename(namePrefix, partitonedEntityName, exportFormat, extension);
+  if (isElectron()) {
+    const path = require('path');
+    filePath = path.join(outDir, outputName);
+  }
 
-    createWriteStream(filePath)
-      .then((ws) => {
-        writeStream = ws;
-        writeStream.on('finish', () => {
-          promiseResolve(filePath);
-        });
-        writeStream.on('error', (err) => {
-          promiseReject(err);
-        });
+  if (isCordova()) {
+    filePath = `${outDir}${outputName}`;
+  }
 
-        streamController = formatter.writeToStream(writeStream);
-      });
-  });
+  logger(`Exporting to ${filePath}.`);
 
-  // Decorate the promise with an abort method that also tears down the
-  // streamController and the writeStream
-  pathPromise.abort = () => {
-    if (streamController) {
-      streamController.abort();
-    }
-    if (writeStream) {
-      writeStream.destroy();
-    }
+  // createWriteStream(filePath)
+  //   .then((ws) => {
+  //     writeStream = ws;
+  //     writeStream.on('finish', () => {
+  //       promiseResolve(filePath);
+  //     });
+  //     writeStream.on('error', (err) => {
+  //       promiseReject(err);
+  //     });
 
-    promiseReject(new UserCancelledExport());
-  };
+  //     streamController = formatter.writeToStream(writeStream);
+  //   });
 
-  return pathPromise;
+  // Test writing to string
+  try {
+    const string = formatter.writeToString();
+    logger(string);
+    await writeFile(filePath, string);
+    logger(`Completed exporting ${namePrefix} as ${exportFormat} to path ${filePath}`);
+    return filePath;
+  } catch (e) {
+    logger(`Failed to write file! ${err}`);
+    throw (e);
+  }
 };
 
 module.exports = exportFile;

src/formatters/csv/attribute-list.js

@@ -98,6 +98,31 @@
   };
 };
 
+const toCSVString = (nodes) => {
+  const attrNames = attributeHeaders(nodes);
+  const headerValue = `${attrNames.map((attr) => sanitizedCellValue(getPrintableAttribute(attr))).join(',')}${csvEOL}`;
+
+  const rows = nodes.map((node) => {
+    const values = attrNames.map((attrName) => {
+      // The primary key and ego id exist at the top-level; all others inside `.attributes`
+      let value;
+      if (
+        attrName === entityPrimaryKeyProperty
+        || attrName === egoProperty
+        || attrName === nodeExportIDProperty
+      ) {
+        value = node[attrName];
+      } else {
+        value = node[entityAttributesProperty][attrName];
+      }
+      return sanitizedCellValue(value);
+    });
+    return `${values.join(',')}${csvEOL}`;
+  });
+
+  return headerValue + rows.join('');
+}
+
 class AttributeListFormatter {
   constructor(data, codebook, exportOptions) {
     this.list = asAttributeList(data, codebook, exportOptions);
@@ -106,6 +131,10 @@
   writeToStream(outStream) {
     return toCSVStream(this.list, outStream);
   }
+
+  writeToString() {
+    return toCSVString(this.list);
+  }
 }
 
 module.exports = {