Skip to content

Commit

Permalink
perf(export): improve memory usage when exporting data (#1029)
Browse files Browse the repository at this point in the history
  • Loading branch information
@DayTF
DayTF authored May 17, 2024
1 parent 72002f9 commit ed6af03
Show file tree
Hide file tree
Showing 6 changed files with 91 additions and 95 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
"body-parser": "^1.20.1",
"compose-middleware": "5.0.1",
"cors": "2.8.5",
"csv-stringify": "1.0.4",
"csv-stringify": "6.5.0",
"express": "^4.18.2",
"express-jwt": "8.4.1",
"forest-ip-utils": "1.0.1",
Expand Down
2 changes: 1 addition & 1 deletion src/routes/associations.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ module.exports = function Associations(app, model, Implementation, integrator, o
.catch(next);
}

function exportCSV(request, response, next) {
async function exportCSV(request, response, next) {
const { params, associationModel } = getContext(request);

const recordsExporter = new Implementation.ResourcesExporter(
Expand Down
2 changes: 1 addition & 1 deletion src/routes/resources.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ module.exports = function Resources(app, model, { configStore } = inject()) {
.catch(next);
};

this.exportCSV = (request, response, next) => {
this.exportCSV = async (request, response, next) => {
const params = request.query;
const recordsExporter = new Implementation.ResourcesExporter(
model,
Expand Down
80 changes: 38 additions & 42 deletions src/services/csv-exporter.js
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,40 @@
const P = require('bluebird');
const moment = require('moment');
const stringify = require('csv-stringify');
// eslint-disable-next-line import/no-unresolved
const { stringify } = require('csv-stringify/sync');
const { inject } = require('@forestadmin/context');
const ParamsFieldsDeserializer = require('../deserializers/params-fields');
const SmartFieldsValuesInjector = require('./smart-fields-values-injector');

// NOTICE: Prevent bad date formatting into timestamps.
const CSV_OPTIONS = {
formatters: {
cast: {
date: (value) => moment(value).format(),
},
};

function CSVExporter(params, response, modelName, recordsExporter) {
const { configStore } = inject();

this.perform = () => {
function getValueForAttribute(record, attribute) {
let value;
if (params.fields[attribute]) {
if (record[attribute]) {
if (params.fields[attribute] && record[attribute][params.fields[attribute]]) {
value = record[attribute][params.fields[attribute]];
} else {
// eslint-disable-next-line
value = record[attribute].id || record[attribute]._id;
}
}
} else {
value = record[attribute];
}

return value || '';
}

// eslint-disable-next-line sonarjs/cognitive-complexity
this.perform = async () => {
const filename = `${params.filename}.csv`;
response.setHeader('Content-Type', 'text/csv; charset=utf-8');
response.setHeader('Content-disposition', `attachment; filename=${filename}`);
Expand All @@ -32,47 +51,24 @@ function CSVExporter(params, response, modelName, recordsExporter) {

const fieldsPerModel = new ParamsFieldsDeserializer(params.fields).perform();

return recordsExporter
.perform((records) => P
.map(records, (record) =>
new SmartFieldsValuesInjector(record, modelName, fieldsPerModel).perform())
.then((recordsWithSmartFieldsValues) =>
new P((resolve) => {
if (configStore.Implementation.Flattener) {
recordsWithSmartFieldsValues = configStore.Implementation.Flattener
.flattenRecordsForExport(modelName, recordsWithSmartFieldsValues);
}
await recordsExporter
.perform(async (records) => {
await Promise.all(
// eslint-disable-next-line max-len
records.map((record) => new SmartFieldsValuesInjector(record, modelName, fieldsPerModel).perform()),
);

const CSVLines = [];
recordsWithSmartFieldsValues.forEach((record) => {
const CSVLine = [];
CSVAttributes.forEach((attribute) => {
let value;
if (params.fields[attribute]) {
if (record[attribute]) {
if (params.fields[attribute] && record[attribute][params.fields[attribute]]) {
value = record[attribute][params.fields[attribute]];
} else {
// eslint-disable-next-line
value = record[attribute].id || record[attribute]._id;
}
}
} else {
value = record[attribute];
}
CSVLine.push(value || '');
});
CSVLines.push(CSVLine);
});
if (configStore.Implementation.Flattener) {
records = configStore.Implementation.Flattener
.flattenRecordsForExport(modelName, records);
}

stringify(CSVLines, CSV_OPTIONS, (error, csv) => {
response.write(csv);
resolve();
});
})))
.then(() => {
response.end();
records.forEach((record) => {
// eslint-disable-next-line max-len
response.write(stringify([CSVAttributes.map((attribute) => getValueForAttribute(record, attribute, params))], CSV_OPTIONS));
});
});
response.end();
};
}

Expand Down
88 changes: 45 additions & 43 deletions src/services/smart-fields-values-injector.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
const _ = require('lodash');
const P = require('bluebird');
const logger = require('./logger');
const Schemas = require('../generators/schemas');

Expand Down Expand Up @@ -74,60 +73,63 @@ function SmartFieldsValuesInjector(
&& fieldsPerModel[modelNameToCheck].indexOf(fieldName) !== -1;
}

this.perform = () =>
P.each(schema.fields, (field) => {
const fieldWasRequested = isRequestedField(requestedField || modelName, field.field);
// eslint-disable-next-line sonarjs/cognitive-complexity
function injectSmartFieldValue(field) {
const fieldWasRequested = isRequestedField(requestedField || modelName, field.field);

if (record && field.isVirtual && (field.get || field.value)) {
if (fieldsPerModel && !fieldWasRequested) {
return null;
}

return setSmartFieldValue(record, field, modelName);
if (record && field.isVirtual && (field.get || field.value)) {
if (fieldsPerModel && !fieldWasRequested) {
return null;
}

if (
!record[field.field]
return setSmartFieldValue(record, field, modelName);
}

if (
!record[field.field]
&& _.isArray(field.type)
&& (field.relationship || field.isVirtual)) {
// Add empty arrays on relation fields so that JsonApiSerializer add the relevant
// `data.x.relationships` section in the response.
//
// The field must match the following condition
// - field is a real or a smart HasMany / BelongsToMany relation
// - field is NOT an 'embedded' relationship (@see mongoose)

record[field.field] = [];
} else if (field.reference && !_.isArray(field.type)) {
// NOTICE: Set Smart Fields values to "belongsTo" associated records.
const modelNameAssociation = getReferencedModelName(field);
const schemaAssociation = Schemas.schemas[modelNameAssociation];

if (schemaAssociation && !_.isArray(field.type)) {
return P.each(schemaAssociation.fields, (fieldAssociation) => {
if (record
// Add empty arrays on relation fields so that JsonApiSerializer add the relevant
// `data.x.relationships` section in the response.
//
// The field must match the following condition
// - field is a real or a smart HasMany / BelongsToMany relation
// - field is NOT an 'embedded' relationship (@see mongoose)

record[field.field] = [];
} else if (field.reference && !_.isArray(field.type)) {
// NOTICE: Set Smart Fields values to "belongsTo" associated records.
const modelNameAssociation = getReferencedModelName(field);
const schemaAssociation = Schemas.schemas[modelNameAssociation];

if (schemaAssociation && !_.isArray(field.type)) {
return Promise.all(schemaAssociation.fields.map((fieldAssociation) => {
if (record
&& record[field.field]
&& fieldAssociation.isVirtual
&& (fieldAssociation.get || fieldAssociation.value)) {
if (fieldsPerModel && !isRequestedField(field.field, fieldAssociation.field)) {
return null;
}

return setSmartFieldValue(
record[field.field],
fieldAssociation,
modelNameAssociation,
);
if (fieldsPerModel && !isRequestedField(field.field, fieldAssociation.field)) {
return null;
}

return null;
});
}
return setSmartFieldValue(
record[field.field],
fieldAssociation,
modelNameAssociation,
);
}

return null;
}));
}
}

return null;
}

return null;
})
.thenReturn(record);
this.perform = async () => Promise.all(
schema.fields.map((field) => injectSmartFieldValue(field)),
);
}

module.exports = SmartFieldsValuesInjector;
12 changes: 5 additions & 7 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3746,12 +3746,10 @@ cssesc@^3.0.0:
resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee"
integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==

csv-stringify@1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-1.0.4.tgz#bc18bab9ad4cef3195fd257980b58b479c42d3e5"
integrity sha1-vBi6ua1M7zGV/SV5gLWLR5xC0+U=
dependencies:
lodash.get "^4.0.0"
csv-stringify@6.5.0:
version "6.5.0"
resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.5.0.tgz#7b1491893c917e018a97de9bf9604e23b88647c2"
integrity sha512-edlXFVKcUx7r8Vx5zQucsuMg4wb/xT6qyz+Sr1vnLrdXqlLD1+UKyWNyZ9zn6mUW1ewmGxrpVwAcChGF0HQ/2Q==

dargs@^7.0.0:
version "7.0.0"
Expand Down Expand Up @@ -6570,7 +6568,7 @@ lodash.escaperegexp@^4.1.2:
resolved "https://registry.yarnpkg.com/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz#64762c48618082518ac3df4ccf5d5886dae20347"
integrity sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==

lodash.get@^4.0.0, lodash.get@^4.4.2:
lodash.get@^4.4.2:
version "4.4.2"
resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=
Expand Down

0 comments on commit ed6af03

Please sign in to comment.