Skip to content

Commit

Permalink
generate() can now serialize deep json objects
Browse files Browse the repository at this point in the history
  • Loading branch information
@koresar
koresar committed Mar 17, 2021
1 parent 75cbab7 commit 5a12541
Show file tree
Hide file tree
Showing 2 changed files with 186 additions and 55 deletions.
105 changes: 73 additions & 32 deletions src/lil-csv.js
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,40 @@
const isArray = Array.isArray;
const isString = (v) => typeof v === "string";
const isNumber = (v) => typeof v === "number";
const isBoolean = (v) => typeof v === "boolean";
const isDate = (v) => v instanceof Date && !isNaN(v.valueOf());
const isObject = (v) => v && typeof v === "object";
const isObject = (v) => v && typeof v === "object" && !isArray(v);
const isFunction = (v) => typeof v === "function";
// function getDeep(obj, path) {
// return path.split(".").reduce((result, curr) => (result == null ? result : result[curr]), obj);
// }
function setDeep(obj, path, value) {
const getDeep = (obj, path) => path.split(".").reduce((result, curr) => (result == null ? result : result[curr]), obj);
const setDeep = (obj, path, value) => {
path.split(".").reduce((result, curr, index, paths) => {
const newVal = index + 1 === paths.length ? value : {};
return isObject(result[curr]) ? result[curr] : (result[curr] = newVal);
}, obj);
}
};

const mergeDeep = (target, ...sources) => {
if (!sources.length) return target;
const source = sources.shift();

if (isObject(target) && isObject(source)) {
for (const [key, value] of Object.entries(source)) {
if (isObject(value)) {
mergeDeep((target[key] = target[key] || {}), value);
} else {
target[key] = value;
}
}
}

return mergeDeep(target, ...sources);
};

const deepKeys = (obj, prefix) =>
Object.entries(obj).reduce((keys, [k, v]) => {
const newKey = prefix ? prefix + "." + k : k;
return keys.concat(isObject(v) ? deepKeys(v, newKey) : newKey);
}, []);

export function parse(str, { header = true, escapeChar = "\\" } = {}) {
const entries = [];
Expand Down Expand Up @@ -76,7 +98,7 @@ export function parse(str, { header = true, escapeChar = "\\" } = {}) {
}

// Auto-construct the headers (JSON objects keys) from the top most line of the file.
if (typeof header === "boolean") header = headerEntry.reduce((o, h) => ((o[h] = true), o), {});
if (isBoolean(header)) header = headerEntry.reduce((o, h) => ((o[h] = true), o), {});

return entries.map((entry) => {
const processedEntry = {};
Expand All @@ -96,48 +118,67 @@ export function parse(str, { header = true, escapeChar = "\\" } = {}) {
});
}

export function generate(rows, { header, lineTerminator = "\n", escapeChar = "\\" } = {}) {
export function generate(rows, { header = true, lineTerminator = "\n", escapeChar = "\\" } = {}) {
const serialise = (v) => {
v = v.replace(/"/g, escapeChar + '"'); // Escape quote character
return v.includes(",") ? '"' + v + '"' : v; // Add quotes if value has commas
};

const valueToString = (v) => {
if (v == null || v === "" || !((isNumber(v) && !isNaN(v)) || isString(v) || isDate(v) || isBoolean(v)))
return ""; // ignore bad data

v = isDate(v) ? v.toISOString() : String(v); // convert any kind of value to string
return serialise(v);
};

let detectedHeaders = null;
if (header) {
if (isBoolean(header)) {
header = Array.from(rows.reduce((all, row) => new Set([...all, ...Object.keys(row)]), new Set()));
} else if (Array.isArray(header)) {
if (!header.every(isString)) throw new Error("If header is array all items must be strings");
if (isObject(rows[0])) {
detectedHeaders = deepKeys(mergeDeep({}, ...rows.filter(isObject)));
if (detectedHeaders.length === 0) throw new Error("Bad header and rows");
} else {
if (!isArray(rows[0])) throw new Error("Can't auto detect header from rows");
}
} else if (isArray(header)) {
if (!header.length || !header.every(isString))
throw new Error("If header is array all items must be strings");
detectedHeaders = header;
} else if (isObject(header)) {
header = Object.entries(header)
detectedHeaders = Object.entries(header)
.filter(([k, v]) => v)
.map(([k]) => k);
} else {
throw new Error("Header must be either boolean, or array, or object");
}

header = header.map((h) => {
h = h.replace(/"/g, escapeChar + '"');
return h.includes(",") ? `"${h}"` : h;
});
}

function valueToString(v) {
if (v == null || v === "" || !((isNumber(v) && !isNaN(v)) || isString(v) || isDate(v) || isBoolean(v)))
return ""; // ignore bad data

v = isDate(v) ? v.toISOString() : String(v); // convert any kind of value to string
v = v.replace(/"/g, escapeChar + '"'); // Escape quote character
if (v.includes(",")) v = '"' + v + '"'; // Add quotes if value has commas
return v;
}

const textHeader = header ? header.join() + lineTerminator : "";
const textHeader = detectedHeaders
? detectedHeaders
.map((h) => {
const dataHeader = isObject(header) ? header[h] : h;
const newHeader = dataHeader.jsonName || (isString(dataHeader) ? dataHeader : h);
return serialise(newHeader);
})
.join() + lineTerminator
: "";
return (
textHeader +
rows
.map((row, i) => {
if (Array.isArray(row)) {
if (header && row.length !== header.length)
throw new Error(`Each row array must have exactly ${header.length} items`);
if (isArray(row)) {
if (detectedHeaders && row.length !== detectedHeaders.length)
throw new Error(`Each row array must have exactly ${detectedHeaders.length} items`);
return row.map(valueToString).join();
}
if (isObject(row)) {
return header.map((h) => valueToString(row[h])).join();
if (!detectedHeaders) throw new Error("Unexpected row object");
return detectedHeaders
.map((h) => {
return valueToString(getDeep(row, h));
})
.join();
}
throw new Error(`Row ${i} must be either array or object`);
})
Expand Down
136 changes: 113 additions & 23 deletions test/lil-csv.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { parse, generate } from "../src/lil-csv";

const assert = require("assert");

describe("parse", () => {
Expand Down Expand Up @@ -124,26 +123,21 @@ describe("parse", () => {
});

describe("generate", () => {
it("should generate and parse back", () => {
const rows = parse(
generate([
[`Column`, `Second Column`, `else`],
["here", " we go ", "false"],
["with,comma", 'with " escaped quotes', "123"],
["", "empty", ""],
]),
{ header: false }
);
assert.deepStrictEqual(rows, [
[`Column`, `Second Column`, `else`],
["here", " we go ", "false"],
["with,comma", 'with " escaped quotes', "123"],
["", "empty", ""],
it("should generate objects", () => {
const text = generate([
{ foo: 1, bar: "baz" },
{ foo: 2, bar: "too" },
]);
assert.deepStrictEqual(
text,
`foo,bar
1,baz
2,too`
);
});

it("should generate with header", () => {
const rows = generate(
const text = generate(
[
["here", " we go ", "false"],
["with,comma", 'with " escaped quotes', "123"],
Expand All @@ -152,7 +146,7 @@ describe("generate", () => {
{ header: [`Column`, `Second Column`, `else`] }
);
assert.deepStrictEqual(
rows,
text,
`Column,Second Column,else
here, we go ,false
"with,comma",with \\" escaped quotes,123
Expand All @@ -161,23 +155,119 @@ here, we go ,false
});

it("should auto format some primitives", () => {
const rows = generate([[new Date("2020-12-12"), 123.123, false]], {
const text = generate([[new Date("2020-12-12"), 123.123, false]], {
header: [`Column`, `Second Column`, `else`],
});
assert.deepStrictEqual(
rows,
text,
`Column,Second Column,else
2020-12-12T00:00:00.000Z,123.123,false`
);
});

it("should generate from deep objects", () => {
const text = generate([{ foo: { deep: { deeper: 1 } }, bar: { deep: 2, deep2: { more: 3 } } }]);
assert.deepStrictEqual(
text,
`foo.deep.deeper,bar.deep,bar.deep2.more
1,2,3`
);
});

it("should generate from deep objects and rename headers", () => {
const rows = [
{
Column: "here",
deep: {
Column2: " we go ",
veryDeep: {
Column3: "false",
},
},
},
{
Column: "with,comma",
deep: {
Column2: 'with " escaped quotes',
veryDeep: {
Column3: "123",
},
},
},
{
Column: "",
deep: {
Column2: "empty",
veryDeep: {
Column3: "",
},
},
},
];
const text = generate(rows, {
header: {
Column: true,
"deep.Column2": "Second Column",
"deep.veryDeep.Column3": "else",
},
});
assert.deepStrictEqual(
text,
`Column,Second Column,else
here, we go ,false
"with,comma",with \\" escaped quotes,123
,empty,`
);
});

it("should ignore bad data", () => {
const rows = generate([[null, undefined, {}, [], () => {}, NaN, "", new Map(), new Set()]]);
assert.deepStrictEqual(rows, `,,,,,,,,`);
const text = generate([[null, undefined, {}, [], () => {}, NaN, "", new Map(), new Set()]], { header: false });
assert.deepStrictEqual(text, `,,,,,,,,`);
});

it("should throw when can't auto detect header", () => {
assert.throws(
() => generate([null]),
(err) => err.message === "Can't auto detect header from rows"
);
});

it("should throw if rows are unprocessable", () => {
assert.throws(
() => generate([{}]),
(err) => err.message === "Bad header and rows"
);
});
});

describe("generate + parse", () => {
it("should generate and parse back arrays", () => {
const rows = parse(
generate([
[`Column`, `Second Column`, `else`],
["here", " we go ", "false"],
["with,comma", 'with " escaped quotes', "123"],
["", "empty", ""],
]),
{ header: false }
);
assert.deepStrictEqual(rows, [
[`Column`, `Second Column`, `else`],
["here", " we go ", "false"],
["with,comma", 'with " escaped quotes', "123"],
["", "empty", ""],
]);
});

it("should generate and parse back objects", () => {
const rows = [
{ a: { deep: "X" }, b: "1" },
{ a: { deep: "Y" }, b: "2" },
];
const rows2 = parse(generate(rows));
assert.deepStrictEqual(rows2, rows);
});

it("should work on fully customised options", () => {
const text = generate(
[
Expand All @@ -190,7 +280,7 @@ describe("generate + parse", () => {
);
const data = parse(text, {
header: {
"A string": { jsonName: "stringX" },
"A string": "stringX",
num: { jsonName: "numberX", parse: (v) => (v && !Number.isNaN(Number(v)) ? Number(v) : "") },
bool: { jsonName: "booleanX", parse: (v) => Boolean(v && v !== "false") },
date: { jsonName: "dateX", parse: (v) => (isNaN(new Date(v).valueOf()) ? "" : new Date(v)) },
Expand Down

0 comments on commit 5a12541

Please sign in to comment.