Merge pull request #5 from erikyo/strip-keys

Strip keys

.github/workflows/node.js.yml

@@ -28,7 +28,7 @@ jobs:
 
     - run: npm run build --if-present
 
-    - run: npm test --coverage
+    - run: npm run test:coverage
 
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v4.0.1

src/block.ts

@@ -5,18 +5,18 @@ import { GetTextTranslation } from 'gettext-parser'
 
 export const matcher: Record<string, RegExp> = {
 	msgid: /^(msgid(?!_))(.*)/,
-	msgstr: /^msgstr(?:\[\d+])?(.*)/, // msgstr or msgstr[0]
-	msgctxt: /^(msgctxt)(.*)/,
-	msgid_plural: /^(msgid_plural)(.*)/,
+	msgstr: /^(msgstr(?:\[\d+])?)\s+(.*)/, // msgstr or msgstr[0]
+	msgctxt: /^(msgctxt)\s+(.*)/,
+	msgid_plural: /^(msgid_plural)\s+(.*)/,
 	extracted: /^(#\.)(.*)/,
-	reference: /^(#:)(.*)/,
-	flag: /^(#,)(.*)/,
-	previous: /^(#\|)(.*)/,
-	translator: /^(#(?![.:,|]))(.*)/, // all # that is not #. #: #, or #|
+	reference: /^(#:)\s+(.*)/,
+	flag: /^(#(?:[^:.,|]|$))\s+(.*)/,
+	previous: /^(#\|)\s+(.*)/,
+	translator: /^(#(?![.:,|]))\s+(.*)/, // OK, I'm lazy! Anyway, this will catch all "#" stuff that is not #. #: #, or #|
 }
 
 export interface Block {
-	msgid?: string // "%s example"
+	msgid: string // "%s example"
 	msgstr?: string[] // ["% esempio", "%s esempi"],
 	msgid_plural?: string // "%s examples"
 	msgctxt?: string // context
@@ -38,9 +38,10 @@ export class Block {
 			this.parseBlock(data)
 		} else if (typeof data === 'object') {
 			for (const key in data as Partial<Block>) {
-				if (key in Block.prototype) {
+				// check if the key exists in the class
+				if (typeof this[key as keyof Block] !== null) {
 					// @ts-ignore
-					this[key] = data[key]
+					this[key as keyof Block] = data[key]
 				}
 			}
 		}
@@ -55,39 +56,72 @@ export class Block {
 		lines.forEach((line) => {
 			if (!line) return
 			if (line.startsWith('"')) {
-				// @ts-ignore
-				rawBlock[currentType].push(line) // Remove quotes and append
+				currentType = currentType || 'msgid' // Assuming the default type is 'msgid' when the line starts with "
+				rawBlock[currentType] = rawBlock[currentType] || [] // Initialize rawBlock[currentType] as an array if not yet defined
+				rawBlock[currentType].push(line.unquote()) // Remove quotes and append
 			} else {
 				// Use the matcher object to find the type id
 				for (const type in matcher) {
-					const regexResult = matcher[type].exec(line)
+					const regexResult = matcher[type].exec(line.unquote())
 					if (regexResult) {
 						currentType = type
+						// Initialize rawBlock[type] as an array if not yet defined
 						if (!rawBlock[type]) {
 							rawBlock[type] = []
 						}
-						rawBlock[type].push(regexResult[0])
+						// Append the matched string to rawBlock[type]
+						rawBlock[type].push(regexResult[2].trim().unquote())
 						break
 					}
 				}
 			}
 		})
 
 		Object.assign(this, {
-			msgid: rawBlock.msgid?.join('\n'),
+			msgid: rawBlock.msgid.join('"\n"') || '',
 			msgid_plural: rawBlock.msgid_plural?.join('\n'),
-			msgstr: rawBlock.msgstr,
+			msgstr: rawBlock.msgstr || [],
 			msgctxt: rawBlock.msgctxt?.join('\n'),
 			comments: {
-				translator: rawBlock.translator,
-				extracted: rawBlock.extracted,
-				reference: rawBlock.reference,
-				flag: rawBlock.flag,
-				previous: rawBlock.previous,
+				translator: rawBlock?.translator,
+				extracted: rawBlock?.extracted,
+				reference: rawBlock?.reference,
+				flag: rawBlock?.flag,
+				previous: rawBlock?.previous,
 			},
 		})
 	}
 
+	/**
+	 * Map an array of strings to a string representation.
+	 *
+	 * @param strings array of strings
+	 * @param prefix string prefix for each line
+	 */
+	mapStrings(strings: string[] = [], prefix = '# '): string {
+		return strings?.map((line) => prefix + line).join('\n')
+	}
+
+	/**
+	 * Extracts a multi-line string from an array of strings.
+	 *
+	 * @param msgstr
+	 * @param prefix
+	 */
+	extractMultiString(msgstr: string[], prefix = 'msgstr'): string {
+		if (msgstr.length > 1) {
+			return msgstr
+				.map(
+					(line, index) =>
+						`${prefix}${msgstr.length > 1 ? '[' + index + ']' : ''} "${splitMultiline(line)}"`
+				)
+				.join('\n')
+		} else if (msgstr.length === 1) {
+			return `${prefix} "${msgstr[0]}"`
+		}
+		return `${prefix} ""`
+	}
+
 	/**
 	 * Converts the object to a string representation.
 	 *
@@ -96,18 +130,18 @@ export class Block {
 	toStr(): string {
 		const { comments, msgid, msgid_plural, msgstr, msgctxt } = this
 		const res = [
-			comments?.translator?.join('\n'),
-			comments?.extracted,
-			comments?.reference?.join('\n'),
-			comments?.flag,
-			comments?.previous?.join('\n'),
-			msgctxt,
-			splitMultiline(msgid),
-			msgid_plural,
-			msgstr?.map((i) => splitMultiline(i))?.join('\n') || '""',
+			this.mapStrings(comments?.translator), // Add key for translator comments
+			this.mapStrings(comments?.extracted, '#. '), // Add key for extracted comments
+			this.mapStrings(comments?.reference, '#: '), // Add key for reference comments
+			comments?.flag ? `#, ${comments.flag}` : undefined, // Add key for flag comments
+			this.mapStrings(comments?.previous, '#| '), // Add key for previous comments
+			msgctxt ? `msgctxt "${msgctxt}"` : undefined, // Add key for msgctxt
+			msgid ? `msgid "${splitMultiline(msgid)}"` : 'msgid ""', // Add key for msgid even if it's empty
+			msgid_plural ? `msgid_plural "${msgid_plural}"` : undefined, // Add key for msgid_plural
+			msgstr ? this.extractMultiString(msgstr) : 'msgstr ""', // Add keys for msgstr even if it's empty
 		]
 			.filter(Boolean)
-			.filter((i) => i?.length)
+			.filter((line) => line?.length)
 
 		return res.join('\n')
 	}
@@ -135,13 +169,15 @@ export class Block {
 	 * @return {number} the hash value generated
 	 */
 	hash(): number {
-		const strToHash = this?.msgctxt || '' + this?.msgid // match only the gettext with the same msgctxt and msgid (context and translation string)
-		let hash = 0
+		const strToHash = (this.msgctxt || '') + '|' + (this.msgid || '') // match only the gettext with the same msgctxt and msgid (context and translation string)
+		let hash = 0x811c9dc5 // FNV offset basis (32-bit)
+
 		for (let i = 0; i < strToHash.length; i++) {
-			const chr = strToHash.charCodeAt(i)
-			hash = ((hash << 5) - hash + chr) | 0
+			hash ^= strToHash.charCodeAt(i) // XOR the hash with the current character code
+			hash *= 0x01000193 // FNV prime (32-bit)
 		}
-		return hash
+
+		return hash >>> 0
 	}
 
 	/**

src/fs.ts

@@ -46,42 +46,12 @@ function readBlocks(lines: string[]): Block[] {
  * @param {string} data - the string to be parsed
  * @return {Block[]} an array of Block objects
  */
-export function parseFile(data: string): Block[] {
+export function parseFileContent(data: string): Block[] {
 	const lines = data.split(/\r?\n/).reverse()
 	const blocks = readBlocks(lines)
 	return blocks.sort(hashCompare)
 }
 
-/**
- * Extracts the header from the given .pot file content.
- *
- * @param {string} potFileContent - the content of the .pot file
- * @return {string} the header extracted from the .pot file content
- */
-export function extractPotHeader(
-	potFileContent: string
-): [Block, string] | [undefined, string] {
-	if (!potFileContent) {
-		return [undefined, '']
-	}
-
-	const lines = potFileContent.split('\n')
-	const firstNonEmptyIndex = lines.findIndex((line) => line.trim() === '')
-	const parsedLines = lines.slice(0, firstNonEmptyIndex)
-
-	if (
-		parsedLines.length === 0 ||
-		!parsedLines.find((line) => line.toLowerCase().includes('project-id-version'))
-	) {
-		return [undefined, potFileContent]
-	}
-
-	return [
-		new Block(parsedLines),
-		lines.slice(firstNonEmptyIndex, lines.length).join('\n'),
-	]
-}
-
 /**
  * Writes the consolidated content of the SetOfBlocks to a file specified by the
  * output parameter.
@@ -99,7 +69,7 @@ export async function writePo(
 	// add the header
 	let consolidated = header ? header.toStr() + '\n\n\n' : ''
 	// consolidate the blocks
-	consolidated += blocks.toStr()
+	consolidated += blocks.cleanup().toStr()
 
 	// TODO: choose whether to override the existing file
 	await fs.writeFile(output, consolidated, { encoding: 'utf8', flag: 'w' })

src/index.ts

@@ -1,8 +1,9 @@
 import fs from 'fs/promises'
 import { SetOfBlocks } from './setOfBlocks.js'
-import { extractPotHeader, parseFile } from './fs.js'
+import { parseFileContent } from './fs.js'
 import { Block } from './block.js'
 import { GetTextComment } from './types.js'
+import { extractPotHeader } from './utils'
 
 /**
  * Merges multiple arrays of blocks into a single set of blocks.
@@ -33,7 +34,7 @@ export async function mergePotFile(filePaths: string[]): Promise<[Block[], SetOf
 			// Store the header in the header array
 			if (header) headers.push(header)
 			// Parse the content and return the SetOfBlocks
-			return new SetOfBlocks(parseFile(content)).blocks
+			return new SetOfBlocks(parseFileContent(content)).blocks
 		})
 	)
 
@@ -51,19 +52,15 @@ export async function mergePotFile(filePaths: string[]): Promise<[Block[], SetOf
  * @return {string} the merged file contents as a single string
  */
 export function mergePotFileContent(fileContents: string[]): string {
-	let response: string = ''
-
 	// merge the files
 	const mergedSet = fileContents.map((content) => {
-		return new SetOfBlocks(parseFile(content)).blocks
+		return new SetOfBlocks(parseFileContent(content)).blocks
 	})
 
 	// Retrieve the current blocks from the mergedSet
 	const currentBlocks = Array.from(mergedSet)
 	// Merge current blocks with the next array of blocks
-	response += mergeBlocks(...currentBlocks).toStr()
-
-	return response
+	return mergeBlocks(...currentBlocks).toStr()
 }
 
 /**
@@ -117,4 +114,4 @@ export function mergeComments(
 	}
 }
 
-export { Block, SetOfBlocks }
+export { Block, SetOfBlocks, extractPotHeader }

src/setOfBlocks.ts

@@ -50,36 +50,66 @@ export class SetOfBlocks {
 		return undefined
 	}
 
+	/**
+	 * Chainable sort function
+	 *
+	 * @param type - sorting type ('alphabetically', 'numerically', etc.)
+	 * @returns {SetOfBlocks} the instance of SetOfBlocks
+	 */
+	sortBlocks(type: string = 'alphabetically'): SetOfBlocks {
+		switch (type) {
+			case 'alphabetically':
+				this.blocks.sort((a, b) => a.msgid.localeCompare(b.msgid))
+				break
+			case 'hash':
+				this.blocks.sort(hashCompare)
+				break
+		}
+		return this
+	}
+
+	/**
+	 * Chainable filter function used to remove blocks without mandatory fields
+	 * Usually you want to fire this function to clean up the blocks without the msgid
+	 *
+	 * @returns {SetOfBlocks} the instance of SetOfBlocks
+	 */
+	cleanup(
+		mandatoryField: keyof Pick<
+			GetTextTranslation,
+			'msgid' | 'msgstr' | 'msgid_plural' | 'msgctxt'
+		> = 'msgid'
+	): SetOfBlocks {
+		this.blocks = this.blocks.filter((b) => !!b[mandatoryField])
+		return this
+	}
+
 	/**
 	 * Convert the blocks to a string representation.
 	 *
 	 * @return {string} the string representation of the blocks
 	 */
 	toStr(): string {
-		return this.blocks
-			.filter((b) => b.msgid)
-			.sort(hashCompare)
-			.reduce((prev, curr) => prev + curr.toStr() + '\n\n', '')
+		return this.blocks.reduce((prev, curr) => prev + curr.toStr() + '\n\n', '')
 	}
 
 	/**
 	 * Convert the blocks to a JSON representation using a compatible format for gettext-parser module
 	 *
-	 * @return {GetTextTranslations['translations']} the JSON representation of the blocks
+	 * @return {Map<string, Map<string, GetTextTranslation>>} the JSON representation of the blocks
 	 */
-	toJson(): GetTextTranslations['translations'] {
-		const newSet: Record<string, { [key: string]: GetTextTranslation }> = {}
+	toJson(): { [key: string]: { [key: string]: GetTextTranslation } } {
+		const jsonObject: GetTextTranslations['translations'] = {}
 
-		this.blocks
-			.filter((b) => b.msgid)
-			.sort(hashCompare)
-			.forEach((b) => {
-				const index = b.msgctxt || ''
-				if (!newSet[index]) newSet[index] = {}
-				newSet[index][b.msgid || ''] = b.toJson()
-			})
+		this.blocks.forEach((block) => {
+			const index = block.msgctxt || ''
+			if (!jsonObject[index]) {
+				jsonObject[index] = {}
+			}
+			jsonObject[index][block.msgid] = block.toJson()
+		})
 
-		return newSet as GetTextTranslations['translations']
+		return jsonObject
 	}
 
 	/**