Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Compose multiplatform strings import & export #2649

Draft
wants to merge 15 commits into
base: main
Choose a base branch
from
Draft
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ enum class ExportFormat(
"application/xml",
defaultFileStructureTemplate = "values-{androidLanguageTag}/strings.{extension}",
),
COMPOSE_XML(
"xml",
"application/xml",
defaultFileStructureTemplate = "values-{androidLanguageTag}/strings.{extension}",
),
FLUTTER_ARB(
"arb",
"application/json",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import StringsdictFileProcessor
import com.fasterxml.jackson.databind.ObjectMapper
import io.tolgee.dtos.dataImport.ImportFileDto
import io.tolgee.exceptions.ImportCannotParseFileException
import io.tolgee.formats.android.`in`.AndroidStringsXmlProcessor
import io.tolgee.formats.apple.`in`.strings.StringsFileProcessor
import io.tolgee.formats.csv.`in`.CsvFileProcessor
import io.tolgee.formats.flutter.`in`.FlutterArbFileProcessor
Expand All @@ -13,6 +12,7 @@ import io.tolgee.formats.json.`in`.JsonFileProcessor
import io.tolgee.formats.po.`in`.PoFileProcessor
import io.tolgee.formats.properties.`in`.PropertiesFileProcessor
import io.tolgee.formats.xliff.`in`.XliffFileProcessor
import io.tolgee.formats.xmlResources.`in`.XmlResourcesProcessor
import io.tolgee.formats.yaml.`in`.YamlFileProcessor
import io.tolgee.service.dataImport.processors.FileProcessorContext
import io.tolgee.service.dataImport.processors.ImportArchiveProcessor
Expand Down Expand Up @@ -58,7 +58,7 @@ class ImportFileProcessorFactory(
ImportFileFormat.STRINGSDICT -> StringsdictFileProcessor(context)
ImportFileFormat.XLIFF -> XliffFileProcessor(context)
ImportFileFormat.PROPERTIES -> PropertiesFileProcessor(context)
ImportFileFormat.XML -> AndroidStringsXmlProcessor(context)
ImportFileFormat.XML -> XmlResourcesProcessor(context)
ImportFileFormat.ARB -> FlutterArbFileProcessor(context, objectMapper)
ImportFileFormat.YAML -> YamlFileProcessor(context, yamlObjectMapper)
ImportFileFormat.CSV -> CsvFileProcessor(context)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.tolgee.formats

import io.tolgee.formats.android.AndroidParsingConstants
import io.tolgee.formats.xmlResources.XmlResourcesParsingConstants

class MobileStringEscaper(
private val string: String,
Expand Down Expand Up @@ -46,7 +46,7 @@ class MobileStringEscaper(
}

private val relevantSpaces =
if (escapeNewLines) AndroidParsingConstants.spacesWithoutNewLines else AndroidParsingConstants.spaces
if (escapeNewLines) XmlResourcesParsingConstants.spacesWithoutNewLines else XmlResourcesParsingConstants.spaces

private fun handleChar(char: Char) {
when (state) {
Expand Down Expand Up @@ -99,7 +99,7 @@ class MobileStringEscaper(
}

State.SPACES -> {
if (char in AndroidParsingConstants.spaces) {
if (char in XmlResourcesParsingConstants.spaces) {
spaces.append(char)
} else {
handleSpacesEnd(char)
Expand Down

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,10 +1,25 @@
package io.tolgee.formats.android.`in`

import io.tolgee.formats.android.AndroidParsingConstants
import io.tolgee.formats.xmlResources.XmlResourcesParsingConstants

class AndroidStringUnescaper(
private val string: Sequence<Char>,
private val isFirst: Boolean,
private val isLast: Boolean,
private val quotationMark: Char = '"',
private val escapeMark: Char = '\\',
private val spacesToTrim: Set<Char> = XmlResourcesParsingConstants.spaces,
private val toUnescape: Map<Char, String> = toUnescapeDefault,
) {
private val initialState
get() = if (isFirst) State.AFTER_SPACE else State.NORMAL

class AndroidStringUnescaper(private val string: String, private val isFirst: Boolean, private val isLast: Boolean) {
companion object {
private val toUnescape =
val defaultFactory = { string: String, isFirst: Boolean, isLast: Boolean ->
AndroidStringUnescaper(string.asSequence(), isFirst, isLast).result
}

private val toUnescapeDefault =
mapOf(
'n' to "\n",
'\'' to "\'",
Expand All @@ -13,94 +28,99 @@ class AndroidStringUnescaper(private val string: String, private val isFirst: Bo
'u' to "\\u",
'\\' to "\\",
)
private val spacesToTrim = AndroidParsingConstants.spaces
}

val unescaped: String by lazy {
string.forEach { char ->
when (char) {
'\\' ->
when (state) {
State.NORMAL -> state = State.ESCAPED
State.ESCAPED -> {
result.append('\\')
resetIgnoreSpace()
state = State.NORMAL
}
}

'"' ->
when (state) {
State.ESCAPED -> {
result.append('"')
resetIgnoreSpace()
state = State.NORMAL
}
var state = initialState

State.NORMAL ->
quotingState =
when (quotingState) {
QuotingState.USING_SPACE, QuotingState.IGNORING_SPACE -> QuotingState.QUOTED
QuotingState.QUOTED -> QuotingState.USING_SPACE
}
}
var space: Char? = null

in spacesToTrim ->
when (quotingState) {
QuotingState.USING_SPACE -> {
result.append(char)
quotingState = QuotingState.IGNORING_SPACE
}
val result: String by lazy {
buildString {
for (char in string) {
state = handleCharacter(state, char)
}

QuotingState.QUOTED -> {
result.append(char)
}
when (state) {
State.NORMAL -> {}
State.AFTER_SPACE -> {
val lastSpace = space
if (lastSpace != null && !isLast) append(lastSpace)
}
State.ESCAPED -> {
// Android deletes the last backslash if it is the last character
}
State.QUOTED -> {
// Quoted text was not closed
}
State.QUOTED_ESCAPED -> {}
}
}
}

QuotingState.IGNORING_SPACE -> {}
private fun StringBuilder.handleCharacter(
state: State,
char: Char,
): State {
return when (state) {
State.NORMAL ->
when (char) {
escapeMark -> State.ESCAPED
quotationMark -> State.QUOTED
in spacesToTrim -> {
space = char
State.AFTER_SPACE
}

else -> {
if (state == State.ESCAPED) {
if (char in toUnescape.keys) {
result.append(toUnescape[char])
} else {
result.append(char)
else -> {
append(char)
State.NORMAL
}
}
State.AFTER_SPACE -> {
when (char) {
in spacesToTrim -> State.AFTER_SPACE
else -> {
val lastSpace = space
if (lastSpace != null) {
append(lastSpace)
space = null
}
state = State.NORMAL
} else {
result.append(char)
handleCharacter(State.NORMAL, char)
}
resetIgnoreSpace()
}
}
State.ESCAPED -> {
append(char.unescape())
when (char) {
in spacesToTrim -> State.AFTER_SPACE
else -> State.NORMAL
}
}
State.QUOTED ->
when (char) {
escapeMark -> State.QUOTED_ESCAPED
quotationMark -> State.NORMAL
else -> {
append(char)
State.QUOTED
}
}
State.QUOTED_ESCAPED -> {
append(char.unescape())
State.QUOTED
}
}

if (quotingState == QuotingState.IGNORING_SPACE && result.lastOrNull() in spacesToTrim && isLast) {
result.deleteCharAt(result.length - 1)
}

result.toString()
}

private fun resetIgnoreSpace() {
if (quotingState == QuotingState.IGNORING_SPACE) {
quotingState = QuotingState.USING_SPACE
}
private fun Char.unescape(): String {
// Android always deletes the backslash even if the escaped character is not valid
return toUnescape.getOrDefault(this, this.toString())
}

enum class State {
NORMAL,
AFTER_SPACE,
ESCAPED,
}

enum class QuotingState {
USING_SPACE,
QUOTED,
IGNORING_SPACE,
QUOTED_ESCAPED,
}

private var state = State.NORMAL
private var quotingState = if (isFirst) QuotingState.IGNORING_SPACE else QuotingState.USING_SPACE

private val result = StringBuilder()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package io.tolgee.formats.compose.`in`

class ComposeStringUnescaper(
private val string: Sequence<Char>,
private val escapeMark: Char = '\\',
private val toUnescape: Map<Char, String> = toUnescapeDefault,
) {
private val initialState
get() = State.NORMAL

companion object {
val defaultFactory = { string: String, isFirst: Boolean, isLast: Boolean ->
ComposeStringUnescaper(string.asSequence()).result
}

private val toUnescapeDefault =
mapOf(
'n' to "\n",
't' to "\t",
'u' to "\\u",
'\\' to "\\",
)
}

val result: String
get() =
buildString {
var state = initialState
for (char in string) {
state =
when (state) {
State.NORMAL ->
when (char) {
escapeMark -> State.ESCAPED
else -> {
append(char)
state
}
}
State.ESCAPED -> {
char.unescape().forEach { append(it) }
State.NORMAL
}
}
}

when (state) {
State.NORMAL -> {}
State.ESCAPED -> append(escapeMark)
}
}

private fun Char.unescape(): String {
return toUnescape.getOrDefault(this, escapeMark + this.toString())
}

enum class State {
NORMAL,
ESCAPED,
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,10 @@ enum class ImportFormat(
ImportFileFormat.XML,
messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { JavaToIcuPlaceholderConvertor() },
),
COMPOSE_XML(
ImportFileFormat.XML,
messageConvertorOrNull = GenericMapPluralImportRawDataConvertor { JavaToIcuPlaceholderConvertor() },
),

FLUTTER_ARB(
ImportFileFormat.ARB,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ import io.tolgee.formats.escapePercentSign
class BaseToCLikePlaceholderConvertor(
private val defaultSpecifier: String = "s",
private val numberSpecifier: String = "d",
numberAllArgs: Boolean = false,
private val argNameStringProvider: (BaseToCLikePlaceholderConvertor.(MessagePatternUtil.ArgNode) -> String)? = null,
) {
private var argIndex = -1
private var wasNumberedArg = false
private var wasNumberedArg = numberAllArgs

fun convert(node: MessagePatternUtil.ArgNode): String {
argIndex++
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ import io.tolgee.formats.MessagePatternUtil
import io.tolgee.formats.escapePercentSign

class IcuToJavaPlaceholderConvertor : FromIcuPlaceholderConvertor {
private val baseToCLikePlaceholderConvertor = BaseToCLikePlaceholderConvertor()
private val baseToCLikePlaceholderConvertor =
BaseToCLikePlaceholderConvertor(
numberAllArgs = true,
)

override fun convert(node: MessagePatternUtil.ArgNode): String {
return baseToCLikePlaceholderConvertor.convert(node)
Expand Down
Loading
Loading