rusty: Work around race conditions when compiling multiple modules

This commit also adds workarounds to make the integration of the generated
code with our online change runtime work well.

wip: almost done with Mutex hashmap

NOTE: Reuse got_indices member from LlvmIndex instead

wip: it works?

src/test_utils: wip: Make these function compile after latest changes

section_mangler: Fix emitted format

wip

codegen custom GOT as a non-external array

wip: running onlinechangexmpl almost works! crashes on signal1()

more hacks

cleanup

cleanup

compiler/plc_driver/Cargo.toml

@@ -16,6 +16,7 @@ plc_index = { path = "../plc_index" }
 
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
+toml = "0.5"
 clap = { version = "3.0", features = ["derive"] }
 rayon = "1.6.1"
 tempfile = "3"

compiler/plc_driver/src/cli.rs

@@ -328,7 +328,9 @@ pub fn get_config_format(name: &str) -> Option<ConfigFormat> {
 
 impl CompileParameters {
     pub fn parse<T: AsRef<OsStr> + AsRef<str>>(args: &[T]) -> Result<CompileParameters, ParameterError> {
-        CompileParameters::try_parse_from(args).and_then(|result| {
+        CompileParameters::try_parse_from(args).and_then(|mut result| {
+            result.got_layout_file = Some(String::from("tmp.json"));
+
             if result.sysroot.len() > result.target.len() {
                 let mut cmd = CompileParameters::command();
                 Err(cmd.error(

compiler/plc_driver/src/pipelines.rs

@@ -1,8 +1,10 @@
 use std::{
+    collections::HashMap,
     env,
     fs::{self, File},
     io::Write,
     path::{Path, PathBuf},
+    sync::Mutex,
 };
 
 use crate::{CompileOptions, LinkOptions};
@@ -33,6 +35,42 @@ use project::{
 use rayon::prelude::*;
 use source_code::{source_location::SourceLocation, SourceContainer};
 
+use serde_json;
+use toml;
+
+pub fn read_got_layout(location: &str, format: ConfigFormat) -> Result<HashMap<String, u64>, Diagnostic> {
+    if !Path::new(location).is_file() {
+        // Assume if the file doesn't exist that there is no existing GOT layout yet. write_got_layout will handle
+        // creating our file when we want to.
+        return Ok(HashMap::new());
+    }
+
+    let s = fs::read_to_string(location)
+        .map_err(|_| Diagnostic::new("GOT layout could not be read from file"))?;
+    match format {
+        ConfigFormat::JSON => serde_json::from_str(&s)
+            .map_err(|_| Diagnostic::new("Could not deserialize GOT layout from JSON")),
+        ConfigFormat::TOML => {
+            toml::de::from_str(&s).map_err(|_| Diagnostic::new("Could not deserialize GOT layout from TOML"))
+        }
+    }
+}
+
+fn write_got_layout(
+    got_entries: HashMap<String, u64>,
+    location: &str,
+    format: ConfigFormat,
+) -> Result<(), Diagnostic> {
+    let s = match format {
+        ConfigFormat::JSON => serde_json::to_string(&got_entries)
+            .map_err(|_| Diagnostic::new("Could not serialize GOT layout to JSON"))?,
+        ConfigFormat::TOML => toml::ser::to_string(&got_entries)
+            .map_err(|_| Diagnostic::new("Could not serialize GOT layout to TOML"))?,
+    };
+
+    fs::write(location, s).map_err(|_| Diagnostic::new("GOT layout could not be written to file"))
+}
+
 ///Represents a parsed project
 ///For this struct to be built, the project would have been parsed correctly and an AST would have
 ///been generated
@@ -234,8 +272,15 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
             .iter()
             .map(|(unit, dependencies, literals)| {
                 let context = CodegenContext::create();
-                self.generate_module(&context, compile_options, unit, dependencies, literals)
-                    .map(|it| it.persist_to_string())
+                self.generate_module(
+                    &context,
+                    compile_options,
+                    unit,
+                    dependencies,
+                    literals,
+                    todo!("GOT layout for codegen_to_string?"),
+                )
+                .map(|it| it.persist_to_string())
             })
             .collect()
     }
@@ -249,7 +294,14 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
             .units
             .iter()
             .map(|(unit, dependencies, literals)| {
-                self.generate_module(context, compile_options, unit, dependencies, literals)
+                self.generate_module(
+                    context,
+                    compile_options,
+                    unit,
+                    dependencies,
+                    literals,
+                    todo!("give GOT layout for single modules?"),
+                )
             })
             .reduce(|a, b| {
                 let a = a?;
@@ -269,6 +321,7 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
         unit: &CompilationUnit,
         dependencies: &FxIndexSet<Dependency>,
         literals: &StringLiterals,
+        got_layout: &Mutex<Option<HashMap<String, u64>>>,
     ) -> Result<GeneratedModule<'ctx>, Diagnostic> {
         let mut code_generator = plc::codegen::CodeGen::new(
             context,
@@ -286,8 +339,9 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
             literals,
             dependencies,
             &self.index,
+            got_layout,
         )?;
-        code_generator.generate(context, unit, &self.annotations, &self.index, &llvm_index)
+        code_generator.generate(context, unit, &self.annotations, &self.index, llvm_index)
     }
 
     pub fn codegen_single_module<'ctx>(
@@ -332,6 +386,15 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
         });
         ensure_compile_dirs(targets, &compile_directory)?;
         let targets = if targets.is_empty() { &[Target::System] } else { targets };
+
+        let got_layout = compile_options
+            .got_layout_file
+            .as_ref()
+            .map(|path| read_got_layout(path, ConfigFormat::JSON))
+            .transpose()?;
+
+        let got_layout = Mutex::new(got_layout);
+
         let res = targets
             .par_iter()
             .map(|target| {
@@ -365,8 +428,15 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
                         };
 
                         let context = CodegenContext::create(); //Create a build location for the generated object files
-                        let module =
-                            self.generate_module(&context, compile_options, unit, dependencies, literals)?;
+                        let module = self.generate_module(
+                            &context,
+                            compile_options,
+                            unit,
+                            dependencies,
+                            literals,
+                            &got_layout,
+                        )?;
+
                         module
                             .persist(
                                 Some(&compile_directory),
@@ -385,6 +455,10 @@ impl<T: SourceContainer + Sync> AnnotatedProject<T> {
             })
             .collect::<Result<Vec<_>, Diagnostic>>()?;
 
+        compile_options.got_layout_file.as_ref().map(|path| {
+            write_got_layout(got_layout.into_inner().unwrap().unwrap(), path, ConfigFormat::JSON)
+        });
+
         Ok(res)
     }
 

compiler/section_mangler/src/lib.rs

@@ -68,6 +68,27 @@ pub struct VariableMangler {
     ty: Type,
 }
 
+pub const RUSTY_PREFIX: &str = "$RUSTY$";
+
+// TODO: How to encode variadics?
+fn mangle_function(FunctionMangler { name, parameters, return_type }: FunctionMangler) -> String {
+    /* FIXME: Is that correct? */
+    let return_type = return_type.unwrap_or(Type::Void);
+
+    let mangled = match parameters.as_slice() {
+        [] => format!("{return_type}[]"),
+        parameters => {
+            parameters.iter().fold(return_type.to_string(), |mangled, arg| format!("{mangled}[{arg}]"))
+        }
+    };
+
+    format!("{name}:{mangled}")
+}
+
+fn mangle_variable(VariableMangler { name, ty }: VariableMangler) -> String {
+    format!("{name}:{ty}")
+}
+
 impl SectionMangler {
     pub fn function<S: Into<String>>(name: S) -> SectionMangler {
         SectionMangler::Function(FunctionMangler { name: name.into(), parameters: vec![], return_type: None })
@@ -111,7 +132,7 @@ impl SectionMangler {
             SectionMangler::Variable(v) => ("var", mangle_variable(v)),
         };
 
-        format!("{prefix}-{content}")
+        format!("{RUSTY_PREFIX}{prefix}-{content}")
     }
 }
 
@@ -249,19 +270,3 @@ impl fmt::Display for Type {
         }
     }
 }
-
-pub const PREFIX: &str = "$RUSTY$";
-
-// TODO: How to encode variadics?
-fn mangle_function(FunctionMangler { name, parameters, return_type }: FunctionMangler) -> String {
-    let mangled = parameters
-        .into_iter()
-        /* FIXME: Is that correct? */
-        .fold(return_type.unwrap_or(Type::Void).to_string(), |mangled, arg| format!("{mangled}[{arg}]"));
-
-    format!("{PREFIX}{name}:{mangled}")
-}
-
-fn mangle_variable(VariableMangler { name, ty }: VariableMangler) -> String {
-    format!("{PREFIX}{name}:{ty}")
-}

compiler/section_mangler/src/parser.rs

@@ -1,4 +1,4 @@
-use crate::{FunctionArgument, SectionMangler, Type};
+use crate::{FunctionArgument, SectionMangler, StringEncoding, Type};
 
 use std::str;
 
@@ -21,7 +21,7 @@ fn parse_prefix(input: &str) -> ParseResult<Prefix> {
     let fn_prefix = tag("fn").map(|_| Prefix::Fn);
     let var_prefix = tag("var").map(|_| Prefix::Var);
 
-    let (input, _) = tag(crate::PREFIX)(input)?;
+    let (input, _) = tag(crate::RUSTY_PREFIX)(input)?;
     let (input, prefix) = alt((fn_prefix, var_prefix))(input)?;
 
     Ok((input, prefix))
@@ -75,8 +75,29 @@ fn type_enum(input: &str) -> ParseResult<Type> {
         .parse(input)
 }
 
+fn string_encoding(input: &str) -> ParseResult<StringEncoding> {
+    let utf8 = tag("8u").map(|_| StringEncoding::Utf8);
+    let utf16 = tag("16u").map(|_| StringEncoding::Utf16);
+
+    alt((utf8, utf16))(input)
+}
+
+fn type_string(input: &str) -> ParseResult<Type> {
+    char('s')
+        .and(string_encoding)
+        .and(number::<usize>)
+        .map(|((_, encoding), size)| Type::String { size, encoding })
+        .parse(input)
+}
+
+fn type_array(input: &str) -> ParseResult<Type> {
+    char('a').and(parse_type).map(|(_, inner_ty)| Type::Array { inner: Box::new(inner_ty) }).parse(input)
+}
+
 fn parse_type(input: &str) -> ParseResult<Type> {
-    alt((type_void, type_integer, type_float, type_pointer, type_struct, type_enum))(input)
+    alt((type_void, type_integer, type_float, type_pointer, type_struct, type_enum, type_string, type_array))(
+        input,
+    )
 }
 
 fn parse_var_content<'i>(input: &'i str, name: &str) -> ParseResult<'i, SectionMangler> {
@@ -188,6 +209,7 @@ mod tests {
         // this needs to be handled by the toplevel parse function
         assert!(type_struct("r0u8u8").is_ok());
         assert!(type_struct("r1u8u8").is_ok());
+        assert!(type_struct("r5s8u1025s8u2049s8u3u64s8u3").is_ok());
 
         // invalid number of elements
         assert!(type_struct("r15u8").is_err());
@@ -235,4 +257,67 @@ mod tests {
 
         assert_eq!(mangled.name(), "Color.red");
     }
+
+    #[test]
+    fn parse_complex1() {
+        let mangled = SectionMangler::from("$RUSTY$var-__File__init:r5s8u1025s8u2049s8u3u64s8u3");
+
+        assert_eq!(mangled.name(), "__File__init");
+    }
+
+    #[test]
+    fn parse_complex2() {
+        let inputs = [
+            "$RUSTY$var-__CosineSignal__init:r4f64i32f64f64",
+            "$RUSTY$var-__File__init:r5s8u1025s8u2049s8u3u64s8u3",
+            "$RUSTY$var-__SineSignal__init:r4f64i32f64f64",
+            "$RUSTY$var-__mainProg_state.Init:e3i32",
+            "$RUSTY$var-__mainProg_state.Running:e3i32",
+            "$RUSTY$var-__mainProg_state.Stopped:e3i32",
+            "$RUSTY$var-__SR__init:r3u8u8u8",
+            "$RUSTY$var-__RS__init:r3u8u8u8",
+            "$RUSTY$var-__CTU__init:r6u8u8i16u8i16u8",
+            "$RUSTY$var-__CTU_INT__init:r6u8u8i16u8i16u8",
+            "$RUSTY$var-__CTU_DINT__init:r6u8u8i32u8i32u8",
+            "$RUSTY$var-__CTU_UDINT__init:r6u8u8u32u8u32u8",
+            "$RUSTY$var-__CTU_LINT__init:r6u8u8i64u8i64u8",
+            "$RUSTY$var-__CTU_ULINT__init:r6u8u8u64u8u64u8",
+            "$RUSTY$var-__CTD__init:r6u8u8i16u8i16u8",
+            "$RUSTY$var-__CTD_INT__init:r6u8u8i16u8i16u8",
+            "$RUSTY$var-__CTD_DINT__init:r6u8u8i32u8i32u8",
+            "$RUSTY$var-__CTD_UDINT__init:r6u8u8u32u8u32u8",
+            "$RUSTY$var-__CTD_LINT__init:r6u8u8i64u8i64u8",
+            "$RUSTY$var-__CTD_ULINT__init:r6u8u8u64u8u64u8",
+            "$RUSTY$var-__CTUD__init:r10u8u8u8u8i16u8u8i16u8u8",
+            "$RUSTY$var-__CTUD_INT__init:r10u8u8u8u8i16u8u8i16u8u8",
+            "$RUSTY$var-__CTUD_DINT__init:r10u8u8u8u8i32u8u8i32u8u8",
+            "$RUSTY$var-__CTUD_UDINT__init:r10u8u8u8u8u32u8u8u32u8u8",
+            "$RUSTY$var-__CTUD_LINT__init:r10u8u8u8u8i64u8u8i64u8u8",
+            "$RUSTY$var-__CTUD_ULINT__init:r10u8u8u8u8u64u8u8u64u8u8",
+            "$RUSTY$var-__R_TRIG__init:r3u8u8u8",
+            "$RUSTY$var-__F_TRIG__init:r3u8u8u8",
+            "$RUSTY$var-__TP__init:r7u8i64u8i64u8u8au8",
+            "$RUSTY$var-__TP_TIME__init:r7u8i64u8i64u8u8au8",
+            "$RUSTY$var-__TP_LTIME__init:r7u8i64u8i64u8u8au8",
+            "$RUSTY$var-__TON__init:r7u8i64u8i64u8u8au8",
+            "$RUSTY$var-__TON_TIME__init:r7u8i64u8i64u8u8au8",
+            "$RUSTY$var-__TON_LTIME__init:r7u8i64u8i64u8u8au8",
+            "$RUSTY$var-__TOF__init:r7u8i64u8i64u8u8au8",
+            "$RUSTY$var-__TOF_TIME__init:r7u8i64u8i64u8u8au8",
+            "$RUSTY$var-__TOF_LTIME__init:r7u8i64u8i64u8u8au8",
+            "$RUSTY$fn-CosineSignal:v[f64][i32][f64]",
+            "$RUSTY$fn-File:v[s8u1025][s8u2049][s8u3]",
+            "$RUSTY$fn-File.Open:v",
+            "$RUSTY$fn-File.Write:v",
+            "$RUSTY$fn-File.Close:v",
+            "$RUSTY$fn-File.Clear:v",
+            "$RUSTY$fn-SineSignal:v[f64][i32][f64]",
+            "$RUSTY$fn-mainProg:v",
+            "$RUSTY$var-mainProg:r7i32r4f64i32f64f64r4f64i32f64f64e3i32r5s8u1025s8u2049s8u3u64s8u3r5s8u1025s8u2049s8u3u64s8u3r5s8u1025s8u2049s8u3u64s8u3"
+        ];
+
+        inputs.into_iter().for_each(|input| {
+            let _ = SectionMangler::from(dbg!(input));
+        });
+    }
 }