From f1a6e66069bbe3da645a4767af70987d9a3762ae Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 22 Jan 2024 22:34:47 +0100 Subject: [PATCH 1/5] split parameter table into a parameter table and a random effect table --- README.md | 14 +++++++++++--- parameters.tsv | 18 +++++++++--------- petab_nlme_example.yaml | 1 + random_effects.tsv | 8 ++++++++ 4 files changed, 29 insertions(+), 12 deletions(-) create mode 100644 random_effects.tsv diff --git a/README.md b/README.md index 41380b5..08419fc 100644 --- a/README.md +++ b/README.md @@ -16,15 +16,23 @@ This table is an addition to the existing PEtab format. It introduces a classifi Further columns may be used to group individuals into different categories. -## Parameter table +## Random effect table -In addition to the existing descriptions of fixed effects, the NLME extension adds the following columns to describe random effects. +In addition to the existing descriptions of fixed effects in the parameter table, the NLME extension adds a random effect table to describe random effects. ### Detailed field description - `randomVariableId` [STRING] - ID of the random variable. If left empty, there is no random effect defined on the parameter of this row. + ID of the random variable. + +- `estimate` [0|1] + + 1 or 0, depending on, if the parameter is estimated (1) or set to zero (to shift a random effect away from zero, see `randomEffectFormula`). + +- `parameterId` [STRING] + + ID of the associated fixed effect. - `randomEffectFormula` [STRING] diff --git a/parameters.tsv b/parameters.tsv index e11a9ab..f88e209 100644 --- a/parameters.tsv +++ b/parameters.tsv @@ -1,9 +1,9 @@ -parameterId parameterScale lowerBound upperBound nominalValue estimate objectivePriorType objectivePriorParameters randomVariableId randomEffectFormula distributionType groupType -k1 log10 0.1 10 1 1 laplace 0; 1 u1 k1*exp(u1) normal individualId -k2 log10 0.1 10 1 1 laplace 0; 1 u2 k2 + u2 normal individualId -k3 log10 0.1 10 1 1 laplace 0; 1 u3 k3 * u3 normal individualId -k4 log10 0.1 10 1 1 laplace 0; 1 u4 k4 + u4 laplace tumorStage -k5 log10 0.1 10 1 1 laplace 0; 1 u5 k5 + u5 laplace tumorStage -k6 log10 0.1 10 1 1 laplace 0; 1 u6 k6 + u6 laplace tumorStage -k7 log10 0.1 10 1 1 laplace 0; 1 u7 k7 + u7 normal sex -k8 log10 0.1 10 1 1 laplace 0; 1 +parameterId parameterScale lowerBound upperBound nominalValue estimate objectivePriorType objectivePriorParameters +k1 log10 0.1 10 1 1 laplace 0; 1 +k2 log10 0.1 10 1 1 laplace 0; 1 +k3 log10 0.1 10 1 1 laplace 0; 1 +k4 log10 0.1 10 1 1 laplace 0; 1 +k5 log10 0.1 10 1 1 laplace 0; 1 +k6 log10 0.1 10 1 1 laplace 0; 1 +k7 log10 0.1 10 1 1 laplace 0; 1 +k8 log10 0.1 10 1 1 laplace 0; 1 diff --git a/petab_nlme_example.yaml b/petab_nlme_example.yaml index 537ff3c..34f8f81 100644 --- a/petab_nlme_example.yaml +++ b/petab_nlme_example.yaml @@ -1,5 +1,6 @@ format_version: 2.0.0 parameter_file: parameters.tsv +random_effect_file: random_effects.tsv problems: - condition_files: - conditions.tsv diff --git a/random_effects.tsv b/random_effects.tsv new file mode 100644 index 0000000..6bce0f2 --- /dev/null +++ b/random_effects.tsv @@ -0,0 +1,8 @@ +randomVariableId estimate parameterId randomEffectFormula distributionType groupType +u1 1 k1 k1*exp(u1) normal individualId +u2 1 k2 k2 + u2 normal individualId +u3 1 k3 k3 * u3 normal individualId +u4 1 k4 k4 + u4 laplace tumorStage +u5 1 k5 k5 + u5 laplace tumorStage +u6 1 k6 k6 + u6 laplace tumorStage +u7 1 k7 k7 + u7 normal sex From bbe5320e79120abd8d54f2e5a1ad1046bf206aa4 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 22 Jan 2024 23:02:01 +0100 Subject: [PATCH 2/5] add example for two random variables in one `randomEffectFormula` --- README.md | 2 +- random_effects.tsv | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 08419fc..4680b26 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ In addition to the existing descriptions of fixed effects in the parameter table - `randomEffectFormula` [STRING] - Random effect function as plain text formula expression. Must contain the `parameterId` and the `randomVariableId` and must not contain any other symbols (or should we allow that???). + Random effect function as plain text formula expression. Must contain the `parameterId` and the `randomVariableId`. May contain other `randomVariableId`s from other distributions. Must not contain any other symbols (or should we allow that???). - `distributionType` [STRING: 'normal' or 'laplace', OPTIONAL] diff --git a/random_effects.tsv b/random_effects.tsv index 6bce0f2..996328b 100644 --- a/random_effects.tsv +++ b/random_effects.tsv @@ -1,8 +1,8 @@ randomVariableId estimate parameterId randomEffectFormula distributionType groupType -u1 1 k1 k1*exp(u1) normal individualId +u1 1 k1 k1*exp(u1+u4) normal individualId u2 1 k2 k2 + u2 normal individualId u3 1 k3 k3 * u3 normal individualId -u4 1 k4 k4 + u4 laplace tumorStage +u4 1 k1 k1*exp(u1+u4) laplace tumorStage u5 1 k5 k5 + u5 laplace tumorStage u6 1 k6 k6 + u6 laplace tumorStage u7 1 k7 k7 + u7 normal sex From 86781055752219652cdf7a71355be0f1789f73f7 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 22 Jan 2024 23:23:32 +0100 Subject: [PATCH 3/5] rename `group` to `distribution` --- ....tsv => covariance_anotherDistribution.tsv | Bin ...vidual.tsv => covariance_distributionA.tsv | 0 ...v => covariance_yetAnotherDistribution.tsv | 0 individuals.tsv | 8 +++--- measurements.tsv | 26 +++++++++--------- petab_nlme_example.yaml | 12 ++++---- random_effects.tsv | 14 +++++----- 7 files changed, 30 insertions(+), 30 deletions(-) rename covariance_tumorStage.tsv => covariance_anotherDistribution.tsv (100%) rename covariance_individual.tsv => covariance_distributionA.tsv (100%) rename covariance_sex.tsv => covariance_yetAnotherDistribution.tsv (100%) diff --git a/covariance_tumorStage.tsv b/covariance_anotherDistribution.tsv similarity index 100% rename from covariance_tumorStage.tsv rename to covariance_anotherDistribution.tsv diff --git a/covariance_individual.tsv b/covariance_distributionA.tsv similarity index 100% rename from covariance_individual.tsv rename to covariance_distributionA.tsv diff --git a/covariance_sex.tsv b/covariance_yetAnotherDistribution.tsv similarity index 100% rename from covariance_sex.tsv rename to covariance_yetAnotherDistribution.tsv diff --git a/individuals.tsv b/individuals.tsv index f25acc8..a524101 100644 --- a/individuals.tsv +++ b/individuals.tsv @@ -1,4 +1,4 @@ -individualId tumorStage sex age -p1 s1 f 60 -p2 s1 m 60 -p3 s2 m 70 +distributionA anotherDistribution yetAnotherDistribution +patient1 stage1 blue +patient2 stage1 brown +patient3 stage2 brown diff --git a/measurements.tsv b/measurements.tsv index 29a5524..f2a34e2 100644 --- a/measurements.tsv +++ b/measurements.tsv @@ -1,13 +1,13 @@ -observableId simulationConditionId individualId measurement time -obs_B high_dose p1 9.4 1 -obs_B high_dose p1 1.5 10 -obs_B low_dose p1 1.8 1 -obs_B low_dose p1 1.06 10 -obs_B high_dose p2 9.6 1 -obs_B high_dose p2 1.52 10 -obs_B low_dose p2 2 1 -obs_B low_dose p2 1.08 10 -obs_B high_dose p3 9 1 -obs_B high_dose p3 1.4 10 -obs_B low_dose p3 1.6 1 -obs_B low_dose p3 1.04 10 +observableId simulationConditionId distributionA anotherDistribution yetAnotherDistribution measurement time +obs_B high_dose patient1 stage1 blue 9.4 1 +obs_B high_dose patient1 stage1 blue 1.5 10 +obs_B low_dose patient1 stage1 blue 1.8 1 +obs_B low_dose patient1 stage1 blue 1.06 10 +obs_B high_dose patient2 stage1 brown 9.6 1 +obs_B high_dose patient2 stage1 brown 1.52 10 +obs_B low_dose patient2 stage1 brown 2 1 +obs_B low_dose patient2 stage1 brown 1.08 10 +obs_B high_dose patient3 stage2 brown 9 1 +obs_B high_dose patient3 stage2 brown 1.4 10 +obs_B low_dose patient3 stage2 brown 1.6 1 +obs_B low_dose patient3 stage2 brown 1.04 10 diff --git a/petab_nlme_example.yaml b/petab_nlme_example.yaml index 34f8f81..4982454 100644 --- a/petab_nlme_example.yaml +++ b/petab_nlme_example.yaml @@ -16,11 +16,11 @@ problems: - individuals.tsv covariance_files: covariance: - group: individual - locaion: covariance_individual.tsv + group: distributionA + locaion: distributionA.tsv covariance: - group: tumorStage - location: covariance_tumorStage.tsv + group: anotherDistribution + location: covariance_anotherDistribution.tsv covariance: - group: sex - location: covariance_sex.tsv + group: yetAnotherDistribution + location: covariance_yetAnotherDistribution.tsv diff --git a/random_effects.tsv b/random_effects.tsv index 996328b..ab48a71 100644 --- a/random_effects.tsv +++ b/random_effects.tsv @@ -1,8 +1,8 @@ randomVariableId estimate parameterId randomEffectFormula distributionType groupType -u1 1 k1 k1*exp(u1+u4) normal individualId -u2 1 k2 k2 + u2 normal individualId -u3 1 k3 k3 * u3 normal individualId -u4 1 k1 k1*exp(u1+u4) laplace tumorStage -u5 1 k5 k5 + u5 laplace tumorStage -u6 1 k6 k6 + u6 laplace tumorStage -u7 1 k7 k7 + u7 normal sex +u1 1 k1 k1*exp(u1+u4) normal distributionA +u2 1 k2 k2 + u2 normal distributionA +u3 1 k3 k3 * u3 normal distributionA +u4 1 k1 k1*exp(u1+u4) laplace anotherDistribution +u5 1 k5 k5 + u5 laplace anotherDistribution +u6 1 k6 k6 + u6 laplace anotherDistribution +u7 1 k7 k7 + u7 normal yetAnotherDistribution From c3bf5ca1bde56ba5450e7be147746958b8549008 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 26 Jan 2024 22:44:00 +0100 Subject: [PATCH 4/5] rename individuals to distributions --- README.md | 9 ++++----- individuals.tsv => distributions.tsv | 0 petab_nlme_example.yaml | 18 +++++++++--------- random_effects.tsv | 16 ++++++++-------- 4 files changed, 21 insertions(+), 22 deletions(-) rename individuals.tsv => distributions.tsv (100%) diff --git a/README.md b/README.md index 4680b26..194829e 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,15 @@ This is a draft for a PEtab NLME extension. It adds the following to the existing PEtab specification. -## Individual table +## Distribution table -This table is an addition to the existing PEtab format. It introduces a classification of individuals (most fine grained grouping) into several (more coarse grained) groups. Random effects can be specific to each individual or to the groups they belong to. +This table is an addition to the existing PEtab format. It can be used to introduce a classification of e.g. individuals (most fine grained grouping) into several (more coarse grained) groups. Random effects can be specific to each individual or to the groups they belong to. ### Detailed field description -- `individualId` [STRING] -- `${groupId}` [STRING, OPTIONAL] +- `${distributionId}` [STRING, OPTIONAL] - Further columns may be used to group individuals into different categories. + Each column lists the possible values for the given distribution. ## Random effect table diff --git a/individuals.tsv b/distributions.tsv similarity index 100% rename from individuals.tsv rename to distributions.tsv diff --git a/petab_nlme_example.yaml b/petab_nlme_example.yaml index 4982454..500d292 100644 --- a/petab_nlme_example.yaml +++ b/petab_nlme_example.yaml @@ -12,15 +12,15 @@ problems: location: model.xml observable_files: - observables.tsv - individual_files: - - individuals.tsv + distribution_files: + - distributions.tsv covariance_files: - covariance: - group: distributionA - locaion: distributionA.tsv - covariance: - group: anotherDistribution + distribution: + distributionId: distributionA + location: distributionA.tsv + distribution: + distributionId: anotherDistribution location: covariance_anotherDistribution.tsv - covariance: - group: yetAnotherDistribution + distribution: + distributionId: yetAnotherDistribution location: covariance_yetAnotherDistribution.tsv diff --git a/random_effects.tsv b/random_effects.tsv index ab48a71..80f6381 100644 --- a/random_effects.tsv +++ b/random_effects.tsv @@ -1,8 +1,8 @@ -randomVariableId estimate parameterId randomEffectFormula distributionType groupType -u1 1 k1 k1*exp(u1+u4) normal distributionA -u2 1 k2 k2 + u2 normal distributionA -u3 1 k3 k3 * u3 normal distributionA -u4 1 k1 k1*exp(u1+u4) laplace anotherDistribution -u5 1 k5 k5 + u5 laplace anotherDistribution -u6 1 k6 k6 + u6 laplace anotherDistribution -u7 1 k7 k7 + u7 normal yetAnotherDistribution +randomVariableId distributionId estimate parameterId randomEffectFormula distributionType +u1 distributionA 1 k1 k1*exp(u1+u4) normal +u2 distributionA 1 k2 k2 + u2 normal +u3 distributionA 1 k3 k3 * u3 normal +u4 anotherDistribution 1 k1 k1*exp(u1+u4) laplace +u5 anotherDistribution 1 k5 k5 + u5 laplace +u6 anotherDistribution 1 k6 k6 + u6 laplace +u7 yetAnotherDistribution 1 k7 k7 + u7 normal From dc0b6d935c61cdac86bba15eb8672c53ba26b8a3 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 26 Jan 2024 23:31:51 +0100 Subject: [PATCH 5/5] remove distrubition table, as this information is better represented in the measurement table (e.g. if there is a `patient` and `tumorStage` distribution, the `tumorStage` might change over time in one and the same patient) --- README.md | 28 +++++++++++----------------- distributions.tsv | 4 ---- petab_nlme_example.yaml | 11 ++++++----- random_effects.tsv | 16 ++++++++-------- 4 files changed, 25 insertions(+), 34 deletions(-) delete mode 100644 distributions.tsv diff --git a/README.md b/README.md index 194829e..8fa8720 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,9 @@ This is a draft for a PEtab NLME extension. It adds the following to the existing PEtab specification. -## Distribution table +## YAML configuration file -This table is an addition to the existing PEtab format. It can be used to introduce a classification of e.g. individuals (most fine grained grouping) into several (more coarse grained) groups. Random effects can be specific to each individual or to the groups they belong to. - -### Detailed field description - -- `${distributionId}` [STRING, OPTIONAL] - - Each column lists the possible values for the given distribution. +In addition to existing attributes, this extension uses `covariance_files` to define distributions of random effects. Each distribution has a `distributionId`, `distributionType` (e.g. normal, laplace) and `covariance` (defined in a TSV file). Distributions have zero mean. The `randomEffectFormula` in the random effect table can be used to shift the location of the distribution. ## Random effect table @@ -25,6 +19,10 @@ In addition to the existing descriptions of fixed effects in the parameter table ID of the random variable. +- `distributionId` [STRING] + + Specifies the Id of the distribution from which this random effect is drawn. Any Id from the distributions listed in the YAML configuration file is allowed. + - `estimate` [0|1] 1 or 0, depending on, if the parameter is estimated (1) or set to zero (to shift a random effect away from zero, see `randomEffectFormula`). @@ -37,17 +35,9 @@ In addition to the existing descriptions of fixed effects in the parameter table Random effect function as plain text formula expression. Must contain the `parameterId` and the `randomVariableId`. May contain other `randomVariableId`s from other distributions. Must not contain any other symbols (or should we allow that???). -- `distributionType` [STRING: 'normal' or 'laplace', OPTIONAL] - - Assumed random effect distribution. Only normal and laplace distributions are currently allowed (but we could allow any term in some ontology for distributions). Defaults to normal. Distributions are assumed to be zero-centered. Covariances are defined in the covariance table. - -- `groupType` [STRING] - - Defines how random effects are grouped together. Any column name from the individual table is allowed, except for covariate (i.e. numeric) columns. - ## Covariance tables -These tables are an addition to the existing PEtab format to describe the covariances of the random effects. Each `groupType` for which covariances are estimated has its own table. Four formats are allowed +These tables are an addition to the existing PEtab format to describe the covariances of the distribtions from which the random effects are drawn. Each `distributionId` has its own table. Four formats are allowed - Matrix format @@ -64,3 +54,7 @@ These tables are an addition to the existing PEtab format to describe the covari - No table If no table is provided, only the variances, but no covariances are estimated. + +## Measurement table + +In addition to existing collumns, the measurement table must contain every `distributionId` from the YAML configuration file as a column. Each data point (i.e. row) is assigned to one entity for each `distributionId`. This allows grouping of data points. As a simple example, the YAML configuration file defines just one `distributionId` that could be called `patient`. Rows that share the same entry in the `patient` column belong to the same patient. Therefore, they must share the same value for all random effects that are drawn from the `patient` distribution. diff --git a/distributions.tsv b/distributions.tsv deleted file mode 100644 index a524101..0000000 --- a/distributions.tsv +++ /dev/null @@ -1,4 +0,0 @@ -distributionA anotherDistribution yetAnotherDistribution -patient1 stage1 blue -patient2 stage1 brown -patient3 stage2 brown diff --git a/petab_nlme_example.yaml b/petab_nlme_example.yaml index 500d292..226c7ce 100644 --- a/petab_nlme_example.yaml +++ b/petab_nlme_example.yaml @@ -12,15 +12,16 @@ problems: location: model.xml observable_files: - observables.tsv - distribution_files: - - distributions.tsv covariance_files: distribution: distributionId: distributionA - location: distributionA.tsv + distributionType: normal + covariance: distributionA.tsv distribution: distributionId: anotherDistribution - location: covariance_anotherDistribution.tsv + distributionType: lognormal + covariance: covariance_anotherDistribution.tsv distribution: distributionId: yetAnotherDistribution - location: covariance_yetAnotherDistribution.tsv + distributionType: laplace + covariance: covariance_yetAnotherDistribution.tsv diff --git a/random_effects.tsv b/random_effects.tsv index 80f6381..377fb2e 100644 --- a/random_effects.tsv +++ b/random_effects.tsv @@ -1,8 +1,8 @@ -randomVariableId distributionId estimate parameterId randomEffectFormula distributionType -u1 distributionA 1 k1 k1*exp(u1+u4) normal -u2 distributionA 1 k2 k2 + u2 normal -u3 distributionA 1 k3 k3 * u3 normal -u4 anotherDistribution 1 k1 k1*exp(u1+u4) laplace -u5 anotherDistribution 1 k5 k5 + u5 laplace -u6 anotherDistribution 1 k6 k6 + u6 laplace -u7 yetAnotherDistribution 1 k7 k7 + u7 normal +randomVariableId distributionId estimate parameterId randomEffectFormula +u1 distributionA 1 k1 k1*exp(u1+u4) +u2 distributionA 1 k2 k2 + u2 +u3 distributionA 1 k3 k3 * u3 +u4 anotherDistribution 1 k1 k1*exp(u1+u4) +u5 anotherDistribution 1 k5 k5 + u5 +u6 anotherDistribution 1 k6 k6 + u6 +u7 yetAnotherDistribution 1 k7 k7 + u7