From 66f40fb07a5c71e9e261b769538bc11553660378 Mon Sep 17 00:00:00 2001 From: darms Date: Mon, 6 Nov 2023 10:25:55 +0100 Subject: [PATCH] feature(k8s.mica): Updated mica helm chart with support for s3 backup; and custom js overwrite --- k8s/mica/README.md | 38 +- k8s/mica/persona/result-parsers.js | 1057 +++++++++++++++++++ k8s/mica/templates/backup-cron.yaml | 5 +- k8s/mica/templates/configmap-custom-js.yaml | 7 + k8s/mica/templates/mica.yaml | 51 +- k8s/mica/values.yaml | 4 +- 6 files changed, 1156 insertions(+), 6 deletions(-) create mode 100644 k8s/mica/persona/result-parsers.js create mode 100644 k8s/mica/templates/configmap-custom-js.yaml diff --git a/k8s/mica/README.md b/k8s/mica/README.md index 21af41e..9e9d772 100644 --- a/k8s/mica/README.md +++ b/k8s/mica/README.md @@ -6,6 +6,42 @@ This creates a volume `{{ .Release.Name }}-template-container-mica` where one can store custom templates. Just copy the freemaker files into the running pod. -`kubcetl cp ~/PycharmProjects/mica-templates/. {{ .Release.Name }}-mongo-0:/usr/share/mica2/webapp/WEB-INF/classes/templates` +`kubcetl cp ~/PycharmProjects/mica-templates/ {{ .Release.Name }}-mica-0:/usr/share/mica2/webapp/WEB-INF/classes/` + k cp _templates/ mica-mica-0:/usr/share/mica2/webapp/WEB-INF/classes/templates/ + +# Load a backup +A backup consists of two parts, a mongodb backup and the contents of `MICA_HOME`. +The latter does not only contain configuration files and caches but also the revision history. +Within the helm chart the `MICA_HOME` is stored within a volume `{{ .Release.Name }}-data-container-mica`. +If configured copies of `MICA_HOME` and the mongodb are regularly stored within the configured s3 location. + +To import a backup into a fresh installation follow the following steps: +1) Start/Install a fresh instance with this chart. +2) Load the backup of `MICA_HOME` into the new volume. + 1) Copy the backup archive into the pod `{{ .Release.Name }}-mica-0` + For example: `k cp mica-src.tar.gz {{ .Release.Name }}-mica-0:/tmp/mica-src.tar.gz ` + 2) Extract the files into the right location + 1) Attach to the running pod e.g. `k exec -it {{ .Release.Name }}-mica-0 -- /bin/bash` + 2) Extract the archive within the pod e.g. `tar -xvzf /tmp/mica-src.tar.gz -C /srv` + 3) Remove the `/srv/work` directory, since the backup was likely generated of a running instance, which results in invalid stated. e.g. `rm -rf /srv/wrok` +3) Load the mongodb backup into the new instance + 1) Copy the backup archive into the pod `{{ .Release.Name }}-mongo-0` + For example: `k cp mica_2023-11-03.archive.gz {{ .Release.Name }}-mica-0:/tmp/mica_2023-11-03.archive.gz` + 2) Obtain the mongodb credentials. + 1) `k get secrets/{{ .Release.Name }}-mongo-secret -o=jsonpath="{.data.username}"| base64 --decode` + 2) `k get secrets/{{ .Release.Name }}-mongo-secret -o=jsonpath="{.data.password}"| base64 --decode` + 3) Load the backup archive. + 1) Attach to the running pod e.g. `k exec -it {{ .Release.Name }}-mongo-0 -- /bin/bash` + 2) Restore the mongodb (Replace $USERNAME and $PW with the mongodb credentials) + e.g. `mongorestore --username=$USERNAME --password=$PW --authenticationDatabase=admin --gzip --drop --archive=/tmp/mica_2023-11-03.archive.gz` + 4) Update mongodb secrets + We just exchanged the authenticationDatabase of the current mongodb with the ones from the backup. Therefore, we need to update the secrets. + e.g. via `k edit secret/{{ .Release.Name }}-mongo-secret`, keep in mind the values are base64 encoded. + 5) Update mica secrets + The configured mica secrets are also invalid and must also be updated with the values valid with the backup. + e.g. via `k edit secret/{{ .Release.Name }}-mica-secret`, keep in mind the values are base64 encoded. +4) If not automatically triggered by k8s. Enforce a restart of mica by deleting the pod `{{ .Release.Name }}-mica-0` + e.g. via `k delete pods/{{ .Release.Name }}-mica-0` +5) Login to mica2 admin UI and drop all caches and reindex everything diff --git a/k8s/mica/persona/result-parsers.js b/k8s/mica/persona/result-parsers.js new file mode 100644 index 0000000..eea2fe2 --- /dev/null +++ b/k8s/mica/persona/result-parsers.js @@ -0,0 +1,1057 @@ +class GraphicsResultParser { + constructor(normalizePath) { + this.normalizePath = normalizePath; + } + + static VALID_CHOROPLETH_COLORSCALE_NAMES = ['Blackbody', 'Bluered', 'Blues', 'Cividis', 'Earth', 'Electric', 'Greens', 'Greys', 'Hot', 'Jet', 'Picnic', 'Portland', 'Rainbow', 'RdBu', 'Reds', 'Viridis', 'YlGnBu', 'YlOrRd']; + + static DEFAULT_GRAPH_PROCESSORS = { + bar: { + /** + * @param input + * @param colors String or Array + */ + processData(input, colors) { + + const x = []; + const y = []; + + input.values.forEach(val => { + x.push(val.count); + y.push(val.title); + }); + + const width = Array(x.length).fill(x.length * 0.1); + + return [{ + type: "bar", + orientation: "h", + marker: { + color: colors + }, + x: x.reverse(), + y: y.reverse(), + customdata: y, + hovertemplate: "(%{x}, %{customdata})", + width + }]; + }, + layoutObject: { + height: 390, + margin: { + t: 20, + b: 40 + }, + xaxis: { + rangemode: 'nonnegative' + }, + yaxis: { + rangemode: 'nonnegative', + automargin: true, + ticksuffix: ' ' + } + } + }, + pie: { + /** + * @param input + * @param colors Array + */ + processData(input, colors) { + const values = []; + const labels = []; + + input.values.forEach(val => { + values.push(val.count); + labels.push(val.title); + }); + + return [{ + type: "pie", + sort: false, + marker: { + colors: colors + }, + hoverinfo: "label+value", + values, + labels + }]; + }, + layoutObject: { + height: 360, + margin: { + t: 50, + b: 40 + } + } + }, + geo: { + /** + * @param input + * @param colors String or Array + */ + processData(input, colors) { + const z = []; + const locations = []; + const text = []; + + input.values.forEach(val => { + z.push(val.count); + locations.push(val.key); + text.push(val.title); + }); + + const trace = { + type: "choropleth", + locations, + text, + z, + zmax: Math.max(...z) || 2, + zmin: 0, + hoverinfo: "text+z", + colorbar: { + thickness: 10, + ypad: 50 + } + } + + if (Array.isArray(colors)) { + trace.colorscale = [[0, "#f3f3f3"]].concat(colors.map((color, index) => [((index + 1) / colors.length), color])); + } else if (GraphicsResultParser.VALID_CHOROPLETH_COLORSCALE_NAMES.indexOf(colors) > -1) { + trace.colorscale = colors; + trace.reversescale = true; + } else { + trace.colorscale = "Blues"; + trace.reversescale = true; + } + + return [trace]; + }, + layoutObject: { + geo: { + showframe: false, + showcoastlines: false, + countrywidth: 0.25, + showcountries: true, + projection: { + type: "robinson", + } + }, + height: 350, + margin: { + t: 0, + r: -20, + b: 0, + l: 0 + } + } + } + }; + + static __isCorrectVocabulary(vocabulary, name) { + return vocabulary && (vocabulary.name === name || vocabulary.attributes.filter(a => a.key === "alias" && a.value === name)[0]); + } + + static __getPlotlyType(type) { + if (type === 'bar' || type === 'horizontalBar') { + return 'bar'; + } else if (type === 'pie' || type === 'doughnut') { + return 'pie'; + } else if (type === 'choropleth') { + return 'geo'; + } + } + + static __parseForChart(chartData, options) { + const studyVocabulary = (options.taxonomy || {vocabularies: []}).vocabularies.filter(vocabulary => GraphicsResultParser.__isCorrectVocabulary(vocabulary, options.agg))[0]; + + if (studyVocabulary) { + const terms = studyVocabulary.terms.map(term => term.name); + chartData.sort((a, b) => { + return terms.indexOf(a.key) - terms.indexOf(b.key); + }); + } + + const processor = GraphicsResultParser.DEFAULT_GRAPH_PROCESSORS[GraphicsResultParser.__getPlotlyType(options.type || 'bar')]; + return [processor.processData({key: options.agg, values: chartData, title: options.title}, options.colors || options.backgroundColor), processor.layoutObject]; + } + + static __parseForTable(vocabulary, chartData, forSubAggData) { + return chartData.filter(term => term.count>0).map(term => { + let row = { + vocabulary: vocabulary.replace(/model-/, ""), + key: term.key, + title: term.title, + count: term.count + }; + + if (forSubAggData) { + const subAgg = term.aggs.filter((agg) => agg.aggregation === forSubAggData.agg)[0]; + row.subAgg = (subAgg[forSubAggData.dataKey] || {data: {}}).data[forSubAggData.data] || 0; + } + + return row; + }); + } + + parse(chartData, chartOptions, totalHits) { + if (!chartData) { + return; + } + + const tr = Vue.filter('translate') || (value => value); + const labelStudies = tr('studies'); + const aggData = chartData[chartOptions.dataKey]; + + let [data, layout] = typeof chartOptions.parseForChart === 'function' + ? chartOptions.parseForChart(aggData, chartOptions, totalHits) + : GraphicsResultParser.__parseForChart(aggData, chartOptions, totalHits); + + const tableCols = [chartOptions.title, labelStudies]; + + if (chartOptions.subAgg) { + tableCols.push(chartOptions.subAgg.title); + } + + const tableRows = typeof chartOptions.parseForTable === 'function' + ? chartOptions.parseForTable(chartOptions.vocabulary, aggData, chartOptions.subAgg, totalHits) + : GraphicsResultParser.__parseForTable(chartOptions.vocabulary, aggData, chartOptions.subAgg, totalHits); + + const plotData = { + data: data, + layout: layout + }; + + return [plotData, {cols: tableCols, rows: tableRows}]; + + } +} + +class VariablesResultParser { + + constructor(normalizePath) { + this.normalizePath = normalizePath; + } + + parse(data, micaConfig, localize, displayOptions, studyTypeSelection) { + const variablesResult = data.variableResultDto; + const tr = Vue.filter('translate') || (value => value); + const taxonomyTitle = Vue.filter('taxonomy-title') || (value => value); + + let columnKey = 'variableColumns'; + if (studyTypeSelection) { + if (studyTypeSelection.study) { + columnKey = 'variableColumnsIndividual'; + } else if(studyTypeSelection.harmonization) { + columnKey = 'variableColumnsHarmonization'; + } + } + + if (!variablesResult) { + throw new Error("No variable results available."); + } + + if (variablesResult.totalHits < 1) return { totalHits: 0 }; + + const result = variablesResult["obiba.mica.DatasetVariableResultDto.result"]; + + if (!result) { + throw new Error("Invalid JSON."); + } + + let parsed = { + data: [], + totalHits: variablesResult.totalHits + } + + const summaries = result.summaries || []; + + summaries.forEach(summary => { + + let path = this.normalizePath(`/variable/${summary.id}`); + let row = []; + + if (displayOptions.showCheckboxes) { + row.push(``); + } + + row.push(`${summary.name}`); + + (displayOptions[columnKey] || displayOptions.variableColumns).forEach(column => { + switch (column) { + case 'label': + case 'label+description': { + let labelElem = marked(localize(summary.variableLabel)); + if (column === 'label+description' && summary.description) { + labelElem = " " + labelElem; + } + row.push(labelElem); + break; + } + case 'valueType': { + row.push(tr(summary.valueType + '-type')); + break; + } + case 'annotations': { + const annotations = (summary.annotations || []).reduce( + (acc, annotation) => + ("" !== acc ? `${acc}
` : "") + " " + + taxonomyTitle.apply(null, [`${annotation.taxonomy}.${annotation.vocabulary}.${annotation.value}`]) + "", + "" + ); + row.push(annotations); + break; + } + case 'type': { + if (micaConfig.isCollectedDatasetEnabled && micaConfig.isHarmonizedDatasetEnabled) { + row.push(tr(summary.variableType.toLowerCase())); + } + break; + } + case 'study': { + if (!micaConfig.isSingleStudyEnabled) { + path = this.normalizePath(`/study/${summary.studyId}`); + row.push(`${localize(summary.studyAcronym)}`); + } + break; + } + + case 'initiative': { + if (!micaConfig.isSingleStudyEnabled) { + path = this.normalizePath(`/study/${summary.studyId}`); + row.push(`${localize(summary.studyAcronym)}`); + } + break; + } + + case 'population': { + path = this.normalizePath(`/study/${summary.studyId}`); + if (summary.populationName) { + row.push(`${localize(summary.populationName)}`); + } else { + row.push('-'); + } + break; + } + case 'dce': + case 'data-collection-event': { + path = this.normalizePath(`/study/${summary.studyId}`); + if (summary.dceName) { + row.push(`${localize(summary.dceName)}`); + } else { + row.push('-'); + } + break; + } + case 'dataset': { + path = this.normalizePath(`https://csh.nfdi4health.de/resource/${summary.datasetId}`); + row.push(`${localize(summary.datasetAcronym)}`); + break; + } + case 'protocol': { + path = this.normalizePath(`/dataset/${summary.datasetId}`); + row.push(`${localize(summary.datasetAcronym)}`); + break; + } + default: + row.push(''); + console.debug('Wrong variable table column: ' + column); + } + }); + + parsed.data.push(row); + }); + + return parsed; + } +} + +class StudiesResultParser { + + constructor(normalizePath, locale) { + this.normalizePath = normalizePath; + this.locale = locale; + } + + static __getNumberOfParticipants(content) { + const numberOfParticipants = content['numberOfParticipants']; + if (numberOfParticipants) { + const participant = numberOfParticipants['participant']; + if (participant) { + return participant.number || '-'; + } + } + + return '-'; + } + + parse(data, micaConfig, localize, displayOptions, studyTypeSelection) { + const studiesResult = data.studyResultDto; + + let columnKey = 'studyColumns'; + if (studyTypeSelection) { + if (studyTypeSelection.study) { + columnKey = 'studyColumnsIndividual'; + } else if(studyTypeSelection.harmonization) { + columnKey = 'studyColumnsHarmonization'; + } + } + + if (!studiesResult) { + throw new Error("No network results available."); + } + + if (studiesResult.totalHits < 1) return { totalHits: 0 }; + + const result = studiesResult["obiba.mica.StudyResultDto.result"]; + + if (!result) { + throw new Error("Invalid JSON."); + } + + let parsed = { + data: [], + totalHits: studiesResult.totalHits + } + + const taxonomyFilter = Vue.filter('taxonomy-title') || (title => title); + const checkIcon = ``; + const summaries = result.summaries || []; + + summaries.forEach(summary => { + + const type = summary.studyResourcePath === 'harmonization-study' + ? taxonomyFilter.apply(null, ['Mica_study.className.HarmonizationStudy']) + : taxonomyFilter.apply(null, ['Mica_study.className.Study']) ; + + const stats = summary['obiba.mica.CountStatsDto.studyCountStats'] || {}; + const content = JSON.parse(summary.content); + const dataSources = summary.dataSources || []; + const hasDatasource = (dataSources, id) => dataSources.indexOf(id) > -1; + const design = summary.design ? taxonomyFilter.apply(null, [`Mica_study.methods-design.${summary.design}`]) : '-'; + let anchor = (type, value, studyType) => + `${value.toLocaleString(this.locale)}`; + + let path = this.normalizePath(`/study/${summary.id}`); + let row = []; + + if (displayOptions.showCheckboxes) { + row.push(``); + } + + row.push(`${localize(summary.acronym)}`); + + (displayOptions[columnKey] || displayOptions.studyColumns).forEach(column => { + switch (column) { + case 'name': { + row.push(localize(summary.name)); + break; + } + case 'type': { + if (micaConfig.isCollectedDatasetEnabled && micaConfig.isHarmonizedDatasetEnabled) { + row.push(type); + } + break; + } + case 'study-design': { + row.push(design); + break; + } + case 'data-sources-available': { + row.push(hasDatasource(dataSources, "questionnaires") ? checkIcon : "-"); + row.push(hasDatasource(dataSources, "physical_measures") ? checkIcon : "-"); + row.push(hasDatasource(dataSources, "biological_samples") ? checkIcon : "-"); + row.push(hasDatasource(dataSources, "cognitive_measures") ? checkIcon : "-"); + row.push(hasDatasource(dataSources, "administratives_databases") ? checkIcon : "-"); + row.push(hasDatasource(dataSources, "others") ? checkIcon : "-"); + break; + } + case 'participants': { + row.push(StudiesResultParser.__getNumberOfParticipants(content)); + break; + } + case 'networks': { + if (micaConfig.isNetworkEnabled && !micaConfig.isSingleNetworkEnabled) { + row.push(stats.networks ? anchor("networks", stats.networks, "") : "-"); + } + break; + } + case 'individual': { + if (micaConfig.isCollectedDatasetEnabled) { + row.push(stats.studyDatasets + ? anchor("datasets", stats.studyDatasets, "Study") + : "-"); + row.push(stats.studyVariables + ? anchor("variables", stats.studyVariables, "Study") + : "-"); + } + break; + } + case 'harmonization': { + if (micaConfig.isHarmonizedDatasetEnabled) { + row.push(stats.harmonizationDatasets + ? anchor("datasets", stats.harmonizationDatasets, "HarmonizationStudy") + : "-"); + row.push(stats.dataschemaVariables + ? anchor("variables", stats.dataschemaVariables, "HarmonizationStudy") + : "-"); + } + break; + } + case 'datasets': { + if (micaConfig.isCollectedDatasetEnabled) { + row.push(stats.studyDatasets + ? anchor("datasets", stats.studyDatasets, "Study") + : "-"); + } + if (micaConfig.isHarmonizedDatasetEnabled) { + row.push(stats.dataschemaDatasets + ? anchor("datasets", stats.dataschemaDatasets, "HarmonizationStudy") + : "-"); + } + break; + } + case 'variables': { + if (micaConfig.isCollectedDatasetEnabled) { + row.push(stats.studyVariables + ? anchor("variables", stats.studyVariables, "Study") + : "-"); + } + if (micaConfig.isHarmonizedDatasetEnabled) { + row.push(stats.dataschemaVariables + ? anchor("variables", stats.dataschemaVariables, "HarmonizationStudy") + : "-"); + } + break; + } + default: + row.push(''); + console.debug('Wrong study table column: ' + column); + } + }); + + parsed.data.push(row); + }); + + return parsed; + } +} + +class DatasetsResultParser { + + constructor(normalizePath, locale) { + this.normalizePath = normalizePath; + this.locale = locale; + } + + parse(data, micaConfig, localize, displayOptions, studyTypeSelection) { + const datasetsResult = data.datasetResultDto; + const tr = Vue.filter('translate') || (value => value); + const taxonomyFilter = Vue.filter('taxonomy-title') || (value => value); + + let columnKey = 'datasetColumns'; + if (studyTypeSelection) { + if (studyTypeSelection.study) { + columnKey = 'datasetColumnsIndividual'; + } else if(studyTypeSelection.harmonization) { + columnKey = 'datasetColumnsHarmonization'; + } + } + + if (!datasetsResult) { + throw new Error("No dataset results available."); + } + + if (datasetsResult.totalHits < 1) return { totalHits: 0}; + + const result = datasetsResult["obiba.mica.DatasetResultDto.result"]; + + if (!result) { + throw new Error("Invalid JSON."); + } + + let parsed = { + data: [], + totalHits: datasetsResult.totalHits + } + + const datasets = result.datasets || []; + + datasets.forEach(dataset => { + + let path = this.normalizePath(`https://csh.nfdi4health.de/resource/${dataset.id}`); + let row = [`${localize(dataset.acronym)}`]; + const type = dataset.variableType === 'Dataschema' + ? taxonomyFilter.apply(null, ['Mica_dataset.className.HarmonizationDataset']) + : taxonomyFilter.apply(null, ['Mica_dataset.className.StudyDataset']) ; + + let opalTable = dataset.variableType === 'Dataschema' + ? (dataset['obiba.mica.HarmonizedDatasetDto.type'] || {}).harmonizationTable + : (dataset['obiba.mica.CollectedDatasetDto.type'] || {}).studyTable; + + const stats = dataset['obiba.mica.CountStatsDto.datasetCountStats'] || {}; + let anchor = (type, value) => `${value.toLocaleString(this.locale)}`; + + (displayOptions[columnKey] || displayOptions.datasetColumns).forEach(column => { + switch (column) { + case 'name': { + row.push(localize(dataset.name)); + break; + } + case 'type': { + if (micaConfig.isCollectedDatasetEnabled && micaConfig.isHarmonizedDatasetEnabled) { + row.push(tr(type.toLowerCase())); + } + break; + } + case 'networks': { + if (micaConfig.isNetworkEnabled && !micaConfig.isSingleNetworkEnabled) { + row.push(stats.networks ? anchor('networks', stats.networks) : '-'); + } + break; + } + case 'studies': { // deprecated + if (!micaConfig.isSingleStudyEnabled) { + row.push(stats.studies ? anchor('studies', stats.studies) : '-'); + } + break; + } + case 'initiatives': { // deprecated + if (!micaConfig.isSingleStudyEnabled) { + row.push(stats.studies ? anchor('studies', stats.studies) : '-'); + } + break; + } + case 'study': { + if (!micaConfig.isSingleStudyEnabled) { + let opalTablePath = path = this.normalizePath(`/study/${opalTable.studySummary.id}`); + row.push(stats.studies ? `${localize(opalTable.studySummary.acronym)}` : '-'); + } + break; + } + case 'initiative': { + if (!micaConfig.isSingleStudyEnabled) { + let opalTablePath = path = this.normalizePath(`/study/${opalTable.studySummary.id}`); + row.push(stats.studies ? `${localize(opalTable.studySummary.acronym)}` : '-'); + } + break; + } + case 'variables': { + row.push(stats.variables ? anchor('variables', stats.variables) : '-'); + break; + } + default: + row.push(''); + console.debug('Wrong dataset table column: ' + column); + } + }); + + parsed.data.push(row); + }); + + return parsed; + } +} + +class NetworksResultParser { + + constructor(normalizePath, locale) { + this.normalizePath = normalizePath; + this.locale = locale; + } + + parse(data, micaConfig, localize, displayOptions, studyTypeSelection) { + const networksResult = data.networkResultDto; + + let columnKey = 'networkColumns'; + if (studyTypeSelection) { + if (studyTypeSelection.study) { + columnKey = 'networkColumnsIndividual'; + } else if(studyTypeSelection.harmonization) { + columnKey = 'networkColumnsHarmonization'; + } + } + + if (!networksResult) { + throw new Error("No network results available."); + } + + if (networksResult.totalHits < 1) return { totalHits: 0}; + + const result = networksResult["obiba.mica.NetworkResultDto.result"]; + + if (!result) { + throw new Error("Invalid JSON."); + } + + let parsed = { + data: [], + totalHits: networksResult.totalHits + } + + const networks = result.networks || []; + + networks.forEach(network => { + const stats = network['obiba.mica.CountStatsDto.networkCountStats'] || {}; + let anchor = (type, value, studyType) => `${value.toLocaleString(this.locale)}`; + + let path = this.normalizePath(`/network/${network.id}`); + let row = []; + + if (displayOptions.showCheckboxes) { + row.push(``); + } + + row.push(`${localize(network.acronym)}`); + + (displayOptions[columnKey] || displayOptions.networkColumns).forEach(column => { + switch (column) { + case 'name': { + row.push(localize(network.name)); + break; + } + case 'studies': { + row.push(stats.studies ? anchor('studies', stats.studies, "Study") : '-'); + break; + } + case 'initiatives': { + row.push(stats.studies ? anchor('studies', stats.studies, "HarmonizationStudy") : '-'); + break; + } + case 'datasets': { + if (micaConfig.isCollectedDatasetEnabled) { + row.push(stats.studyDatasets ? anchor('datasets', stats.studyDatasets, 'Study') : '-'); + } + if (micaConfig.isHarmonizedDatasetEnabled) { + row.push(stats.harmonizationDatasets ? anchor('datasets', stats.harmonizationDatasets, 'HarmonizationStudy') : '-'); + } + break; + } + case 'variables': { + if (micaConfig.isCollectedDatasetEnabled) { + row.push(stats.studyVariables ? anchor('variables', stats.studyVariables, 'Study') : '-'); + } + if (micaConfig.isHarmonizedDatasetEnabled) { + row.push(stats.dataschemaVariables ? anchor('variables', stats.dataschemaVariables, 'HarmonizationStudy') : '-'); + } + break; + } + case 'individual': { + if (micaConfig.isCollectedDatasetEnabled) { + row.push(stats.studies ? anchor('studies', stats.studies, "Study") : '-'); + row.push(stats.studyDatasets + ? anchor("datasets", stats.studyDatasets, "Study") + : "-"); + row.push(stats.studyVariables + ? anchor("variables", stats.studyVariables, "Study") + : "-"); + } + break; + } + case 'harmonization': { + if (micaConfig.isHarmonizedDatasetEnabled) { + row.push(stats.studies ? anchor('studies', stats.studies, "HarmonizationStudy") : '-'); + row.push(stats.harmonizationDatasets + ? anchor("datasets", stats.harmonizationDatasets, "HarmonizationStudy") + : "-"); + row.push(stats.dataschemaVariables + ? anchor("variables", stats.dataschemaVariables, "HarmonizationStudy") + : "-"); + } + break; + } + default: + row.push(''); + console.debug('Wrong network table column: ' + column); + } + }); + + parsed.data.push(row); + }); + + return parsed; + } +} + +class IdSplitter { + constructor(bucket, result, normalizePath) { + this.bucket = bucket; + this.result = result; + this.normalizePath = normalizePath; + this.rowSpans = {}; + this.minMax = {}; + this.currentYear = new Date().getFullYear(); + this.currentMonth = new Date().getMonth() + 1; + this.currentYearMonth = this.currentYear + '-' + this.currentMonth; + this.currentDate = this.__toTime(this.currentYearMonth, true); + } + + static BUCKET_TYPES = { + STUDY: 'studyId', + DCE: 'dceId', + DATASET: 'datasetId', + } + + __getBucketUrl(bucket, id) { + switch (bucket) { + case IdSplitter.BUCKET_TYPES.STUDY: + case IdSplitter.BUCKET_TYPES.DCE: + return this.normalizePath(`/study/${id}`); + case IdSplitter.BUCKET_TYPES.DATASET: + return this.normalizePath(`/dataset/${id}`) + } + + return this.normalizePath(''); + } + + __appendRowSpan(id) { + let rowSpan; + if (!this.rowSpans[id]) { + rowSpan = 1; + this.rowSpans[id] = 1; + } else { + rowSpan = 0; + this.rowSpans[id] = this.rowSpans[id] + 1; + } + return rowSpan; + } + + __appendMinMax(id, start, end) { + if (this.minMax[id]) { + if (start < this.minMax[id][0]) { + this.minMax[id][0] = start; + } + if (end > this.minMax[id][1]) { + this.minMax[id][1] = end; + } + } else { + this.minMax[id] = [start, end]; + } + } + + __toTime(yearMonth, start) { + let res; + if (yearMonth) { + if (yearMonth.indexOf('-') > 0) { + let ym = yearMonth.split('-'); + if (!start) { + let m = parseInt(ym[1]); + if (m < 12) { + ym[1] = m + 1; + } else { + ym[0] = parseInt(ym[0]) + 1; + ym[1] = 1; + } + } + let ymStr = ym[0] + '/' + ym[1] + '/01'; + res = Date.parse(ymStr); + } else { + res = start ? Date.parse(yearMonth + '/01/01') : Date.parse(yearMonth + '/12/31'); + } + } + return res; + } + + __getProgress(startYearMonth, endYearMonth) { + let start = this.__toTime(startYearMonth, true); + let end = endYearMonth ? this.__toTime(endYearMonth, false) : this.currentDate; + let current = end < this.currentDate ? end : this.currentDate; + if (end === start) { + return 100; + } else { + return Math.round(startYearMonth ? 100 * (current - start) / (end - start) : 0); + } + } + + splitIds(micaConfig, locale) { + + let cols = { + colSpan: this.bucket.startsWith('dce') ? (micaConfig.isSingleStudyEnabled ? 2 : 3) : 1, + ids: {} + }; + + let odd = true; + let groupId; + + this.result.rows.forEach((row, i) => { + row.hitsTitles = row.hits.map(function (hit) { + return hit.toLocaleString(locale); + }); + cols.ids[row.value] = []; + if (this.bucket.startsWith('dce')) { + let ids = row.value.split(':'); + let isHarmo = row.className.indexOf('Harmonization') > -1 || ids[2] === '.'; // would work for both HarmonizationDataset and HarmonizationStudy + let titles = row.title.split(':'); + let descriptions = row.description.split(':'); + let rowSpan; + let id; + + // study + id = ids[0]; + if (!groupId) { + groupId = id; + } else if (id !== groupId) { + odd = !odd; + groupId = id; + } + rowSpan = this.__appendRowSpan(id); + this.__appendMinMax(id, row.start || this.currentYearMonth, row.end || this.currentYearMonth); + const studyUrl = this.__getBucketUrl(this.bucket, id); + + cols.ids[row.value].push({ + id: id, + url: studyUrl, + title: titles[0], + description: descriptions[0], + rowSpan: rowSpan, + index: i++ + }); + + // population + id = ids[0] + ':' + ids[1]; + const populationUrl = `${studyUrl}#/population/${id}`; + + rowSpan = this.__appendRowSpan(id); + cols.ids[row.value].push({ + id: isHarmo ? '-' : id, + url: populationUrl, + title: titles[1], + description: descriptions[1], + rowSpan: rowSpan, + index: i++ + }); + + // dce + cols.ids[row.value].push({ + id: isHarmo ? '-' : row.value, + title: titles[2], + description: descriptions[2], + start: row.start, + current: this.currentYearMonth, + end: row.end, + progressClass: odd ? 'info' : 'warning', + url: isHarmo ? studyUrl : `${populationUrl}/data-collection-event/${row.value}`, + rowSpan: 1, + index: i++ + }); + } else { + cols.ids[row.value].push({ + id: row.value, + url: this.__getBucketUrl(this.bucket, row.value), + title: row.title, + description: row.description, + min: row.start, + start: row.start, + current: this.currentYear, + end: row.end, + max: row.end, + progressStart: 0, + progress: this.__getProgress(row.start ? row.start + '-01' : this.currentYearMonth, row.end ? row.end + '-12' : this.currentYearMonth), + progressClass: odd ? 'info' : 'warning', + rowSpan: 1, + index: i++ + }); + odd = !odd; + } + }); + + // adjust the rowspans and the progress + if (this.bucket.startsWith('dce')) { + this.result.rows.forEach((row, i) => { + row.hitsTitles = row.hits.map(function (hit) { + return hit.toLocaleString(locale); + }); + if (cols.ids[row.value][0].rowSpan > 0) { + cols.ids[row.value][0].rowSpan = this.rowSpans[cols.ids[row.value][0].id]; + } + if (cols.ids[row.value][1].rowSpan > 0) { + cols.ids[row.value][1].rowSpan = this.rowSpans[cols.ids[row.value][1].id]; + } + let ids = row.value.split(':'); + if (this.minMax[ids[0]]) { + let min = this.minMax[ids[0]][0]; + let max = this.minMax[ids[0]][1]; + let start = cols.ids[row.value][2].start || this.currentYearMonth; + let end = cols.ids[row.value][2].end || this.currentYearMonth; + let diff = this.__toTime(max, false) - this.__toTime(min, true); + // set the DCE min and max dates of the study + cols.ids[row.value][2].min = min; + cols.ids[row.value][2].max = max; + // compute the progress + cols.ids[row.value][2].progressStart = 100 * (this.__toTime(start, true) - this.__toTime(min, true)) / diff; + cols.ids[row.value][2].progress = 100 * (this.__toTime(end, false) - this.__toTime(start, true)) / diff; + cols.ids[row.value].index = i; + } + }); + } + + return cols; + } + +} + +class CoverageResultParser { + + constructor(micaConfig, locale, normalizePath) { + this.micaConfig = micaConfig; + this.locale = locale; + this.normalizePath = normalizePath; + } + + decorateVocabularyHeaders(headers, vocabularyHeaders) { + let count = 0, i = 0; + for (let j = 0; j < vocabularyHeaders.length; j++) { + if (count >= headers[i].termsCount) { + i++; + count = 0; + } + + count += vocabularyHeaders[j].termsCount; + vocabularyHeaders[j].taxonomyName = headers[i].entity.name; + } + } + + decorateTermHeaders(headers, termHeaders, attr) { + let idx = 0; + return headers.reduce(function (result, h) { + result[h.entity.name] = termHeaders.slice(idx, idx + h.termsCount).map(function (t) { + if (h.termsCount > 1 && attr === 'vocabularyName') { + t.canRemove = true; + } + + t[attr] = h.entity.name; + + return t; + }); + + idx += h.termsCount; + return result; + }, {}); + } + + parseHeaders(bucket, result) { + let table = { cols: [] }; + let vocabulariesTermsMap = {}; + + if (result && result.rows) { + var tableTmp = result; + tableTmp.cols = new IdSplitter(bucket, result, this.normalizePath).splitIds(this.micaConfig, this.locale); + table = tableTmp; + + // TODO let filteredRows = []; + // TODO let nextFilteredRowsPage = 0; + // TODO $scope.loadMoreRows(); + + vocabulariesTermsMap = this.decorateTermHeaders(table.vocabularyHeaders, table.termHeaders, 'vocabularyName'); + this.decorateTermHeaders(table.taxonomyHeaders, table.termHeaders, 'taxonomyName'); + this.decorateVocabularyHeaders(table.taxonomyHeaders, table.vocabularyHeaders); + // TODO $scope.isFullCoverageImpossibleOrCoverageAlreadyFull(); + } + + return { table, vocabulariesTermsMap}; + } + + parse(data) { + return data; + } +} diff --git a/k8s/mica/templates/backup-cron.yaml b/k8s/mica/templates/backup-cron.yaml index a18142b..3136837 100644 --- a/k8s/mica/templates/backup-cron.yaml +++ b/k8s/mica/templates/backup-cron.yaml @@ -1,9 +1,10 @@ +{{- if eq .Values.backup.enabled true }} apiVersion: batch/v1 kind: CronJob metadata: name: {{ .Release.Name }}-mongodb-backup spec: - schedule: "30 7 * * *" + schedule: {{ .Values.backup.schedule }} successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 concurrencyPolicy: Forbid @@ -63,4 +64,4 @@ type: Opaque data: key: {{ .Values.backup.s3.key | b64enc| quote}} keyid: {{ .Values.backup.s3.keyid |b64enc| quote}} - +{{- end }} \ No newline at end of file diff --git a/k8s/mica/templates/configmap-custom-js.yaml b/k8s/mica/templates/configmap-custom-js.yaml new file mode 100644 index 0000000..f29c9cc --- /dev/null +++ b/k8s/mica/templates/configmap-custom-js.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-mica-result-parsers +data: + result-parsers.js: > + {{ .Files.Get "persona/result-parsers.js" | nindent 4 }} \ No newline at end of file diff --git a/k8s/mica/templates/mica.yaml b/k8s/mica/templates/mica.yaml index 4301840..a4bd59b 100644 --- a/k8s/mica/templates/mica.yaml +++ b/k8s/mica/templates/mica.yaml @@ -54,8 +54,52 @@ spec: volumeMounts: - mountPath: /srv name: {{ .Release.Name }}-data-container-mica - - mountPath: /usr/share/mica2/webapp/WEB-INF/classes/templates + - mountPath: /usr/share/mica2/webapp/WEB-INF/classes/_templates name: {{ .Release.Name }}-template-container + - mountPath: /usr/share/mica2/webapp/assets/js/vue-mica-search/libs/result-parsers.js + name: {{ .Release.Name }}-mica-result-parsers + subPath: result-parsers.js +{{- if eq .Values.backup.enabled true }} + - name: backup + image: alpine + env: + - name: S3_KEY + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-s3-backup-secret + key: key + - name: S3_KEYID + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-s3-backup-secret + key: keyid + - name: S3_HOST + value: {{ .Values.backup.s3.host }} + - name: S3_BUCKET + value: {{ .Values.backup.s3.bucket }} + - name: CRON_SCHEDULE + value: {{ .Values.backup.schedule }} + - name: RELEASE_NAME + value: {{ .Release.Name }} + command: [ "/bin/sh", "-c" ] + args: [ 'apk add s3cmd; +rm -f script.sh; +touch script.sh; +echo "tar -czvf /tmp/${RELEASE_NAME}_$(date +%Y-%m-%d).mica.archive.gz /srv" >> script.sh; +echo "s3cmd --host=${S3_HOST} --access_key=${S3_KEYID} --secret_key=${S3_KEY} --host-bucket=${S3_BUCKET}.${S3_HOST} put /tmp/${RELEASE_NAME}_$(date +%Y-%m-%d).mica.archive.gz s3://${S3_BUCKET}/${RELEASE_NAME}_$(date +%Y-%m-%d).mica.archive.gz" >> script.sh; +echo "echo \"done\"" >> script.sh; +chmod +x script.sh; +crontab -l | grep -v -F "${CMD}"; echo "$CRON_SCHEDULE /script.sh > /dev/stdout" | crontab - ; +crond -f -l 8' ] + resources: + limits: + memory: 512Mi + requests: + memory: 264Mi + volumeMounts: + - mountPath: /srv + name: {{ .Release.Name }}-data-container-mica +{{- end }} restartPolicy: Always volumes: - name: {{ .Release.Name }}-data-container-mica @@ -64,6 +108,10 @@ spec: - name: {{ .Release.Name }}-template-container persistentVolumeClaim: claimName: {{ .Release.Name }}-template-container-mica + - name: {{ .Release.Name }}-mica-result-parsers + configMap: + name: {{ .Release.Name }}-mica-result-parsers + defaultMode: 0544 --- apiVersion: v1 kind: PersistentVolumeClaim @@ -114,4 +162,3 @@ data: admin_pw: {{ randAlphaNum 20 |b64enc| quote }} user_pw: {{ randAlphaNum 20 |b64enc| quote }} {{ end -}} - diff --git a/k8s/mica/values.yaml b/k8s/mica/values.yaml index b790383..4bb7df9 100644 --- a/k8s/mica/values.yaml +++ b/k8s/mica/values.yaml @@ -3,11 +3,13 @@ ingress: enableSSL: false certIssuer: backup: + enabled: false + schedule: "30 7 * * *" s3: host: bucket: key: keyid: -image: obiba/mica:5.2.3 +image: obiba/mica:5.2 mongo: image: mongo:7.0.2 \ No newline at end of file