Skip to content

Commit

Permalink
Added support for null values
Browse files Browse the repository at this point in the history
  • Loading branch information
@Tomansion
Tomansion committed Aug 19, 2024
1 parent 69cb6ed commit c9033d8
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 26 deletions.
2 changes: 0 additions & 2 deletions debiaiServer/swagger.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1238,8 +1238,6 @@ paths:
type: array
items:
type: array
items:
type: number

responses:
200:
Expand Down
48 changes: 34 additions & 14 deletions frontend/src/components/debiai/dataAnalysis/widgets/Statistics/Statistics.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
<th v-if="selectedColumn.type !== String">Average</th>
<th v-if="selectedColumn.type !== String">Std</th>
<th v-if="selectedColumn.type !== String">Variance</th>
<th v-if="nbNullValues">Missing values</th>
</tr>
</thead>
<tbody>
Expand All @@ -88,6 +89,11 @@
<td v-if="selectedColumn.type !== String">{{ average }}</td>
<td v-if="selectedColumn.type !== String">{{ std }}</td>
<td v-if="selectedColumn.type !== String">{{ variance }}</td>
<td v-if="nbNullValues">
{{ nbNullValues }} ({{
Math.round((nbNullValues / nbSelectedSamplesAtUpdate) * 100)
}}%)
</td>
</tr>
</tbody>
</table>
Expand Down Expand Up @@ -120,6 +126,7 @@
<th v-if="selectedColumn.type !== String">Average</th>
<th v-if="selectedColumn.type !== String">Std</th>
<th v-if="selectedColumn.type !== String">Variance</th>
<th v-if="nbNullValues">Missing values</th>
</tr>
</thead>
<tbody>
Expand Down Expand Up @@ -175,6 +182,12 @@
({{ Math.round((colVal.frequency / colVal.repartition) * 100) }}%)
</span>
</td>
<!-- Null values -->
<td v-if="nbNullValues">
{{ colVal.nbNullValues }} ({{
Math.round((colVal.nbNullValues / colVal.repartition) * 100)
}}%)
</td>
</tr>
</tbody>
</table>
Expand Down Expand Up @@ -208,6 +221,7 @@ export default {
// Statistics
nbSelectedSamplesAtUpdate: null,
nbNullValues: null,
// Class
top: null,
Expand Down Expand Up @@ -250,15 +264,21 @@ export default {
},
updateStatistics() {
let values;
let valuesOriginal; // With null values
let valuesText;
this.nbNullValues = 0;
if (this.selectedColumn.type === String) {
values = this.data.selectedData.map((sId) => this.selectedColumn.valuesIndex[sId]);
valuesOriginal = values;
valuesText = this.data.selectedData.map((sId) => this.selectedColumn.values[sId]);
this.nbNullValues = values.filter((v) => v === null).length;
} else {
values = this.data.selectedData.map((sId) => this.selectedColumn.values[sId]);
valuesOriginal = this.data.selectedData.map((sId) => this.selectedColumn.values[sId]);
values = valuesOriginal.filter((v) => v !== null);
if (this.absolute) values = values.map((v) => Math.abs(v));
this.nbNullValues = this.data.selectedData.length - values.length;
}
this.nbSelectedSamplesAtUpdate = values.length;
this.nbSelectedSamplesAtUpdate = this.data.selectedData.length;
// Average or frequency
if (this.selectedColumn.type === String) {
Expand All @@ -275,14 +295,12 @@ export default {
// Min
this.min = null;
if (this.selectedColumn.type !== String)
if (!this.absolute) this.min = dataOperations.humanize(this.selectedColumn.min);
else this.min = dataOperations.humanize(dataOperations.getMin(values));
this.min = dataOperations.humanize(dataOperations.getMin(values));
// Max
this.max = null;
if (this.selectedColumn.type !== String)
if (!this.absolute) this.max = dataOperations.humanize(this.selectedColumn.max);
else this.max = dataOperations.humanize(dataOperations.getMax(values));
this.max = dataOperations.humanize(dataOperations.getMax(values));
// Std & variance
this.std = null;
Expand All @@ -308,7 +326,8 @@ export default {
);
this.statByColor = groupedValues.map((sampleIds, i) => {
let gpValues = sampleIds.map((sId) => values[sId]);
const gpValues = sampleIds.map((sId) => valuesOriginal[sId]);
const gpValuesNonNull = gpValues.filter((v) => v !== null);
// Stats
// Numbers
Expand All @@ -321,19 +340,19 @@ export default {
let top;
let frequency;
if (gpValues.length) {
if (gpValuesNonNull.length) {
if (this.selectedColumn.type === String) {
let mode = dataOperations.mode(gpValues);
let mode = dataOperations.mode(gpValuesNonNull);
top = this.selectedColumn.uniques[mode.top];
frequency = mode.frequency;
} else {
average = dataOperations.humanize(dataOperations.mean(gpValues));
average = dataOperations.humanize(dataOperations.mean(gpValuesNonNull));
// Std & variance
colStd = dataOperations.humanize(std(gpValues));
colVariance = dataOperations.humanize(variance(gpValues));
min = dataOperations.humanize(dataOperations.getMin(gpValues));
max = dataOperations.humanize(dataOperations.getMax(gpValues));
colStd = dataOperations.humanize(std(gpValuesNonNull));
colVariance = dataOperations.humanize(variance(gpValuesNonNull));
min = dataOperations.humanize(dataOperations.getMin(gpValuesNonNull));
max = dataOperations.humanize(dataOperations.getMax(gpValuesNonNull));
}
}
return {
Expand All @@ -347,6 +366,7 @@ export default {
std: colStd,
variance: colVariance,
repartition: gpValues.length,
nbNullValues: gpValues.length - gpValuesNonNull.length,
};
});
Expand Down
5 changes: 4 additions & 1 deletion frontend/src/services/statistics/data.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -498,13 +498,16 @@ class Column {
this.uniques = [...new Set(this.originalValues)];
this.nbOccurrence = this.uniques.length;

const isAllDictionaries = (arr) => {
return arr.every((item) => item !== null && typeof item === "object" && !Array.isArray(item));
};
// Checking if the this.umn is type text, number or got undefined values
if (!(this.uniques.findIndex((v) => !Array.isArray(v)) >= 0)) {
// If all the values are arrays
this.type = Array;
this.typeText = "Array";
this.arrayColumnSizeNumber = this.getArrayColumnSizeNumber();
} else if (this.uniques.findIndex((v) => typeof v !== "object")) {
} else if (isAllDictionaries(this.uniques)) {
// If all the values are dictionaries
this.type = Object;
this.typeText = "Dict";
Expand Down
24 changes: 15 additions & 9 deletions frontend/src/services/statistics/dataOperations.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,19 @@ const MAX_STRING_LENGTH = 30;

// create data for the plotly par. coord.
const columnsCreation = function (columns, selectedSamplesIds) {
let plotlyColumns = columns.map((col) => {
const plotlyColumns = columns.map((col) => {
if (col.type == String) {
let tickvals = [];
const tickvals = [...Array(col.nbOccurrence).keys()];
let ticktext = [];

tickvals = [...Array(col.nbOccurrence).keys()];

if (col.nbOccurrence > MAX_STRING_DISPLAYED)
// No text on the axis
ticktext = tickvals.map(() => "");
else
ticktext = col.uniques.map((v) =>
v.length > MAX_STRING_LENGTH ? v.substring(0, MAX_STRING_LENGTH) + "..." : v
);
ticktext = col.uniques.map((v) => {
if (v === null) return "null";
return v.length > MAX_STRING_LENGTH ? v.substring(0, MAX_STRING_LENGTH) + "..." : v;
});

return {
label: col.label,
Expand All @@ -29,10 +28,15 @@ const columnsCreation = function (columns, selectedSamplesIds) {
ticktext: ticktext,
};
} else {
const values = selectedSamplesIds.map((sId) => {
const v = col.values[sId];
if (v === null) return undefined;
return v;
});
return {
type: "int",
label: col.label,
values: selectedSamplesIds.map((sId) => col.values[sId]),
values: values,
};
}
});
Expand Down Expand Up @@ -151,7 +155,9 @@ const getRepartition = function (x, interval, min, max) {
for (let i = 0; i < interval + 1; i++) {
xSections.push(sectionLength * i + min);
repartition.push(
x.filter((v) => v >= sectionLength * i + min && v < sectionLength * (i + 1) + min).length
x.filter(
(v) => v >= sectionLength * i + min && v < sectionLength * (i + 1) + min && v !== null
).length
);
}

Expand Down

0 comments on commit c9033d8

Please sign in to comment.