Skip to content

Commit

Permalink
Fix Data generation
Browse files Browse the repository at this point in the history
  • Loading branch information
sebt3 committed Feb 8, 2024
1 parent c482dca commit d8a0cc1
Show file tree
Hide file tree
Showing 25 changed files with 393,478 additions and 52 deletions.
13,038 changes: 13,037 additions & 1 deletion data/certmanager.json

Large diffs are not rendered by default.

21,702 changes: 21,701 additions & 1 deletion data/cnpg.json

Large diffs are not rendered by default.

14,663 changes: 14,662 additions & 1 deletion data/fluxcd.json

Large diffs are not rendered by default.

44,449 changes: 44,448 additions & 1 deletion data/jaegertracing.json

Large diffs are not rendered by default.

78,117 changes: 78,116 additions & 1 deletion data/k8s.json

Large diffs are not rendered by default.

26,795 changes: 26,794 additions & 1 deletion data/k8up.json

Large diffs are not rendered by default.

30,146 changes: 30,145 additions & 1 deletion data/mariadb.json

Large diffs are not rendered by default.

1,213 changes: 1,212 additions & 1 deletion data/mongodb.json

Large diffs are not rendered by default.

74,864 changes: 74,863 additions & 1 deletion data/monitoring.json

Large diffs are not rendered by default.

25,158 changes: 25,157 additions & 1 deletion data/opentelemetry.json

Large diffs are not rendered by default.

2,629 changes: 2,628 additions & 1 deletion data/oracle.json

Large diffs are not rendered by default.

8,916 changes: 8,915 additions & 1 deletion data/projectcalico.json

Large diffs are not rendered by default.

12,851 changes: 12,850 additions & 1 deletion data/rabbitmq.json

Large diffs are not rendered by default.

17,655 changes: 17,654 additions & 1 deletion data/redis.json

Large diffs are not rendered by default.

1,099 changes: 1,098 additions & 1 deletion data/secretgenerator.json

Large diffs are not rendered by default.

16,975 changes: 16,974 additions & 1 deletion data/traefik.json

Large diffs are not rendered by default.

1,000 changes: 999 additions & 1 deletion data/vynil.json

Large diffs are not rendered by default.

2,178 changes: 2,177 additions & 1 deletion data/zalando.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion front/components/navigation/MainMenu.vue
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ function toggleLeftDrawer() {
<q-drawer show-if-above v-model="leftDrawerOpen" side="left" bordered :mini="miniState" :width="400" :breakpoint="500" @mouseover="miniState = false" @mouseout="miniState = true" mini-to-overlay>
<q-scroll-area class="fit" :horizontal-thumb-style="{ opacity: '0' }">
<q-list padding v-if="router.currentRoute.value.matched.length>1">
<MainMenuLinks
<MainMenuLinks class="q-mr-xs"
v-for="link in links"
:key="`${link.title}-${router.currentRoute.value.matched[router.currentRoute.value.matched.length-1].path}`"
v-bind="link">
Expand Down
21 changes: 17 additions & 4 deletions front/libs/k8s/custom.ts
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
// noGramoGenerator
import { QTableColumn } from 'quasar'
import { tableColumnAlign } from '../core'
export const k8sIcon = 'apps';
export const k8sIcon = 'fas fa-dharmachakra';
export const k8sTitle = 'Standard';
export const iconConfigMap = 'album';
export const iconConfigMap = 'settings_suggest';
export const colorConfigMap = '';
export const descriptionConfigMap = '';
export const shortConfigMap = '';
export const iconEndpoints = 'album';
export const iconEndpoints = 'fas fa-network-wired';
export const colorEndpoints = '';
export const descriptionEndpoints = '';
export const shortEndpoints = '';
export const iconLimitRange = 'album';
export const colorLimitRange = '';
export const descriptionLimitRange = '';
export const shortLimitRange = '';
export const iconNamespace = 'album';
export const iconNamespace = 'fas fa-object-ungroup';
export const colorNamespace = '';
export const descriptionNamespace = '';
export const shortNamespace = '';
Expand Down Expand Up @@ -63,6 +63,10 @@ export const iconAPIService = 'album';
export const colorAPIService = '';
export const descriptionAPIService = '';
export const shortAPIService = '';
export const iconControllerRevision = 'album';
export const colorControllerRevision = '';
export const descriptionControllerRevision = '';
export const shortControllerRevision = 'ControllerRevision';
export const iconDaemonSet = 'album';
export const colorDaemonSet = '';
export const descriptionDaemonSet = '';
Expand Down Expand Up @@ -216,6 +220,9 @@ export const extraServiceAccountColumns:Array<QTableColumn> = [
export const extraAPIServiceColumns:Array<QTableColumn> = [
// {name: 'Name', label: 'Name', field: row => row.metadata.name, sortable: true, align: tableColumnAlign.left},
];
export const extraControllerRevisionColumns:Array<QTableColumn> = [
// {name: 'Name', label: 'Name', field: row => row.metadata.name, sortable: true, align: tableColumnAlign.left},
];
export const extraDaemonSetColumns:Array<QTableColumn> = [
// {name: 'Name', label: 'Name', field: row => row.metadata.name, sortable: true, align: tableColumnAlign.left},
];
Expand Down Expand Up @@ -387,6 +394,12 @@ export const APIServiceReadExcludes = [
];
export const APIServiceSimpleExcludes = [
];
export const ControllerRevisionListExcludes = [
];
export const ControllerRevisionReadExcludes = [
];
export const ControllerRevisionSimpleExcludes = [
];
export const DaemonSetListExcludes = [
];
export const DaemonSetReadExcludes = [
Expand Down
4 changes: 2 additions & 2 deletions front/libs/vynil/custom.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
// noGramoGenerator
import { QTableColumn } from 'quasar'
import { tableColumnAlign } from '../core'
export const vynilIcon = 'apps';
export const vynilIcon = 'album';
export const vynilTitle = 'Vynil';
export const iconDistrib = 'album';
export const iconDistrib = 'speaker_group';
export const colorDistrib = '';
export const descriptionDistrib = '';
export const shortDistrib = '';
Expand Down
2 changes: 1 addition & 1 deletion utils/generator/front/grp.pages.list.vue.hbs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ onResult((res) => {
})
</script>
<template>
<div class="row q-mb-sm q-ml-sm" v-if="isNamespaced()">
<div class="row q-mb-sm q-ml-sm" v-if="!isNamespaced()">
{{#each objects}}
{{#if namespaced}}
<div class="col-lg-4" v-if="ready && {{ short }}ByTs.length>1" >
Expand Down
2 changes: 1 addition & 1 deletion utils/generator/hb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ HB.registerHelper('haveQueries', (objects)=>(objects.filter(o=>!o['namespaced'])
HB.registerHelper('capitalizeFirstLetter', (str)=>new HB.SafeString(capitalizeFirstLetter(str)))
HB.registerHelper('lower', (str)=>new HB.SafeString(str.toLowerCase()))
HB.registerHelper('minimizeFirstLetter', (str)=>new HB.SafeString(minimizeFirstLetter(str)))
HB.registerHelper('toJson', v => new HB.SafeString(JSON.stringify(v)));
HB.registerHelper('toJson', v => new HB.SafeString(JSON.stringify(v, null,2)));
HB.registerHelper('getGQLType', (name,obj)=>new HB.SafeString(Object.entries(obj).filter(([n])=>name==n).map(([_,v])=>v)[0] as string))
HB.registerHelper('genParameterQuery', (obj)=>new HB.SafeString(
Object.entries(obj['gqlDefs'])
Expand Down
46 changes: 24 additions & 22 deletions utils/generator/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ export function rmdir(directory:string){
fs.rmdirSync(directory)
}
}
export function saveTo(filename:string,content:any){
fs.writeFileSync(filename, JSON.stringify(content));
export function saveTo(filename:string,content:object){
fs.writeFileSync(filename, JSON.stringify(content, null, 2));
}
export function LoadFrom(filename:string){
return JSON.parse(fs.readFileSync(filename).toLocaleString())
Expand Down Expand Up @@ -119,35 +119,26 @@ export const enhenceObject = (group:string, obj:unspeciedObject) => {
export const finalizeObject = (obj:k8sObject, all:k8sObject[]) => {
return {
...obj,
alternatives: obj['alternatives'].length>1?obj['alternatives']:[],
category: categoryMappingShort[obj.short]!=undefined?categoryMappingShort[obj.short]:categoryMappingGroup[obj.group]!=undefined?categoryMappingGroup[obj.group]:'varia',
resolvers: obj.resolvers.map(r=>{return {...r, properties: all.filter(o=>o.group==r['resultGroup']&&o.short==r['resultShort']).map(o=>o.readProperties)[0]}})
}
}
export const getObjFQN = (c: k8sDefinitionProperties) => c.spec.group.split('.').reverse().join('.')+'.'+getTargetVersion(c.spec.versions)+'.'+c.spec.names.kind

export const replaceRefWithDef = (defs: [string, openapiDefinition][]) => {
const excluded = [
'io.k8s.apimachinery.pkg.apis.meta.v1.Time',
'io.k8s.apimachinery.pkg.util.intstr.IntOrString',
'io.k8s.apimachinery.pkg.api.resource.Quantity',
'io.k8s.api.core.v1.ConfigMapVolumeSource',
'io.k8s.api.core.v1.ConfigMapProjection',
'io.k8s.api.core.v1.DownwardAPIVolumeSource',
'io.k8s.api.core.v1.DownwardAPIProjection',
'io.k8s.api.core.v1.SecretProjection',
'io.k8s.api.core.v1.SecretVolumeSource',
'io.k8s.apimachinery.pkg.apis.meta.v1.MicroTime',
'io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceSubresourceStatus',
'io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.JSONSchemaProps']
const excluded: string[] = [
'io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.JSONSchemaProps'
]
const queue = structuredClone(defs);
const ret:Map<string, openapiDefinitionPropertiesDef> = new Map();
const deref = (data) => {
const res = structuredClone(data);
for (const [key, val] of Object.entries(data.properties as HashMap<openapiDefinitionPropertiesDef>)) {
if (val['$ref'] != undefined && !excluded.includes(val['$ref'].split('/')[2])) {
if (data.properties != undefined) {for (const [key, val] of Object.entries(data.properties as HashMap<openapiDefinitionPropertiesDef>)) {
if (val['$ref'] != undefined && val['$ref'].split('/').length>2 && !excluded.includes(val['$ref'].split('/')[2])) {
res.properties[key] = ret[val['$ref'].split('/')[2]]
}
if (val['type'] == 'array' && val['items'] != undefined && val['items']['$ref'] != undefined && !excluded.includes(val['items']['$ref'].split('/')[2])) {
if (val['type'] == 'array' && val['items'] != undefined && val['items']['$ref'] != undefined && val['items']['$ref'].split('/').length>2 && !excluded.includes(val['items']['$ref'].split('/')[2])) {
res.properties[key]['items'] = ret[val['items']['$ref'].split('/')[2]]
}
if (val['type'] == 'object' && val.properties != undefined) {
Expand All @@ -157,37 +148,48 @@ export const replaceRefWithDef = (defs: [string, openapiDefinition][]) => {
}
}
}
}
}}
return res
}
let counter=0;
while (queue.length>0) {
const item = queue.shift();
if (item===undefined) continue;
let missing = false;
for (const [key, val] of Object.entries(item[1].properties)) {
if (item[1].properties != undefined) {for (const [key, val] of Object.entries(item[1].properties)) {
missing = missing || (val['$ref'] != undefined && key !='metadata' && !excluded.includes(val['$ref'].split('/')[2]) && !Object.keys(ret).includes(val['$ref'].split('/')[2]));
//if ((val['$ref'] != undefined && key !='metadata' && !excluded.includes(val['$ref'].split('/')[2]) && !Object.keys(ret).includes(val['$ref'].split('/')[2]))) console.log('missing',val['$ref'].split('/')[2])
if (val['type'] == 'array') {
missing = missing || (val['items']['$ref'] != undefined && !excluded.includes(val['items']['$ref'].split('/')[2]) && !Object.keys(ret).includes(val['items']['$ref'].split('/')[2]));
//if (val['items']['$ref'] != undefined && !excluded.includes(val['items']['$ref'].split('/')[2]) && !Object.keys(ret).includes(val['items']['$ref'].split('/')[2])) console.log('missing',val['items']['$ref'].split('/')[2])
}
if (val['type'] == 'object' && typeof val.properties == 'object') {
for (const [, v2] of Object.entries(val.properties)) {
missing = missing || ((v2 as object)['$ref'] != undefined && !excluded.includes((v2 as object)['$ref'].split('/')[2]) && !Object.keys(ret).includes((v2 as object)['$ref'].split('/')[2]));
//if ((v2 as object)['$ref'] != undefined && !excluded.includes((v2 as object)['$ref'].split('/')[2]) && !Object.keys(ret).includes((v2 as object)['$ref'].split('/')[2]))console.log('missing',(v2 as object)['$ref'].split('/')[2]);
}
}
if (missing) break;
}
}}
if (missing) {
queue.push(item)
counter++;
if (counter>queue.length) {
console.log('leaving with',
Object.fromEntries(queue.map(([n,v])=>[n,
Object.fromEntries(
Object.entries(v['properties']).filter(([_,val])=>val['$ref']!=null).map(([name,val])=>[name,val['$ref']])
.concat(Object.entries(v['properties']).filter(([_,val])=>val['type'] == 'array'&& val['items'] != null && val['items']['$ref']!=null).map(([name,val])=>[name,val['items']['$ref']]))
)
])),
queue.length)
break;
}
//console.log(queue.length,Object.keys(ret).length)
continue;
}
counter=0;
ret[item[0]] = deref(item[1])
//if (item[0]=='io.k8s.api.core.v1.ServiceSpec') console.log(ret[item[0]]["properties"])
}
return ret
}
Expand Down
5 changes: 2 additions & 3 deletions utils/prepareData.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Promise.all([getClusterByPath('openapi/v2'), getClusterByPath('apis'), getCluste
// and group them according to their defined apiGroup and final super group
// Group all swagger definitions marked as a kubernetes object into their apiGroup,
// add its CRD if found and start to refine the object properties
const definitions = replaceRefWithDef(Object.entries((data_in as openapi).definitions).filter(e => e[1].properties!=undefined && e[1].properties['items']==undefined))
const definitions = replaceRefWithDef(Object.entries((data_in as openapi).definitions))//.filter(e => e[1].properties!=undefined && e[1].properties['items']==undefined))
const known_data = ((apis_in as object)['groups'] as Array<object>).map(i => { return {
name: i['name'],
...getShort(i['name']),
Expand All @@ -29,7 +29,7 @@ Promise.all([getClusterByPath('openapi/v2'), getClusterByPath('apis'), getCluste
})})
}});
// List the remaining objects not matched in a defined apiGroup (they belong to the group '' which is kubernetes main objects)
const excluded_objects = Object.entries((data_in as openapi).definitions).filter(([n, i]) =>
const excluded_objects = Object.entries(definitions).filter(([n, i]) =>
Object.keys(i).includes('x-kubernetes-group-version-kind') && i['x-kubernetes-group-version-kind'].length==1 &&
!known_data.map(d=>d.name).includes(i['x-kubernetes-group-version-kind'][0]['group']) &&
!/List$/.test(n) && !/^io.k8s.apimachinery.pkg.apis/.test(n)
Expand All @@ -46,7 +46,6 @@ Promise.all([getClusterByPath('openapi/v2'), getClusterByPath('apis'), getCluste
// Deduplicate same objects in distinct apiGroup and mark the alternative options
return subgroups.map(s=>s.group).filter(uniq).map(g => { return {
name: g,
subs: subgroups.filter(s=>s.group==g && s.objects.length>0),
objects: subgroups.filter(s=>s.group==g && s.objects.length>0).map(s=>s.objects).flat().map(o=>o.short).filter(uniq).map((short) => {return {
alternatives: subgroups.filter(s=>s.group==g && s.objects.length>0).map(s=>s.objects.filter(o=>o.short==short)).flat()
}}).map(tmp=>{return finalizeObject({...tmp,...tmp.alternatives.sort((a,b)=>a.apiVersion.length>b.apiVersion.length?-1:a.apiVersion.length==b.apiVersion.length?a.apiVersion<b.apiVersion?-1:1:1).reverse()[0]} as k8sObject,allObjects)})
Expand Down

0 comments on commit d8a0cc1

Please sign in to comment.