Skip to content

Commit

Permalink
Support MODULEDIR schema (#440)
Browse files Browse the repository at this point in the history 
* support new schema

* upedate schema

* update

* remove comments

* update

* update

* update formatting

* fix issue

* use readJson instead readJSON to be consistent with old code
  • Loading branch information
@adashen
adashen authored May 14, 2019
1 parent f61ca13 commit c06d4c2
Show file tree
Hide file tree
Showing 10 changed files with 127 additions and 47 deletions.
2 changes: 1 addition & 1 deletion assets/solution/deployment.template.json
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"$schema-template": "1.0.0",
"$schema-template": "2.0.0",
"modulesContent": {
"$edgeAgent": {
"properties.desired": {
Expand Down
11 changes: 10 additions & 1 deletion src/common/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ export class Constants {
public static groupIDPlaceholder = "%GROUP_ID%";
public static repositoryPlaceholder = "%REPOSITORY%";
public static dllPlaceholder = "%DLLNAME%";
public static imagePlaceholderPattern: RegExp = new RegExp(/\${MODULES\..+}/g);
public static externalModulePlaceholderPattern: RegExp = new RegExp(/\${MODULEDIR<(.+)>(\..+)?}/g);
public static imagePlaceholderPattern: RegExp = new RegExp(/\${MODULES\..+}|\${MODULEDIR<(.+)>(\..+)?}/g);
public static versionPlaceholderPattern: RegExp = new RegExp(/\${VERSION\..+}/g);
public static assetsFolder = "assets";
public static solutionFolder = "solution";
Expand Down Expand Up @@ -170,6 +171,14 @@ export class Constants {

public static openSampleEvent = "openSample";
public static openSampleUrlEvent = "openSampleUrl";

public static subModuleKeyPrefixTemplate(name: string): string {
return `MODULES.${name}`;
}

public static extModuleKeyPrefixTemplate(dir: string): string {
return `MODULEDIR<${dir}>`;
}
}

export enum ContainerState {
Expand Down
109 changes: 75 additions & 34 deletions src/common/utility.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,8 @@ export class Utility {
return JSON.parse(expandedContent);
}

public static expandModules(input: string, moduleMap: Map<string, string>): string {
public static expandModules(inputJSON: any, moduleMap: Map<string, string>): string {
const input = JSON.stringify(inputJSON, null, 2);
return Utility.expandPlacesHolders(Constants.imagePlaceholderPattern, input, moduleMap);
}

Expand Down Expand Up @@ -204,12 +205,8 @@ export class Utility {
return moduleFolderName.replace(pattern, "_");
}

public static getModuleKey(name: string, platform: string): string {
return `MODULES.${name}.${platform}`;
}

public static getModuleKeyNoPlatform(name: string, isDebug: boolean): string {
return isDebug ? `MODULES.${name}.debug` : `MODULES.${name}`;
public static getDefaultModuleKey(keyPrefix: string, isDebug: boolean) {
return isDebug ? `${keyPrefix}.debug` : keyPrefix;
}

public static getImage(repo: string, version: string, platform: string): string {
Expand Down Expand Up @@ -237,22 +234,25 @@ export class Utility {
}
}

public static async setSlnModulesMap(slnPath: string,
public static async setSlnModulesMap(templateFilePath: string,
moduleToImageMap: Map<string, string>,
imageToBuildSettings?: Map<string, BuildSettings>): Promise<void> {
const modulesPath: string = path.join(slnPath, Constants.moduleFolder);
if (!await fse.pathExists(modulesPath)) {
return;
}
const stat: fse.Stats = await fse.lstat(modulesPath);
if (!stat.isDirectory()) {
return;
}

const moduleDirs: string[] = await Utility.getSubDirectories(modulesPath);
const slnPath: string = path.dirname(templateFilePath);
const moduleDirs: string[] = await Utility.getSubModules(slnPath);
await Promise.all(
moduleDirs.map(async (module) => {
await this.setModuleMap(module, moduleToImageMap, imageToBuildSettings);
moduleDirs.map(async (modulePath) => {
const keyPrefix = Constants.subModuleKeyPrefixTemplate(path.basename(modulePath));
await Utility.setModuleMap(modulePath, keyPrefix, moduleToImageMap, imageToBuildSettings);
}),
);
const externalModuleDirs: string[] = await Utility.getExternalModules(templateFilePath);
await Promise.all(
externalModuleDirs.map(async (module) => {
if (module) {
const moduleFullPath = path.resolve(slnPath, module);
const keyPrefix = Constants.extModuleKeyPrefixTemplate(module);
await Utility.setModuleMap(moduleFullPath, keyPrefix, moduleToImageMap, imageToBuildSettings);
}
}),
);
}
Expand All @@ -267,34 +267,29 @@ export class Utility {
return new BuildSettings(dockerFilePath, context, optionArray);
}

public static async setModuleMap(modulePath: string,
public static async setModuleMap(moduleFullPath: string,
moduleKeyPrefix: string,
moduleToImageMap: Map<string, string>,
imageToBuildSettings?: Map<string, BuildSettings>): Promise<void> {
const moduleFile = path.join(modulePath, Constants.moduleManifest);
const name: string = path.basename(modulePath);
const moduleFile = path.join(moduleFullPath, Constants.moduleManifest);
if (await fse.pathExists(moduleFile)) {
const module = await Utility.readJsonAndExpandEnv(moduleFile, Constants.moduleSchemaVersion);
const platformKeys: string[] = Object.keys(module.image.tag.platforms);
const repo: string = module.image.repository;
const version: string = module.image.tag.version;
platformKeys.map((platform) => {
const moduleKey: string = Utility.getModuleKey(name, platform);
const image: string = Utility.getImage(repo, version, platform);
moduleToImageMap.set(moduleKey, image);
const imageKeys: string[] = Utility.getModuleKeyFromPlatform(moduleKeyPrefix, platform);
imageKeys.map((key) => {
moduleToImageMap.set(key, image);
});
if (imageToBuildSettings !== undefined) {
const dockerFilePath = path.resolve(modulePath, module.image.tag.platforms[platform]);
const dockerFilePath = path.resolve(moduleFullPath, module.image.tag.platforms[platform]);
imageToBuildSettings.set(
image,
Utility.getBuildSettings(modulePath,
Utility.getBuildSettings(moduleFullPath,
dockerFilePath, module.image.buildOptions, module.image.contextPath));
}

const defaultPlatform: Platform = Platform.getDefaultPlatform();
if (platform === defaultPlatform.platform) {
moduleToImageMap.set(Utility.getModuleKeyNoPlatform(name, false), image);
} else if (platform === `${defaultPlatform.platform}.debug`) {
moduleToImageMap.set(Utility.getModuleKeyNoPlatform(name, true), image);
}
});
}
}
Expand Down Expand Up @@ -618,6 +613,52 @@ export class Utility {
validateFunc, Constants.moduleNameDft);
}

private static async getSubModules(slnPath: string): Promise<string[]> {
const modulesPath: string = path.join(slnPath, Constants.moduleFolder);
if (!await fse.pathExists(modulesPath)) {
return [];
}
const stat: fse.Stats = await fse.lstat(modulesPath);
if (!stat.isDirectory()) {
return [];
}
return await Utility.getSubDirectories(modulesPath);
}

private static async getExternalModules(templateFilePath: string): Promise<string[]> {
const modules = [];
if (!templateFilePath || !await fse.pathExists(templateFilePath)) {
return modules;
}

const input: string = JSON.stringify(await fse.readJson(templateFilePath), null, 2);
const externalModules: string[] = input.match(Constants.externalModulePlaceholderPattern);

if (externalModules) {
externalModules.map((placeholder) => {
if (placeholder) {
const start = "${MODULEDIR<".length;
const end = placeholder.lastIndexOf(">");
placeholder.substring(start, end);
modules.push(placeholder.substring(start, end).trim());
}
});
}
return modules;
}

private static getModuleKeyFromPlatform(keyPrefix: string, platform: string): string[] {
const keys: string[] = [`${keyPrefix}.${platform}`];
const defaultPlatform: Platform = Platform.getDefaultPlatform();
if (platform !== defaultPlatform.platform && platform !== `${defaultPlatform.platform}.debug`) {
return keys;
}

const isDebug: boolean = (platform === `${defaultPlatform.platform}.debug`);
keys.push(isDebug ? `${keyPrefix}.debug` : keyPrefix);
return keys;
}

private static async validateSolutionName(name: string, parentPath?: string): Promise<string | undefined> {
if (!name || name.trim() === "") {
return "The name could not be empty";
Expand Down
5 changes: 3 additions & 2 deletions src/container/containerManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export class ContainerManager {
if (moduleConfigFilePath) {
const directory = path.dirname(moduleConfigFilePath);
await Utility.loadEnv(path.join(directory, "..", "..", Constants.envFile));
await Utility.loadEnv(path.join(directory, Constants.envFile));
const moduleConfig = await Utility.readJsonAndExpandEnv(moduleConfigFilePath, Constants.moduleSchemaVersion);
const platforms = moduleConfig.image.tag.platforms;
const platform = await vscode.window.showQuickPick(Object.keys(platforms), { placeHolder: Constants.selectPlatform, ignoreFocusOut: true });
Expand Down Expand Up @@ -72,7 +73,7 @@ export class ContainerManager {
const imageToBuildSettings: Map<string, BuildSettings> = new Map();
const slnPath: string = path.dirname(templateFile);
await Utility.loadEnv(path.join(slnPath, Constants.envFile));
await Utility.setSlnModulesMap(slnPath, moduleToImageMap, imageToBuildSettings);
await Utility.setSlnModulesMap(templateFile, moduleToImageMap, imageToBuildSettings);
const configPath: string = path.join(slnPath, Constants.outputConfig);
const deployment: any = await this.generateDeploymentInfo(templateFile, configPath, moduleToImageMap);
const dpManifest: any = deployment.manifestObj;
Expand Down Expand Up @@ -105,7 +106,7 @@ export class ContainerManager {
private async generateDeploymentInfo(templateFile: string,
configPath: string,
moduleToImageMap: Map<string, string>): Promise<any> {
const data: string = await fse.readFile(templateFile, "utf8");
const data: any = await fse.readJson(templateFile);
const moduleExpanded: string = Utility.expandModules(data, moduleToImageMap);
const exceptStr = ["$edgeHub", "$edgeAgent", "$upstream", Constants.SchemaTemplate];
const generatedDeployFile: string = Utility.expandEnv(moduleExpanded, ...exceptStr);
Expand Down
9 changes: 5 additions & 4 deletions src/edge/edgeManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ export class EdgeManager {
default:
break;
}
const debugImageName = `\${${Utility.getModuleKeyNoPlatform(moduleName, true)}}`;
const debugImageName = `\${${Utility.getDefaultModuleKey(Constants.subModuleKeyPrefixTemplate(moduleName), true)}}`;
return { debugImageName, debugCreateOptions };
}

Expand Down Expand Up @@ -558,6 +558,7 @@ export class EdgeManager {
let debugImageName: string = "";
let debugCreateOptions: any = {};
let env: object = null;
const moduleKeyPrefix = Constants.subModuleKeyPrefixTemplate(module);
const thirdPartyModuleTemplate = this.get3rdPartyModuleTemplateByName(template);
if (template === Constants.ACR_MODULE) {
const acrManager = new AcrManager();
Expand All @@ -582,11 +583,11 @@ export class EdgeManager {
if (thirdPartyModuleTemplate.command && thirdPartyModuleTemplate.command.includes(Constants.repositoryNameSubstitution)) {
repositoryName = await this.inputRepository(module);
}
imageName = `\${${Utility.getModuleKeyNoPlatform(module, false)}}`;
debugImageName = `\${${Utility.getModuleKeyNoPlatform(module, true)}}`;
imageName = `\${${Utility.getDefaultModuleKey(moduleKeyPrefix, false)}}`;
debugImageName = `\${${Utility.getDefaultModuleKey(moduleKeyPrefix, true)}}`;
} else {
repositoryName = await this.inputRepository(module);
imageName = `\${${Utility.getModuleKeyNoPlatform(module, false)}}`;
imageName = `\${${Utility.getDefaultModuleKey(moduleKeyPrefix, false)}}`;
const debugSettings = await this.generateDebugCreateOptions(module, template);
debugImageName = debugSettings.debugImageName;
debugCreateOptions = debugSettings.debugCreateOptions;
Expand Down
2 changes: 1 addition & 1 deletion src/intelliSense/configCompletionItemProvider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ export class ConfigCompletionItemProvider implements vscode.CompletionItemProvid

if (IntelliSenseUtility.locationMatch(location, Constants.imgDeploymentManifestJsonPath)) {
const moduleToImageMap: Map<string, string> = new Map();
await Utility.setSlnModulesMap(path.dirname(document.uri.fsPath), moduleToImageMap);
await Utility.setSlnModulesMap(document.uri.fsPath, moduleToImageMap);
return this.getCompletionItems(Array.from(moduleToImageMap.keys()), document, position, location);
}

Expand Down
2 changes: 1 addition & 1 deletion src/intelliSense/configDiagnosticProvider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ export class ConfigDiagnosticProvider {
const moduleToImageMap: Map<string, string> = new Map();

try {
await Utility.setSlnModulesMap(path.dirname(document.uri.fsPath), moduleToImageMap);
await Utility.setSlnModulesMap(document.uri.fsPath, moduleToImageMap);

const rootNode: parser.Node = parser.parseTree(document.getText());
const moduleJsonPath: string[] = Constants.moduleDeploymentManifestJsonPath.slice(0, - 1); // remove the trailing "*" element
Expand Down
2 changes: 1 addition & 1 deletion src/intelliSense/intelliSenseUtility.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export class IntelliSenseUtility {
const imageToBuildSettingsMap: Map<string, BuildSettings> = new Map();

try {
await Utility.setSlnModulesMap(path.dirname(document.uri.fsPath), moduleToImageMap, imageToBuildSettingsMap);
await Utility.setSlnModulesMap(document.uri.fsPath, moduleToImageMap, imageToBuildSettingsMap);

const node: parser.Node = location.previousNode;
const imagePlaceholder: string = Utility.unwrapImagePlaceholder(node.value);
Expand Down
22 changes: 20 additions & 2 deletions test/utility.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ suite("utility tests", () => {
}).timeout(60 * 1000);

test("expandModules", async () => {
const input: string = await fse.readFile(path.resolve(__dirname, "../../testResources/deployment.template.json"), "utf8");
const input = await fse.readJson(path.resolve(__dirname, "../../testResources/deployment.template.json"));
const mapObj: Map<string, string> = new Map<string, string>();
mapObj.set("MODULES.SampleModule.amd64", "test.az.io/filter:0.0.1-amd64");
const generated: string = Utility.expandModules(input, mapObj);
Expand Down Expand Up @@ -136,12 +136,30 @@ suite("utility tests", () => {
const moduleDir = path.resolve(__dirname, "../../testResources/module1");
const moduleToImageMap: Map<string, string> = new Map();
const imageToBuildSettings: Map<string, BuildSettings> = new Map();
await Utility.setModuleMap(moduleDir, moduleToImageMap, imageToBuildSettings);
await Utility.setModuleMap(moduleDir, Constants.subModuleKeyPrefixTemplate(path.basename(moduleDir)), moduleToImageMap, imageToBuildSettings);
assert.equal(moduleToImageMap.size, 7);
assert.equal(moduleToImageMap.get("MODULES.module1"), "localhost:5000/samplemodule:0.0.1-arm32v7");
assert.equal(moduleToImageMap.get("MODULES.module1.debug"), "localhost:5000/samplemodule:0.0.1-arm32v7.debug");
assert.equal(imageToBuildSettings.size, 5);
assert.equal(imageToBuildSettings.get("localhost:5000/samplemodule:0.0.1-amd64").options.length, 8);
sinon.restore();
}).timeout(60 * 1000);

test("setSlnModulesMap", async () => {
sinon.stub(Platform, "getDefaultPlatform").callsFake(() => {
return new Platform("arm32v7", "camera");
});
const slnDir = path.resolve(__dirname, "../../testResources");
const templateFile = path.join(slnDir, "deployment.template.json");
const moduleToImageMap: Map<string, string> = new Map();
const imageToBuildSettings: Map<string, BuildSettings> = new Map();
await Utility.setSlnModulesMap(templateFile, moduleToImageMap, imageToBuildSettings);
assert.equal(moduleToImageMap.size, 7);
assert.equal(moduleToImageMap.get("MODULEDIR<./module1>"), "localhost:5000/samplemodule:0.0.1-arm32v7");
assert.equal(moduleToImageMap.get("MODULEDIR<./module1>.debug"), "localhost:5000/samplemodule:0.0.1-arm32v7.debug");
assert.equal(imageToBuildSettings.size, 5);
assert.equal(imageToBuildSettings.get("localhost:5000/samplemodule:0.0.1-amd64").options.length, 8);
sinon.restore();
}).timeout(60 * 1000);

test("getDisplayName", () => {
Expand Down
10 changes: 10 additions & 0 deletions testResources/deployment.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,16 @@
"image": "${MODULES.SampleModule.amd64}",
"createOptions": "{}"
}
},
"externalmodule": {
"version": "1.0",
"type": "docker",
"status": "running",
"restartPolicy": "always",
"settings": {
"image": "${MODULEDIR<./module1>.debug}",
"createOptions": {}
}
}
}
}
Expand Down

0 comments on commit c06d4c2

Please sign in to comment.