Merge pull request #39 from w3f/gitConfigLoader

added gitConfigLoader

README.md

@@ -62,3 +62,10 @@ validatorAddress: "<validator-001-stash-address>"
 
 # Optional - Deep Check
 This is an optional parameter you can add to force a scan starting from the last 84 era, rather then relying on the onchain account ledger information. Keep it disabled for normal operations.
+
+
+# Optional - Validators from Git
+
+The list of target addresses can be dynamically retrieved (app startup/restart) from a Git file. Check the [GitConfigLoader](src/gitConfigLoader) implementation.  
+
+- [GitLab API](https://docs.gitlab.com/ee/api/repository_files.html)

charts/polkadot-k8s-payouts/Chart.yaml

@@ -1,5 +1,5 @@
 description: Polkadot K8s Payouts
 name: polkadot-k8s-payouts
-version: v1.1.5
-appVersion: v1.1.5
+version: v1.2.0
+appVersion: v1.2.0
 apiVersion: v2

charts/polkadot-k8s-payouts/values.yaml

@@ -6,10 +6,17 @@ image:
 
 config:
   logLevel: info
-  wsEndpoint: "wss://rpc.polkadot.io/"
-  targets:
-  - alias: ALESSIO
-    validatorAddress: "16cdSZUq7kxq6mtoVMWmYXo62FnNGT9jzWjVRUg87CpL9pxP"
+  wsEndpoint: "wss://kusama-rpc.polkadot.io/"
+  # targetsFromGit: #optional
+  #   enabled: false
+  #   targets:
+  #   - platform: GitHub1kv
+  #     private:
+  #       enabled: false
+  #       apiToken: xxx
+  #     network: kusama
+  #     url: https://github.com/w3f/1k-validators-be/raw/master/helmfile.d/config/kusama/otv-backend-prod.yaml.gotmpl
+  targets: [] #optional
   deepCheck: 
     enabled: false
   claim:
@@ -31,9 +38,9 @@ resources:
   requests:
     cpu: "10m"
     memory: "50Mi"
-  limits:
-    cpu: "150m"
-    memory: "150Mi" 
+  # limits:
+  #   cpu: "150m"
+  #   memory: "150Mi" 
 
 cronjob:
   schedule: "0 12 */2 * *"

config/main.sample.yaml

@@ -1,6 +1,21 @@
 logLevel: info
 wsEndpoint: "wss://kusama-rpc.polkadot.io/"
-targets:
+targetsFromGit: #optional
+  enabled: false
+  targets:
+  - platform: GitHub1kv
+    private:
+      enabled: false
+      apiToken: xxx
+    network: kusama
+    url: https://github.com/w3f/1k-validators-be/raw/master/helmfile.d/config/kusama/otv-backend-prod.yaml.gotmpl
+  - platform: GitLab
+    private:
+      enabled: false
+      apiToken: xxx
+    network: kusama
+    url: https://your.gitlab.domain/api/v4/projects/number/repository/files/accounts.yaml/raw?ref=main  
+targets: #optional  
 - alias: validator-000
     validatorAddress: "<validator-000-stash-address>"
 - alias: validator-001

package.json

@@ -1,6 +1,6 @@
 {
   "name": "polkadot-payouts",
-  "version": "1.1.5",
+  "version": "1.2.0",
   "description": "Automated transfers among accounts",
   "repository": "git@github.com:w3f/accountant.git",
   "author": "W3F Infrastructure Team <devops@web3.foundation>",
@@ -25,7 +25,9 @@
     "@w3f/logger": "^0.4.3",
     "async-wait-until": "^1.2.6",
     "bn.js": "^5.1.3",
-    "commander": "^4.1.1"
+    "commander": "^4.1.1",
+    "node-fetch": "^2.6.6",
+    "yaml": "^2.2.1"
   },
   "devDependencies": {
     "@types/chai": "^4.3.0",

src/actions/start.ts

@@ -1,15 +1,37 @@
 import { Client } from '../client';
 import { Config } from '@w3f/config';
 
-import { InputConfig } from '../types';
+import { InputConfig, Target } from '../types';
 import { LoggerSingleton } from '../logger';
 import { Claimer } from '../claimer';
+import { GitConfigLoaderFactory } from '../gitConfigLoader/gitConfigLoaderFactory';
 
+const _loadConfig = async (config: any): Promise<InputConfig> =>{
+    const cfg = new Config<InputConfig>().parse(config);
+    const gitLoaders = new GitConfigLoaderFactory(cfg).makeGitConfigLoaders()
+
+    const gitTargets: Array<Target> = []
+    for (const l of gitLoaders) {
+        const t = await l.downloadAndLoad()
+        gitTargets.push(...t)
+    }
+
+    const seen = new Set();
+    if(!cfg.targets) cfg.targets = []
+    const filteredArr = [...cfg.targets,...gitTargets].filter(el=>{ //priority given to locals over downloaded ones
+        const isDuplicate = seen.has(el.alias);
+        seen.add(el.alias)
+        return !isDuplicate
+    })
+    cfg.targets = filteredArr
+    return cfg
+}
 
 export async function startAction(cmd): Promise<void> {
-    const cfg = new Config<InputConfig>().parse(cmd.config);
+    const cfg = await _loadConfig(cmd.config)
 
     const logger = LoggerSingleton.getInstance(cfg.logLevel)
+    logger.info(`loaded ${cfg.targets.length} targets`)
 
     const api = await new Client(cfg).connect()
 

src/gitConfigLoader/disabled.ts

@@ -0,0 +1,8 @@
+import { GitConfigLoader } from "./gitConfigLoaderInterface";
+import { Target } from "../types";
+
+export class Disabled implements GitConfigLoader {
+  async downloadAndLoad(): Promise<Array<Target>> {    
+    return []
+  }
+}

src/gitConfigLoader/gitConfigLoaderFactory.ts

@@ -0,0 +1,32 @@
+import { InputConfig } from "../types"
+import { Disabled } from "./disabled"
+import { GitConfigLoader } from "./gitConfigLoaderInterface"
+import { GitHub1kv } from "./gitHub1kv"
+import { GitLabPrivate } from "./gitLabPrivate"
+
+export class GitConfigLoaderFactory {
+  constructor(private readonly cfg: InputConfig){}
+  makeGitConfigLoaders = (): Array<GitConfigLoader> => {
+
+    const gitConfig = this.cfg.targetsFromGit
+
+    if(!gitConfig?.enabled)
+      return [new Disabled()]
+
+    const result: Array<GitConfigLoader> = []
+    for (const target of gitConfig.targets) {
+      switch (target.platform.toLowerCase()) {
+
+        case "github1kv":
+          result.push(new GitHub1kv(target.url))
+          break;
+        case "gitlab":
+          result.push(new GitLabPrivate(target.url,target.private.apiToken,target.network))
+          break;
+        default:
+          result.push(new Disabled())
+      }
+    }
+    return result 
+  }
+}

src/gitConfigLoader/gitConfigLoaderInterface.ts

@@ -0,0 +1,5 @@
+import { Target } from "../types";
+
+export interface GitConfigLoader {
+  downloadAndLoad(): Promise<Array<Target>>;
+}

src/gitConfigLoader/gitHub1kv.ts

@@ -0,0 +1,28 @@
+import { GitConfigLoader } from "./gitConfigLoaderInterface";
+import fetch from 'node-fetch';
+import { Target } from "../types";
+import { TargetFromGit1kv } from "./types";
+import { parse } from 'yaml'
+
+export class GitHub1kv implements GitConfigLoader {
+
+  constructor(
+    protected readonly url: string
+    ) { }
+
+  async downloadAndLoad(): Promise<Array<Target>> {
+    const response = await fetch(this.url);
+    let data = await response.text();
+    // based on the shape of https://github.com/w3f/1k-validators-be/blob/master/helmfile.d/config/kusama/otv-backend-prod.yaml.gotmpl
+    data = data.replace(/{{.*}}/gm, '')
+    const candidates: Array<TargetFromGit1kv> = JSON.parse(parse(data)["config"])["scorekeeper"]["candidates"]
+
+    return candidates.map(c=>{
+      const target: Target = {
+        alias: c.name,
+        validatorAddress: c.stash
+      } 
+      return target
+    })
+  }
+}

src/gitConfigLoader/gitLabPrivate.ts

@@ -0,0 +1,48 @@
+import { GitConfigLoader } from "./gitConfigLoaderInterface";
+import fetch from 'node-fetch';
+import fs from 'fs';
+import { Config } from '@w3f/config';
+import { Target } from "../types";
+import { GitLabTarget, InputConfigFromGitLabPrivate } from "./types";
+
+export class GitLabPrivate implements GitConfigLoader {
+
+  constructor(
+    protected readonly url: string, 
+    protected readonly apiToken: string, 
+    protected readonly network: string
+    ) { }
+
+  async downloadAndLoad(): Promise<Array<Target>> {
+    const response = await fetch(this.url, {
+    headers: {
+        'PRIVATE-TOKEN': this.apiToken
+    }
+    });
+    const data = await response.text();
+    if(!response.ok) throw new Error("git config download probelm: " + data)
+
+    fs.writeFileSync("./tmp.yaml",data)
+    const cfg = new Config<InputConfigFromGitLabPrivate>().parse("./tmp.yaml");
+    fs.rmSync("./tmp.yaml")
+
+    let tmp: Array<GitLabTarget> = [];
+    switch (this.network.toLowerCase()) {
+      case "kusama":
+        tmp = cfg.Kusama
+        break;
+      case "polkadot":
+        tmp = cfg.Polkadot
+        break;
+      default:
+        throw new Error("unexpected configuration")
+    }
+    return tmp.map(t=>{
+      const target: Target = {
+        alias: t.name,
+        validatorAddress: t.address
+      } 
+      return target
+    })
+  }
+}

src/gitConfigLoader/types.ts

@@ -0,0 +1,19 @@
+//https://raw.githubusercontent.com/w3f/1k-validators-be/master/helmfile.d/config/kusama/otv-backend-prod.yaml.gotmpl
+export interface TargetFromGit1kv {
+  name: string;
+  stash: string;
+}
+
+/*********************/
+
+export interface GitLabTarget {
+  name: string;
+  address: string;
+}
+
+export interface InputConfigFromGitLabPrivate {
+  Kusama: Array<GitLabTarget>;
+  Polkadot: Array<GitLabTarget>;
+}
+
+/*********************/

src/types.ts

@@ -9,7 +9,7 @@ export interface Target {
 }
 
 export interface ClaimerInputConfig {
-    targets: Array<Target>;
+    targets?: Array<Target>;
     deepCheck: {
       enabled: boolean;
     };
@@ -19,6 +19,18 @@ export interface ClaimerInputConfig {
       batchSize: number;
       claimerKeystore: Keystore;
     };
+    targetsFromGit?: {
+      enabled: boolean;
+      targets: Array<{
+        platform: string;
+        private: {
+            enabled: boolean;
+            apiToken: string;
+        };
+        network?: string;
+        url: string;
+      }>;
+    };
 }
 
 export interface InputConfig extends ClaimerInputConfig {