Merge sync node to develop

.github/workflows/publish.yml

@@ -29,9 +29,11 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install python-setuptools
+
       - name: Set Versions
         run: |
           bash ./scripts/set_versions_ga.sh
+
       - name: Set release
         run: |
           if [[ "$BRANCH" == "stable" ]]; then
@@ -56,7 +58,8 @@ jobs:
         run: |
           echo "::set-output name=version::$VERSION"
           echo "::set-output name=branch::$BRANCH"
-  build_and_publish:
+
+  build_and_publish_normal:
     if: github.event.pull_request.merged
     needs: create_release
     name: Build and publish for ${{ matrix.os }}
@@ -81,11 +84,11 @@ jobs:
     - name: Checkout submodules
       run: git submodule update --init
 
-    - name: Build binary
+    - name: Build normal binary
       run: |
         mkdir ./dist
         docker build . -t node-cli-builder
-        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }}
+        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} normal
         ls -altr /home/ubuntu/dist/
         docker rm -f $(docker ps -aq)
 
@@ -114,3 +117,62 @@ jobs:
         asset_path: /home/ubuntu/dist/sha512sum
         asset_name: ${{ matrix.asset_name }}.sha512
         asset_content_type: text/plain
+
+  build_and_publish_sync:
+    if: github.event.pull_request.merged
+    needs: create_release
+    name: Build and publish for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-20.04
+            asset_name: skale-${{ needs.create_release.outputs.version }}-Linux-x86_64-sync
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.11
+
+    - name: Install ubuntu dependencies
+      if: matrix.os == 'ubuntu-20.04'
+      run: |
+        sudo apt-get update
+
+    - name: Checkout submodules
+      run: git submodule update --init
+
+    - name: Build sync release binary
+      run: |
+        mkdir ./dist
+        docker build . -t node-cli-builder
+        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} sync
+        ls -altr /home/ubuntu/dist/
+        docker rm -f $(docker ps -aq)
+
+    - name: Save sha512sum
+      run: |
+        sudo sha512sum /home/ubuntu/dist/${{ matrix.asset_name }} | sudo tee > /dev/null /home/ubuntu/dist/sha512sum
+
+    - name: Upload release sync CLI
+      id: upload-sync-release-asset
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.create_release.outputs.upload_url }}
+        asset_path: /home/ubuntu/dist/${{ matrix.asset_name }}
+        asset_name: ${{ matrix.asset_name }}
+        asset_content_type: application/octet-stream
+
+    - name: Upload release sync CLI checksum
+      id: upload-sync-release-checksum
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.create_release.outputs.upload_url }}
+        asset_path: /home/ubuntu/dist/sha512sum
+        asset_name: ${{ matrix.asset_name }}.sha512
+        asset_content_type: text/plain

.github/workflows/test.yml

@@ -40,18 +40,34 @@ jobs:
       run: |
         mkdir ./dist
         docker build . -t node-cli-builder
-        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test
+        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test normal
         docker rm -f $(docker ps -aq)
 
-    - name: Check build
+    - name: Check build - normal
       run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64
 
     - name: Build binary in Ubuntu 20.04 environment
       run: |
         scripts/build.sh test test
 
-    - name: Check build
+    - name: Check build - normal
       run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64
+    - name: Build sync binary in Ubuntu 18.04 environment
+      run: |
+        mkdir ./dist
+        docker build . -t node-cli-builder
+        docker run  -v /home/ubuntu/dist:/app/dist node-cli-builder scripts/build.sh test test sync
+        docker rm -f $(docker ps -aq)
+
+    - name: Check build - sync
+      run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync
+
+    - name: Build sync binary in Ubuntu 20.04 environment
+      run: |
+        scripts/build.sh test test sync
+
+    - name: Check build - sync
+      run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync
 
     - name: Run tests
       run: bash ./scripts/run_tests.sh

.gitignore

@@ -116,4 +116,7 @@ meta.json
 disk_mountpoint.txt
 sgx_server_url.txt
 resource_allocation.json
-conf.json
+conf.json
+test-env
+
+nginx.conf

README.md

@@ -19,8 +19,11 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line
     2.7 [Logs](#logs-commands)  
     2.8 [Resources allocation](#resources-allocation-commands)  
     2.9 [Validate](#validate-commands)  
-3.  [Exit codes](#exit-codes)
-4.  [Development](#development)
+3.  [Sync CLI usage](#sync-cli-usage)  
+    3.1 [Top level commands](#top-level-commands-sync)  
+    3.2 [Sync node commands](#sync-node-commands)  
+4.  [Exit codes](#exit-codes)
+5.  [Development](#development)
 
 ## Installation
 
@@ -34,10 +37,10 @@ Ensure that the following package is installed: **docker**, **docker-compose** (
 VERSION_NUM={put the version number here} && sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m` >  /usr/local/bin/skale"
 ```
 
-For versions `<1.1.0`:
+For Sync node version:
 
 ```shell
-VERSION_NUM=0.0.0 && sudo -E bash -c "curl -L https://skale-cli.sfo2.cdn.digitaloceanspaces.com/skale-$VERSION_NUM-`uname -s`-`uname -m` >  /usr/local/bin/skale"
+VERSION_NUM={put the version number here} && sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m`-sync >  /usr/local/bin/skale"
 ```
 
 -   Apply executable permissions to the downloaded binary:
@@ -104,7 +107,7 @@ skale node init [ENV_FILE]
 
 Arguments:
 
-- `ENV_FILE` - path to .env file (required parameters are listed in the `skale init` command)
+- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command)
 
 You should specify the following environment variables:
 
@@ -117,6 +120,7 @@ You should specify the following environment variables:
 -   `MANAGER_CONTRACTS_ABI_URL` - URL to SKALE Manager contracts ABI and addresses
 -   `IMA_CONTRACTS_ABI_URL` - URL to IMA contracts ABI and addresses
 -   `FILEBEAT_URL` - URL to the Filebeat log server
+-   `ENV_TYPE` - environement type (mainnet, testnet, etc)
 
 
 Optional variables:
@@ -136,7 +140,7 @@ skale node restore [BACKUP_PATH] [ENV_FILE]
 Arguments:
 
 - `BACKUP_PATH` - path to the archive with backup data generated by `skale node backup` command
-- `ENV_FILE` - path to .env file (required parameters are listed in the `skale init` command)
+- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command)
 
 #### Node backup
 
@@ -177,7 +181,7 @@ skale node update [ENV_FILEPATH]
 
 Options:
 
--   `--yes` - remove without additional confirmation
+-   `--yes` - update without additional confirmation
 
 Arguments:
 
@@ -197,7 +201,7 @@ skale node turn-off
 Options:
 
 -   `--maintenance-on` - set SKALE node into maintenance mode before turning off
--   `--yes` - remove without additional confirmation
+-   `--yes` - turn off without additional confirmation
 
 #### Node turn-on
 
@@ -210,7 +214,7 @@ skale node turn-on [ENV_FILEPATH]
 Options:
 
 -   `--maintenance-off` - turn off maintenance mode after turning on the node
--   `--yes` - remove without additional confirmation
+-   `--yes` - turn on without additional confirmation
 
 Arguments:
 
@@ -477,7 +481,7 @@ skale resources-allocation generate [ENV_FILE]
 
 Arguments:
 
-- `ENV_FILE` - path to .env file (required parameters are listed in the `skale init` command)
+- `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command)
 
 Options:
 
@@ -500,6 +504,83 @@ Options:
 
 -   `--json` - show validation result in json format 
 
+
+## Sync CLI usage
+
+### Top level commands sync
+
+#### Info
+
+Print build info
+
+```shell
+skale info
+```
+
+#### Version
+
+Print version number
+
+```shell
+skale version
+```
+
+Options:
+
+-   `--short` - prints version only, without additional text.
+
+### Sync node commands
+
+> Prefix: `skale sync-node`
+
+#### Sync node initialization
+
+Initialize full sync SKALE node on current machine
+
+```shell
+skale sync-node init [ENV_FILE]
+```
+
+Arguments:
+
+- `ENV_FILE` - path to .env file (required parameters are listed in the `skale sync-node init` command)
+
+You should specify the following environment variables:
+
+-   `DISK_MOUNTPOINT` - disk mount point for storing sChains data
+-   `DOCKER_LVMPY_STREAM` - stream of `docker-lvmpy` to use
+-   `CONTAINER_CONFIGS_STREAM` - stream of `skale-node` to use
+-   `ENDPOINT` - RPC endpoint of the node in the network where SKALE Manager is deployed
+-   `MANAGER_CONTRACTS_ABI_URL` - URL to SKALE Manager contracts ABI and addresses
+-   `IMA_CONTRACTS_ABI_URL` - URL to IMA contracts ABI and addresses
+-   `SCHAIN_NAME` - name of the SKALE chain to sync
+-   `ENV_TYPE` - environement type (mainnet, testnet, etc)
+
+
+Options:
+
+-   `--archive` - Run sync node in an archive node (disable block rotation)
+-   `--historic-state` - Enable historic state (works only in pair with --archive flag)
+-   `--catchup` - Add a flag to start sync node in catchup mode
+
+#### Sync node update
+
+Update full sync SKALE node on current machine
+
+```shell
+skale sync-node update [ENV_FILEPATH]
+```
+
+Options:
+
+- `--yes` - update without additional confirmation
+
+Arguments:
+
+- `ENV_FILEPATH` - path to env file where parameters are defined
+
+> NOTE: You can just update a file with environment variables used during `skale sync-node init`.
+
 ## Exit codes
 
 Exit codes conventions for SKALE CLI tools

node_cli/cli/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.3.1'
+__version__ = '2.4.0'
 
 if __name__ == "__main__":
     print(__version__)

node_cli/cli/node.py

@@ -2,7 +2,7 @@
 #
 #   This file is part of node-cli
 #
-#   Copyright (C) 2019 SKALE Labs
+#   Copyright (C) 2019-Present SKALE Labs
 #
 #   This program is free software: you can redistribute it and/or modify
 #   it under the terms of the GNU Affero General Public License as published by

node_cli/cli/ssl.py

@@ -21,9 +21,9 @@
 from terminaltables import SingleTable
 
 from node_cli.utils.exit_codes import CLIExitCodes
-from node_cli.utils.helper import get_request, safe_load_texts, error_exit
-from node_cli.configs import DEFAULT_SSL_CHECK_PORT, SSL_CERT_FILEPATH, SSL_KEY_FILEPATH
-from node_cli.core.ssl import check_cert, upload_cert
+from node_cli.utils.helper import safe_load_texts, error_exit
+from node_cli.configs.ssl import DEFAULT_SSL_CHECK_PORT, SSL_CERT_FILEPATH, SSL_KEY_FILEPATH
+from node_cli.core.ssl import check_cert, upload_cert, cert_status
 
 
 TEXTS = safe_load_texts()
@@ -42,10 +42,7 @@ def ssl():
 
 @ssl.command(help="Status of the SSL certificates on the node")
 def status():
-    status, payload = get_request(
-        blueprint=BLUEPRINT_NAME,
-        method='status'
-    )
+    status, payload = cert_status()
     if status == 'ok':
         if payload.get('is_empty'):
             print(TEXTS['ssl']['no_cert'])
@@ -79,7 +76,7 @@ def upload(key_path, cert_path, force):
     if status == 'ok':
         print(TEXTS['ssl']['uploaded'])
     else:
-        error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE)
+        error_exit(payload, exit_code=CLIExitCodes.FAILURE)
 
 
 @ssl.command(help="Check certificates")