Updates CI job to ensure docs for modules are up to date (#171)

Updates CI job to ensure docs for modules  are up to date

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.0
+current_version = 2.0.1
 commit = True
 message = Bumps version to {new_version}
 tag = False

.gitignore

@@ -101,3 +101,7 @@ htmlcov
 *.tfstate.backup
 .terraform/
 terraform.log
+
+
+# Templated module MD
+MAIN.md

.travis.yml

@@ -10,27 +10,23 @@ env:
 jobs:
   include:
     - stage: test
-      env:
-        - JOB="Terraform Format Verification"
+      name: "Terraform Format Verification"
       script:
         - make terraform/lint
     - stage: test
-      env:
-        - JOB="JSON Lint/Format"
+      name: "JSON Lint/Format"
       script:
         - make json/lint
     - stage: test
-      env:
-        - JOB="CFN Lint"
+      name: "CFN Lint"
       install:
         - pip install -r _requirements/cfn-lint.txt
         - cfn-lint --version
       language: python
       script:
         - make cfn/lint
     - stage: test
-      env:
-        - JOB="EditorConfig Lint"
+      name: "EditorConfig Lint"
       install:
         - npm install -g eclint
         - eclint --version
@@ -39,18 +35,27 @@ jobs:
       script:
         - make eclint/lint
     - stage: test
-      env:
-        - JOB="CFN Version Validation"
+      name: "CFN Version Validation"
       install:
         - pip install -r _requirements/cfn-version-check.txt
         - yq --version
       language: python
       script:
         - make cfn/version VERSION=$RELEASE_VERSION
+    - stage: test
+      name: "Module Docs Lint"
+      install:
+        - pip install -r _requirements/jinja2-cli.txt
+        - make terraform-docs/install
+        - make terraform/install
+        - terraform-docs --version
+      language: python
+      script:
+        - make docs/lint/modules
     - stage: deploy
+      name: "Deploy to GitHub Releases"
       if: branch = env(RELEASE_BRANCH) AND tag IS blank
       env:
-        - JOB="Deploy to GitHub Releases"
         - PRIOR_VERSION=$(git describe --abbrev=0 --tags)
         - RELEASE_BODY="* [$REPO v$RELEASE_VERSION changes](https://github.com/$OWNER/$REPO/compare/$PRIOR_VERSION...$RELEASE_VERSION)"
       before_script:

Makefile

@@ -6,6 +6,8 @@ XARGS ?= xargs -I {}
 BIN_DIR ?= ${HOME}/bin
 PATH := $(BIN_DIR):$(PATH)
 
+FIND_EXCLUDES ?= -not \( -name .terraform -prune \)
+
 VERSION ?= $$(grep -E '^current_version' .bumpversion.cfg | sed 's/^.*= //')
 
 MAKEFLAGS += --no-print-directory
@@ -15,38 +17,80 @@ SHELL := bash
 
 .PHONY: %/lint %/format %/install
 
+GITHUB_ACCESS_TOKEN ?= 4224d33b8569bec8473980bb1bdb982639426a92
+# Macro to return the download url for a github release
+# For latest release, use version=latest
+# To pin a release, use version=tags/<tag>
+# $(call parse_github_download_url,owner,repo,version,asset select query)
+parse_github_download_url = $(CURL) https://api.github.com/repos/$(1)/$(2)/releases/$(3)?access_token=$(GITHUB_ACCESS_TOKEN) | jq --raw-output  '.assets[] | select($(4)) | .browser_download_url'
+
+# Macro to download a github binary release
+# $(call download_github_release,file,owner,repo,version,asset select query)
+download_github_release = $(CURL) -o $(1) $(shell $(call parse_github_download_url,$(2),$(3),$(4),$(5)))
+
+# Macro to download a hashicorp archive release
+# $(call download_hashicorp_release,file,app,version)
+download_hashicorp_release = $(CURL) -o $(1) https://releases.hashicorp.com/$(2)/$(3)/$(2)_$(3)_$(OS)_$(ARCH).zip
+
 # Macro to download a hashicorp archive release
 # $(call download_hashicorp_release,file,app,version)
 download_hashicorp_release = $(CURL) -o $(1) https://releases.hashicorp.com/$(2)/$(3)/$(2)_$(3)_$(OS)_$(ARCH).zip
 
+guard/env/%:
+	@ _="$(or $($*),$(error Make/environment variable '$*' not present))"
+
 guard/program/%:
 	@ which $* > /dev/null || $(MAKE) $*/install
 
-terraform/install: TERRAFORM_VERSION ?= $(shell $(CURL) https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')
-terraform/install: | $(BIN_DIR)
+$(BIN_DIR):
+	@ echo "[make]: Creating directory '$@'..."
+	mkdir -p $@
+
+install/gh-release/%: guard/env/FILENAME guard/env/OWNER guard/env/REPO guard/env/VERSION guard/env/QUERY
+install/gh-release/%:
+	@ echo "[$@]: Installing $*..."
+	$(call download_github_release,$(FILENAME),$(OWNER),$(REPO),$(VERSION),$(QUERY))
+	chmod +x $(FILENAME)
+	$* --version
+	@ echo "[$@]: Completed successfully!"
+
+terraform/install: TERRAFORM_VERSION_LATEST := $(CURL) https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version' | sed 's/^v//'
+terraform/install: TERRAFORM_VERSION ?= $(shell $(TERRAFORM_VERSION_LATEST))
+terraform/install: | $(BIN_DIR) guard/program/jq
 	@ echo "[$@]: Installing $(@D)..."
 	$(call download_hashicorp_release,$(@D).zip,$(@D),$(TERRAFORM_VERSION))
 	unzip $(@D).zip && rm -f $(@D).zip && chmod +x $(@D)
 	mv $(@D) "$(BIN_DIR)"
 	$(@D) --version
 	@ echo "[$@]: Completed successfully!"
 
-jq/install: JQ_VERSION ?= jq-1.5
-jq/install: JQ_URL ?= https://github.com/stedolan/jq/releases/download/$(JQ_VERSION)/jq-linux64
+tfdocs-awk/install: $(BIN_DIR)
+tfdocs-awk/install: ARCHIVE := https://github.com/plus3it/tfdocs-awk/archive/master.tar.gz
+tfdocs-awk/install:
+	$(CURL) $(ARCHIVE) | tar -C $(BIN_DIR) --strip-components=1 --wildcards '*.sh' --wildcards '*.awk' -xzvf -
+
+terraform-docs/install: TFDOCS_VERSION ?= latest
+terraform-docs/install: | $(BIN_DIR) guard/program/jq
+	@ $(MAKE) install/gh-release/$(@D) FILENAME="$(BIN_DIR)/$(@D)" OWNER=segmentio REPO=$(@D) VERSION=$(TFDOCS_VERSION) QUERY='.name | endswith("$(OS)-$(ARCH)")'
+
+jq/install: JQ_VERSION ?= latest
 jq/install: | $(BIN_DIR)
-	@ echo "[$@]: Installing $(@D)..."
-	@ echo "[$@]: JQ_URL=$(JQ_URL)"
-	$(CURL) -o $(BIN_DIR)/$(@D) "$(JQ_URL)"
-	chmod +x $(BIN_DIR)/$(@D)
+	@ $(MAKE) install/gh-release/$(@D) FILENAME="$(BIN_DIR)/$(@D)" OWNER=stedolan REPO=$(@D) VERSION=$(JQ_VERSION) QUERY='.name | endswith("$(OS)64")'
+
+shellcheck/install: SHELLCHECK_VERSION ?= latest
+shellcheck/install: SHELLCHECK_URL ?= https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz
+shellcheck/install: $(BIN_DIR) guard/program/xz
+	$(CURL) $(SHELLCHECK_URL) | tar -xJv
+	mv $(@D)-*/$(@D) $(BIN_DIR)
+	rm -rf $(@D)-*
 	$(@D) --version
-	@ echo "[$@]: Completed successfully!"
 
 terraform/lint: | guard/program/terraform
 	@ echo "[$@]: Linting Terraform files..."
 	terraform fmt -check=true -diff=true
 	@ echo "[$@]: Terraform files PASSED lint test!"
 
-json/%: FIND_JSON := find . -name '*.json' -type f
+json/%: FIND_JSON := find . $(FIND_EXCLUDES) -name '*.json' -type f
 json/lint: | guard/program/jq
 	@ echo "[$@]: Linting JSON files..."
 	$(FIND_JSON) | $(XARGS) bash -c 'cmp {} <(jq --indent 4 -S . {}) || (echo "[{}]: Failed JSON Lint Test"; exit 1)'
@@ -57,14 +101,16 @@ json/format: | guard/program/jq
 	$(FIND_JSON) | $(XARGS) bash -c 'echo "$$(jq --indent 4 -S . "{}")" > "{}"'
 	@ echo "[$@]: Successfully formatted JSON files!"
 
-sh/%: FIND_SH ?= find . -name '*.sh' -type f
+sh/%: FIND_SH := find . $(FIND_EXCLUDES) -name '*.sh' -type f -print0
 sh/lint: | guard/program/shellcheck
+	@ echo "[$@]: Linting shell scripts..."
 	$(FIND_SH) | $(XARGS) shellcheck {}
+	@ echo "[$@]: Shell scripts PASSED lint test!"
 
 yaml/lint: | guard/program/yamllint
 	yamllint --strict .
 
-cfn/%: FIND_CFN ?= find . -name '*.template.cfn.*' -type f
+cfn/%: FIND_CFN ?= find . $(FIND_EXCLUDES) -name '*.template.cfn.*' -type f
 cfn/lint: | guard/program/cfn-lint
 	$(FIND_CFN) | $(XARGS) cfn-lint -t {}
 
@@ -75,3 +121,39 @@ eclint/lint: | guard/program/eclint
 	@ echo "[$@]: Linting all files with eclint..."
 	eclint check
 	@ echo "[$@]: All files PASSED lint test!"
+
+docs/%: FOUND_DIR ?= find ${@F} -type d \( -name "lx*" -o -name "win*" \)
+docs/%: README_DIR ?= .
+docs/%: README_BASE ?= _docs/MAIN.md.jinja
+docs/%: README_MAIN := $(README_DIR)/MAIN.md
+docs/%: README_PARTS ?= $(README_MAIN) <(echo) <($(BIN_DIR)/terraform-docs.sh markdown table $(README_DIR))
+docs/%: README_FILE ?= $(README_DIR)/README.md
+
+docs/template:
+	@ echo "[$@/$(README_DIR)]: Templating doc..."
+	jinja2 -D name=$(README_DIR) $(README_BASE) > $(README_MAIN)
+
+docs/lint: | guard/program/terraform-docs
+	@ echo "[$@/$(README_DIR)]: Linting documentation file.."
+	diff $(README_FILE) <(cat $(README_PARTS))
+	@ echo "[$@/$(README_DIR)]: Documentation file PASSED lint test!"
+
+docs/generate: | guard/program/terraform-docs
+	@ echo "[$@/$(README_DIR)]: Creating documentation file.."
+	cat $(README_PARTS) > $(README_FILE)
+	@ echo "[$@/$(README_DIR)]: Completed successfully!"
+
+docs/template/%: | %
+	@ echo "[$@]: Templating documentation main file.."
+	$(FOUND_DIR) | $(XARGS) $(MAKE) docs/template README_DIR={}
+	@ echo "[$@]: Completed successfully!"
+
+docs/lint/%: | docs/template/% tfdocs-awk/install
+	@ echo "[$@]: Starting documentation linting for $(@F).."
+	$(FOUND_DIR) | $(XARGS) $(MAKE) docs/lint README_DIR={}
+	@ echo "[$@]: Linting documents for $(@F) complete!"
+
+docs/generate/%: | docs/template/% tfdocs-awk/install
+	@ echo "[$@]: Start generating documentations for $(@F).."
+	$(FOUND_DIR) | $(XARGS) $(MAKE) docs/generate README_DIR={}
+	@ echo "[$@]: Generate documentations for $(@F) complete!"

_docs/MAIN.md.jinja

@@ -0,0 +1,5 @@
+{%- set modules = {'modules/lx-autoscale': 'Linux AutoScaling Group', 'modules/lx-instance': 'Linux Instance', 'modules/win-autoscale':'Windows AutoScaling Group', 'modules/win-instance': 'Windows Instance'} -%}
+# Terraform Module - Watchmaker {{ modules[name] }}
+
+This module utilizes the Terraform CloudFormation resource in conjunction
+with an AWS CloudFormation template to deploy a Watchmaker {{ modules[name] }}.

_requirements/jinja2-cli.txt

@@ -0,0 +1 @@
+jinja2-cli==0.7.0

modules/lx-autoscale/README.md

@@ -1,4 +1,47 @@
 # Terraform Module - Watchmaker Linux AutoScaling Group
 
-This module utilizes the Terraform Cloudformation data source in conjunction with an AWS Cloudformation template
-to deploy a Watchmaker Linux Autoscaling group.
+This module utilizes the Terraform CloudFormation resource in conjunction
+with an AWS CloudFormation template to deploy a Watchmaker Linux AutoScaling Group.
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| AmiDistro | (Required) Linux distro of the AMI | string | n/a | yes |
+| AmiId | (Required) ID of the AMI to launch | string | n/a | yes |
+| AppScriptParams | (Optional) Parameter string to pass to the application script. Ignored if AppScriptUrl is blank | string | `"null"` | no |
+| AppScriptShell | (Optional) Shell with which to execute the application script. Ignored if AppScriptUrl is blank | string | `"bash"` | no |
+| AppScriptUrl | (Optional) S3 URL to the application script in an S3 bucket (s3://). Leave blank to launch without an application script. If specified, an appropriate InstanceRole is required | string | `"null"` | no |
+| AppVolumeDevice | (Optional) Device to mount an extra EBS volume. Leave blank to launch without an extra application volume | string | `"null"` | no |
+| AppVolumeMountPath | (Optional) Filesystem path to mount the extra app volume. Ignored if AppVolumeDevice is blank | string | `"/opt/data"` | no |
+| AppVolumeSize | (Optional) Size in GB of the EBS volume to create. Ignored if AppVolumeDevice is blank | string | `"1"` | no |
+| AppVolumeType | (Optional) Type of EBS volume to create. Ignored if AppVolumeDevice is blank | string | `"gp2"` | no |
+| CfnEndpointUrl | (Optional) URL to the CloudFormation Endpoint. e.g. https://cloudformation.us-east-1.amazonaws.com | string | `"https://cloudformation.us-east-1.amazonaws.com"` | no |
+| CfnGetPipUrl | (Optional) URL to get-pip.py | string | `"https://bootstrap.pypa.io/2.6/get-pip.py"` | no |
+| EnableRepos | (Optional) Specify repos to be enabled by yum-config-manager | string | `"null"` | no |
+| InstanceRole | (Optional) IAM instance role to apply to the instance | string | `"null"` | no |
+| InstanceType | (Optional) Amazon EC2 instance type | string | `"t2.micro"` | no |
+| KeyPairName | (Required) Public/private key pairs allow you to securely connect to your instance after it launches | string | n/a | yes |
+| Name | (Required) Name of CloudFormation Stack | string | n/a | yes |
+| NoPublicIp | (Optional) Controls whether to assign the instance a public IP. Recommended to leave at true _unless_ launching in a public subnet | bool | `"true"` | no |
+| NoReboot | (Optional) Controls whether to reboot the instance as the last step of cfn-init execution | bool | `"false"` | no |
+| NoUpdates | (Optional) Controls whether to run yum update during a stack update (On the initial instance launch, Watchmaker _always_ installs updates) | bool | `"false"` | no |
+| PrivateIp | (Optional) Set a static, primary private IP. Leave blank to auto-select a free IP | string | `"null"` | no |
+| PypiIndexUrl | (Optional) URL to the PyPi Index | string | `"https://pypi.org/simple"` | no |
+| SecurityGroupIds | (Required) List of security groups to apply to the instance | string | n/a | yes |
+| SubnetId | (Required) ID of the subnet to assign to the instance | string | n/a | yes |
+| ToggleCfnInitUpdate | (Optional) A/B toggle that forces a change to instance metadata, triggering the cfn-init update sequence | string | `"A"` | no |
+| WatchmakerAdminGroups | (Optional) Colon-separated list of domain groups that should have admin permissions on the EC2 instance | string | `"null"` | no |
+| WatchmakerAdminUsers | (Optional) Colon-separated list of domain users that should have admin permissions on the EC2 instance | string | `"null"` | no |
+| WatchmakerComputerName | (Optional) Sets the hostname/computername within the OS | string | `"null"` | no |
+| WatchmakerConfig | (Optional) URL to a Watchmaker config file | string | `"null"` | no |
+| WatchmakerEnvironment | (Optional) Environment in which the instance is being deployed | string | `"null"` | no |
+| WatchmakerOuPath | (Optional) DN of the OU to place the instance when joining a domain. If blank and WatchmakerEnvironment enforces a domain join, the instance will be placed in a default container. Leave blank if not joining a domain, or if WatchmakerEnvironment is false | string | `"null"` | no |
+| WatchmakerS3Source | (Optional) Flag that tells watchmaker to use its instance role to retrieve watchmaker content from S3 | bool | `"false"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| watchmaker-lx-instance-stack | CloudFormation stack object for watchmaker-lx-instance-stack |
+

modules/lx-autoscale/watchmaker-lx-autoscale.template.cfn.yaml

@@ -205,7 +205,7 @@ Metadata:
         default: Force Cfn Init Update
       ToggleNewInstances:
         default: Force New Instances
-  Version: 2.0.0
+  Version: 2.0.1
 Outputs:
   ScaleDownScheduledAction:
     Condition: UseScheduledAction

modules/lx-instance/README.md

@@ -1,4 +1,47 @@
 # Terraform Module - Watchmaker Linux Instance
 
-This module utilizes the Terraform Cloudformation data source in conjunction with an AWS Cloudformation template
-to deploy a Watchmaker Linux instance.
+This module utilizes the Terraform CloudFormation resource in conjunction
+with an AWS CloudFormation template to deploy a Watchmaker Linux Instance.
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| AmiDistro | (Required) Linux distro of the AMI | string | n/a | yes |
+| AmiId | (Required) ID of the AMI to launch | string | n/a | yes |
+| AppScriptParams | (Optional) Parameter string to pass to the application script. Ignored if AppScriptUrl is blank | string | `"null"` | no |
+| AppScriptShell | (Optional) Shell with which to execute the application script. Ignored if AppScriptUrl is blank | string | `"bash"` | no |
+| AppScriptUrl | (Optional) S3 URL to the application script in an S3 bucket (s3://). Leave blank to launch without an application script. If specified, an appropriate InstanceRole is required | string | `"null"` | no |
+| AppVolumeDevice | (Optional) Device to mount an extra EBS volume. Leave blank to launch without an extra application volume | string | `"null"` | no |
+| AppVolumeMountPath | (Optional) Filesystem path to mount the extra app volume. Ignored if AppVolumeDevice is blank | string | `"/opt/data"` | no |
+| AppVolumeSize | (Optional) Size in GB of the EBS volume to create. Ignored if AppVolumeDevice is blank | string | `"1"` | no |
+| AppVolumeType | (Optional) Type of EBS volume to create. Ignored if AppVolumeDevice is blank | string | `"gp2"` | no |
+| CfnEndpointUrl | (Optional) URL to the CloudFormation Endpoint. e.g. https://cloudformation.us-east-1.amazonaws.com | string | `"https://cloudformation.us-east-1.amazonaws.com"` | no |
+| CfnGetPipUrl | (Optional) URL to get-pip.py | string | `"https://bootstrap.pypa.io/2.6/get-pip.py"` | no |
+| EnableRepos | (Optional) Specify repos to be enabled by yum-config-manager | string | `"null"` | no |
+| InstanceRole | (Optional) IAM instance role to apply to the instance | string | `"null"` | no |
+| InstanceType | (Optional) Amazon EC2 instance type | string | `"t2.micro"` | no |
+| KeyPairName | (Required) Public/private key pairs allow you to securely connect to your instance after it launches | string | n/a | yes |
+| Name | (Required) Name of CloudFormation Stack | string | n/a | yes |
+| NoPublicIp | (Optional) Controls whether to assign the instance a public IP. Recommended to leave at true _unless_ launching in a public subnet | bool | `"true"` | no |
+| NoReboot | (Optional) Controls whether to reboot the instance as the last step of cfn-init execution | bool | `"false"` | no |
+| NoUpdates | (Optional) Controls whether to run yum update during a stack update (On the initial instance launch, Watchmaker _always_ installs updates) | bool | `"false"` | no |
+| PrivateIp | (Optional) Set a static, primary private IP. Leave blank to auto-select a free IP | string | `"null"` | no |
+| PypiIndexUrl | (Optional) URL to the PyPi Index | string | `"https://pypi.org/simple"` | no |
+| SecurityGroupIds | (Required) List of security groups to apply to the instance | string | n/a | yes |
+| SubnetId | (Required) ID of the subnet to assign to the instance | string | n/a | yes |
+| ToggleCfnInitUpdate | (Optional) A/B toggle that forces a change to instance metadata, triggering the cfn-init update sequence | string | `"A"` | no |
+| WatchmakerAdminGroups | (Optional) Colon-separated list of domain groups that should have admin permissions on the EC2 instance | string | `"null"` | no |
+| WatchmakerAdminUsers | (Optional) Colon-separated list of domain users that should have admin permissions on the EC2 instance | string | `"null"` | no |
+| WatchmakerComputerName | (Optional) Sets the hostname/computername within the OS | string | `"null"` | no |
+| WatchmakerConfig | (Optional) URL to a Watchmaker config file | string | `"null"` | no |
+| WatchmakerEnvironment | (Optional) Environment in which the instance is being deployed | string | `"null"` | no |
+| WatchmakerOuPath | (Optional) DN of the OU to place the instance when joining a domain. If blank and WatchmakerEnvironment enforces a domain join, the instance will be placed in a default container. Leave blank if not joining a domain, or if WatchmakerEnvironment is false | string | `"null"` | no |
+| WatchmakerS3Source | (Optional) Flag that tells watchmaker to use its instance role to retrieve watchmaker content from S3 | bool | `"false"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| watchmaker-lx-instance-stack | CloudFormation stack object for watchmaker-lx-instance-stack |
+

modules/lx-instance/watchmaker-lx-instance.template.cfn.yaml

@@ -156,7 +156,7 @@ Metadata:
     ParameterLabels:
       ToggleCfnInitUpdate:
         default: Force Cfn Init Update
-  Version: 2.0.0
+  Version: 2.0.1
 Outputs:
   WatchmakerInstanceId:
     Description: Instance ID