v1.0.1

pentestfail · Apr 18, 2018 · 56f6261 · 56f6261
1 parent fe66280
commit 56f6261
Show file tree

Hide file tree

Showing 1,239 changed files with 200 additions and 70,278 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-GitHub Addon
+Github Addon
 ========================
 Provides modular inputs & framework to ingest JSON data from Github APIs.
 
@@ -22,6 +22,14 @@ Name of repository. Example: https://api.github.com/owner/[REPOSITORY]
 - Repository Issues (https://developer.github.com/v3/issues/#list-issues-for-a-repository)
 
 ## Release Notes:
+
+###v1.0.1
+Maintenance & Updates for Github.com API
+- Adds input parameter and logic to handle differences beteween Github's public and enterprise API paths.
+- Additional logic for API "still processing" (202 response) for stats API
+- Improved logging and error handling.
+
+###v1.0.0
 Initial release. Documentation will be included in future releases.
 
 ## Submit issues or requests via Github:

diff --git a/README.txt b/README.txt
@@ -1,4 +1,4 @@
-GitHub Addon
+Github Addon
 ========================
 Provides modular inputs & framework to ingest JSON data from Github APIs.
 
@@ -22,7 +22,15 @@ Name of repository. Example: https://api.github.com/owner/[REPOSITORY]
 - Repository Issues (https://developer.github.com/v3/issues/#list-issues-for-a-repository)
 
 ## Release Notes:
+
+###v1.0.1
+Maintenance & Updates for Github.com API
+- Adds input parameter and logic to handle differences beteween Github's public and enterprise API paths.
+- Additional logic for API "still processing" (202 response) for stats API
+- Improved logging and error handling.
+
+###v1.0.0
 Initial release. Documentation will be included in future releases.
 
 ## Submit issues or requests via Github:
-TA-Github: https://github.com/pentestfail/TA-Github
+TA-Github: https://github.com/pentestfail/TA-Github
diff --git a/README/ta_github_account.conf.spec b/README/ta_github_account.conf.spec
@@ -1,4 +1,5 @@
 [<name>]
 username = 
 password = 
-github_instance =
+github_instance = 
+enterprise = 
diff --git a/app.manifest b/app.manifest
@@ -5,7 +5,7 @@
     "id": {
       "group": null,
       "name": "TA-Github",
-      "version": "1.0.0"
+      "version": "1.0.1"
     },
     "author": [
       {

diff --git a/appserver/static/js/build/globalConfig.json b/appserver/static/js/build/globalConfig.json
@@ -43,6 +43,12 @@
                             ], 
                             "help": "Default domain path to use for accessing your Github server.  Example: https://[INSTANCE]/owner/repo", 
                             "field": "github_instance"
+                        }, 
+                        {
+                            "label": "Enterprise API", 
+                            "type": "checkbox", 
+                            "help": "Configures input to use Github Enterprise API v3 endpoints (/api/v3/) instead of Github.com public API.",
+                            "field": "enterprise"
                         },
                         {
                             "label": "Username", 
@@ -94,6 +100,10 @@
                                 "label": "Github Instance", 
                                 "field": "github_instance"
                             }, 
+                            {
+                                "label": "Enterprise", 
+                                "field": "enterprise"
+                            }, 
                             {
                                 "label": "Username", 
                                 "field": "username"
@@ -400,6 +410,16 @@
                             ], 
                             "field": "index"
                         }, 
+                        {
+                            "label": "Github Credentials", 
+                            "type": "singleSelect", 
+                            "options": {
+                                "referenceName": "account"
+                            }, 
+                            "required": true, 
+                            "help": "Select stored credentials with appropriate access to specified Github repository.", 
+                            "field": "github_creds"
+                        }, 
                         {
                             "label": "Owner or Organization", 
                             "type": "text", 
@@ -429,16 +449,6 @@
                             ], 
                             "help": "Name of repository. Example: https://api.github.com/owner/[REPOSITORY]", 
                             "field": "github_repo"
-                        }, 
-                        {
-                            "label": "Github Credentials", 
-                            "type": "singleSelect", 
-                            "options": {
-                                "referenceName": "account"
-                            }, 
-                            "required": true, 
-                            "help": "Select stored credentials with appropriate access to specified Github repository.", 
-                            "field": "github_creds"
                         }
                     ]
                 }, 
@@ -500,6 +510,16 @@
                                 }
                             ], 
                             "field": "index"
+                        }, 
+                        {
+                            "label": "Github Credentials", 
+                            "type": "singleSelect", 
+                            "options": {
+                                "referenceName": "account"
+                            }, 
+                            "required": true, 
+                            "help": "Select stored credentials with appropriate access to specified Github repository.", 
+                            "field": "github_creds"
                         },  
                         {
                             "label": "Owner or Organization", 
@@ -530,16 +550,6 @@
                             ], 
                             "help": "Name of repository. Example: https://api.github.com/owner/[REPOSITORY]", 
                             "field": "github_repo"
-                        }, 
-                        {
-                            "label": "Github Credentials", 
-                            "type": "singleSelect", 
-                            "options": {
-                                "referenceName": "account"
-                            }, 
-                            "required": true, 
-                            "help": "Select stored credentials with appropriate access to specified Github repository.", 
-                            "field": "github_creds"
                         }
                     ]
                 }

diff --git a/bin/TA_Github_rh_account.py b/bin/TA_Github_rh_account.py
@@ -24,6 +24,13 @@
             max_len=8192, 
         )
     ),
+    field.RestField(
+        'enterprise',
+        required=False,
+        encrypted=False,
+        default=None,
+        validator=None
+    ),
     field.RestField(
         'username',
         required=True,

diff --git a/bin/input_module_github_api_repos_commits.py b/bin/input_module_github_api_repos_commits.py
@@ -26,26 +26,26 @@ def collect_events(helper, ew):
     git_repo = helper.get_arg('github_repo')
     git_username = helper.get_arg('github_creds')['username']
     git_password = helper.get_arg('github_creds')['password']
+    git_enterprise = bool(helper.get_arg('github_creds')['enterprise'])
     git_pagesize = helper.get_arg('pagesize') or 50 #Page size of results
     git_daysago = helper.get_arg('days_ago') or 356 #Max days ago since commit
     inputname = helper.get_input_stanza_names()
-    inputsource = helper.get_input_type() + ":" + inputname
-    helper.log_info("input_type=github_api_repos_commits input={0:s} message='Collecting events.'".format(inputname))
+    inputtype = helper.get_input_type()
+    inputsource = inputtype + ":" + inputname
+    helper.log_info("input_type={0:s} input={1:s} message='Collecting events.'".format(inputtype,inputname))
+
+    # Create checkpoint key
+    opt_checkpoint = "{0:s}-{1:s}".format(inputtype,inputname)
 
     # Create initial time to query for commits in last 365days
     initial_status = (datetime.now() - timedelta(git_daysago)).strftime("%Y-%m-%d")
-    #initial_status = "2017-01-01"
-
-    # Create checkpoint key
-    opt_checkpoint = "github_api_repos_commits-{0:s}".format(inputname)
-    #helper.delete_check_point(opt_checkpoint)
 
     #Check for last query execution data in kvstore & generate if not present
     try:
         last_status = helper.get_check_point(opt_checkpoint) or initial_status
-        helper.log_debug("input_type=github_api_repos_commits input={0:s} message='Last successful checkpoint time.' last_status={1:s}".format(inputname,json.dumps(last_status)))
+        helper.log_debug("input_type={0:s} input={1:s} message='Last successful checkpoint time.' last_status={2:s}".format(inputtype,inputname,json.dumps(last_status)))
     except Exception as e:
-        helper.log_error("input_type=github_api_repos_commits input={0:s} message='Unable to retrieve last execution checkpoint!'".format(inputname))
+        helper.log_error("input_type={0:s} input={1:s} message='Unable to retrieve last execution checkpoint!'".format(inputtype,inputname))
         raise e
 
     # Create API request parameters    
@@ -54,8 +54,20 @@ def collect_events(helper, ew):
     parameter = {}
     parameter['since'] = last_status
     parameter['per_page'] = git_pagesize
-    url = "https://{0}/repos/{1}/{2}/commits".format(git_instance,git_owner,git_repo)
     method = 'GET'
+
+    # Determine API schema to use
+    if git_instance=="api.github.com":
+        url = "https://{0}/repos/{1}/{2}/commits".format(git_instance,git_owner,git_repo)
+        helper.log_debug("input_type={0:s} input={1:s} message='Github.com identified as instance. Using api subdomain.' url='{2:s}'".format(inputtype,inputname,url))
+        header['Accept'] = 'application/vnd.github.v3+json'
+    elif git_enterprise is True:
+        url = "https://{0}/api/v3/repos/{1}/{2}/commits".format(git_instance,git_owner,git_repo)
+        helper.log_debug("input_type={0:s} input={1:s} message='Github Enterprise specified in input configuration. Using /api/v3/repos/ path instead of subdomain.' url='{2:s}'".format(inputtype,inputname,url))
+    else:
+        url = "https://{0}/repos/{1}/{2}/commits".format(git_instance,git_owner,git_repo)
+        header['Accept'] = 'application/vnd.github.v3+json'
+        helper.log_error("input_type={0:s} input={1:s} message='Github instance not configured as enterprise & doesn't match public API domain! WTF!? Defaulting to public API path (/repos/).' url='{2:s}'".format(inputtype,inputname,url))
 
     try:
         has_results = True
@@ -64,22 +76,22 @@ def collect_events(helper, ew):
         while has_results:
             # Leverage helper function to send http request
             response = helper.send_http_request(url, method, parameters=parameter, payload=None, headers=header, cookies=None, verify=True,     cert=None, timeout=25, use_proxy=True)
-            helper.log_debug("input_type=github_api_repos_commits input={0:s} message='Requesting commit data from Github API.' url='{1:s}' parameters='{2:s}'".format(inputname,url,json.dumps(parameter)))
+            helper.log_debug("input_type={0:s} input={1:s} message='Requesting commit data from Github API.' url='{2:s}' parameters='{3:s}'".format(inputtype,inputname,url,json.dumps(parameter)))
 
             # Return API response code
             r_status = response.status_code
             # Return API request status_code
             if r_status is 202:
-                helper.log_info("input_type=github_api_repos_commits input={0:s} message='API still processing request. Will retry in 10 seconds.' status_code={1:d}".format(inputname,r_status))
+                helper.log_info("input_type={0:s} input={1:s} message='API still processing request. Will retry in 10 seconds.' status_code={2:d}".format(inputtype,inputname,r_status))
                 time.sleep(10)
             elif r_status is not 200:
-                helper.log_error("input_type=github_api_repos_commits input={0:s} message='API request unsuccessful.' status_code={1:d}".format    (inputname,r_status))
+                helper.log_error("input_type={0:s} input={1:s} message='API request unsuccessful.' status_code={2:d}".format(inputtype,inputname,r_status))
                 response.raise_for_status()
             # Return API request as JSON
             obj = response.json()
 
             if obj is None:
-                helper.log_info("input_type=github_api_repos_commits input={0:s} message='No records retrieved from Github API.'".format    (inputname))
+                helper.log_info("input_type={0:s} input={1:s} message='No records retrieved from Github API.'".format(inputtype,inputname))
                 has_results = False
 
             #page_count = len(obj) #Count of items in the results from page.
@@ -104,20 +116,20 @@ def collect_events(helper, ew):
                 i+=1
                 #helper.log_debug("input_type=github_api_repos_commits input={0:s} processed={1:d} total={2:d}".format(inputname,i,total))
 
-            helper.log_debug("input_type=github_api_repos_commits input={0:s} processed={1:d}".format(inputname,i))
+            helper.log_debug("input_type={0:s} input={1:s} processed={2:d}".format(inputtype,inputname,i))
 
             if has_results:
-                helper.log_debug("input_type=github_api_repos_commits input={0:s} message='Getting next page.' link_next='{1:s}'".format(inputname,url))
+                helper.log_debug("input_type={0:s} input={1:s} message='Getting next page.' link_next='{2:s}'".format(inputtype,inputname,url))
                 response = helper.send_http_request(url, method, parameters=None, payload=None, headers=header, cookies=None, verify=True, cert=None, timeout=25, use_proxy=True)
             else:
-                helper.log_debug("input_type=github_api_repos_commits input={0:s} message='No additional pages.'".format(inputname))
+                helper.log_debug("input_type={0:s} input={1:s} message='No additional pages.'".format(inputtype,inputname))
 
         #Update last completed execution time
         updated = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) #Add meta value for troubleshooting
         helper.save_check_point(opt_checkpoint,updated)
-        helper.log_info("input_type=github_api_repos_commits input={0:s} message='Collection complete.' indexed={1:d}".format(inputname,i))
-        helper.log_debug("input_type=github_api_repos_commits input={0:s} message='Storing checkpoint.' updated={1:s}".format(inputname,updated))
+        helper.log_info("input_type={0:s} input={1:s} message='Collection complete.' indexed={2:d}".format(inputtype,inputname,i))
+        helper.log_debug("input_type={0:s} input={1:s} message='Storing checkpoint.' updated={2:s}".format(inputtype,inputname,updated))
 
     except Exception as error:
-        helper.log_error("input_type=github_api_repos_commits input={0:s} message='An unknown error occurred!'".format(inputname))
-        raise error
+        helper.log_error("input_type={0:s} input={1:s} message='An unknown error occurred!'".format(inputtype,inputname))
+        raise error