App troubleshooting additions (#284)

* App troubleshooting updates

* Add idempotent PR behaviour

codebundles/k8s-app-troubleshoot/runbook.robot

@@ -80,13 +80,12 @@ Troubleshoot `${WORKLOAD_NAME}` Application Logs
     ...    include_in_history=False
     ...    env=${env}
     ...    secret_file__kubeconfig=${kubeconfig}
-
+    ${app_repo}=    RW.K8sApplications.Clone Repo    ${REPO_URI}    ${REPO_AUTH_TOKEN}    ${NUM_OF_COMMITS}
     # ${test_data}=    RW.K8sApplications.Get Test Data
     ${proc_list}=    RW.K8sApplications.Format Process List    ${proc_list.stdout}
     ${serialized_env}=    RW.K8sApplications.Serialize env    ${printenv.stdout}
     ${parsed_exceptions}=    RW.K8sApplications.Parse Exceptions    ${logs.stdout}
     # ${parsed_exceptions}=    RW.K8sApplications.Parse Exceptions    ${test_data}
-    ${app_repo}=    RW.K8sApplications.Clone Repo    ${REPO_URI}    ${REPO_AUTH_TOKEN}    ${NUM_OF_COMMITS}
     ${repos}=    Create List    ${app_repo}
     ${ts_results}=    RW.K8sApplications.Troubleshoot Application
     ...    repos=${repos}

libraries/RW/K8sApplications/k8s_applications.py

@@ -1,4 +1,4 @@
-import logging, hashlib
+import logging, hashlib, yaml
 from dataclasses import dataclass, field
 from thefuzz import process as fuzzprocessor
 from datetime import datetime
@@ -9,6 +9,7 @@
     PythonStackTraceParse,
     DRFStackTraceParse,
     GoogleDRFStackTraceParse,
+    CSharpStackTraceParse,
 )
 from .repository import (
     Repository,
@@ -57,6 +58,15 @@ def test(git_uri, git_token, k8s_env, process_list, *args, **kwargs):
     return report
 
 
+def test_search(
+    repo: Repository, exceptions: list[StackTraceData]
+) -> list[RepositorySearchResult]:
+    rr = []
+    for excep in exceptions:
+        rr += repo.search(search_files=excep.files)
+    return rr
+
+
 def get_test_data():
     data = ""
     with open(f"{THIS_DIR}/test_logs.txt", "r") as fh:
@@ -78,6 +88,7 @@ def parse_exceptions(
     parsers: list[BaseStackTraceParse] = [
         GoogleDRFStackTraceParse,
         PythonStackTraceParse,
+        CSharpStackTraceParse,
     ]
     # TODO: support multiline parsing
     for log in logs:
@@ -95,7 +106,7 @@ def parse_exceptions(
 
 def clone_repo(git_uri, git_token, number_of_commits_history: int = 10) -> Repository:
     repo = Repository(source_uri=git_uri, auth_token=git_token)
-    repo.clone_repo(number_of_commits_history)
+    repo.clone_repo(number_of_commits_history, cache=True)
     return repo
 
 
@@ -200,3 +211,59 @@ def create_github_issue(repo: Repository, content: str) -> str:
         return f"Here's a link to an open GitHub issue for application exceptions: {report_url}"
     else:
         return "No related GitHub issue could be found."
+
+
+def scale_up_hpa(
+    infra_repo: Repository,
+    manifest_file_path: str,
+    increase_value: int = 1,
+    set_value: int = -1,
+    max_allowed_replicas: int = 10,
+) -> dict:
+    working_branch = "infra-scale-hpa"
+    manifest_file: RepositoryFile = infra_repo.files.files[manifest_file_path]
+    logger.info(manifest_file.content)
+    manifest_object = yaml.safe_load(manifest_file.content)
+    max_replicas = manifest_object.get("spec", {}).get("maxReplicas", None)
+    if not max_replicas:
+        raise Exception(f"manifest does not contain a maxReplicas {manifest_object}")
+    max_replicas += increase_value
+    if set_value > 0:
+        max_replicas = set_value
+    max_replicas = min(max_allowed_replicas, max_replicas)
+    manifest_object["spec"]["maxReplicas"] = max_replicas
+    manifest_file.content = yaml.safe_dump(manifest_object)
+    manifest_file.write_content()
+    manifest_file.git_add()
+    infra_repo.git_commit(
+        branch_name=working_branch,
+        comment=f"Update maxReplicas in {manifest_file.basename}",
+    )
+    infra_repo.git_push_branch(working_branch)
+    pr_body = f"""
+# HorizontalPodAutoscaler Update
+Due to insufficient scaling, we've recommended the following change:
+- Updates maxReplicas to {max_replicas} in {manifest_file.basename}
+"""
+    rsp = infra_repo.git_pr(
+        title=f"{RUNWHEN_ISSUE_KEYWORD} Update maxReplicas in {manifest_file.basename}",
+        branch=working_branch,
+        body=pr_body,
+    )
+    pr_url = None
+    if "html_url" in rsp:
+        pr_url = rsp["html_url"]
+    report = f"""
+A change request could not be generated for this manifest. Consider running additional troubleshooting or contacting the service owner.
+"""
+    if pr_url:
+        report = f"""
+The following change request was made in the repository {infra_repo.repo_name}
+
+{pr_url}
+
+Next Steps:
+- Review and merge the change request at {pr_url} 
+"""
+
+    return report

libraries/RW/K8sApplications/parsers.py

@@ -128,6 +128,15 @@ def extract_sentences(text) -> list[str]:
         return results
 
 
+class CSharpStackTraceParse(BaseStackTraceParse):
+    @staticmethod
+    def parse_log(log) -> StackTraceData:
+        if ".Exception" in log:
+            return BaseStackTraceParse.parse_log(log)
+        else:
+            return None
+
+
 class PythonStackTraceParse(BaseStackTraceParse):
     @staticmethod
     def parse_log(log) -> StackTraceData:

libraries/RW/K8sApplications/repository.py

@@ -16,6 +16,7 @@
 from RW.Core import Core
 
 logger = logging.getLogger(__name__)
+RUNWHEN_PR_KEYWORD: str = "[RunWhen]"
 
 
 @dataclass
@@ -101,6 +102,7 @@ class RepositoryFile:
     git_file_url: str
     relative_file_path: str
     absolute_file_path: str
+    basename: str
     line_count: int
     git_url_base: str
     filesystem_basepath: str
@@ -111,6 +113,7 @@ def __init__(
         self, absolute_filepath, filesystem_basepath, repo_base_url, branch="main"
     ) -> None:
         self.absolute_file_path = str(absolute_filepath)
+        self.basename = os.path.basename(self.absolute_file_path)
         self.filesystem_basepath = filesystem_basepath
         self.git_url_base = str(repo_base_url)
         self.branch = branch
@@ -133,6 +136,23 @@ def search(self, search_term: str) -> RepositorySearchResult:
         # skip_regex = r"^[ \t\n\r#*,(){}\[\]\"\'\':]*$"
         return None
 
+    def git_add(self):
+        try:
+            add_stdout = subprocess.run(
+                ["git", "add", self.relative_file_path],
+                text=True,
+                capture_output=True,
+                check=True,
+                cwd=self.filesystem_basepath,
+            ).stdout
+            logger.info(f"File {self.relative_file_path} has been added to staging.")
+        except subprocess.CalledProcessError as e:
+            logger.error(f"An error occurred: {e}")
+
+    def write_content(self) -> None:
+        with open(self.absolute_file_path, "w") as fp:
+            fp.write(self.content)
+
 
 @dataclass
 class RepositoryFiles:
@@ -148,6 +168,18 @@ def add_source_file(self, src_file: RepositoryFile) -> None:
     def file_paths(self) -> list[str]:
         return self.files.keys()
 
+    @property
+    def all_files(self) -> list[RepositoryFile]:
+        return self.files.values()
+
+    @property
+    def all_basenames(self) -> list[str]:
+        basenames: list[str] = []
+        all_files = self.all_files
+        for repo_file in all_files:
+            basenames.append(repo_file.basename)
+        return basenames
+
 
 class Repository:
     EXCLUDE_PATHS: list[str] = [
@@ -164,6 +196,7 @@ class Repository:
     clone_directory: str
     branch: str
     commit_history: list[GitCommit] = []
+    local_branch_lookup: dict = {}
 
     def __str__(self):
         repo_summary = f"Repository: {self.repo_owner}/{self.repo_name}\nRepository URI: {self.source_uri}"
@@ -251,6 +284,107 @@ def clone_repo(
         )
         return self.clone_directory
 
+    def git_commit(
+        self,
+        branch_name: str,
+        comment: str,
+    ) -> None:
+        current_datetime = datetime.now()
+        timestamp = int(current_datetime.timestamp())
+        unique_branch_name = f"{branch_name}-{timestamp}"
+        self.local_branch_lookup[branch_name] = unique_branch_name
+        checkout_b = subprocess.run(
+            [
+                "git",
+                "checkout",
+                "-b",
+                f"{unique_branch_name}",
+            ],
+            check=True,
+            cwd=self.clone_directory,
+            capture_output=True,
+        )
+        logger.info(f"Checked out branch {unique_branch_name}")
+        commitcmd = subprocess.run(
+            [
+                "git",
+                "commit",
+                "-m",
+                f"{comment}",
+            ],
+            check=True,
+            cwd=self.clone_directory,
+            capture_output=True,
+        )
+        logger.info(f"Comitting... {commitcmd.stdout}")
+        checkout_default = subprocess.run(
+            [
+                "git",
+                "checkout",
+                f"{self.branch}",
+            ],
+            check=True,
+            cwd=self.clone_directory,
+            capture_output=True,
+        )
+        logger.info(f"Returning to default branch {checkout_default.stdout}")
+
+    def git_push_branch(self, branch: str, remote: str = "origin") -> None:
+        true_branch_name = self.local_branch_lookup[branch]
+        gitpushcmd = subprocess.run(
+            [
+                "git",
+                "push",
+                f"{remote}",
+                f"{true_branch_name}",
+            ],
+            check=True,
+            capture_output=True,
+            cwd=self.clone_directory,
+        )
+
+    def git_pr(
+        self,
+        title: str,
+        branch: str,
+        body: str,
+        check_open: bool = True,
+    ) -> dict:
+        is_already_open: bool = False
+        true_branch_name = self.local_branch_lookup[branch]
+        url = f"https://api.github.com/repos/{self.repo_owner}/{self.repo_name}/pulls"
+        headers = {
+            "Authorization": f"token {self.auth_token}",
+            "Accept": "application/vnd.github.v3+json",
+        }
+        if check_open:
+            rsp = requests.get(url, headers=headers)
+            if rsp.status_code < 200 or rsp.status_code >= 300:
+                logger.warning(f"Error: {rsp.status_code}, {rsp.text}")
+                return {}
+            rsp = rsp.json()
+            for pr in rsp:
+                title = pr["title"]
+                state = pr["state"]
+                if state == "open" and RUNWHEN_PR_KEYWORD in title:
+                    is_already_open = True
+
+        if is_already_open:
+            return {}
+        data = {
+            "title": title,
+            "head": branch,
+            "body": body,
+            "base": "main",
+        }
+        response = requests.post(url, headers=headers, json=data)
+
+        if response.status_code >= 300:
+            logger.warning(f"Error: {response.status_code}, {response.text}")
+            return {}
+
+        return response.json()
+
     def get_repo_base_url(self) -> str:
         remote_url = self.source_uri
         if remote_url.startswith("git@"):
@@ -290,27 +424,52 @@ def create_file_list(self) -> None:
 
     def search(
         self,
-        search_words: list[str],
-        search_files: list[str],
+        search_words: list[str] = [],
+        search_files: list[str] = [],
         # max_results_per_word: int = 5,
         # search_match_score_min: int = 90,
     ) -> [RepositorySearchResult]:
+        if not search_files and not search_words:
+            logger.warning(f"You must provide files and/or words to search with")
+            return []
         logger.info(f"Performing search with words: {search_words}")
+        logger.info(f"Performing search with files: {search_files}")
         # check both paths starting with / and without - this is a bit of hackery but
         # needed to get around a weakness in regex parsing
         file_paths: list[str] = self.files.file_paths
+        # add any basename matches, for case when build paths differ from source paths
+        # potentially causes false positives but should mostly work for now
+        # TODO: revisit build vs src paths
+        repo_file_bases: list[str] = self.files.all_basenames
+        search_files_bases: list[str] = [os.path.basename(sfp) for sfp in search_files]
+        repo_basename_mapping = {}
+        for sfb in search_files_bases:
+            if sfb in self.files.all_basenames:
+                for repo_file in self.files.all_files:
+                    if sfb == repo_file.basename:
+                        repo_basename_mapping[
+                            repo_file.basename
+                        ] = repo_file.relative_file_path
+
+        search_files_bases = set([fp for fp in repo_basename_mapping.values()])
+
         slashed_file_paths: list[str] = [f"/{fp}" for fp in file_paths]
         logger.info(search_files)
         files_to_examine: list[str] = set(search_files).intersection(file_paths)
         slashed_files_to_examine: list[str] = set(search_files).intersection(
             slashed_file_paths
         )
+
         logger.info(
-            f"Searching files: {files_to_examine} and {slashed_files_to_examine}"
+            f"Searching files: {files_to_examine}, {search_files_bases} and {slashed_files_to_examine}"
         )
         # recombine, remove slashes and unique entries
         search_in: list[str] = list(
-            set(list(files_to_examine) + [fp[1:] for fp in slashed_files_to_examine])
+            set(
+                list(files_to_examine)
+                + list(search_files_bases)
+                + [fp[1:] for fp in slashed_files_to_examine]
+            )
         )
         logger.info(f"SEARCH FILES: {search_in}")
         results: list[RepositorySearchResult] = []