diff --git a/INSTALL.md b/INSTALL.md index d02980335..9efb4aa3b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -415,4 +415,41 @@ You can get a Bitbucket token for your repository by following Repository Settin Please contact if you're interested in a hosted BitBucket app solution that provides full functionality including PR reviews and comment handling. It's based on the [bitbucket_app.py](https://github.com/Codium-ai/pr-agent/blob/main/pr_agent/git_providers/bitbucket_provider.py) implmentation. +### Bitbucket Server and Data Center + +Login into your on-prem instance of Bitbucket with your service account username and password. +Navigate to `Manage account`, `HTTP Access tokens`, `Create Token`. +Generate the token and add it to .secret.toml under `bitbucket_server` section + +```toml +[bitbucket_server] +bearer_token = "" +``` + +#### Run it as CLI + +Modify `configuration.toml`: + +```toml +git_provider="bitbucket_server" +``` + +and pass the Pull request URL: +```shell +python cli.py --pr_url https://git.onpreminstanceofbitbucket.com/projects/PROJECT/repos/REPO/pull-requests/1 review +``` + +#### Run it as service + +To run pr-agent as webhook, build the docker image: +``` +docker build . -t codiumai/pr-agent:bitbucket_server_webhook --target bitbucket_server_webhook -f docker/Dockerfile +docker push codiumai/pr-agent:bitbucket_server_webhook # Push to your Docker repository +``` + +Navigate to `Projects` or `Repositories`, `Settings`, `Webhooks`, `Create Webhook`. +Fill the name and URL, Authentication None select the Pull Request Opened checkbox to receive that event as webhook. + +The url should be ends with `/webhook`, example: https://domain.com/webhook + ======= diff --git a/docker/Dockerfile b/docker/Dockerfile index 951f846c3..0f669e89b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,6 +14,10 @@ FROM base as bitbucket_app ADD pr_agent pr_agent CMD ["python", "pr_agent/servers/bitbucket_app.py"] +FROM base as bitbucket_server_webhook +ADD pr_agent pr_agent +CMD ["python", "pr_agent/servers/bitbucket_server_webhook.py"] + FROM base as github_polling ADD pr_agent pr_agent CMD ["python", "pr_agent/servers/github_polling.py"] diff --git a/pr_agent/git_providers/__init__.py b/pr_agent/git_providers/__init__.py index 968f0dfc7..14103a951 100644 --- a/pr_agent/git_providers/__init__.py +++ b/pr_agent/git_providers/__init__.py @@ -1,5 +1,6 @@ from pr_agent.config_loader import get_settings from pr_agent.git_providers.bitbucket_provider import BitbucketProvider +from pr_agent.git_providers.bitbucket_server_provider import BitbucketServerProvider from pr_agent.git_providers.codecommit_provider import CodeCommitProvider from pr_agent.git_providers.github_provider import GithubProvider from pr_agent.git_providers.gitlab_provider import GitLabProvider @@ -12,6 +13,7 @@ 'github': GithubProvider, 'gitlab': GitLabProvider, 'bitbucket': BitbucketProvider, + 'bitbucket_server': BitbucketServerProvider, 'azure': AzureDevopsProvider, 'codecommit': CodeCommitProvider, 'local' : LocalGitProvider, diff --git a/pr_agent/git_providers/bitbucket_server_provider.py b/pr_agent/git_providers/bitbucket_server_provider.py new file mode 100644 index 000000000..44347850b --- /dev/null +++ b/pr_agent/git_providers/bitbucket_server_provider.py @@ -0,0 +1,351 @@ +import json +from typing import Optional, Tuple +from urllib.parse import urlparse + +import requests +from atlassian.bitbucket import Bitbucket +from starlette_context import context + +from .git_provider import FilePatchInfo, GitProvider, EDIT_TYPE +from ..algo.pr_processing import find_line_number_of_relevant_line_in_file +from ..algo.utils import load_large_diff +from ..config_loader import get_settings +from ..log import get_logger + + +class BitbucketServerProvider(GitProvider): + def __init__( + self, pr_url: Optional[str] = None, incremental: Optional[bool] = False + ): + s = requests.Session() + try: + bearer = context.get("bitbucket_bearer_token", None) + s.headers["Authorization"] = f"Bearer {bearer}" + except Exception: + s.headers[ + "Authorization" + ] = f'Bearer {get_settings().get("BITBUCKET_SERVER.BEARER_TOKEN", None)}' + + s.headers["Content-Type"] = "application/json" + self.headers = s.headers + self.bitbucket_server_url = None + self.workspace_slug = None + self.repo_slug = None + self.repo = None + self.pr_num = None + self.pr = None + self.pr_url = pr_url + self.temp_comments = [] + self.incremental = incremental + self.diff_files = None + self.bitbucket_pull_request_api_url = pr_url + + self.bitbucket_server_url = self._parse_bitbucket_server(url=pr_url) + self.bitbucket_client = Bitbucket(url=self.bitbucket_server_url, + token=get_settings().get("BITBUCKET_SERVER.BEARER_TOKEN", None)) + + if pr_url: + self.set_pr(pr_url) + + def get_repo_settings(self): + try: + url = (f"{self.bitbucket_server_url}/projects/{self.workspace_slug}/repos/{self.repo_slug}/src/" + f"{self.pr.destination_branch}/.pr_agent.toml") + response = requests.request("GET", url, headers=self.headers) + if response.status_code == 404: # not found + return "" + contents = response.text.encode('utf-8') + return contents + except Exception: + return "" + + def publish_code_suggestions(self, code_suggestions: list) -> bool: + """ + Publishes code suggestions as comments on the PR. + """ + post_parameters_list = [] + for suggestion in code_suggestions: + body = suggestion["body"] + relevant_file = suggestion["relevant_file"] + relevant_lines_start = suggestion["relevant_lines_start"] + relevant_lines_end = suggestion["relevant_lines_end"] + + if not relevant_lines_start or relevant_lines_start == -1: + if get_settings().config.verbosity_level >= 2: + get_logger().exception( + f"Failed to publish code suggestion, relevant_lines_start is {relevant_lines_start}" + ) + continue + + if relevant_lines_end < relevant_lines_start: + if get_settings().config.verbosity_level >= 2: + get_logger().exception( + f"Failed to publish code suggestion, " + f"relevant_lines_end is {relevant_lines_end} and " + f"relevant_lines_start is {relevant_lines_start}" + ) + continue + + if relevant_lines_end > relevant_lines_start: + post_parameters = { + "body": body, + "path": relevant_file, + "line": relevant_lines_end, + "start_line": relevant_lines_start, + "start_side": "RIGHT", + } + else: # API is different for single line comments + post_parameters = { + "body": body, + "path": relevant_file, + "line": relevant_lines_start, + "side": "RIGHT", + } + post_parameters_list.append(post_parameters) + + try: + self.publish_inline_comments(post_parameters_list) + return True + except Exception as e: + if get_settings().config.verbosity_level >= 2: + get_logger().error(f"Failed to publish code suggestion, error: {e}") + return False + + def is_supported(self, capability: str) -> bool: + if capability in ['get_issue_comments', 'get_labels', 'gfm_markdown']: + return False + return True + + def set_pr(self, pr_url: str): + self.workspace_slug, self.repo_slug, self.pr_num = self._parse_pr_url(pr_url) + self.pr = self._get_pr() + + def get_file(self, path: str, commit_id: str): + file_content = "" + try: + file_content = self.bitbucket_client.get_content_of_file(self.workspace_slug, + self.repo_slug, + path, + commit_id) + except requests.HTTPError as e: + get_logger().debug(f"File {path} not found at commit id: {commit_id}") + return file_content + + def get_files(self): + changes = self.bitbucket_client.get_pull_requests_changes(self.workspace_slug, self.repo_slug, self.pr_num) + diffstat = [change["path"]['toString'] for change in changes] + return diffstat + + def get_diff_files(self) -> list[FilePatchInfo]: + if self.diff_files: + return self.diff_files + + commits_in_pr = self.bitbucket_client.get_pull_requests_commits( + self.workspace_slug, + self.repo_slug, + self.pr_num + ) + + commit_list = list(commits_in_pr) + base_sha, head_sha = commit_list[0]['parents'][0]['id'], commit_list[-1]['id'] + + diff_files = [] + original_file_content_str = "" + new_file_content_str = "" + + changes = self.bitbucket_client.get_pull_requests_changes(self.workspace_slug, self.repo_slug, self.pr_num) + for change in changes: + file_path = change['path']['toString'] + match change['type']: + case 'ADD': + edit_type = EDIT_TYPE.ADDED + new_file_content_str = self.get_file(file_path, head_sha) + if isinstance(new_file_content_str, (bytes, bytearray)): + new_file_content_str = new_file_content_str.decode("utf-8") + original_file_content_str = "" + case 'DELETE': + edit_type = EDIT_TYPE.DELETED + new_file_content_str = "" + original_file_content_str = self.get_file(file_path, base_sha) + if isinstance(original_file_content_str, (bytes, bytearray)): + original_file_content_str = original_file_content_str.decode("utf-8") + case 'RENAME': + edit_type = EDIT_TYPE.RENAMED + case _: + edit_type = EDIT_TYPE.MODIFIED + original_file_content_str = self.get_file(file_path, base_sha) + if isinstance(original_file_content_str, (bytes, bytearray)): + original_file_content_str = original_file_content_str.decode("utf-8") + new_file_content_str = self.get_file(file_path, head_sha) + if isinstance(new_file_content_str, (bytes, bytearray)): + new_file_content_str = new_file_content_str.decode("utf-8") + + patch = load_large_diff(file_path, new_file_content_str, original_file_content_str) + + diff_files.append( + FilePatchInfo( + original_file_content_str, + new_file_content_str, + patch, + file_path, + edit_type=edit_type, + ) + ) + + self.diff_files = diff_files + return diff_files + + def publish_comment(self, pr_comment: str, is_temporary: bool = False): + if not is_temporary: + self.bitbucket_client.add_pull_request_comment(self.workspace_slug, self.repo_slug, self.pr_num, pr_comment) + + def remove_initial_comment(self): + try: + for comment in self.temp_comments: + self.remove_comment(comment) + except ValueError as e: + get_logger().exception(f"Failed to remove temp comments, error: {e}") + + def remove_comment(self, comment): + pass + + # funtion to create_inline_comment + def create_inline_comment(self, body: str, relevant_file: str, relevant_line_in_file: str): + position, absolute_position = find_line_number_of_relevant_line_in_file( + self.get_diff_files(), + relevant_file.strip('`'), + relevant_line_in_file + ) + if position == -1: + if get_settings().config.verbosity_level >= 2: + get_logger().info(f"Could not find position for {relevant_file} {relevant_line_in_file}") + subject_type = "FILE" + else: + subject_type = "LINE" + path = relevant_file.strip() + return dict(body=body, path=path, position=absolute_position) if subject_type == "LINE" else {} + + def publish_inline_comment(self, comment: str, from_line: int, file: str): + payload = { + "text": comment, + "severity": "NORMAL", + "anchor": { + "diffType": "EFFECTIVE", + "path": file, + "lineType": "ADDED", + "line": from_line, + "fileType": "TO" + } + } + + response = requests.post(url=self._get_pr_comments_url(), json=payload, headers=self.headers) + return response + + def generate_link_to_relevant_line_number(self, suggestion) -> str: + try: + relevant_file = suggestion['relevant file'].strip('`').strip("'") + relevant_line_str = suggestion['relevant line'] + if not relevant_line_str: + return "" + + diff_files = self.get_diff_files() + position, absolute_position = find_line_number_of_relevant_line_in_file \ + (diff_files, relevant_file, relevant_line_str) + + if absolute_position != -1 and self.pr_url: + link = f"{self.pr_url}/#L{relevant_file}T{absolute_position}" + return link + except Exception as e: + if get_settings().config.verbosity_level >= 2: + get_logger().info(f"Failed adding line link, error: {e}") + + return "" + + def publish_inline_comments(self, comments: list[dict]): + for comment in comments: + self.publish_inline_comment(comment['body'], comment['position'], comment['path']) + + def get_title(self): + return self.pr.title + + def get_languages(self): + return {"yaml": 0} # devops LOL + + def get_pr_branch(self): + return self.pr.fromRef['displayId'] + + def get_pr_description_full(self): + return self.pr.description + + def get_user_id(self): + return 0 + + def get_issue_comments(self): + raise NotImplementedError( + "Bitbucket provider does not support issue comments yet" + ) + + def add_eyes_reaction(self, issue_comment_id: int) -> Optional[int]: + return True + + def remove_reaction(self, issue_comment_id: int, reaction_id: int) -> bool: + return True + + @staticmethod + def _parse_bitbucket_server(url: str) -> str: + parsed_url = urlparse(url) + return f"{parsed_url.scheme}://{parsed_url.netloc}" + + @staticmethod + def _parse_pr_url(pr_url: str) -> Tuple[str, str, int]: + parsed_url = urlparse(pr_url) + path_parts = parsed_url.path.strip("/").split("/") + if len(path_parts) < 6 or path_parts[4] != "pull-requests": + raise ValueError( + "The provided URL does not appear to be a Bitbucket PR URL" + ) + + workspace_slug = path_parts[1] + repo_slug = path_parts[3] + try: + pr_number = int(path_parts[5]) + except ValueError as e: + raise ValueError("Unable to convert PR number to integer") from e + + return workspace_slug, repo_slug, pr_number + + def _get_repo(self): + if self.repo is None: + self.repo = self.bitbucket_client.get_repo(self.workspace_slug, self.repo_slug) + return self.repo + + def _get_pr(self): + pr = self.bitbucket_client.get_pull_request(self.workspace_slug, self.repo_slug, pull_request_id=self.pr_num) + return type('new_dict', (object,), pr) + + def _get_pr_file_content(self, remote_link: str): + return "" + + def get_commit_messages(self): + def get_commit_messages(self): + raise NotImplementedError("Get commit messages function not implemented yet.") + # bitbucket does not support labels + def publish_description(self, pr_title: str, description: str): + payload = json.dumps({ + "description": description, + "title": pr_title + }) + + response = requests.put(url=self.bitbucket_pull_request_api_url, headers=self.headers, data=payload) + return response + + # bitbucket does not support labels + def publish_labels(self, pr_types: list): + pass + + # bitbucket does not support labels + def get_labels(self): + pass + + def _get_pr_comments_url(self): + return f"{self.bitbucket_server_url}/rest/api/latest/projects/{self.workspace_slug}/repos/{self.repo_slug}/pull-requests/{self.pr_num}/comments" diff --git a/pr_agent/servers/bitbucket_server_webhook.py b/pr_agent/servers/bitbucket_server_webhook.py new file mode 100644 index 000000000..c6ce83536 --- /dev/null +++ b/pr_agent/servers/bitbucket_server_webhook.py @@ -0,0 +1,64 @@ +import json + +import uvicorn +from fastapi import APIRouter, FastAPI +from fastapi.encoders import jsonable_encoder +from starlette import status +from starlette.background import BackgroundTasks +from starlette.middleware import Middleware +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette_context.middleware import RawContextMiddleware + +from pr_agent.agent.pr_agent import PRAgent +from pr_agent.config_loader import get_settings +from pr_agent.log import get_logger + +router = APIRouter() + + +def handle_request(background_tasks: BackgroundTasks, url: str, body: str, log_context: dict): + log_context["action"] = body + log_context["event"] = "pull_request" if body == "review" else "comment" + log_context["api_url"] = url + with get_logger().contextualize(**log_context): + background_tasks.add_task(PRAgent().handle_request, url, body) + + +@router.post("/webhook") +async def handle_webhook(background_tasks: BackgroundTasks, request: Request): + log_context = {"server_type": "bitbucket_server"} + data = await request.json() + get_logger().info(json.dumps(data)) + + pr_id = data['pullRequest']['id'] + repository_name = data['pullRequest']['toRef']['repository']['slug'] + project_name = data['pullRequest']['toRef']['repository']['project']['key'] + bitbucket_server = get_settings().get("BITBUCKET_SERVER.URL") + pr_url = f"{bitbucket_server}/projects/{project_name}/repos/{repository_name}/pull-requests/{pr_id}" + + log_context["api_url"] = pr_url + log_context["event"] = "pull_request" + + handle_request(background_tasks, pr_url, "review", log_context) + return JSONResponse(status_code=status.HTTP_200_OK, content=jsonable_encoder({"message": "success"})) + + +@router.get("/") +async def root(): + return {"status": "ok"} + + +def start(): + bitbucket_server_url = get_settings().get("BITBUCKET_SERVER.URL", None) + if not bitbucket_server_url: + raise ValueError("BITBUCKET_SERVER.URL is not set") + get_settings().config.git_provider = "bitbucket_server" + middleware = [Middleware(RawContextMiddleware)] + app = FastAPI(middleware=middleware) + app.include_router(router) + uvicorn.run(app, host="0.0.0.0", port=3000) + + +if __name__ == '__main__': + start() diff --git a/tests/unittest/test_bitbucket_provider.py b/tests/unittest/test_bitbucket_provider.py index 3bb64a0c1..e17a26ce8 100644 --- a/tests/unittest/test_bitbucket_provider.py +++ b/tests/unittest/test_bitbucket_provider.py @@ -1,3 +1,4 @@ +from pr_agent.git_providers import BitbucketServerProvider from pr_agent.git_providers.bitbucket_provider import BitbucketProvider @@ -8,3 +9,10 @@ def test_parse_pr_url(self): assert workspace_slug == "WORKSPACE_XYZ" assert repo_slug == "MY_TEST_REPO" assert pr_number == 321 + + def test_bitbucket_server_pr_url(self): + url = "https://git.onpreminstance.com/projects/AAA/repos/my-repo/pull-requests/1" + workspace_slug, repo_slug, pr_number = BitbucketServerProvider._parse_pr_url(url) + assert workspace_slug == "AAA" + assert repo_slug == "my-repo" + assert pr_number == 1