From 65294ac724b5c008b9807e16691bd841d1ee783e Mon Sep 17 00:00:00 2001
From: Eli <43382407+eli64s@users.noreply.github.com>
Date: Mon, 25 Sep 2023 21:15:16 -0500
Subject: [PATCH] Codebase Clean Up (#54)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* 🐳 Remove tree installation from image.
* maintenance: Clean up codebase, improve readability.
---
...build-docker-image.yml => build_image.yml} | 0
...h-pypi-package.yml => publish_package.yml} | 0
CHANGELOG.md | 11 +++++
Dockerfile | 5 +--
README.md | 1 +
poetry.lock | 15 ++++++-
pyproject.toml | 45 ++++++++++---------
readmeai/builder.py | 39 ++++++++--------
readmeai/conf/conf.toml | 4 +-
readmeai/conf/ignore_files.toml | 2 +
readmeai/main.py | 44 ++++++++++--------
readmeai/model.py | 4 +-
readmeai/utils.py | 3 +-
13 files changed, 106 insertions(+), 67 deletions(-)
rename .github/workflows/{build-docker-image.yml => build_image.yml} (100%)
rename .github/workflows/{publish-pypi-package.yml => publish_package.yml} (100%)
diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build_image.yml
similarity index 100%
rename from .github/workflows/build-docker-image.yml
rename to .github/workflows/build_image.yml
diff --git a/.github/workflows/publish-pypi-package.yml b/.github/workflows/publish_package.yml
similarity index 100%
rename from .github/workflows/publish-pypi-package.yml
rename to .github/workflows/publish_package.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1dad5f69..85b5e9b8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,17 @@ All notable changes to the *readme-ai* project will be documented in this file.
---
+## [v0.1.2] - *2023-09-25*
+
+### 🔐 Security
+
+- Implement custom directory tree method using pure Python [#53](https://github.com/eli64s/readme-ai/pull/53)
+ - Removes dependency on the tree command line tool.
+ - Improves security by removing the subprocess module.
+ - More details on these risk can be found [here](https://bandit.readthedocs.io/en/latest/plugins/b607_start_process_with_partial_path.html#b607-start-process-with-partial-path)
+
+---
+
## [v0.1.1] - *2023-09-24*
### 🚀 Features
diff --git a/Dockerfile b/Dockerfile
index 1011e1c1..14126128 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,9 +8,8 @@ WORKDIR /app
ENV GIT_PYTHON_REFRESH=quiet
# Install system dependencies and clean up apt cache
-RUN apt-get update && apt-get install -y \
- git \
- tree && \
+RUN apt-get update && apt-get install -y \
+ git && \
rm -rf /var/lib/apt/lists/*
# Create a non-root user with a specific UID and GID (i.e. 1000 in this case)
diff --git a/README.md b/README.md
index 97c4c2c2..2bce1ee1 100644
--- a/README.md
+++ b/README.md
@@ -399,6 +399,7 @@ To generate a *README.md* file, use the `readmeai` command in your terminal, alo
| Short Flag | Long Flag | Description | Status |
|------------|----------------|---------------------------------------------------|--------------|
| `-k` | `--api-key` | Your OpenAI API secret key. | Optional |
+| `-c` | `--encoding` | Encodings specify how text is converted into tokens.| Optional |
| `-e` | `--engine` | OpenAI GPT language model engine (gpt-3.5-turbo) | Optional |
| `-f` | `--offline-mode`| Run offline without calling the OpenAI API. | Optional |
| `-o` | `--output` | The output path for your README.md file. | Optional |
diff --git a/poetry.lock b/poetry.lock
index f27b3d1e..c8818bf5 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -154,6 +154,19 @@ files = [
{file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"},
]
+[[package]]
+name = "asyncio"
+version = "3.4.3"
+description = "reference implementation of PEP 3156"
+optional = false
+python-versions = "*"
+files = [
+ {file = "asyncio-3.4.3-cp33-none-win32.whl", hash = "sha256:b62c9157d36187eca799c378e572c969f0da87cd5fc42ca372d92cdb06e7e1de"},
+ {file = "asyncio-3.4.3-cp33-none-win_amd64.whl", hash = "sha256:c46a87b48213d7464f22d9a497b9eef8c1928b68320a2fa94240f969f6fec08c"},
+ {file = "asyncio-3.4.3-py3-none-any.whl", hash = "sha256:c4d18b22701821de07bd6aea8b53d21449ec0ec5680645e5317062ea21817d2d"},
+ {file = "asyncio-3.4.3.tar.gz", hash = "sha256:83360ff8bc97980e4ff25c964c7bd3923d333d177aa4f7fb736b019f26c7cb41"},
+]
+
[[package]]
name = "attrs"
version = "23.1.0"
@@ -1581,4 +1594,4 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = "^3.8.1"
-content-hash = "c358cb0829781affc50e8638048a3210fc5308e4d55aab86774b3fda61c0d3cb"
+content-hash = "8d6c464acf3f7bf05c14a53cb28ee12a5d6c522cfde5b1847165fbdc871e6e63"
diff --git a/pyproject.toml b/pyproject.toml
index a3898509..f7046197 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "readmeai"
-version = "0.3.083"
+version = "0.3.084"
description = "🚀 Generate beautiful README.md files from the terminal. Powered by OpenAI's GPT LLMs 💫"
authors = ["Eli <0x.eli.64s@gmail.com>"]
license = "MIT"
@@ -12,26 +12,26 @@ readme = "README.md"
homepage = "https://github.com/eli64s/readme-ai"
documentation = "https://github.com/eli64s/readme-ai/blob/main/README.md"
keywords = [
- 'python',
- 'markdown',
- 'readme',
- 'documentation',
- 'ai',
- 'readme-badges',
- 'openai',
- 'readme-template',
- 'shieldsio',
- 'readme-md',
- 'awesome-readme',
- 'readme-generator',
- 'readme-generation',
- 'gpt-3',
- 'openai-api',
- 'automated-readme',
- 'auto-readme',
- 'readme-md-generator',
- 'gpt-4',
- 'llm-agent'
+ "python",
+ "cli",
+ "readme",
+ "ai",
+ "readme-badges",
+ "openai",
+ "language-model",
+ "hacktoberfest",
+ "readme-md",
+ "readme-generator",
+ "readme-generation",
+ "gpt-3",
+ "openai-api",
+ "automated-readme",
+ "gpt-4",
+ "chatgpt",
+ "openaiapi",
+ "gpt-35-turbo",
+ "llm-agent",
+ "openai-api-python"
]
include = ["readmeai", "readmeai.*"]
@@ -55,6 +55,8 @@ toml = "^0.10.2"
pydantic = "^1.10.9"
click = "^8.1.6"
tornado = "^6.3.3"
+asyncio = "^3.4.3"
+aiohttp = "^3.8.5"
[tool.poetry.dev-dependencies]
black = "*"
@@ -65,6 +67,7 @@ pytest-cov = "*"
pre-commit = "*"
[tool.isort]
+profile = "black"
line_length = 88
known_third_party = ["tenacity"]
multi_line_output = 3
diff --git a/readmeai/builder.py b/readmeai/builder.py
index f449c99d..17389a59 100644
--- a/readmeai/builder.py
+++ b/readmeai/builder.py
@@ -12,14 +12,14 @@
logger = logger.Logger(__name__)
-def build_markdown_file(
+def build_readme_file(
config: conf.AppConfig,
helper: conf.ConfigHelper,
packages: list,
- summaries: tuple,
+ code_summary: tuple,
) -> None:
"""Builds the README Markdown file for your codebase."""
- readme_sections = create_markdown_sections(config, helper, packages, summaries)
+ readme_sections = build_markdown_sections(config, helper, packages, code_summary)
readme_file = "\n".join(readme_sections)
readme_path = Path(config.paths.readme)
@@ -28,11 +28,11 @@ def build_markdown_file(
logger.info(f"README file generated at: {readme_path}")
-def create_markdown_sections(
+def build_markdown_sections(
config: conf.AppConfig,
helper: conf.ConfigHelper,
packages: list,
- summaries: tuple,
+ code_summary: tuple,
) -> List[str]:
"""Constructs each section of the README file."""
name = config.git.name
@@ -51,10 +51,10 @@ def create_markdown_sections(
else markdown_badges
)
- markdown_setup_guide = create_setup_guide(config, helper, summaries)
+ markdown_setup_guide = create_setup_guide(config, helper, code_summary)
- if not config.api.offline_mode:
- tables = create_markdown_tables(summaries)
+ if config.api.offline_mode is False:
+ tables = create_markdown_tables(config.md.default, code_summary)
config.md.tables = create_tables(tables, config.md.dropdown, user_repo)
markdown_sections = [
@@ -145,11 +145,17 @@ def create_setup_guide(
return (default_install_command, default_run_command, default_test_command)
-def create_markdown_tables(summaries: Tuple[str, str]) -> List[Tuple[str, str]]:
- """Formats the generated code summaries into a list."""
+def create_markdown_tables(
+ placeholder: str, code_summary: Tuple[str, str]
+) -> List[Tuple[str, str]]:
+ """Formats the generated code code_summary into a list."""
summary_list = []
- for module, summary in summaries:
- summary_list.append((module, summary))
+ for summary in code_summary:
+ if isinstance(summary, tuple) and len(summary) == 2:
+ module, summary_text = summary
+ else:
+ module, summary_text = summary, placeholder
+ summary_list.append((module, summary_text))
return summary_list
@@ -203,7 +209,7 @@ def create_table(data: List[Tuple[str, str]], user_repo_name: str) -> str:
return "\n".join(formatted_lines)
-def generate_code_summary_table(base_url: str, directory: Path, level=0) -> str:
+def build_recursive_tables(base_url: str, directory: Path, placeholder) -> str:
"""Creates a Markdown table structure for the given directory."""
markdown = ""
markdown += "| File | Summary |\n"
@@ -211,17 +217,14 @@ def generate_code_summary_table(base_url: str, directory: Path, level=0) -> str:
for item in sorted(directory.iterdir()):
if item.is_file():
- relative_path = os.path.relpath(item, start=directory)
- url_path = urllib.parse.quote(relative_path)
- full_url = urllib.parse.urljoin(base_url, url_path)
- markdown += f"| [{item.name}]({full_url}) | Summary of {item.name} |\n"
+ markdown += f"| [{item.name}]({item.name}) | {placeholder} |\n"
for item in sorted(directory.iterdir()):
if item.is_dir():
# If it is a sub-directory, create a collapsible section
markdown += f"\n{item.name}
\n\n"
# Recursive call for sub-directory
- markdown += generate_code_summary_table(base_url, item, level + 1)
+ markdown += build_recursive_tables(base_url, item, placeholder)
# Close the collapsible section
markdown += "\n{}
\n\n{}\n\n