-
+
diff --git a/src/templates/github.jinja b/src/templates/git.jinja
similarity index 97%
rename from src/templates/github.jinja
rename to src/templates/git.jinja
index c373367..62def5c 100644
--- a/src/templates/github.jinja
+++ b/src/templates/git.jinja
@@ -6,7 +6,7 @@
data-message="{{ error_message }}">{{ error_message }}
{% endif %}
{% with is_index=true, show_examples=false %}
- {% include 'components/github_form.jinja' %}
+ {% include 'components/git_form.jinja' %}
{% endwith %}
{% if loading %}
diff --git a/src/templates/index.jinja b/src/templates/index.jinja
index b1d42b2..e490a38 100644
--- a/src/templates/index.jinja
+++ b/src/templates/index.jinja
@@ -48,10 +48,10 @@
data-message="{{ error_message }}">{{ error_message }}
{% endif %}
{% with is_index=true, show_examples=true %}
- {% include 'components/github_form.jinja' %}
+ {% include 'components/git_form.jinja' %}
{% endwith %}
- You can also replace 'hub' with 'ingest' in any Github URL
+ You can also replace 'hub' with 'ingest' in any GitHub URL.
{% include 'components/result.jinja' %}
{% endblock %}
diff --git a/tests/query_parser/test_git_host_agnostic.py b/tests/query_parser/test_git_host_agnostic.py
new file mode 100644
index 0000000..8e86355
--- /dev/null
+++ b/tests/query_parser/test_git_host_agnostic.py
@@ -0,0 +1,81 @@
+""" Tests to verify that the query parser is Git host agnostic. """
+
+import pytest
+
+from gitingest.query_parser import parse_query
+
+
+@pytest.mark.parametrize(
+ "urls, expected_user, expected_repo, expected_url",
+ [
+ (
+ [
+ "https://github.com/tiangolo/fastapi",
+ "github.com/tiangolo/fastapi",
+ "tiangolo/fastapi",
+ ],
+ "tiangolo",
+ "fastapi",
+ "https://github.com/tiangolo/fastapi",
+ ),
+ (
+ [
+ "https://gitlab.com/gitlab-org/gitlab-runner",
+ "gitlab.com/gitlab-org/gitlab-runner",
+ "gitlab-org/gitlab-runner",
+ ],
+ "gitlab-org",
+ "gitlab-runner",
+ "https://gitlab.com/gitlab-org/gitlab-runner",
+ ),
+ (
+ [
+ "https://bitbucket.org/na-dna/llm-knowledge-share",
+ "bitbucket.org/na-dna/llm-knowledge-share",
+ "na-dna/llm-knowledge-share",
+ ],
+ "na-dna",
+ "llm-knowledge-share",
+ "https://bitbucket.org/na-dna/llm-knowledge-share",
+ ),
+ (
+ [
+ "https://gitea.com/xorm/xorm",
+ "gitea.com/xorm/xorm",
+ "xorm/xorm",
+ ],
+ "xorm",
+ "xorm",
+ "https://gitea.com/xorm/xorm",
+ ),
+ (
+ [
+ "https://codeberg.org/forgejo/forgejo",
+ "codeberg.org/forgejo/forgejo",
+ "forgejo/forgejo",
+ ],
+ "forgejo",
+ "forgejo",
+ "https://codeberg.org/forgejo/forgejo",
+ ),
+ ],
+)
+@pytest.mark.asyncio
+async def test_parse_query_without_host(
+ urls: list[str],
+ expected_user: str,
+ expected_repo: str,
+ expected_url: str,
+) -> None:
+ for url in urls:
+ result = await parse_query(url, max_file_size=50, from_web=True)
+ # Common assertions for all cases
+ assert result["user_name"] == expected_user
+ assert result["repo_name"] == expected_repo
+ assert result["url"] == expected_url
+ assert result["slug"] == f"{expected_user}-{expected_repo}"
+ assert result["id"] is not None
+ assert result["subpath"] == "/"
+ assert result["branch"] is None
+ assert result["commit"] is None
+ assert result["type"] is None
diff --git a/tests/test_query_parser.py b/tests/query_parser/test_query_parser.py
similarity index 62%
rename from tests/test_query_parser.py
rename to tests/query_parser/test_query_parser.py
index 97a829d..0db65d3 100644
--- a/tests/test_query_parser.py
+++ b/tests/query_parser/test_query_parser.py
@@ -5,12 +5,12 @@
import pytest
from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
-from gitingest.query_parser import _parse_patterns, _parse_url, parse_query
+from gitingest.query_parser import _parse_patterns, _parse_repo_source, parse_query
-def test_parse_url_valid_https() -> None:
+async def test_parse_url_valid_https() -> None:
"""
- Test `_parse_url` with valid HTTPS URLs from supported platforms (GitHub, GitLab, Bitbucket).
+ Test `_parse_repo_source` with valid HTTPS URLs from supported platforms (GitHub, GitLab, Bitbucket, Gitea).
Verifies that user and repository names are correctly extracted.
"""
test_cases = [
@@ -19,15 +19,15 @@ def test_parse_url_valid_https() -> None:
"https://bitbucket.org/user/repo",
]
for url in test_cases:
- result = _parse_url(url)
+ result = await _parse_repo_source(url)
assert result["user_name"] == "user"
assert result["repo_name"] == "repo"
assert result["url"] == url
-def test_parse_url_valid_http() -> None:
+async def test_parse_url_valid_http() -> None:
"""
- Test `_parse_url` with valid HTTP URLs from supported platforms.
+ Test `_parse_repo_source` with valid HTTP URLs from supported platforms.
Verifies that user and repository names, as well as the slug, are correctly extracted.
"""
test_cases = [
@@ -36,88 +36,88 @@ def test_parse_url_valid_http() -> None:
"http://bitbucket.org/user/repo",
]
for url in test_cases:
- result = _parse_url(url)
+ result = await _parse_repo_source(url)
assert result["user_name"] == "user"
assert result["repo_name"] == "repo"
assert result["slug"] == "user-repo"
-def test_parse_url_invalid() -> None:
+async def test_parse_url_invalid() -> None:
"""
- Test `_parse_url` with an invalid URL that does not include a repository structure.
+ Test `_parse_repo_source` with an invalid URL that does not include a repository structure.
Verifies that a ValueError is raised with an appropriate error message.
"""
- url = "https://only-domain.com"
+ url = "https://github.com"
with pytest.raises(ValueError, match="Invalid repository URL"):
- _parse_url(url)
+ await _parse_repo_source(url)
-def test_parse_query_basic() -> None:
+async def test_parse_query_basic() -> None:
"""
Test `parse_query` with basic inputs including valid repository URLs.
Verifies that user and repository names, URL, and ignore patterns are correctly parsed.
"""
test_cases = ["https://github.com/user/repo", "https://gitlab.com/user/repo"]
for url in test_cases:
- result = parse_query(url, max_file_size=50, from_web=True, ignore_patterns="*.txt")
+ result = await parse_query(url, max_file_size=50, from_web=True, ignore_patterns="*.txt")
assert result["user_name"] == "user"
assert result["repo_name"] == "repo"
assert result["url"] == url
assert "*.txt" in result["ignore_patterns"]
-def test_parse_query_mixed_case() -> None:
+async def test_parse_query_mixed_case() -> None:
"""
Test `parse_query` with mixed case URLs.
"""
url = "Https://GitHub.COM/UsEr/rEpO"
- result = parse_query(url, max_file_size=50, from_web=True)
+ result = await parse_query(url, max_file_size=50, from_web=True)
assert result["user_name"] == "user"
assert result["repo_name"] == "repo"
-def test_parse_query_include_pattern() -> None:
+async def test_parse_query_include_pattern() -> None:
"""
Test `parse_query` with an include pattern.
Verifies that the include pattern is set correctly and default ignore patterns are applied.
"""
url = "https://github.com/user/repo"
- result = parse_query(url, max_file_size=50, from_web=True, include_patterns="*.py")
+ result = await parse_query(url, max_file_size=50, from_web=True, include_patterns="*.py")
assert result["include_patterns"] == ["*.py"]
assert set(result["ignore_patterns"]) == set(DEFAULT_IGNORE_PATTERNS)
-def test_parse_query_invalid_pattern() -> None:
+async def test_parse_query_invalid_pattern() -> None:
"""
Test `parse_query` with an invalid pattern containing special characters.
Verifies that a ValueError is raised with an appropriate error message.
"""
url = "https://github.com/user/repo"
with pytest.raises(ValueError, match="Pattern.*contains invalid characters"):
- parse_query(url, max_file_size=50, from_web=True, include_patterns="*.py;rm -rf")
+ await parse_query(url, max_file_size=50, from_web=True, include_patterns="*.py;rm -rf")
-def test_parse_url_with_subpaths() -> None:
+async def test_parse_url_with_subpaths() -> None:
"""
- Test `_parse_url` with a URL containing a branch and subpath.
+ Test `_parse_repo_source` with a URL containing a branch and subpath.
Verifies that user name, repository name, branch, and subpath are correctly extracted.
"""
url = "https://github.com/user/repo/tree/main/subdir/file"
- result = _parse_url(url)
+ result = await _parse_repo_source(url)
assert result["user_name"] == "user"
assert result["repo_name"] == "repo"
assert result["branch"] == "main"
assert result["subpath"] == "/subdir/file"
-def test_parse_url_invalid_repo_structure() -> None:
+async def test_parse_url_invalid_repo_structure() -> None:
"""
- Test `_parse_url` with an invalid repository structure in the URL.
+ Test `_parse_repo_source` with an invalid repository structure in the URL.
Verifies that a ValueError is raised with an appropriate error message.
"""
url = "https://github.com/user"
with pytest.raises(ValueError, match="Invalid repository URL"):
- _parse_url(url)
+ await _parse_repo_source(url)
def test_parse_patterns_valid() -> None:
@@ -140,35 +140,35 @@ def test_parse_patterns_invalid_characters() -> None:
_parse_patterns(patterns)
-def test_parse_query_with_large_file_size() -> None:
+async def test_parse_query_with_large_file_size() -> None:
"""
Test `parse_query` with a very large file size limit.
Verifies that the file size limit and default ignore patterns are set correctly.
"""
url = "https://github.com/user/repo"
- result = parse_query(url, max_file_size=10**9, from_web=True)
+ result = await parse_query(url, max_file_size=10**9, from_web=True)
assert result["max_file_size"] == 10**9
assert result["ignore_patterns"] == DEFAULT_IGNORE_PATTERNS
-def test_parse_query_empty_patterns() -> None:
+async def test_parse_query_empty_patterns() -> None:
"""
Test `parse_query` with empty include and ignore patterns.
Verifies that the include patterns are set to None and default ignore patterns are applied.
"""
url = "https://github.com/user/repo"
- result = parse_query(url, max_file_size=50, from_web=True, include_patterns="", ignore_patterns="")
+ result = await parse_query(url, max_file_size=50, from_web=True, include_patterns="", ignore_patterns="")
assert result["include_patterns"] is None
assert result["ignore_patterns"] == DEFAULT_IGNORE_PATTERNS
-def test_parse_query_include_and_ignore_overlap() -> None:
+async def test_parse_query_include_and_ignore_overlap() -> None:
"""
Test `parse_query` with overlapping include and ignore patterns.
Verifies that overlapping patterns are removed from the ignore patterns.
"""
url = "https://github.com/user/repo"
- result = parse_query(
+ result = await parse_query(
url,
max_file_size=50,
from_web=True,
@@ -180,50 +180,50 @@ def test_parse_query_include_and_ignore_overlap() -> None:
assert "*.txt" in result["ignore_patterns"]
-def test_parse_query_local_path() -> None:
+async def test_parse_query_local_path() -> None:
"""
Test `parse_query` with a local file path.
Verifies that the local path is set, a unique ID is generated, and the slug is correctly created.
"""
path = "/home/user/project"
- result = parse_query(path, max_file_size=100, from_web=False)
+ result = await parse_query(path, max_file_size=100, from_web=False)
tail = Path("home/user/project")
assert result["local_path"].parts[-len(tail.parts) :] == tail.parts
assert result["id"] is not None
assert result["slug"] == "user/project"
-def test_parse_query_relative_path() -> None:
+async def test_parse_query_relative_path() -> None:
"""
Test `parse_query` with a relative file path.
Verifies that the local path and slug are correctly resolved.
"""
path = "./project"
- result = parse_query(path, max_file_size=100, from_web=False)
+ result = await parse_query(path, max_file_size=100, from_web=False)
tail = Path("project")
assert result["local_path"].parts[-len(tail.parts) :] == tail.parts
assert result["slug"].endswith("project")
-def test_parse_query_empty_source() -> None:
+async def test_parse_query_empty_source() -> None:
"""
Test `parse_query` with an empty source input.
Verifies that a ValueError is raised with an appropriate error message.
"""
with pytest.raises(ValueError, match="Invalid repository URL"):
- parse_query("", max_file_size=100, from_web=True)
+ await parse_query("", max_file_size=100, from_web=True)
-def test_parse_url_branch_and_commit_distinction() -> None:
+async def test_parse_url_branch_and_commit_distinction() -> None:
"""
- Test `_parse_url` with URLs containing either a branch name or a commit hash.
+ Test `_parse_repo_source` with URLs containing either a branch name or a commit hash.
Verifies that the branch and commit are correctly distinguished.
"""
url_branch = "https://github.com/user/repo/tree/main"
url_commit = "https://github.com/user/repo/tree/abcd1234abcd1234abcd1234abcd1234abcd1234"
- result_branch = _parse_url(url_branch)
- result_commit = _parse_url(url_commit)
+ result_branch = await _parse_repo_source(url_branch)
+ result_commit = await _parse_repo_source(url_commit)
assert result_branch["branch"] == "main"
assert result_branch["commit"] is None
@@ -232,23 +232,45 @@ def test_parse_url_branch_and_commit_distinction() -> None:
assert result_commit["commit"] == "abcd1234abcd1234abcd1234abcd1234abcd1234"
-def test_parse_query_uuid_uniqueness() -> None:
+async def test_parse_query_uuid_uniqueness() -> None:
"""
Test `parse_query` to ensure that each call generates a unique UUID for the query result.
"""
path = "/home/user/project"
- result1 = parse_query(path, max_file_size=100, from_web=False)
- result2 = parse_query(path, max_file_size=100, from_web=False)
+ result1 = await parse_query(path, max_file_size=100, from_web=False)
+ result2 = await parse_query(path, max_file_size=100, from_web=False)
assert result1["id"] != result2["id"]
-def test_parse_url_with_query_and_fragment() -> None:
+async def test_parse_url_with_query_and_fragment() -> None:
"""
- Test `_parse_url` with a URL containing query parameters and a fragment.
+ Test `_parse_repo_source` with a URL containing query parameters and a fragment.
Verifies that the URL is cleaned and other fields are correctly extracted.
"""
url = "https://github.com/user/repo?arg=value#fragment"
- result = _parse_url(url)
+ result = await _parse_repo_source(url)
assert result["user_name"] == "user"
assert result["repo_name"] == "repo"
assert result["url"] == "https://github.com/user/repo" # URL should be cleaned
+
+
+async def test_parse_url_unsupported_host() -> None:
+ url = "https://only-domain.com"
+ with pytest.raises(ValueError, match="Unknown domain 'only-domain.com' in URL"):
+ await _parse_repo_source(url)
+
+
+async def test_parse_query_with_branch() -> None:
+ url = "https://github.com/pandas-dev/pandas/blob/2.2.x/.github/ISSUE_TEMPLATE/documentation_improvement.yaml"
+ result = await parse_query(url, max_file_size=10**9, from_web=True)
+ assert result["user_name"] == "pandas-dev"
+ assert result["repo_name"] == "pandas"
+ assert result["url"] == "https://github.com/pandas-dev/pandas"
+ assert result["slug"] == "pandas-dev-pandas"
+ assert result["id"] is not None
+ print('result["subpath"]', result["subpath"])
+ print("/.github/ISSUE_TEMPLATE/documentation_improvement.yaml")
+ assert result["subpath"] == "/.github/ISSUE_TEMPLATE/documentation_improvement.yaml"
+ assert result["branch"] == "2.2.x"
+ assert result["commit"] is None
+ assert result["type"] == "blob"
diff --git a/tests/test_repository_clone.py b/tests/test_repository_clone.py
index 892bd04..3bfa3b2 100644
--- a/tests/test_repository_clone.py
+++ b/tests/test_repository_clone.py
@@ -204,8 +204,9 @@ async def test_clone_repo_commit_without_branch() -> None:
@pytest.mark.asyncio
async def test_check_repo_exists_with_redirect() -> None:
"""
- Test the `_check_repo_exists` function for handling HTTP redirects (302 Found).
- Verifies that it correctly identifies the repository's existence.
+ Test the `_check_repo_exists` function when the repository URL returns a redirect response.
+
+ Verifies that the function returns False when a 302 Found response is received.
"""
url = "https://github.com/user/repo"
with patch("asyncio.create_subprocess_exec", new_callable=AsyncMock) as mock_exec:
@@ -214,4 +215,21 @@ async def test_check_repo_exists_with_redirect() -> None:
mock_process.returncode = 0 # Simulate successful request
mock_exec.return_value = mock_process
+ assert await _check_repo_exists(url) is False
+
+
+@pytest.mark.asyncio
+async def test_check_repo_exists_with_permanent_redirect() -> None:
+ """
+ Test the `_check_repo_exists` function when the repository URL returns a redirect response.
+
+ Verifies that the function returns True when a 301 Found response is received.
+ """
+ url = "https://github.com/user/repo"
+ with patch("asyncio.create_subprocess_exec", new_callable=AsyncMock) as mock_exec:
+ mock_process = AsyncMock()
+ mock_process.communicate.return_value = (b"HTTP/1.1 301 Found\n", b"")
+ mock_process.returncode = 0 # Simulate successful request
+ mock_exec.return_value = mock_process
+
assert await _check_repo_exists(url)