Checa se site está disponível e se tem bloqueio a robôs

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: debug-statements
+-   repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
+    hooks:
+    -   id: flake8
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+    -   id: isort
+-   repo: https://github.com/ambv/black
+    rev: 23.3.0
+    hooks:
+    -   id: black
+        language_version: python3.8

README.md

@@ -8,3 +8,9 @@ Monitora portais da transparência de acordo com a LAI e LRF
 
 Lucas da Feira foi um herói do povo negro.
 Saiba mais [aqui](https://www.feiradesantana.ba.gov.br/secom/noticias.asp?idn=14154) e [aqui](https://www.feiradesantana.ba.gov.br/servicos.asp?s=a&link=secom/webtv.asp&idv=27031).
+
+## Desenvolvimento
+
+Dependências:
+
+* [playwright](https://playwright.dev/python/docs/intro)

lucas/checks.py

@@ -0,0 +1,38 @@
+import requests
+
+
+def _has_recaptcha(html):  # TODO add typing
+    """Check if recaptcha is on.
+
+    Docs: https://developers.google.com/recaptcha/docs/v3
+    """
+    possibilities = [
+        html.find("iframe[title^='reCAPTCHA']"),
+        html.find("input#recaptcha-token"),
+        html.find("div.g-recaptcha"),
+        html.find("textarea#g-recaptcha-response"),
+        html.find("textarea.g-recaptcha-response"),
+        html.find("script[src*='www.google.com/recaptcha']"),
+    ]
+    return any(possibilities)
+
+
+def has_robot_blocker(body):
+    """Verify if robot blockers, like recaptcha, are configured."""
+    return _has_recaptcha(body)
+
+
+def is_reachable(url):
+    result = {
+        "reachable": True,
+        "robot_friendly": True,
+    }
+    response = requests.get(url, timeout=(3.05, 27))
+    try:
+        response.raise_for_status()
+    except requests.RequestException:
+        result["reachable"] = False
+    else:
+        if has_robot_blocker(response.text):
+            result["robot_friendly"] = False
+    return result

lucas/cli.py

@@ -0,0 +1,22 @@
+from lucas.checks import is_reachable
+
+config = {
+    "portal": "http://www.transparencia.feiradesantana.ba.gov.br/",
+    "despesas": "http://www.transparencia.feiradesantana.ba.gov.br/index.php?view=despesa",
+    "receitas": "http://www.transparencia.feiradesantana.ba.gov.br/index.php?view=receita",
+    "licitacoes": "http://www.transparencia.feiradesantana.ba.gov.br/index.php?view=licitacoes",
+    "contratos": "http://www.transparencia.feiradesantana.ba.gov.br/index.php?view=contratos",
+}
+
+
+def main():
+    print("Acessível?")
+    for key, url in config.items():
+        print(f"{key} - {url}")
+        result = is_reachable(url)
+        for criteria, criteria_result in result.items():
+            print(f"- {criteria} {'✅' if criteria_result else '❌'}")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -6,13 +6,28 @@ authors = ["Ana Paula Gomes <[email protected]>"]
 license = "MIT"
 readme = "README.md"
 
+[tool.poetry.scripts]
+lucas = "lucas.cli:main"
+
 [tool.poetry.dependencies]
 python = "^3.8"
 requests = "^2.28.2"
 bs4 = "^0.0.1"
 ipython = "^8.12.0"
+pytest-playwright = "^0.3.2"
+
 
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.3.1"
+pytest-mock = "^3.10.0"
+requests-mock = "^1.10.0"
+pre-commit = "^3.2.2"
 
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+markers = [
+    "rules: marca um teste como uma regra (pule-os com '-m \"not rules\"')",
+]