+ {{ label }} +
++ {{ subLabel }} +
++ Performance Schema is not enabled on database server +
++ Please reach out to + support + to enable it +
+Process Killed
+ + + diff --git a/dashboard/src2/components/devtools/database/DatabaseTableSchemaDialog.vue b/dashboard/src2/components/devtools/database/DatabaseTableSchemaDialog.vue index 6e88a09148..5bd9143cd6 100644 --- a/dashboard/src2/components/devtools/database/DatabaseTableSchemaDialog.vue +++ b/dashboard/src2/components/devtools/database/DatabaseTableSchemaDialog.vue @@ -20,7 +20,10 @@ v-if="selectedSchema" /> -- Using MariaDB Client + Using MariaDB CLI
Run this command in your terminal to access MariaDB console @@ -55,12 +55,43 @@ computer.
SSL Info
++ You need to use SSL to connect to the database. +
+ +Need Help?
++ Please check out the + documentation + to get guides on troubleshooting. +
+${this.selectedQuery}
+ `,
+ primaryAction: {
+ label: 'Run Query',
+ variant: 'solid',
+ onClick: ({ hide }) => {
+ this.runSQLQuery(false, true);
hide();
}
}
diff --git a/dashboard/src2/router.js b/dashboard/src2/router.js
index 3113d083bd..19afb7b733 100644
--- a/dashboard/src2/router.js
+++ b/dashboard/src2/router.js
@@ -291,6 +291,11 @@ let router = createRouter({
component: () =>
import('./pages/devtools/database/DatabaseSQLPlayground.vue')
},
+ {
+ path: '/database-analyzer',
+ name: 'DB Analyzer',
+ component: () => import('./pages/devtools/database/DatabaseAnalyzer.vue')
+ },
{
path: '/log-browser/:mode?/:docName?/:logId?',
name: 'Log Browser',
diff --git a/package.json b/package.json
index 6b0c346679..969e90b6e2 100644
--- a/package.json
+++ b/package.json
@@ -20,7 +20,10 @@
"@tailwindcss/typography": "^0.5.1",
"autoprefixer": "^10.4.2",
"postcss": "^8.4.6",
- "tailwindcss": "^3.2"
+ "tailwindcss": "^3.2",
+ "@commitlint/config-conventional": "^19.6.0",
+ "commitlint": "^19.6.1",
+ "cspell": "^8.17.1"
},
"dependencies": {
"dayjs": "^1.11.10",
diff --git a/press/agent.py b/press/agent.py
index 34ea88f61f..b102b736cf 100644
--- a/press/agent.py
+++ b/press/agent.py
@@ -1161,13 +1161,18 @@ def get_site_apps(self, site):
apps: list[str] = [line.split()[0] for line in raw_apps_list["data"].splitlines() if line]
return apps
- def fetch_database_table_schema(self, site):
+ def fetch_database_table_schema(
+ self, site, include_table_size: bool = False, include_index_info: bool = False
+ ):
return self.create_agent_job(
"Fetch Database Table Schema",
f"benches/{site.bench}/sites/{site.name}/database/schema",
bench=site.bench,
site=site.name,
- data={},
+ data={
+ "include_table_size": include_table_size,
+ "include_index_info": include_index_info,
+ },
reference_doctype="Site",
reference_name=site.name,
)
@@ -1178,6 +1183,48 @@ def run_sql_query_in_database(self, site, query, commit):
data={"query": query, "commit": commit, "as_dict": False},
)
+ def get_summarized_performance_report_of_database(self, site):
+ return self.post(
+ f"benches/{site.bench}/sites/{site.name}/database/performance-report",
+ data={"mariadb_root_password": get_mariadb_root_password(site)},
+ )
+
+ def analyze_slow_queries(self, site, normalized_queries: list[dict]):
+ """
+ normalized_queries format:
+ [
+ {
+ "example": "",
+ "normalized" : "",
+ }
+ ]
+ """
+ return self.create_agent_job(
+ "Analyze Slow Queries",
+ f"benches/{site.bench}/sites/{site.name}/database/analyze-slow-queries",
+ data={
+ "queries": normalized_queries,
+ "mariadb_root_password": get_mariadb_root_password(site),
+ },
+ site=site.name,
+ )
+
+ def fetch_database_processes(self, site):
+ return self.post(
+ f"benches/{site.bench}/sites/{site.name}/database/processes",
+ data={
+ "mariadb_root_password": get_mariadb_root_password(site),
+ },
+ )
+
+ def kill_database_process(self, site, id):
+ return self.post(
+ f"benches/{site.bench}/sites/{site.name}/database/kill-process/{id}",
+ data={
+ "mariadb_root_password": get_mariadb_root_password(site),
+ },
+ )
+
class AgentCallbackException(Exception):
pass
diff --git a/press/api/__init__.py b/press/api/__init__.py
index 1a47323294..ad1fcc1c79 100644
--- a/press/api/__init__.py
+++ b/press/api/__init__.py
@@ -1,9 +1,10 @@
import frappe
-from press.utils import get_minified_script, get_minified_script_2
+from press.api.client import dashboard_whitelist
from press.saas.doctype.product_trial_request.product_trial_request import (
get_app_trial_page_url,
)
+from press.utils import get_full_chain_cert_of_domain, get_minified_script, get_minified_script_2, log_error
@frappe.whitelist(allow_guest=True)
@@ -20,3 +21,18 @@ def script_2():
def handle_suspended_site_redirection():
frappe.local.response["type"] = "redirect"
frappe.local.response["location"] = get_app_trial_page_url() or "/dashboard"
+
+
+@dashboard_whitelist()
+def download_ssl_cert(domain: str):
+ if (
+ not (domain.endswith("frappe.cloud") or domain.endswith("frappecloud.com"))
+ and not frappe.conf.developer_mode
+ ):
+ frappe.throw("Invalid domain provided")
+
+ try:
+ return get_full_chain_cert_of_domain(domain)
+ except Exception as e:
+ log_error("Error downloading SSL certificate", data=e)
+ frappe.throw("Failed to download SSL certificate. Please try again later.")
diff --git a/press/api/analytics.py b/press/api/analytics.py
index a00d11ecad..63e491f16a 100644
--- a/press/api/analytics.py
+++ b/press/api/analytics.py
@@ -24,9 +24,6 @@
from press.agent import Agent
from press.api.site import protected
from press.press.doctype.site_plan.site_plan import get_plan_config
-from press.press.report.binary_log_browser.binary_log_browser import (
- convert_user_timezone_to_utc,
-)
from press.press.report.binary_log_browser.binary_log_browser import (
get_data as get_binary_log_data,
)
@@ -654,7 +651,7 @@ def get_current_cpu_usage_for_sites_on_server(server):
metric = row["usage"]["counter"]["top"]
if metric:
result[site] = metric[0]["metrics"]["json.request.counter"]
- return result
+ return result
@frappe.whitelist()
@@ -767,28 +764,19 @@ def mariadb_slow_queries(
@frappe.whitelist()
@protected("Site")
-def deadlock_report(site, start, end, max_lines=20):
- from press.press.report.mariadb_deadlock_browser.mariadb_deadlock_browser import (
- post_process,
- )
-
- server = frappe.db.get_value("Site", site, "server")
- db_server_name = frappe.db.get_value("Server", server, "database_server")
- database_server = frappe.get_doc("Database Server", db_server_name)
- agent = Agent(database_server.name, "Database Server")
-
- data = {
- "private_ip": database_server.private_ip,
- "mariadb_root_password": database_server.get_password("mariadb_root_password"),
- "database": database_server.name,
- "start_datetime": convert_user_timezone_to_utc(start),
- "stop_datetime": convert_user_timezone_to_utc(end),
- "max_lines": max_lines,
- }
-
- results = agent.post("database/deadlocks", data=data)
+def deadlock_report(name, start_datetime, stop_datetime, max_log_size=500):
+ from press.press.report.mariadb_deadlock_browser.mariadb_deadlock_browser import execute
- return post_process(results)
+ meta = frappe._dict(
+ {
+ "site": name,
+ "start_datetime": start_datetime,
+ "stop_datetime": stop_datetime,
+ "max_log_size": max_log_size,
+ }
+ )
+ _, data = execute(filters=meta)
+ return data
# MARKETPLACE - Plausible
diff --git a/press/api/dboptimize.py b/press/api/dboptimize.py
deleted file mode 100644
index 37209e0408..0000000000
--- a/press/api/dboptimize.py
+++ /dev/null
@@ -1,176 +0,0 @@
-import json
-
-import frappe
-
-from press.api.site import protected
-from press.press.report.mariadb_slow_queries.db_optimizer import (
- ColumnStat,
- DBExplain,
- DBOptimizer,
- DBTable,
-)
-from press.press.report.mariadb_slow_queries.mariadb_slow_queries import (
- OptimizeDatabaseQuery,
- _fetch_column_stats,
- _fetch_table_stats,
-)
-from press.utils import log_error
-
-
-@frappe.whitelist()
-@protected("Site")
-def mariadb_analyze_query(name, row):
- return analyze_query(row=row, site=name)
-
-
-def analyze_query(row, site):
- # if mariadb_analyze_query_already_exists(site, row["query"]):
- # frappe.throw("The query seems to have already been optimized")
- doc = frappe.get_doc(
- {
- "doctype": "MariaDB Analyze Query",
- "site": site,
- "tables_in_query": [],
- }
- )
- doc.status = "Running"
-
- query = row["example"]
- doc.query = query
- doc.normalized_query = row["query"]
-
- if not query.lower().startswith(("select", "update", "delete")):
- doc.status = "Failure"
- doc.save(ignore_permissions=True)
- frappe.db.commit()
- return None
-
- doc.save(ignore_permissions=True)
- frappe.db.commit()
-
- analyzer = OptimizeDatabaseQuery(site, query)
- explain_output = analyzer.fetch_explain() or []
- doc.explain_output = json.dumps(explain_output)
- explain_output = [DBExplain.from_frappe_ouput(e) for e in explain_output]
-
- optimizer = DBOptimizer(query=analyzer.query, explain_plan=explain_output)
- for table in optimizer.tables_examined:
- stats = _fetch_table_stats(analyzer.site, table)
- doc.append("tables_in_query", {"table": table, "table_statistics": json.dumps(stats)})
-
- if not stats:
- # Old framework version
- doc.status = "Failure"
- doc.save(ignore_permissions=True)
- frappe.db.commit()
- return None
-
- # This is an agent job. Remaining is processed in the callback.
- _fetch_column_stats(analyzer.site, table, doc.get_title())
-
- doc.save(ignore_permissions=True)
- return doc.status
-
-
-def check_if_all_fetch_column_stats_was_successful(doc):
- return all(item.status == "Success" for item in doc.tables_in_query)
-
-
-def fetch_column_stats_update(job, response_data):
- request_data_json = json.loads(job.request_data)
- doc_name = request_data_json["doc_name"]
- table = request_data_json["table"]
-
- if job.status == "Success":
- column_statistics = response_data["steps"][0]["data"]["output"]
- doc = frappe.get_doc("MariaDB Analyze Query", doc_name)
- for item in doc.tables_in_query:
- if item.table == table:
- item.column_statistics = column_statistics
- item.status = "Success"
- doc.save()
- frappe.db.commit()
- if check_if_all_fetch_column_stats_was_successful(doc):
- doc.status = "Success"
- doc.save()
- frappe.db.commit()
- # Persists within doctype
- save_suggested_index(doc)
- elif job.status == "Failure":
- doc = frappe.get_doc("MariaDB Analyze Query", doc_name)
- for item in doc.tables_in_query:
- if item.table == table:
- item.status = "Failure"
- doc.save()
-
- doc.status = "Failure"
- doc.save()
- frappe.db.commit()
-
-
-def save_suggested_index(doc):
- explain_output = json.loads(doc.explain_output)
- explain_output = [DBExplain.from_frappe_ouput(e) for e in explain_output]
- optimizer = DBOptimizer(query=doc.query, explain_plan=explain_output)
- for item in doc.tables_in_query:
- stats = json.loads(item.table_statistics)
- if not stats:
- # Old framework version
- return
- db_table = DBTable.from_frappe_ouput(stats)
- column_stats = json.loads(item.column_statistics)
- column_stats = [ColumnStat.from_frappe_ouput(c) for c in column_stats]
- db_table.update_cardinality(column_stats)
- optimizer.update_table_data(db_table)
- index = optimizer.suggest_index()
- doc.suggested_index = f"{index.table}.{index.column}"
- doc.save()
-
-
-@frappe.whitelist()
-@protected("Site")
-def get_status_of_mariadb_analyze_query(name, query):
- filters = {"site": name, "query": query}
- doc = frappe.get_all(
- "MariaDB Analyze Query",
- filters=filters,
- fields=["status", "suggested_index"],
- limit=1,
- )
- if doc:
- return doc[0]
- return None
-
-
-def mariadb_analyze_query_already_exists(site, normalized_query):
- if frappe.db.exists("MariaDB Analyze Query", {"site": site, "normalized_query": normalized_query}):
- return True
- return False
-
-
-@frappe.whitelist()
-@protected("Site")
-def mariadb_analyze_query_already_running_for_site(name):
- if frappe.db.exists("MariaDB Analyze Query", {"site": name, "status": "Running"}):
- return True
- return False
-
-
-@frappe.whitelist()
-@protected("Site")
-def get_suggested_index(name, normalized_query):
- return frappe.get_value(
- "MariaDB Analyze Query",
- {"site": name, "status": "Success", "normalized_query": normalized_query},
- ["site", "normalized_query", "suggested_index"],
- as_dict=True,
- )
-
-
-def delete_all_occurences_of_mariadb_analyze_query(job):
- try:
- if job.status == "Success" or job.status == "Failure":
- frappe.db.delete("MariaDB Analyze Query", {"site": job.site})
- frappe.db.commit()
- except Exception as e:
- log_error("Deleting all occurrences of MariaDB Analyze Query Failed", data=e)
diff --git a/press/api/server.py b/press/api/server.py
index df37dbd7ab..e36746ef47 100644
--- a/press/api/server.py
+++ b/press/api/server.py
@@ -300,6 +300,42 @@ def analytics(name, query, timezone, duration):
f"""node_memory_MemTotal_bytes{{instance="{name}",job="node"}} - node_memory_MemFree_bytes{{instance="{name}",job="node"}} - (node_memory_Cached_bytes{{instance="{name}",job="node"}} + node_memory_Buffers_bytes{{instance="{name}",job="node"}})""",
lambda x: "Used",
),
+ "database_uptime": (
+ f"""mysql_up{{instance="{name}",job="mariadb"}}""",
+ lambda x: "Uptime",
+ ),
+ "database_commands_count": (
+ f"""sum(round(increase(mysql_global_status_commands_total{{instance='{name}', command=~"select|update|insert|delete|begin|commit|rollback"}}[{timegrain}s]))) by (command)""",
+ lambda x: x["command"],
+ ),
+ "database_connections": (
+ f"""{{__name__=~"mysql_global_status_threads_connected|mysql_global_variables_max_connections", instance="{name}"}}""",
+ lambda x: "Max Connections"
+ if x["__name__"] == "mysql_global_variables_max_connections"
+ else "Connected Clients",
+ ),
+ "innodb_bp_size": (
+ f"""mysql_global_variables_innodb_buffer_pool_size{{instance='{name}'}}""",
+ lambda x: "Buffer Pool Size",
+ ),
+ "innodb_bp_size_of_total_ram": (
+ f"""avg by (instance) ((mysql_global_variables_innodb_buffer_pool_size{{instance=~"{name}"}} * 100)) / on (instance) (avg by (instance) (node_memory_MemTotal_bytes{{instance=~"{name}"}}))""",
+ lambda x: "Buffer Pool Size of Total Ram",
+ ),
+ "innodb_bp_miss_percent": (
+ f"""
+avg by (instance) (
+ rate(mysql_global_status_innodb_buffer_pool_reads{{instance=~"{name}"}}[{timegrain}s])
+ /
+ rate(mysql_global_status_innodb_buffer_pool_read_requests{{instance=~"{name}"}}[{timegrain}s])
+)
+""",
+ lambda x: "Buffer Pool Miss Percentage",
+ ),
+ "innodb_avg_row_lock_time": (
+ f"""(rate(mysql_global_status_innodb_row_lock_time{{instance="{name}"}}[{timegrain}s]) / 1000)/rate(mysql_global_status_innodb_row_lock_waits{{instance="{name}"}}[{timegrain}s])""",
+ lambda x: "Avg Row Lock Time",
+ ),
}
return prometheus_query(query_map[query][0], query_map[query][1], timezone, timespan, timegrain)
diff --git a/press/fixtures/agent_job_type.json b/press/fixtures/agent_job_type.json
index b6b044d9e4..a4b52ccdaa 100644
--- a/press/fixtures/agent_job_type.json
+++ b/press/fixtures/agent_job_type.json
@@ -2260,5 +2260,35 @@
"step_name": "Modify Database User Permissions"
}
]
+ },
+ {
+ "disabled_auto_retry": 1,
+ "docstatus": 0,
+ "doctype": "Agent Job Type",
+ "max_retry_count": 1,
+ "modified": "2024-10-28 14:49:19.894247",
+ "name": "Fetch Database Table Schema",
+ "request_method": "POST",
+ "request_path": "/benches/{bench}/sites/{site}/database/schema",
+ "steps": [
+ {
+ "step_name": "Fetch Database Table Schema"
+ }
+ ]
+ },
+ {
+ "disabled_auto_retry": 1,
+ "docstatus": 0,
+ "doctype": "Agent Job Type",
+ "max_retry_count": 3,
+ "modified": "2024-12-19 17:21:14.136650",
+ "name": "Analyze Slow Queries",
+ "request_method": "POST",
+ "request_path": "/benches/{bench}/sites/{site}/database/analyze-slow-queries",
+ "steps": [
+ {
+ "step_name": "Analyze Slow Queries"
+ }
+ ]
}
]
\ No newline at end of file
diff --git a/press/press/doctype/account_request/account_request.py b/press/press/doctype/account_request/account_request.py
index 3cf9287121..f907bca7a5 100644
--- a/press/press/doctype/account_request/account_request.py
+++ b/press/press/doctype/account_request/account_request.py
@@ -10,7 +10,7 @@
from frappe.model.document import Document
from frappe.utils import get_url, random_string
-from press.utils import get_country_info
+from press.utils import get_country_info, is_valid_email_address
from press.utils.telemetry import capture
@@ -63,6 +63,17 @@ class AccountRequest(Document):
# end: auto-generated types
def before_insert(self):
+ # This pre-verification is only beneficial for SaaS signup
+ # because, in general flow we already have e-mail link/otp based verification
+ if (
+ not frappe.conf.developer_mode
+ and frappe.db.get_single_value("Press Settings", "enable_email_pre_verification")
+ and self.saas
+ and not self.oauth_signup
+ and not is_valid_email_address(self.email)
+ ):
+ frappe.throw(f"{self.email} is not a valid email address")
+
if not self.team:
self.team = self.email
diff --git a/press/press/doctype/agent_job/agent_job.py b/press/press/doctype/agent_job/agent_job.py
index 69ab83a916..f0c7589f62 100644
--- a/press/press/doctype/agent_job/agent_job.py
+++ b/press/press/doctype/agent_job/agent_job.py
@@ -879,10 +879,6 @@ def process_job_updates(job_name: str, response_data: dict | None = None): # no
start = now_datetime()
try:
- from press.api.dboptimize import (
- delete_all_occurences_of_mariadb_analyze_query,
- fetch_column_stats_update,
- )
from press.press.doctype.agent_job.agent_job_notifications import (
send_job_failure_notification,
)
@@ -912,6 +908,7 @@ def process_job_updates(job_name: str, response_data: dict | None = None): # no
process_archive_site_job_update,
process_complete_setup_wizard_job_update,
process_create_user_job_update,
+ process_fetch_database_table_schema_job_update,
process_install_app_site_job_update,
process_migrate_site_job_update,
process_move_site_to_bench_job_update,
@@ -923,9 +920,6 @@ def process_job_updates(job_name: str, response_data: dict | None = None): # no
process_uninstall_app_site_job_update,
)
from press.press.doctype.site_backup.site_backup import process_backup_site_job_update
- from press.press.doctype.site_database_table_schema.site_database_table_schema import (
- SiteDatabaseTableSchema,
- )
from press.press.doctype.site_domain.site_domain import process_new_host_job_update
from press.press.doctype.site_update.site_update import (
process_update_site_job_update,
@@ -1001,23 +995,8 @@ def process_job_updates(job_name: str, response_data: dict | None = None): # no
AppPatch.process_patch_app(job)
elif job.job_type == "Run Remote Builder":
DeployCandidate.process_run_build(job, response_data)
- elif job.job_type == "Column Statistics":
- frappe.enqueue(
- fetch_column_stats_update,
- queue="default",
- timeout=None,
- is_async=True,
- now=False,
- job_name="Fetch Column Updates Through Enque",
- enqueue_after_commit=False,
- at_front=False,
- job=job,
- response_data=response_data,
- )
elif job.job_type == "Create User":
process_create_user_job_update(job)
- elif job.job_type == "Add Database Index":
- delete_all_occurences_of_mariadb_analyze_query(job)
elif job.job_type == "Complete Setup Wizard":
process_complete_setup_wizard_job_update(job)
elif job.job_type == "Update Bench In Place":
@@ -1025,7 +1004,7 @@ def process_job_updates(job_name: str, response_data: dict | None = None): # no
elif job.job_type == "Recover Update In Place":
Bench.process_recover_update_inplace(job)
elif job.job_type == "Fetch Database Table Schema":
- SiteDatabaseTableSchema.process_job_update(job)
+ process_fetch_database_table_schema_job_update(job)
elif job.job_type in [
"Create Database User",
"Remove Database User",
diff --git a/press/press/doctype/bench/bench.py b/press/press/doctype/bench/bench.py
index 29bd5b4bf2..b3ee8b2cdc 100644
--- a/press/press/doctype/bench/bench.py
+++ b/press/press/doctype/bench/bench.py
@@ -598,6 +598,14 @@ def allocate_workers(
round(self.workload / server_workload * max_gunicorn_workers),
), # min 2 max 36
)
+ if self.gunicorn_threads_per_worker:
+ # Allocate fewer workers if threaded workers are used
+ # Roughly workers / threads_per_worker = total number of workers
+ # 1. At least one worker
+ # 2. Slightly more workers than required
+ self.gunicorn_workers = frappe.utils.ceil(
+ self.gunicorn_workers / self.gunicorn_threads_per_worker
+ )
self.background_workers = min(
max_bg or MAX_BACKGROUND_WORKERS,
max(
diff --git a/press/press/doctype/invoice/invoice.py b/press/press/doctype/invoice/invoice.py
index 3c4c0d5d6c..143c29c97c 100644
--- a/press/press/doctype/invoice/invoice.py
+++ b/press/press/doctype/invoice/invoice.py
@@ -506,11 +506,23 @@ def validate_dates(self):
def update_item_descriptions(self):
for item in self.items:
- if not item.description and item.document_type == "Site" and item.plan:
- site_name = item.document_name.split(".archived")[0]
- plan = frappe.get_cached_value("Site Plan", item.plan, "plan_title")
+ if not item.description:
how_many_days = f"{cint(item.quantity)} day{'s' if item.quantity > 1 else ''}"
- item.description = f"{site_name} active for {how_many_days} on {plan} plan"
+ if item.document_type == "Site" and item.plan:
+ site_name = item.document_name.split(".archived")[0]
+ plan = frappe.get_cached_value("Site Plan", item.plan, "plan_title")
+ item.description = f"{site_name} active for {how_many_days} on {plan} plan"
+ elif item.document_type in ["Server", "Database Server"]:
+ server_title = frappe.get_cached_value(item.document_type, item.document_name, "title")
+ if item.plan == "Add-on Storage plan":
+ item.description = f"{server_title} Storage Add-on for {how_many_days}"
+ else:
+ item.description = f"{server_title} active for {how_many_days}"
+ elif item.document_type == "Marketplace App":
+ app_title = frappe.get_cached_value("Marketplace App", item.document_name, "title")
+ item.description = f"Marketplace app {app_title} active for {how_many_days}"
+ else:
+ item.description = "Prepaid Credits"
def add_usage_record(self, usage_record):
if self.type != "Subscription":
diff --git a/press/press/doctype/mariadb_analyze_query/__init__.py b/press/press/doctype/mariadb_analyze_query/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/press/press/doctype/mariadb_analyze_query/mariadb_analyze_query.js b/press/press/doctype/mariadb_analyze_query/mariadb_analyze_query.js
deleted file mode 100644
index f6228a4cd9..0000000000
--- a/press/press/doctype/mariadb_analyze_query/mariadb_analyze_query.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2024, Frappe and contributors
-// For license information, please see license.txt
-
-// frappe.ui.form.on("MariaDB Analyze Query", {
-// refresh(frm) {
-
-// },
-// });
diff --git a/press/press/doctype/mariadb_analyze_query/mariadb_analyze_query.json b/press/press/doctype/mariadb_analyze_query/mariadb_analyze_query.json
deleted file mode 100644
index 17e02dffe0..0000000000
--- a/press/press/doctype/mariadb_analyze_query/mariadb_analyze_query.json
+++ /dev/null
@@ -1,92 +0,0 @@
-{
- "actions": [],
- "allow_rename": 1,
- "creation": "2024-07-21 16:44:56.074012",
- "doctype": "DocType",
- "engine": "InnoDB",
- "field_order": [
- "site",
- "normalized_query",
- "column_break_holt",
- "query",
- "status",
- "section_break_cdwy",
- "explain_output",
- "tables_in_query",
- "suggested_index"
- ],
- "fields": [
- {
- "fieldname": "query",
- "fieldtype": "Long Text",
- "label": "Query",
- "reqd": 1
- },
- {
- "fieldname": "site",
- "fieldtype": "Data",
- "in_list_view": 1,
- "label": "Site",
- "reqd": 1
- },
- {
- "fieldname": "tables_in_query",
- "fieldtype": "Table",
- "label": "Tables In Query",
- "options": "MariaDB Analyze Query Tables"
- },
- {
- "fieldname": "suggested_index",
- "fieldtype": "Data",
- "label": "Suggested Index"
- },
- {
- "fieldname": "explain_output",
- "fieldtype": "Code",
- "label": "Explain Output"
- },
- {
- "fieldname": "status",
- "fieldtype": "Data",
- "label": "Status"
- },
- {
- "fieldname": "column_break_holt",
- "fieldtype": "Column Break"
- },
- {
- "fieldname": "normalized_query",
- "fieldtype": "Long Text",
- "label": "Normalized Query",
- "reqd": 1
- },
- {
- "fieldname": "section_break_cdwy",
- "fieldtype": "Section Break"
- }
- ],
- "index_web_pages_for_search": 1,
- "links": [],
- "modified": "2024-07-29 17:32:21.578471",
- "modified_by": "Administrator",
- "module": "Press",
- "name": "MariaDB Analyze Query",
- "owner": "Administrator",
- "permissions": [
- {
- "create": 1,
- "delete": 1,
- "email": 1,
- "export": 1,
- "print": 1,
- "read": 1,
- "report": 1,
- "role": "System Manager",
- "share": 1,
- "write": 1
- }
- ],
- "sort_field": "creation",
- "sort_order": "DESC",
- "states": []
-}
\ No newline at end of file
diff --git a/press/press/doctype/mariadb_analyze_query/mariadb_analyze_query.py b/press/press/doctype/mariadb_analyze_query/mariadb_analyze_query.py
deleted file mode 100644
index cb963a42af..0000000000
--- a/press/press/doctype/mariadb_analyze_query/mariadb_analyze_query.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright (c) 2024, Frappe and contributors
-# For license information, please see license.txt
-
-# import frappe
-from frappe.model.document import Document
-
-
-class MariaDBAnalyzeQuery(Document):
- # begin: auto-generated types
- # This code is auto-generated. Do not modify anything in this block.
-
- from typing import TYPE_CHECKING
-
- if TYPE_CHECKING:
- from frappe.types import DF
- from press.press.doctype.mariadb_analyze_query_tables.mariadb_analyze_query_tables import (
- MariaDBAnalyzeQueryTables,
- )
-
- explain_output: DF.Code | None
- normalized_query: DF.LongText
- query: DF.LongText
- site: DF.Data
- status: DF.Data | None
- suggested_index: DF.Data | None
- tables_in_query: DF.Table[MariaDBAnalyzeQueryTables]
- # end: auto-generated types
-
- pass
diff --git a/press/press/doctype/mariadb_analyze_query/test_mariadb_analyze_query.py b/press/press/doctype/mariadb_analyze_query/test_mariadb_analyze_query.py
deleted file mode 100644
index 082e41b5b7..0000000000
--- a/press/press/doctype/mariadb_analyze_query/test_mariadb_analyze_query.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright (c) 2024, Frappe and Contributors
-# See license.txt
-
-# import frappe
-from frappe.tests.utils import FrappeTestCase
-
-
-class TestMariaDBAnalyzeQuery(FrappeTestCase):
- pass
diff --git a/press/press/doctype/mariadb_analyze_query_tables/__init__.py b/press/press/doctype/mariadb_analyze_query_tables/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/press/press/doctype/mariadb_analyze_query_tables/mariadb_analyze_query_tables.json b/press/press/doctype/mariadb_analyze_query_tables/mariadb_analyze_query_tables.json
deleted file mode 100644
index dc8bb19960..0000000000
--- a/press/press/doctype/mariadb_analyze_query_tables/mariadb_analyze_query_tables.json
+++ /dev/null
@@ -1,52 +0,0 @@
-{
- "actions": [],
- "allow_rename": 1,
- "creation": "2024-07-21 16:49:47.144539",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
- "table",
- "table_statistics",
- "column_statistics",
- "status"
- ],
- "fields": [
- {
- "fieldname": "table",
- "fieldtype": "Data",
- "in_list_view": 1,
- "label": "Table"
- },
- {
- "fieldname": "column_statistics",
- "fieldtype": "Code",
- "in_list_view": 1,
- "label": "Column Statistics"
- },
- {
- "fieldname": "table_statistics",
- "fieldtype": "Code",
- "in_list_view": 1,
- "label": "Table Statistics"
- },
- {
- "fieldname": "status",
- "fieldtype": "Data",
- "in_list_view": 1,
- "label": "Status"
- }
- ],
- "index_web_pages_for_search": 1,
- "istable": 1,
- "links": [],
- "modified": "2024-07-24 11:46:38.741427",
- "modified_by": "Administrator",
- "module": "Press",
- "name": "MariaDB Analyze Query Tables",
- "owner": "Administrator",
- "permissions": [],
- "sort_field": "creation",
- "sort_order": "DESC",
- "states": []
-}
\ No newline at end of file
diff --git a/press/press/doctype/mariadb_analyze_query_tables/mariadb_analyze_query_tables.py b/press/press/doctype/mariadb_analyze_query_tables/mariadb_analyze_query_tables.py
deleted file mode 100644
index c0a058d29d..0000000000
--- a/press/press/doctype/mariadb_analyze_query_tables/mariadb_analyze_query_tables.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright (c) 2024, Frappe and contributors
-# For license information, please see license.txt
-
-# import frappe
-from frappe.model.document import Document
-
-
-class MariaDBAnalyzeQueryTables(Document):
- # begin: auto-generated types
- # This code is auto-generated. Do not modify anything in this block.
-
- from typing import TYPE_CHECKING
-
- if TYPE_CHECKING:
- from frappe.types import DF
-
- column_statistics: DF.Code | None
- parent: DF.Data
- parentfield: DF.Data
- parenttype: DF.Data
- status: DF.Data | None
- table: DF.Data | None
- table_statistics: DF.Code | None
- # end: auto-generated types
-
- pass
diff --git a/press/press/doctype/press_settings/press_settings.json b/press/press/doctype/press_settings/press_settings.json
index 71c8c7e90a..56f2644310 100644
--- a/press/press/doctype/press_settings/press_settings.json
+++ b/press/press/doctype/press_settings/press_settings.json
@@ -193,6 +193,7 @@
"column_break_rdlr",
"disable_auto_retry",
"disable_agent_job_deduplication",
+ "enable_email_pre_verification",
"section_break_jstu",
"enable_app_grouping",
"default_apps",
@@ -1249,8 +1250,8 @@
"fieldtype": "Link",
"label": "Press Trial Plan",
"options": "Site Plan"
- },
- {
+ },
+ {
"fieldname": "section_break_jstu",
"fieldtype": "Section Break"
},
@@ -1265,11 +1266,17 @@
"fieldtype": "Table",
"label": "Default Apps",
"options": "App Group"
+ },
+ {
+ "default": "0",
+ "fieldname": "enable_email_pre_verification",
+ "fieldtype": "Check",
+ "label": "Enable Email Pre-Verification"
}
],
"issingle": 1,
"links": [],
- "modified": "2024-10-14 21:02:12.948747",
+ "modified": "2024-12-10 15:07:57.494659",
"modified_by": "Administrator",
"module": "Press",
"name": "Press Settings",
diff --git a/press/press/doctype/press_settings/press_settings.py b/press/press/doctype/press_settings/press_settings.py
index cedb08f91c..bc4c1a559b 100644
--- a/press/press/doctype/press_settings/press_settings.py
+++ b/press/press/doctype/press_settings/press_settings.py
@@ -64,6 +64,7 @@ class PressSettings(Document):
domain: DF.Link | None
eff_registration_email: DF.Data
enable_app_grouping: DF.Check
+ enable_email_pre_verification: DF.Check
enable_google_oauth: DF.Check
enable_site_pooling: DF.Check
enforce_storage_limits: DF.Check
diff --git a/press/press/doctype/proxy_server/proxy_server.json b/press/press/doctype/proxy_server/proxy_server.json
index 6b1b73e31d..cb7a053fd9 100644
--- a/press/press/doctype/proxy_server/proxy_server.json
+++ b/press/press/doctype/proxy_server/proxy_server.json
@@ -18,6 +18,9 @@
"is_self_hosted",
"team",
"public",
+ "storage_section",
+ "auto_add_storage_min",
+ "auto_add_storage_max",
"section_break_8",
"ip",
"enabled_default_routing",
@@ -397,10 +400,31 @@
"fieldname": "public",
"fieldtype": "Check",
"label": "Public"
+ },
+ {
+ "fieldname": "storage_section",
+ "fieldtype": "Section Break",
+ "label": "Storage"
+ },
+ {
+ "default": "10",
+ "description": "Minimum storage to add automatically each time",
+ "fieldname": "auto_add_storage_min",
+ "fieldtype": "Int",
+ "label": "Auto Add Storage Min",
+ "non_negative": 1
+ },
+ {
+ "default": "50",
+ "description": "Maximum storage to add automatically each time",
+ "fieldname": "auto_add_storage_max",
+ "fieldtype": "Int",
+ "label": "Auto Add Storage Max",
+ "non_negative": 1
}
],
"links": [],
- "modified": "2024-09-10 15:44:10.989216",
+ "modified": "2024-12-26 13:20:05.120010",
"modified_by": "Administrator",
"module": "Press",
"name": "Proxy Server",
diff --git a/press/press/doctype/proxy_server/proxy_server.py b/press/press/doctype/proxy_server/proxy_server.py
index f2d77c5f81..6a517662da 100644
--- a/press/press/doctype/proxy_server/proxy_server.py
+++ b/press/press/doctype/proxy_server/proxy_server.py
@@ -1,7 +1,6 @@
-# -*- coding: utf-8 -*-
# Copyright (c) 2020, Frappe and contributors
# For license information, please see license.txt
-
+from __future__ import annotations
from typing import TYPE_CHECKING
@@ -9,13 +8,13 @@
from frappe.utils import unique
from press.agent import Agent
-from press.press.doctype.root_domain.root_domain import RootDomain
from press.press.doctype.server.server import BaseServer
from press.runner import Ansible
from press.utils import log_error
if TYPE_CHECKING:
from press.press.doctype.bench.bench import Bench
+ from press.press.doctype.root_domain.root_domain import RootDomain
class ProxyServer(BaseServer):
@@ -26,11 +25,12 @@ class ProxyServer(BaseServer):
if TYPE_CHECKING:
from frappe.types import DF
- from press.press.doctype.proxy_server_domain.proxy_server_domain import (
- ProxyServerDomain,
- )
+
+ from press.press.doctype.proxy_server_domain.proxy_server_domain import ProxyServerDomain
agent_password: DF.Password | None
+ auto_add_storage_max: DF.Int
+ auto_add_storage_min: DF.Int
cluster: DF.Link | None
disable_agent_job_auto_retry: DF.Check
domain: DF.Link | None
@@ -83,7 +83,7 @@ def validate_domains(self):
code_servers = [row.code_server for row in self.domains]
# Always include self.domain in the domains child table
# Remove duplicates
- domains = unique([self.domain] + domains)
+ domains = unique([self.domain, *domains])
self.domains = []
for i, domain in enumerate(domains):
if not frappe.db.exists(
@@ -133,23 +133,17 @@ def _setup_server(self):
"TLS Certificate", {"wildcard": True, "domain": self.domain}, "name"
)
certificate = frappe.get_doc("TLS Certificate", certificate_name)
- monitoring_password = frappe.get_doc("Cluster", self.cluster).get_password(
- "monitoring_password"
- )
+ monitoring_password = frappe.get_doc("Cluster", self.cluster).get_password("monitoring_password")
log_server = frappe.db.get_single_value("Press Settings", "log_server")
if log_server:
- kibana_password = frappe.get_doc("Log Server", log_server).get_password(
- "kibana_password"
- )
+ kibana_password = frappe.get_doc("Log Server", log_server).get_password("kibana_password")
else:
kibana_password = None
try:
ansible = Ansible(
- playbook="self_hosted_proxy.yml"
- if getattr(self, "is_self_hosted", False)
- else "proxy.yml",
+ playbook="self_hosted_proxy.yml" if getattr(self, "is_self_hosted", False) else "proxy.yml",
server=self,
user=self.ssh_user or "root",
port=self.ssh_port or 22,
@@ -181,9 +175,7 @@ def _setup_server(self):
self.save()
def _install_exporters(self):
- monitoring_password = frappe.get_doc("Cluster", self.cluster).get_password(
- "monitoring_password"
- )
+ monitoring_password = frappe.get_doc("Cluster", self.cluster).get_password("monitoring_password")
try:
ansible = Ansible(
playbook="proxy_exporters.yml",
@@ -201,9 +193,7 @@ def _install_exporters(self):
@frappe.whitelist()
def setup_ssh_proxy(self):
- frappe.enqueue_doc(
- self.doctype, self.name, "_setup_ssh_proxy", queue="long", timeout=1200
- )
+ frappe.enqueue_doc(self.doctype, self.name, "_setup_ssh_proxy", queue="long", timeout=1200)
def _setup_ssh_proxy(self):
settings = frappe.db.get_value(
@@ -238,9 +228,7 @@ def _setup_ssh_proxy(self):
def setup_fail2ban(self):
self.status = "Installing"
self.save()
- frappe.enqueue_doc(
- self.doctype, self.name, "_setup_fail2ban", queue="long", timeout=1200
- )
+ frappe.enqueue_doc(self.doctype, self.name, "_setup_fail2ban", queue="long", timeout=1200)
def _setup_fail2ban(self):
try:
@@ -261,9 +249,7 @@ def _setup_fail2ban(self):
@frappe.whitelist()
def setup_proxysql(self):
- frappe.enqueue_doc(
- self.doctype, self.name, "_setup_proxysql", queue="long", timeout=1200
- )
+ frappe.enqueue_doc(self.doctype, self.name, "_setup_proxysql", queue="long", timeout=1200)
def _setup_proxysql(self):
try:
@@ -294,9 +280,7 @@ def _setup_proxysql(self):
def setup_replication(self):
self.status = "Installing"
self.save()
- frappe.enqueue_doc(
- self.doctype, self.name, "_setup_replication", queue="long", timeout=1200
- )
+ frappe.enqueue_doc(self.doctype, self.name, "_setup_replication", queue="long", timeout=1200)
def _setup_replication(self):
self._setup_secondary()
@@ -351,9 +335,7 @@ def trigger_failover(self):
return
self.status = "Installing"
self.save()
- frappe.enqueue_doc(
- self.doctype, self.name, "_trigger_failover", queue="long", timeout=3600
- )
+ frappe.enqueue_doc(self.doctype, self.name, "_trigger_failover", queue="long", timeout=3600)
def stop_primary(self):
primary = frappe.get_doc("Proxy Server", self.primary)
@@ -387,9 +369,7 @@ def remove_primarys_access(self):
playbook="failover_remove_primary_access.yml",
server=self,
variables={
- "primary_public_key": frappe.db.get_value(
- "Proxy Server", self.primary, "frappe_public_key"
- )
+ "primary_public_key": frappe.db.get_value("Proxy Server", self.primary, "frappe_public_key")
},
)
ansible.run()
@@ -445,9 +425,7 @@ def add_ssh_users_for_existing_benches(self):
bench.add_ssh_user()
def update_app_servers(self):
- frappe.db.set_value(
- "Server", {"proxy_server": self.primary}, "proxy_server", self.name
- )
+ frappe.db.set_value("Server", {"proxy_server": self.primary}, "proxy_server", self.name)
def switch_primary(self):
frappe.db.set_value("Proxy Server", self.primary, "is_primary", False)
@@ -458,9 +436,7 @@ def switch_primary(self):
@frappe.whitelist()
def setup_proxysql_monitor(self):
- frappe.enqueue_doc(
- self.doctype, self.name, "_setup_proxysql_monitor", queue="long", timeout=1200
- )
+ frappe.enqueue_doc(self.doctype, self.name, "_setup_proxysql_monitor", queue="long", timeout=1200)
def _setup_proxysql_monitor(self):
try:
@@ -486,9 +462,7 @@ def _setup_proxysql_monitor(self):
@frappe.whitelist()
def setup_wireguard(self):
if not self.private_ip_interface_id:
- play = frappe.get_last_doc(
- "Ansible Play", {"play": "Ping Server", "server": self.name}
- )
+ play = frappe.get_last_doc("Ansible Play", {"play": "Ping Server", "server": self.name})
task = frappe.get_doc("Ansible Task", {"play": play.name, "task": "Gather Facts"})
import json
@@ -497,9 +471,7 @@ def setup_wireguard(self):
if task_res[i]["ipv4"]["address"] == self.private_ip:
self.private_ip_interface_id = task_res[i]["device"]
self.save()
- frappe.enqueue_doc(
- self.doctype, self.name, "_setup_wireguard", queue="long", timeout=1200
- )
+ frappe.enqueue_doc(self.doctype, self.name, "_setup_wireguard", queue="long", timeout=1200)
def _setup_wireguard(self):
try:
@@ -516,7 +488,7 @@ def _setup_wireguard(self):
"wireguard_private_key": False,
"wireguard_public_key": False,
"peers": "",
- "reload_wireguard": True if self.is_wireguard_setup else False,
+ "reload_wireguard": bool(self.is_wireguard_setup),
},
)
play = ansible.run()
@@ -536,9 +508,7 @@ def _setup_wireguard(self):
@frappe.whitelist()
def reload_wireguard(self):
- frappe.enqueue_doc(
- "Proxy Server", self.name, "_reload_wireguard", queue="default", timeout=1200
- )
+ frappe.enqueue_doc("Proxy Server", self.name, "_reload_wireguard", queue="default", timeout=1200)
def _reload_wireguard(self):
import json
diff --git a/press/press/doctype/server/server.py b/press/press/doctype/server/server.py
index 6631e40da9..cbdf82d9c9 100644
--- a/press/press/doctype/server/server.py
+++ b/press/press/doctype/server/server.py
@@ -15,6 +15,7 @@
from frappe.core.utils import find, find_all
from frappe.installer import subprocess
from frappe.model.document import Document
+from frappe.utils import cint
from frappe.utils.user import is_system_user
from press.agent import Agent
@@ -634,7 +635,7 @@ def create_subscription_for_storage(self, increment: int) -> None:
"Subscription",
existing_subscription.name,
"additional_storage",
- increment + int(existing_subscription.additional_storage),
+ increment + cint(existing_subscription.additional_storage),
)
else:
frappe.get_doc(
diff --git a/press/press/doctype/site/site.py b/press/press/doctype/site/site.py
index 39fd53254d..c6360ef912 100644
--- a/press/press/doctype/site/site.py
+++ b/press/press/doctype/site/site.py
@@ -13,6 +13,8 @@
import dateutil.parser
import frappe
+import frappe.data
+import frappe.utils
import pytz
import requests
from frappe import _
@@ -68,6 +70,9 @@
from press.press.doctype.site_activity.site_activity import log_site_activity
from press.press.doctype.site_analytics.site_analytics import create_site_analytics
from press.press.doctype.site_plan.site_plan import get_plan_config
+from press.press.report.mariadb_slow_queries.mariadb_slow_queries import (
+ get_doctype_name,
+)
from press.utils import (
convert,
fmt_timedelta,
@@ -2265,10 +2270,92 @@ def get_update_information(self):
out.update_available = any([app["update_available"] for app in out.apps])
return out
- @frappe.whitelist()
- def optimize_tables(self):
+ def fetch_running_optimize_tables_job(self):
+ return frappe.db.exists(
+ "Agent Job",
+ {
+ "site": self.name,
+ "job_type": "Optimize Tables",
+ "status": ["in", ["Undelivered", "Running", "Pending"]],
+ },
+ )
+
+ @dashboard_whitelist()
+ def optimize_tables(self, ignore_checks: bool = False):
+ if not ignore_checks:
+ # check for running `Optimize Tables` agent job
+ if job := self.fetch_running_optimize_tables_job():
+ return {
+ "success": True,
+ "message": "Optimize Tables job is already running on this site.",
+ "job_name": job,
+ }
+ # check if `Optimize Tables` has run in last 1 hour
+ recent_agent_job_name = frappe.db.exists(
+ "Agent Job",
+ {
+ "site": self.name,
+ "job_type": "Optimize Tables",
+ "status": ["not in", ["Failure", "Delivery Failure"]],
+ "creation": [">", frappe.utils.add_to_date(frappe.utils.now_datetime(), hours=-1)],
+ },
+ )
+ if recent_agent_job_name:
+ return {
+ "success": False,
+ "message": "Optimize Tables job has already run in the last 1 hour. Try later.",
+ "job_name": None,
+ }
+
agent = Agent(self.server)
- agent.optimize_tables(self)
+ job_name = agent.optimize_tables(self).name
+ return {
+ "success": True,
+ "message": "Optimize Tables has been triggered on this site.",
+ "job_name": job_name,
+ }
+
+ @dashboard_whitelist()
+ def get_database_performance_report(self):
+ from press.press.report.mariadb_slow_queries.mariadb_slow_queries import get_data as get_slow_queries
+
+ agent = Agent(self.server)
+ result = agent.get_summarized_performance_report_of_database(self)
+ # fetch slow queries of last 7 days
+ slow_queries = get_slow_queries(
+ frappe._dict(
+ {
+ "database": self.database_name,
+ "start_datetime": frappe.utils.add_to_date(None, days=-7),
+ "stop_datetime": frappe.utils.now_datetime(),
+ "search_pattern": ".*",
+ "max_lines": 2000,
+ "normalize_queries": True,
+ }
+ )
+ )
+ # remove `parent` & `creation` indexes from unused_indexes
+ result["unused_indexes"] = [
+ index
+ for index in result.get("unused_indexes", [])
+ if index["index_name"] not in ["parent", "creation"]
+ ]
+
+ # convert all the float to int
+ for query in slow_queries:
+ for key, value in query.items():
+ if isinstance(value, float):
+ query[key] = int(value)
+ # sort the slow queries by `rows_examined`
+ result["slow_queries"] = sorted(slow_queries, key=lambda x: x["rows_examined"], reverse=True)
+ result["is_performance_schema_enabled"] = False
+ if database_server := frappe.db.get_value("Server", self.server, "database_server"):
+ result["is_performance_schema_enabled"] = frappe.db.get_value(
+ "Database Server",
+ database_server,
+ "is_performance_schema_enabled",
+ )
+ return result
@property
def server_logs(self):
@@ -2625,19 +2712,66 @@ def forcefully_remove_site(self, bench):
@dashboard_whitelist()
def fetch_database_table_schema(self, reload=False):
- if not frappe.db.exists("Site Database Table Schema", {"site": self.name}):
- frappe.get_doc({"doctype": "Site Database Table Schema", "site": self.name}).insert(
- ignore_permissions=True
- )
+ """
+ Store dump in redis cache
+ """
+ key_for_schema = f"database_table_schema__data:{self.name}"
+ key_for_schema_status = (
+ f"database_table_schema__status:{self.name}" # 1 - loading, 2 - done, None - not available
+ )
+
+ if reload:
+ frappe.cache().delete_value(key_for_schema)
+ frappe.cache().delete_value(key_for_schema_status)
+
+ status = frappe.utils.cint(frappe.cache().get_value(key_for_schema_status))
+ if status:
+ if status == 1:
+ return {
+ "loading": True,
+ "data": [],
+ }
+ if status == 2:
+ return {
+ "loading": False,
+ "data": json.loads(frappe.cache().get_value(key_for_schema)),
+ }
- doc = frappe.get_doc("Site Database Table Schema", {"site": self.name})
- loading, data = doc.fetch(reload)
+ # Check if any agent job is created within 5 minutes and in pending/running condition
+ # Checks to prevent duplicate agent job creation due to race condition
+ if not frappe.db.exists(
+ "Agent Job",
+ {
+ "job_type": "Fetch Database Table Schema",
+ "site": self.name,
+ "status": ["in", ["Undelivered", "Pending", "Running"]],
+ "creation": (">", frappe.utils.add_to_date(None, minutes=-5)),
+ },
+ ):
+ # create the agent job and put it in loading state
+ frappe.cache().set_value(key_for_schema_status, 1, expires_in_sec=600)
+ Agent(self.server).fetch_database_table_schema(
+ self, include_index_info=True, include_table_size=True
+ )
return {
- "loading": loading,
- "data": data,
- "last_updated": doc.last_updated,
+ "loading": True,
+ "data": [],
}
+ @dashboard_whitelist()
+ def fetch_database_processes(self):
+ agent = Agent(self.server)
+ if agent.should_skip_requests():
+ return None
+ return agent.fetch_database_processes(self)
+
+ @dashboard_whitelist()
+ def kill_database_process(self, id):
+ agent = Agent(self.server)
+ if agent.should_skip_requests():
+ return None
+ return agent.kill_database_process(self, id)
+
@dashboard_whitelist()
def run_sql_query_in_database(self, query: str, commit: bool):
if not query:
@@ -2656,6 +2790,88 @@ def run_sql_query_in_database(self, query: str, commit: bool):
doc.insert(ignore_permissions=True)
return response
+ @dashboard_whitelist()
+ def suggest_database_indexes(self):
+ from press.press.report.mariadb_slow_queries.mariadb_slow_queries import get_data as get_slow_queries
+
+ existing_agent_job_name = frappe.db.exists(
+ "Agent Job",
+ {
+ "site": self.name,
+ "status": ("not in", ("Failure", "Delivery Failure")),
+ "job_type": "Analyze Slow Queries",
+ "creation": (
+ ">",
+ frappe.utils.add_to_date(None, minutes=-30),
+ ),
+ "retry_count": 0,
+ },
+ )
+
+ if existing_agent_job_name:
+ existing_agent_job = frappe.get_doc("Agent Job", existing_agent_job_name)
+ if existing_agent_job.status == "Success":
+ return {
+ "loading": False,
+ "data": json.loads(existing_agent_job.data).get("result", []),
+ }
+ return {
+ "loading": True,
+ "data": [],
+ }
+
+ # fetch slow queries of last 7 days
+ slow_queries = get_slow_queries(
+ frappe._dict(
+ {
+ "database": self.database_name,
+ "start_datetime": frappe.utils.add_to_date(None, days=-7),
+ "stop_datetime": frappe.utils.now_datetime(),
+ "search_pattern": ".*",
+ "max_lines": 1000,
+ "normalize_queries": True,
+ }
+ )
+ )
+ slow_queries = [{"example": x["example"], "normalized": x["query"]} for x in slow_queries]
+ if len(slow_queries) == 0:
+ return {
+ "loading": False,
+ "data": [],
+ }
+ agent = Agent(self.server)
+ agent.analyze_slow_queries(self, slow_queries)
+
+ return {
+ "loading": True,
+ "data": [],
+ }
+
+ @dashboard_whitelist()
+ def add_database_index(self, table, column):
+ record = frappe.db.exists(
+ "Agent Job",
+ {
+ "site": self.name,
+ "status": ["in", ["Undelivered", "Running", "Pending"]],
+ "job_type": "Add Database Index",
+ },
+ )
+ if record:
+ return {
+ "success": False,
+ "message": "There is already a job running for adding database index. Please wait until finished.",
+ "job_name": record,
+ }
+ doctype = get_doctype_name(table)
+ agent = Agent(self.server)
+ job = agent.add_database_index(self, doctype=doctype, columns=[column])
+ return {
+ "success": True,
+ "message": "Database index will be added on site.",
+ "job_name": job.name,
+ }
+
def site_cleanup_after_archive(site):
delete_site_domains(site)
@@ -2689,6 +2905,51 @@ def release_name(name):
frappe.rename_doc("Site", name, new_name)
+def process_fetch_database_table_schema_job_update(job):
+ key_for_schema = f"database_table_schema__data:{job.site}"
+ key_for_schema_status = (
+ f"database_table_schema__status:{job.site}" # 1 - loading, 2 - done, None - not available
+ )
+
+ if job.status in ["Failure", "Delivery Failure"]:
+ frappe.cache().delete_value(key_for_schema)
+ frappe.cache().delete_value(key_for_schema_status)
+ return
+
+ if job.status == "Success":
+ """
+ Support old agent versions
+ Remove this once all agents are updated
+ """
+ data = json.loads(job.data)
+ is_old_agent = False
+
+ if len(data) > 0 and isinstance(data[next(iter(data.keys()))], list):
+ is_old_agent = True
+
+ if is_old_agent:
+ data_copy = data.copy()
+ data = {}
+ for key, value in data_copy.items():
+ data[key] = {
+ "columns": value,
+ "size": {
+ "data_length": 0,
+ "index_length": 0,
+ "total_size": 0,
+ }, # old agent api doesn't have size info
+ }
+ for column in data[key]["columns"]:
+ column["index_info"] = {
+ "index_usage": {x: 0 for x in column["indexes"]}, # just fill some dummy value
+ "indexes": column["indexes"],
+ "is_indexed": len(column["indexes"]) > 0,
+ }
+
+ frappe.cache().set_value(key_for_schema, json.dumps(data), expires_in_sec=6000)
+ frappe.cache().set_value(key_for_schema_status, 2, expires_in_sec=6000)
+
+
def process_new_site_job_update(job): # noqa: C901
site_status = frappe.get_value("Site", job.site, "status", for_update=True)
diff --git a/press/press/doctype/site_database_table_schema/__init__.py b/press/press/doctype/site_database_table_schema/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/press/press/doctype/site_database_table_schema/site_database_table_schema.js b/press/press/doctype/site_database_table_schema/site_database_table_schema.js
deleted file mode 100644
index 4747337e12..0000000000
--- a/press/press/doctype/site_database_table_schema/site_database_table_schema.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2024, Frappe and contributors
-// For license information, please see license.txt
-
-// frappe.ui.form.on("Site Database Table Schema", {
-// refresh(frm) {
-
-// },
-// });
diff --git a/press/press/doctype/site_database_table_schema/site_database_table_schema.json b/press/press/doctype/site_database_table_schema/site_database_table_schema.json
deleted file mode 100644
index 7917ec3755..0000000000
--- a/press/press/doctype/site_database_table_schema/site_database_table_schema.json
+++ /dev/null
@@ -1,91 +0,0 @@
-{
- "actions": [],
- "creation": "2024-10-28 11:45:28.634612",
- "doctype": "DocType",
- "engine": "InnoDB",
- "field_order": [
- "site",
- "column_break_mufi",
- "agent_job",
- "section_break_heok",
- "schema_json"
- ],
- "fields": [
- {
- "fieldname": "site",
- "fieldtype": "Link",
- "in_list_view": 1,
- "in_standard_filter": 1,
- "label": "Site",
- "options": "Site",
- "reqd": 1,
- "unique": 1
- },
- {
- "fieldname": "agent_job",
- "fieldtype": "Link",
- "in_list_view": 1,
- "in_standard_filter": 1,
- "label": "Agent Job",
- "options": "Agent Job"
- },
- {
- "fieldname": "column_break_mufi",
- "fieldtype": "Column Break"
- },
- {
- "fieldname": "section_break_heok",
- "fieldtype": "Section Break"
- },
- {
- "default": "{}",
- "fieldname": "schema_json",
- "fieldtype": "Long Text",
- "label": "Schema JSON",
- "reqd": 1
- }
- ],
- "index_web_pages_for_search": 1,
- "links": [],
- "modified": "2024-10-28 15:43:58.768773",
- "modified_by": "Administrator",
- "module": "Press",
- "name": "Site Database Table Schema",
- "owner": "Administrator",
- "permissions": [
- {
- "create": 1,
- "delete": 1,
- "email": 1,
- "export": 1,
- "print": 1,
- "read": 1,
- "report": 1,
- "role": "System Manager",
- "share": 1,
- "write": 1
- },
- {
- "email": 1,
- "export": 1,
- "print": 1,
- "read": 1,
- "report": 1,
- "role": "Press Admin",
- "share": 1
- },
- {
- "email": 1,
- "export": 1,
- "print": 1,
- "read": 1,
- "report": 1,
- "role": "Press Member",
- "share": 1
- }
- ],
- "sort_field": "creation",
- "sort_order": "DESC",
- "states": [],
- "title_field": "site"
-}
\ No newline at end of file
diff --git a/press/press/doctype/site_database_table_schema/site_database_table_schema.py b/press/press/doctype/site_database_table_schema/site_database_table_schema.py
deleted file mode 100644
index cf13d99e5c..0000000000
--- a/press/press/doctype/site_database_table_schema/site_database_table_schema.py
+++ /dev/null
@@ -1,95 +0,0 @@
-# Copyright (c) 2024, Frappe and contributors
-# For license information, please see license.txt
-from __future__ import annotations
-
-import json
-from typing import TYPE_CHECKING
-
-import frappe
-from frappe.model.document import Document
-
-from press.agent import Agent
-
-if TYPE_CHECKING:
- from press.press.doctype.site_migration.site_migration import AgentJob
-
-
-class SiteDatabaseTableSchema(Document):
- # begin: auto-generated types
- # This code is auto-generated. Do not modify anything in this block.
-
- from typing import TYPE_CHECKING
-
- if TYPE_CHECKING:
- from frappe.types import DF
-
- agent_job: DF.Link | None
- schema_json: DF.LongText
- site: DF.Link
- # end: auto-generated types
-
- def fetch(self, reload=False) -> tuple[bool, dict]:
- """
- This function will return the schema of the database table
-
- Args:
- reload: bool - if True, it will fetch the schema from the server again
-
- Returns:
- tuple[bool, list]
- - 1st element: bool - Loading status
- - 2nd element: dict - Dictionary of table schemas
- Example -
- {
- "__Auth": [
- {
- "column": "doctype",
- "data_type": "varchar",
- "default": "NULL",
- "indexes": [
- "PRIMARY"
- ],
- "is_nullable": false
- },
- ....
- ],
- ....
- }
- """
- if len(self.schema) > 0 and not reload:
- return False, self.schema
-
- if self.agent_job is not None and frappe.get_value("Agent Job", self.agent_job, "status") in [
- "Undelivered",
- "Pending",
- "Running",
- ]:
- return True, {}
-
- self.schema_json = "{}"
- site = frappe.get_doc("Site", self.site)
- self.agent_job = Agent(site.server).fetch_database_table_schema(site).name
- self.save(ignore_permissions=True)
-
- return True, {}
-
- @property
- def last_updated(self) -> str:
- return self.modified or self.creation
-
- @property
- def schema(self) -> dict:
- try:
- return json.loads(self.schema_json)
- except frappe.DoesNotExistError:
- return {}
-
- @staticmethod
- def process_job_update(job: "AgentJob"):
- if job.status != "Success":
- return
- response_data = json.loads(job.data) or {}
- if response_data and frappe.db.exists("Site Database Table Schema", {"site": job.site}):
- doc = frappe.get_doc("Site Database Table Schema", {"site": job.site})
- doc.schema_json = json.dumps(response_data)
- doc.save(ignore_permissions=True)
diff --git a/press/press/doctype/site_database_table_schema/test_site_database_table_schema.py b/press/press/doctype/site_database_table_schema/test_site_database_table_schema.py
deleted file mode 100644
index b75f628d81..0000000000
--- a/press/press/doctype/site_database_table_schema/test_site_database_table_schema.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright (c) 2024, Frappe and Contributors
-# See license.txt
-
-# import frappe
-from frappe.tests import UnitTestCase
-
-# On IntegrationTestCase, the doctype test records and all
-# link-field test record depdendencies are recursively loaded
-# Use these module variables to add/remove to/from that list
-EXTRA_TEST_RECORD_DEPENDENCIES = [] # eg. ["User"]
-IGNORE_TEST_RECORD_DEPENDENCIES = [] # eg. ["User"]
-
-
-class TestSiteDatabaseTableSchema(UnitTestCase):
- """
- Unit tests for SiteDatabaseTableSchema.
- Use this class for testing individual functions and methods.
- """
-
- pass
diff --git a/press/press/doctype/subscription/subscription.py b/press/press/doctype/subscription/subscription.py
index e85364a2be..fd707a178b 100644
--- a/press/press/doctype/subscription/subscription.py
+++ b/press/press/doctype/subscription/subscription.py
@@ -7,6 +7,7 @@
import rq
from frappe.model.document import Document
from frappe.query_builder.functions import Coalesce, Count
+from frappe.utils import cint
from press.overrides import get_permission_query_conditions_for_doctype
from press.press.doctype.site_plan.site_plan import SitePlan
@@ -127,12 +128,12 @@ def disable(self):
frappe.log_error(title="Disable Subscription Error")
@frappe.whitelist()
- def create_usage_record(self):
+ def create_usage_record(self, date: DF.Date | None = None):
cannot_charge = not self.can_charge_for_subscription()
if cannot_charge:
return None
- if self.is_usage_record_created():
+ if self.is_usage_record_created(date):
return None
team = frappe.get_cached_doc("Team", self.team)
@@ -151,7 +152,7 @@ def create_usage_record(self):
if self.additional_storage:
price = plan.price_inr if team.currency == "INR" else plan.price_usd
price_per_day = price / plan.period # no rounding off to avoid discrepancies
- amount = price_per_day * int(self.additional_storage)
+ amount = price_per_day * cint(self.additional_storage)
else:
amount = plan.get_price_for_interval(self.interval, team.currency)
@@ -163,6 +164,7 @@ def create_usage_record(self):
plan_type=self.plan_type,
plan=plan.name,
amount=amount,
+ date=date,
subscription=self.name,
interval=self.interval,
site=(
@@ -186,7 +188,7 @@ def can_charge_for_subscription(self):
return True
- def is_usage_record_created(self):
+ def is_usage_record_created(self, date=None):
filters = {
"team": self.team,
"document_type": self.document_type,
@@ -197,7 +199,8 @@ def is_usage_record_created(self):
}
if self.interval == "Daily":
- filters.update({"date": frappe.utils.today()})
+ date = date or frappe.utils.today()
+ filters.update({"date": date})
if self.interval == "Monthly":
date = frappe.utils.getdate()
diff --git a/press/press/doctype/team/team.py b/press/press/doctype/team/team.py
index 546d9cb06d..542b989e64 100644
--- a/press/press/doctype/team/team.py
+++ b/press/press/doctype/team/team.py
@@ -111,6 +111,7 @@ class Team(Document):
"is_developer",
"enable_performance_tuning",
"enable_inplace_updates",
+ "servers_enabled",
)
def get_doc(self, doc):
@@ -1094,7 +1095,9 @@ def suspend_sites(self, reason=None):
def get_sites_to_suspend(self):
plan = frappe.qb.DocType("Site Plan")
query = (
- frappe.qb.from_(plan).select(plan.name).where((plan.enabled == 1) & ((plan.is_frappe_plan == 1) | (plan.is_trial_plan == 1)))
+ frappe.qb.from_(plan)
+ .select(plan.name)
+ .where((plan.enabled == 1) & ((plan.is_frappe_plan == 1) | (plan.is_trial_plan == 1)))
).run(as_dict=True)
frappe_plans = [d.name for d in query]
diff --git a/press/press/doctype/virtual_disk_snapshot/virtual_disk_snapshot.py b/press/press/doctype/virtual_disk_snapshot/virtual_disk_snapshot.py
index 2ba94babf8..55c88c9505 100644
--- a/press/press/doctype/virtual_disk_snapshot/virtual_disk_snapshot.py
+++ b/press/press/doctype/virtual_disk_snapshot/virtual_disk_snapshot.py
@@ -184,6 +184,7 @@ def sync_all_snapshots_from_aws():
if _should_skip_snapshot(snapshot):
continue
try:
+ delete_duplicate_snapshot_docs(snapshot)
if _update_snapshot_if_exists(snapshot, random_snapshot):
continue
tag_name = next(tag["Value"] for tag in snapshot["Tags"] if tag["Key"] == "Name")
@@ -238,6 +239,23 @@ def _should_skip_snapshot(snapshot):
return False
+def delete_duplicate_snapshot_docs(snapshot):
+ # Delete all except one snapshot document
+ # It doesn't matter which one we keep
+ snapshot_id = snapshot["SnapshotId"]
+ snapshot_count = frappe.db.count("Virtual Disk Snapshot", {"snapshot_id": snapshot_id})
+ if snapshot_count > 1:
+ frappe.db.sql(
+ """
+ DELETE
+ FROM `tabVirtual Disk Snapshot`
+ WHERE snapshot_id=%s
+ LIMIT %s
+ """,
+ (snapshot_id, snapshot_count - 1),
+ )
+
+
def _update_snapshot_if_exists(snapshot, random_snapshot):
snapshot_id = snapshot["SnapshotId"]
if frappe.db.exists("Virtual Disk Snapshot", {"snapshot_id": snapshot_id}):
diff --git a/press/press/report/mariadb_deadlock_browser/mariadb_deadlock_browser.py b/press/press/report/mariadb_deadlock_browser/mariadb_deadlock_browser.py
index d0afa62572..3c58a3e5ac 100644
--- a/press/press/report/mariadb_deadlock_browser/mariadb_deadlock_browser.py
+++ b/press/press/report/mariadb_deadlock_browser/mariadb_deadlock_browser.py
@@ -248,7 +248,7 @@ def deadlock_summary(transactions: list[DatabaseTransactionLog]) -> list[dict]:
def execute(filters=None):
- frappe.only_for(["System Manager", "Site Manager"])
+ frappe.only_for(["System Manager", "Site Manager", "Press Admin", "Press Member"])
filters.database = frappe.db.get_value("Site", filters.site, "database_name")
if not filters.database:
frappe.throw(
diff --git a/press/press/report/mariadb_slow_queries/db_optimizer.py b/press/press/report/mariadb_slow_queries/db_optimizer.py
deleted file mode 100644
index 36f6ad5afb..0000000000
--- a/press/press/report/mariadb_slow_queries/db_optimizer.py
+++ /dev/null
@@ -1,352 +0,0 @@
-"""Basic DB optimizer for Frappe Framework based app.
-
-This is largely based on heuristics and known good practices for indexing.
-"""
-
-from collections import defaultdict
-from dataclasses import dataclass
-from typing import Literal
-import re
-import frappe
-from frappe.utils import cint, cstr, flt
-from sql_metadata import Parser
-
-# Any index that reads more than 30% table on average is not "useful"
-INDEX_SCORE_THRESHOLD = 0.3
-# Anything reading less than this percent of table is considered optimal
-OPTIMIZATION_THRESHOLD = 0.1
-
-
-@dataclass
-class DBExplain:
- # refer: https://mariadb.com/kb/en/explain/
- # Anything not explicitly encoded here is likely not supported.
- select_type: Literal["SIMPLE", "PRIMARY", "SUBQUERY", "UNION", "DERIVED"]
- table: str
- scan_type: Literal[ # What type of scan will be performed
- "ALL", # Full table scan
- "CONST", # Single row will be read
- "EQ_REF", # A single row is found from *unique* index
- "REF", # Index is used, but MIGHT hit more than 1 rows as it's non-unique
- "RANGE", # The table will be accessed with a key over one or more value ranges.
- "INDEX_MERGE", # multiple indexes are used and merged smartly. Equivalent to RANGE
- "INDEX_SUBQUERY",
- "INDEX", # Full index scan is performed. Similar to full table scan in case of large number of rows.
- "REF_OR_NULL",
- "UNIQUE_SUBQUERY",
- "FULLTEXT", # Full text index is used,
- ]
- possible_keys: list[str] | None = None # possible indexes that can be used
- key: str | None = None # This index is being used
- key_len: int | None = None # How many prefix bytes from index are being used
- ref: str | None = None # is reference constant or some other column
- rows: int = 0 # roughly how many rows will be examined
- extra: str | None = None
-
- @classmethod
- def from_frappe_ouput(cls, data) -> "DBExplain":
- return cls(
- select_type=cstr(data["select_type"]).upper(),
- table=data["table"],
- scan_type=cstr(data["type"]).upper(),
- possible_keys=data["possible_keys"],
- key=data["key"],
- key_len=cint(data["key_len"]) if data["key_len"] else None,
- ref=data["ref"],
- rows=cint(data["rows"]),
- extra=data.get("Extra"),
- )
-
-
-@dataclass
-class DBColumn:
- name: str
- cardinality: int | None
- is_nullable: bool
- default: str
- data_type: str
-
- @classmethod
- def from_frappe_ouput(cls, data) -> "DBColumn":
- "Parse DBColumn from output of describe-database-table command in Frappe"
- return cls(
- name=data["column"],
- cardinality=data.get("cardinality"),
- is_nullable=data["is_nullable"],
- default=data["default"],
- data_type=data["type"],
- )
-
-
-@dataclass
-class DBIndex:
- name: str
- column: str
- table: str
- unique: bool | None = None
- cardinality: int | None = None
- sequence: int = 1
- nullable: bool = True
- _score: float = 0.0
-
- def __eq__(self, other: "DBIndex") -> bool:
- return (
- self.column == other.column
- and self.sequence == other.sequence
- and self.table == other.table
- )
-
- def __repr__(self):
- return f"DBIndex(`{self.table}`.`{self.column}`)"
-
- @classmethod
- def from_frappe_ouput(cls, data, table) -> "DBIndex":
- "Parse DBIndex from output of describe-database-table command in Frappe"
- return cls(
- name=data["name"],
- table=table,
- unique=data["unique"],
- cardinality=data["cardinality"],
- sequence=data["sequence"],
- nullable=data["nullable"],
- column=data["column"],
- )
-
-
-@dataclass
-class ColumnStat:
- column_name: str
- avg_frequency: float
- avg_length: float
- nulls_ratio: float | None = None
- histogram: list[float] = None
-
- def __post_init__(self):
- if not self.histogram:
- self.histogram = []
-
- @classmethod
- def from_frappe_ouput(cls, data) -> "ColumnStat":
- return cls(
- column_name=data["column_name"],
- avg_frequency=data["avg_frequency"],
- avg_length=data["avg_length"],
- nulls_ratio=data["nulls_ratio"],
- histogram=[flt(bin) for bin in data["histogram"].split(",")]
- if data["histogram"]
- else [],
- )
-
-
-@dataclass
-class DBTable:
- name: str
- total_rows: int
- schema: list[DBColumn] | None = None
- indexes: list[DBIndex] | None = None
-
- def __post_init__(self):
- if not self.schema:
- self.schema = []
- if not self.indexes:
- self.indexes = []
-
- def update_cardinality(self, column_stats: list[ColumnStat]) -> None:
- """Estimate cardinality using mysql.column_stat"""
- for column_stat in column_stats:
- for col in self.schema:
- if (
- col.name == column_stat.column_name
- and not col.cardinality
- and column_stat.avg_frequency
- ):
- # "hack" or "math" - average frequency is on average how frequently a row value appears.
- # Avg = total_rows / cardinality, so...
- col.cardinality = self.total_rows / column_stat.avg_frequency
-
- @classmethod
- def from_frappe_ouput(cls, data) -> "DBTable":
- "Parse DBTable from output of describe-database-table command in Frappe"
- table_name = data["table_name"]
- return cls(
- name=table_name,
- total_rows=data["total_rows"],
- schema=[DBColumn.from_frappe_ouput(c) for c in data["schema"]],
- indexes=[DBIndex.from_frappe_ouput(i, table_name) for i in data["indexes"]],
- )
-
- def has_column(self, column: str) -> bool:
- for col in self.schema:
- if col.name == column:
- return True
- return False
-
-
-@dataclass
-class DBOptimizer:
- query: str # raw query in string format
- explain_plan: list[DBExplain] = None
- tables: dict[str, DBTable] = None
- parsed_query: Parser = None
-
- def __post_init__(self):
- if not self.explain_plan:
- self.explain_plan = []
- if not self.tables:
- self.tables = {}
- for explain_entry in self.explain_plan:
- explain_entry.select_type = explain_entry.select_type.upper()
- explain_entry.scan_type = explain_entry.scan_type.upper()
- self.parsed_query = Parser(re.sub(r'"(\S+)"', r"'\1'", self.query))
-
- @property
- def tables_examined(self) -> list[str]:
- return self.parsed_query.tables
-
- def update_table_data(self, table: DBTable):
- self.tables[table.name] = table
-
- def potential_indexes(self) -> list[DBIndex]:
- """Get all columns that can potentially be indexed to speed up this query."""
-
- possible_indexes = []
-
- # Where claus columns using these operators benefit from index
- # 1. = (equality)
- # 2. >, <, >=, <=
- # 3. LIKE 'xyz%' (Prefix search)
- # 4. BETWEEN (for date[time] fields)
- # 5. IN (similar to equality)
- if where_columns := self.parsed_query.columns_dict.get("where"):
- # TODO: Apply some heuristics here, not all columns in where clause are actually useful
- possible_indexes.extend(where_columns)
-
- # Join clauses - Both sides of join should ideally be indexed. One will *usually* be primary key.
- if join_columns := self.parsed_query.columns_dict.get("join"):
- possible_indexes.extend(join_columns)
-
- # Top N query variant - Order by column can possibly speed up the query
- if order_by_columns := self.parsed_query.columns_dict.get("order_by"):
- if self.parsed_query.limit_and_offset:
- possible_indexes.extend(order_by_columns)
-
- possible_db_indexes = [self._convert_to_db_index(i) for i in possible_indexes]
- possible_db_indexes = [
- i for i in possible_db_indexes if i.column not in ("*", "name")
- ]
- possible_db_indexes.sort(key=lambda i: (i.table, i.column))
-
- return self._remove_existing_indexes(possible_db_indexes)
-
- def _convert_to_db_index(self, column: str) -> DBIndex:
- column_name, table = None, None
-
- if "." in column:
- table, column_name = column.split(".")
- else:
- column_name = column
- for table_name, db_table in self.tables.items():
- if db_table.has_column(column):
- table = table_name
- break
- return DBIndex(column=column_name, name=column_name, table=table)
-
- def _remove_existing_indexes(self, potential_indexes: list[DBIndex]) -> list[DBIndex]:
- """Given list of potential index candidates remove the ones that already exist.
-
- This also removes multi-column indexes for parts that are applicable to query.
- Example: If multi-col index A+B+C exists and query utilizes A+B then
- A+B are removed from potential indexes.
- """
-
- def remove_maximum_indexes(idx: list[DBIndex]):
- """Try to remove entire index from potential indexes, if not possible, reduce one part and try again until no parts are left."""
- if not idx:
- return
- matched_sub_index = []
- for idx_part in list(idx):
- matching_part = [
- i
- for i in potential_indexes
- if i.column == idx_part.column and i.table == idx_part.table
- ]
- if not matching_part:
- # pop and recurse
- idx.pop()
- return remove_maximum_indexes(idx)
- else:
- matched_sub_index.extend(matching_part)
-
- # Every part matched now, lets remove those parts
- for i in matched_sub_index:
- potential_indexes.remove(i)
-
- # Reconstruct multi-col index
- for table in self.tables.values():
- merged_indexes = defaultdict(list)
- for index in table.indexes:
- merged_indexes[index.name].append(index)
-
- for idx in merged_indexes.values():
- idx.sort(key=lambda x: x.sequence)
-
- for idx in merged_indexes.values():
- remove_maximum_indexes(idx)
- return potential_indexes
-
- def suggest_index(self) -> DBIndex | None:
- """Suggest best possible column to index given query and table stats."""
- if missing_tables := (set(self.tables_examined) - set(self.tables.keys())):
- frappe.throw("DBTable infomation missing for: " + ", ".join(missing_tables))
-
- potential_indexes = self.potential_indexes()
-
- for index in list(potential_indexes):
- table = self.tables[index.table]
-
- # Data type is not easily indexable - skip
- column = [c for c in table.schema if c.name == index.column][0]
- if "text" in column.data_type.lower() or "json" in column.data_type.lower():
- potential_indexes.remove(index)
- # Update cardinality from column so scoring can be done
- index.cardinality = column.cardinality
-
- for index in potential_indexes:
- index._score = self.index_score(index)
-
- potential_indexes.sort(key=lambda i: i._score)
- if (
- potential_indexes
- and (best_index := potential_indexes[0])
- and best_index._score < INDEX_SCORE_THRESHOLD
- ):
- return best_index
-
- def index_score(self, index: DBIndex) -> float:
- """Score an index from 0 to 1 based on usefulness.
-
- A score of 0.5 indicates on average this index will read 50% of the table. (e.g. checkboxes)"""
- table = self.tables[index.table]
-
- cardinality = index.cardinality or 2
- total_rows = table.total_rows or cardinality or 1
-
- # We assume most unique values are evenly distributed, this is
- # definitely not the case IRL but it should be good enough assumptions
- # Score is rouhgly what percentage of table we will end up reading on typical query
- rows_fetched_on_average = (table.total_rows or cardinality) / cardinality
- return rows_fetched_on_average / total_rows
-
- def can_be_optimized(self) -> bool:
- """Return true if it's worth optimizeing.
-
- Few cases can not be optimized any further. E.g. ref/eq_ref/cost type
- of queries. Assume that anything that reads <10% of table already is
- not possible to truly optimize with these heuristics."""
- for explain in self.explain_plan:
- for table in self.tables.values():
- if table.name != explain.table:
- continue
- if (explain.rows / table.total_rows) > OPTIMIZATION_THRESHOLD:
- return True
- return False
diff --git a/press/press/report/mariadb_slow_queries/mariadb_slow_queries.json b/press/press/report/mariadb_slow_queries/mariadb_slow_queries.json
index 5897aaffd9..d32316c08f 100644
--- a/press/press/report/mariadb_slow_queries/mariadb_slow_queries.json
+++ b/press/press/report/mariadb_slow_queries/mariadb_slow_queries.json
@@ -1,15 +1,15 @@
{
"add_total_row": 0,
"columns": [],
- "creation": "2021-11-01 19:16:08.357082",
- "disable_prepared_report": 0,
+ "creation": "2024-12-23 11:36:40.301426",
"disabled": 0,
"docstatus": 0,
"doctype": "Report",
"filters": [],
"idx": 0,
"is_standard": "Yes",
- "modified": "2022-11-08 17:10:41.382656",
+ "letterhead": null,
+ "modified": "2024-12-23 11:36:40.301426",
"modified_by": "Administrator",
"module": "Press",
"name": "MariaDB Slow Queries",
@@ -25,5 +25,6 @@
{
"role": "Site Manager"
}
- ]
+ ],
+ "timeout": 0
}
\ No newline at end of file
diff --git a/press/press/report/mariadb_slow_queries/mariadb_slow_queries.py b/press/press/report/mariadb_slow_queries/mariadb_slow_queries.py
index 902c4cd70c..a546e699db 100644
--- a/press/press/report/mariadb_slow_queries/mariadb_slow_queries.py
+++ b/press/press/report/mariadb_slow_queries/mariadb_slow_queries.py
@@ -3,28 +3,16 @@
from __future__ import annotations
-import json
import re
from collections import defaultdict
-from dataclasses import dataclass
import frappe
import requests
import sqlparse
from frappe.core.doctype.access_log.access_log import make_access_log
from frappe.utils import convert_utc_to_timezone, get_system_timezone
-from frappe.utils.caching import redis_cache
from frappe.utils.password import get_decrypted_password
-from press.agent import Agent
-from press.api.site import protected
-from press.press.report.mariadb_slow_queries.db_optimizer import (
- DBExplain,
- DBIndex,
- DBOptimizer,
- DBTable,
-)
-
def execute(filters=None):
frappe.only_for(["System Manager", "Site Manager", "Press Admin", "Press Member"])
@@ -89,15 +77,6 @@ def execute(filters=None):
},
)
- if filters.analyze:
- columns.append(
- {
- "fieldname": "suggested_index",
- "label": frappe._("Suggest Index"),
- "fieldtype": "Data",
- },
- )
-
data = get_data(filters)
return columns, data
@@ -120,13 +99,13 @@ def get_data(filters):
get_system_timezone(),
)
+ # Filter out queries starting with `SET`
+ dql_stmt = ["SELECT", "UPDATE", "DELETE", "INSERT"]
+ rows = [x for x in rows if any(x["query"].startswith(stmt) for stmt in dql_stmt)]
+
if filters.normalize_queries:
rows = summarize_by_query(rows)
- # You can not analyze a query unless it has been normalized.
- if filters.analyze:
- rows = analyze_queries(rows, filters.site)
-
return rows
@@ -211,111 +190,5 @@ def summarize_by_query(data):
return result
-def analyze_queries(data, site):
- # TODO: handle old framework and old agents and general failures
- for row in data:
- query = row["example"]
- if not query.lower().startswith(("select", "update", "delete")):
- continue
- analyzer = OptimizeDatabaseQuery(site, query)
- if index := analyzer.analyze():
- row["suggested_index"] = f"{index.table}.{index.column}"
- return data
-
-
-@dataclass
-class OptimizeDatabaseQuery:
- site: str
- query: str
-
- def analyze(self) -> DBIndex | None:
- explain_output = self.fetch_explain() or []
-
- explain_output = [DBExplain.from_frappe_ouput(e) for e in explain_output]
- optimizer = DBOptimizer(query=self.query, explain_plan=explain_output)
- tables = optimizer.tables_examined
-
- for table in tables:
- stats = _fetch_table_stats(self.site, table)
- if not stats:
- # Old framework version
- return None
- db_table = DBTable.from_frappe_ouput(stats)
- column_stats = _fetch_column_stats(self.site, table)
- if not column_stats:
- # Failing due to large size, TODO: move this to a job
- return None
- db_table.update_cardinality(column_stats)
- optimizer.update_table_data(db_table)
-
- return optimizer.suggest_index()
-
- def fetch_explain(self) -> list[dict]:
- site = frappe.get_cached_doc("Site", self.site)
- db_server_name = frappe.db.get_value("Server", site.server, "database_server", cache=True)
- database_server = frappe.get_cached_doc("Database Server", db_server_name)
- agent = Agent(database_server.name, "Database Server")
-
- data = {
- "schema": site.database_name,
- "query": self.query,
- "private_ip": database_server.private_ip,
- "mariadb_root_password": database_server.get_password("mariadb_root_password"),
- }
-
- return agent.post("database/explain", data=data)
-
-
-@redis_cache(ttl=60 * 5)
-def _fetch_table_stats(site: str, table: str):
- site = frappe.get_cached_doc("Site", site)
- agent = Agent(site.server)
- return agent.describe_database_table(
- site,
- doctype=get_doctype_name(table),
- columns=[],
- )
-
-
-@redis_cache(ttl=60 * 5)
-def _fetch_column_stats(site, table, doc_name):
- site = frappe.get_cached_doc("Site", site)
- db_server_name = frappe.db.get_value("Server", site.server, "database_server", cache=True)
- database_server = frappe.get_cached_doc("Database Server", db_server_name)
- agent = Agent(database_server.name, "Database Server")
-
- data = {
- # "site": site,
- "doc_name": doc_name,
- "schema": site.database_name,
- "table": table,
- "private_ip": database_server.private_ip,
- "mariadb_root_password": database_server.get_password("mariadb_root_password"),
- }
- agent.create_agent_job("Column Statistics", "/database/column-stats", data)
-
-
def get_doctype_name(table_name: str) -> str:
return table_name.removeprefix("tab")
-
-
-@frappe.whitelist()
-@protected("Site")
-def add_suggested_index(name, indexes):
- if isinstance(indexes, str):
- indexes = json.loads(indexes)
- frappe.enqueue(_add_suggested_index, indexes=indexes, site_name=name)
-
-
-def _add_suggested_index(site_name, indexes):
- if not indexes:
- frappe.throw("No index suggested")
-
- for index in indexes:
- table, column = index.split(".")
- doctype = get_doctype_name(table)
-
- site = frappe.get_cached_doc("Site", site_name)
- agent = Agent(site.server)
- agent.add_database_index(site, doctype=doctype, columns=[column])
- frappe.msgprint(f"Index {index} added on site {site_name} successfully", realtime=True)
diff --git a/press/press/report/mariadb_slow_queries/test_db_optimizer.py b/press/press/report/mariadb_slow_queries/test_db_optimizer.py
deleted file mode 100644
index 65eead5535..0000000000
--- a/press/press/report/mariadb_slow_queries/test_db_optimizer.py
+++ /dev/null
@@ -1,618 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
-# License: MIT. See LICENSE
-
-import json
-
-from frappe.tests.utils import FrappeTestCase
-
-from press.press.report.mariadb_slow_queries.db_optimizer import (
- DBExplain,
- DBOptimizer,
- DBTable,
-)
-
-
-class TestDBOptimizer(FrappeTestCase):
- def test_basic_index_existence_analysis(self):
- def possible_indexes(q):
- user = DBTable.from_frappe_ouput(USER_TABLE)
- has_role = DBTable.from_frappe_ouput(HAS_ROLE_TABLE)
- return [
- i.column
- for i in DBOptimizer(
- query=q,
- tables={"tabUser": user, "tabHas Role": has_role},
- ).potential_indexes()
- ]
-
- self.assertEqual(
- ["creation"],
- possible_indexes("select `name` from `tabUser` order by `creation` desc limit 1"),
- )
-
- self.assertEqual(
- ["full_name"],
- possible_indexes("select `name` from `tabUser` where full_name = 'xyz'"),
- )
-
- self.assertIn(
- "user",
- possible_indexes(
- "select `name` from `tabUser` u join `tabHas Role` h on h.user = u.name"
- ),
- )
-
- def test_suggestion_using_table_stats(self):
- user = DBTable.from_frappe_ouput(USER_TABLE)
- has_role = DBTable.from_frappe_ouput(HAS_ROLE_TABLE)
-
- tables = {"tabUser": user, "tabHas Role": has_role}
- self.assertEqual(user.total_rows, 92)
-
- # This should suggest adding api_key as it definitely has highest cardinality.
- optimizer = DBOptimizer(
- query="select name from tabUser where enabled = 1 and api_key = 'xyz'", tables=tables
- )
- self.assertIn("api_key", [i.column for i in optimizer.potential_indexes()])
-
- index = optimizer.suggest_index()
- self.assertEqual(index.column, "api_key")
-
- # This should suggest nothing as modified is already indexed
- optimizer = DBOptimizer(
- query="select name from tabUser order by modified asc",
- tables=tables,
- )
- self.assertIsNone(optimizer.suggest_index())
-
- # This should suggest nothing as modified is already indexed
- optimizer = DBOptimizer(
- query="select name from tabUser u join `tabHas Role` r on r.parent = u.name where r.role='System Manager'",
- tables=tables,
- )
- index = optimizer.suggest_index()
- self.assertEqual(index.column, "role")
- self.assertEqual(index.table, "tabHas Role")
-
- def test_complex_sub_query_aliases(self):
- """Check if table identification is correct for subqueries."""
-
- q = """SELECT *,
- (SELECT COUNT(*) FROM `tabHD Ticket Comment` WHERE `tabHD Ticket Comment`.`reference_ticket`=`tabHD Ticket`.`name`) `count_comment`,
- (SELECT COUNT(*) FROM `tabCommunication` WHERE `tabCommunication`.`reference_doctype`='HD Ticket' AND `tabCommunication`.`reference_name`=`tabHD Ticket`.`name`) `count_msg`,
- FROM `tabHD Ticket`
- WHERE `agent_group`='L2'
- ORDER BY `modified` DESC
- LIMIT 20
- """
- explain = [DBExplain.from_frappe_ouput(e) for e in json.loads(EXPLAIN_OUTPUT)]
- optimizer = DBOptimizer(query=q, explain_plan=explain)
- optimizer.update_table_data(DBTable.from_frappe_ouput(HD_TICKET_TABLE))
- optimizer.update_table_data(DBTable.from_frappe_ouput(HD_TICKET_COMMENT_TABLE))
- optimizer.update_table_data(DBTable.from_frappe_ouput(COMMUNICATION_TABLE))
-
- self.assertTrue(optimizer.can_be_optimized())
- index = optimizer.suggest_index()
- self.assertEqual(index.table, "tabHD Ticket Comment")
- self.assertEqual(index.column, "reference_ticket")
-
-
-# Table stats extracted using describe-database-table for testing.
-
-USER_TABLE = {
- "table_name": "tabUser",
- "total_rows": 92,
- "schema": [
- {
- "column": "name",
- "type": "varchar(140)",
- "is_nullable": False,
- "default": None,
- "cardinality": 91,
- },
- {"column": "creation", "type": "datetime(6)", "is_nullable": True, "default": None},
- {
- "column": "modified",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- "cardinality": 91,
- },
- {
- "column": "modified_by",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- },
- {"column": "owner", "type": "varchar(140)", "is_nullable": True, "default": None},
- {"column": "docstatus", "type": "int(1)", "is_nullable": False, "default": "0"},
- {"column": "idx", "type": "int(8)", "is_nullable": False, "default": "0"},
- {"column": "enabled", "type": "int(1)", "is_nullable": False, "default": "1"},
- {"column": "email", "type": "varchar(140)", "is_nullable": False, "default": ""},
- {
- "column": "first_name",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- "cardinality": 88,
- },
- {
- "column": "reset_password_key",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- "cardinality": 84,
- },
- {
- "column": "user_type",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": "System User",
- "cardinality": 2,
- },
- {
- "column": "api_key",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- "cardinality": 70,
- },
- {"column": "api_secret", "type": "text", "is_nullable": True, "default": None},
- {"column": "_user_tags", "type": "text", "is_nullable": True, "default": None},
- {"column": "_comments", "type": "text", "is_nullable": True, "default": None},
- {"column": "_assign", "type": "text", "is_nullable": True, "default": None},
- {"column": "_liked_by", "type": "text", "is_nullable": True, "default": None},
- ],
- "indexes": [
- {
- "unique": True,
- "cardinality": 91,
- "name": "PRIMARY",
- "sequence": 1,
- "nullable": False,
- "column": "name",
- "type": "BTREE",
- },
- {
- "unique": True,
- "cardinality": 91,
- "name": "username",
- "sequence": 1,
- "nullable": True,
- "column": "username",
- "type": "BTREE",
- },
- {
- "unique": False,
- "cardinality": 91,
- "name": "modified",
- "sequence": 1,
- "nullable": True,
- "column": "modified",
- "type": "BTREE",
- },
- {
- "unique": False,
- "cardinality": 91,
- "name": "reset_password_key_index",
- "sequence": 1,
- "nullable": True,
- "column": "reset_password_key",
- "type": "BTREE",
- },
- ],
-}
-
-
-HAS_ROLE_TABLE = {
- "table_name": "tabHas Role",
- "total_rows": 96,
- "schema": [
- {
- "column": "name",
- "type": "varchar(140)",
- "is_nullable": "NO",
- "default": None,
- "cardinality": 92,
- },
- {"column": "creation", "type": "datetime(6)", "is_nullable": "YES", "default": None},
- {"column": "modified", "type": "datetime(6)", "is_nullable": "YES", "default": None},
- {
- "column": "modified_by",
- "type": "varchar(140)",
- "is_nullable": "YES",
- "default": None,
- },
- {"column": "owner", "type": "varchar(140)", "is_nullable": "YES", "default": None},
- {"column": "docstatus", "type": "int(1)", "is_nullable": "NO", "default": "0"},
- {"column": "idx", "type": "int(8)", "is_nullable": "NO", "default": "0"},
- {
- "column": "role",
- "type": "varchar(140)",
- "is_nullable": "YES",
- "default": None,
- "cardinality": 78,
- },
- {
- "column": "parent",
- "type": "varchar(140)",
- "is_nullable": "YES",
- "default": None,
- "cardinality": 92,
- },
- {
- "column": "parentfield",
- "type": "varchar(140)",
- "is_nullable": "YES",
- "default": None,
- },
- {
- "column": "parenttype",
- "type": "varchar(140)",
- "is_nullable": "YES",
- "default": None,
- },
- ],
- "indexes": [
- {
- "unique": True,
- "cardinality": 92,
- "name": "PRIMARY",
- "sequence": 1,
- "nullable": "",
- "column": "name",
- "type": "BTREE",
- },
- {
- "unique": False,
- "cardinality": 92,
- "name": "parent",
- "sequence": 1,
- "nullable": "YES",
- "column": "parent",
- "type": "BTREE",
- },
- ],
-}
-
-
-HD_TICKET_TABLE = {
- "table_name": "tabHD Ticket",
- "total_rows": 3820,
- "schema": [
- {
- "column": "name",
- "type": "bigint(20)",
- "is_nullable": False,
- "default": None,
- "cardinality": 3529,
- },
- {"column": "creation", "type": "datetime(6)", "is_nullable": True, "default": None},
- {
- "column": "modified",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- "cardinality": 3529,
- },
- {
- "column": "modified_by",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- },
- {"column": "owner", "type": "varchar(140)", "is_nullable": True, "default": None},
- {"column": "docstatus", "type": "int(1)", "is_nullable": False, "default": "0"},
- {"column": "idx", "type": "int(8)", "is_nullable": False, "default": "0"},
- {"column": "subject", "type": "varchar(140)", "is_nullable": True, "default": None},
- {"column": "raised_by", "type": "varchar(140)", "is_nullable": True, "default": None},
- {
- "column": "status",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": "Open",
- "cardinality": 8,
- },
- {"column": "priority", "type": "varchar(140)", "is_nullable": True, "default": None},
- {
- "column": "ticket_type",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "agent_group",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": "L1",
- "cardinality": 9,
- },
- {
- "column": "ticket_split_from",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- },
- {"column": "description", "type": "longtext", "is_nullable": True, "default": None},
- {"column": "template", "type": "varchar(140)", "is_nullable": True, "default": None},
- {"column": "sla", "type": "varchar(140)", "is_nullable": True, "default": None},
- {
- "column": "response_by",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "response_by_variance",
- "type": "decimal(21,9)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "agreement_status",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "resolution_by",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "resolution_by_variance",
- "type": "decimal(21,9)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "service_level_agreement_creation",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "on_hold_since",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "total_hold_time",
- "type": "decimal(21,9)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "first_response_time",
- "type": "decimal(21,9)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "first_responded_on",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "avg_response_time",
- "type": "decimal(21,9)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "resolution_details",
- "type": "longtext",
- "is_nullable": True,
- "default": None,
- },
- {"column": "opening_date", "type": "date", "is_nullable": True, "default": None},
- {"column": "opening_time", "type": "time(6)", "is_nullable": True, "default": None},
- {
- "column": "resolution_date",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "resolution_time",
- "type": "decimal(21,9)",
- "is_nullable": True,
- "default": None,
- },
- {
- "column": "user_resolution_time",
- "type": "decimal(21,9)",
- "is_nullable": True,
- "default": None,
- },
- {"column": "contact", "type": "varchar(140)", "is_nullable": True, "default": None},
- {"column": "customer", "type": "varchar(140)", "is_nullable": True, "default": None},
- {
- "column": "email_account",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- },
- {"column": "attachment", "type": "text", "is_nullable": True, "default": None},
- {"column": "_user_tags", "type": "text", "is_nullable": True, "default": None},
- {"column": "_comments", "type": "text", "is_nullable": True, "default": None},
- {"column": "_assign", "type": "text", "is_nullable": True, "default": None},
- {"column": "_liked_by", "type": "text", "is_nullable": True, "default": None},
- {"column": "_seen", "type": "text", "is_nullable": True, "default": None},
- ],
- "indexes": [
- {
- "unique": True,
- "cardinality": 3529,
- "name": "PRIMARY",
- "sequence": 1,
- "nullable": False,
- "column": "name",
- "type": "BTREE",
- },
- {
- "unique": False,
- "cardinality": 8,
- "name": "status",
- "sequence": 1,
- "nullable": True,
- "column": "status",
- "type": "BTREE",
- },
- {
- "unique": False,
- "cardinality": 3529,
- "name": "modified",
- "sequence": 1,
- "nullable": True,
- "column": "modified",
- "type": "BTREE",
- },
- ],
-}
-
-
-HD_TICKET_COMMENT_TABLE = {
- "table_name": "tabHD Ticket Comment",
- "total_rows": 2683,
- "schema": [
- {
- "column": "name",
- "type": "varchar(140)",
- "is_nullable": False,
- "default": None,
- "cardinality": 2683,
- },
- {"column": "creation", "type": "datetime(6)", "is_nullable": True, "default": None},
- {
- "column": "modified",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- "cardinality": 2345,
- },
- {
- "column": "reference_ticket",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- "cardinality": 1379,
- },
- {
- "column": "commented_by",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- },
- {"column": "content", "type": "longtext", "is_nullable": True, "default": None},
- {"column": "is_pinned", "type": "int(1)", "is_nullable": False, "default": "0"},
- ],
- "indexes": [
- {
- "unique": True,
- "cardinality": 2345,
- "name": "PRIMARY",
- "sequence": 1,
- "nullable": False,
- "column": "name",
- "type": "BTREE",
- },
- {
- "unique": False,
- "cardinality": 2345,
- "name": "modified",
- "sequence": 1,
- "nullable": True,
- "column": "modified",
- "type": "BTREE",
- },
- ],
-}
-
-
-COMMUNICATION_TABLE = {
- "table_name": "tabCommunication",
- "total_rows": 20727,
- "schema": [
- {
- "column": "name",
- "type": "varchar(140)",
- "is_nullable": False,
- "default": None,
- "cardinality": 19713,
- },
- {"column": "creation", "type": "datetime(6)", "is_nullable": True, "default": None},
- {
- "column": "modified",
- "type": "datetime(6)",
- "is_nullable": True,
- "default": None,
- "cardinality": 19713,
- },
- {
- "column": "reference_doctype",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- "cardinality": 1,
- },
- {
- "column": "reference_name",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- "cardinality": 3798,
- },
- {
- "column": "reference_owner",
- "type": "varchar(140)",
- "is_nullable": True,
- "default": None,
- "cardinality": 1314,
- },
- ],
- "indexes": [
- {
- "unique": True,
- "cardinality": 19713,
- "name": "PRIMARY",
- "sequence": 1,
- "nullable": False,
- "column": "name",
- "type": "BTREE",
- },
- {
- "unique": False,
- "cardinality": 19713,
- "name": "modified",
- "sequence": 1,
- "nullable": True,
- "column": "modified",
- "type": "BTREE",
- },
- {
- "unique": False,
- "cardinality": 2,
- "name": "reference_doctype_reference_name_index",
- "sequence": 1,
- "nullable": True,
- "column": "reference_doctype",
- "type": "BTREE",
- },
- {
- "unique": False,
- "cardinality": 9856,
- "name": "reference_doctype_reference_name_index",
- "sequence": 2,
- "nullable": True,
- "column": "reference_name",
- "type": "BTREE",
- },
- ],
-}
-
-
-EXPLAIN_OUTPUT = """[{"Extra": "", "id": 1, "key": "modified", "key_len": "9", "possible_keys": null, "ref": null, "rows": "20", "select_type": "PRIMARY", "table": "tabHD Ticket", "type": "index"}, {"Extra": "Using index condition; Using where", "id": 4, "key": "reference_doctype_reference_name_index", "key_len": "563", "possible_keys": "reference_doctype_reference_name_index", "ref": "const", "rows": "10236", "select_type": "DEPENDENT SUBQUERY", "table": "tabCommunication", "type": "ref"}, {"Extra": "Using index condition; Using where", "id": 3, "key": "reference_doctype_reference_name_index", "key_len": "563", "possible_keys": "reference_doctype_reference_name_index", "ref": "const", "rows": "10236", "select_type": "DEPENDENT SUBQUERY", "table": "tabCommunication", "type": "ref"}, {"Extra": "Using where; Using index", "id": 2, "key": "reference_ticket_index", "key_len": "563", "possible_keys": "reference_ticket_index", "ref": null, "rows": "2823", "select_type": "DEPENDENT SUBQUERY", "table": "tabHD Ticket Comment", "type": "index"}]"""
diff --git a/press/utils/__init__.py b/press/utils/__init__.py
index 5d40ae330c..4f05b1190d 100644
--- a/press/utils/__init__.py
+++ b/press/utils/__init__.py
@@ -6,21 +6,29 @@
import functools
import json
import re
+import socket
+import ssl
import time
from datetime import datetime, timedelta
from pathlib import Path
from typing import TypedDict, TypeVar
from urllib.parse import urljoin
+from urllib.request import urlopen
import frappe
import pytz
import requests
import wrapt
from babel.dates import format_timedelta
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.x509.oid import ExtensionOID
from frappe.utils import get_datetime, get_system_timezone
from frappe.utils.caching import site_cache
from pymysql.err import InterfaceError
+from press.utils.email_validator import validate_email
+
class SupervisorProcess(TypedDict):
program: str
@@ -840,3 +848,51 @@ def get_mariadb_root_password(site):
field = "mariadb_root_password"
return get_decrypted_password(doctype, name, field)
+
+
+def is_valid_email_address(email) -> bool:
+ if frappe.cache.exists(f"email_validity:{email}"):
+ return bool(frappe.utils.data.cint(frappe.cache.get_value(f"email_validity:{email}")))
+ try:
+ is_valid = bool(validate_email(email=email, check_mx=True, verify=True, smtp_timeout=10))
+ frappe.cache.set_value(f"email_validity:{email}", int(is_valid), expires_in_sec=3600)
+ if not is_valid:
+ log_error("Invalid email address on signup", data=email)
+ return bool(is_valid)
+ except Exception as e:
+ log_error("Email validation error on signup", data=e)
+ frappe.cache.set_value(f"email_validity:{email}", 0, expires_in_sec=3600)
+ return False
+
+
+def get_full_chain_cert_of_domain(domain: str) -> str:
+ cert_chain = []
+
+ # Get initial certificate
+ context = ssl.create_default_context()
+ with socket.create_connection((domain, 443)) as sock: # noqa: SIM117
+ with context.wrap_socket(sock, server_hostname=domain) as ssl_socket:
+ cert_pem = ssl.DER_cert_to_PEM_cert(ssl_socket.getpeercert(True))
+ cert = x509.load_pem_x509_certificate(cert_pem.encode(), default_backend())
+ cert_chain.append(cert_pem)
+
+ # Walk up the chain via certificate authority information access (AIA)
+ while True:
+ try:
+ aia = cert.extensions.get_extension_for_oid(ExtensionOID.AUTHORITY_INFORMATION_ACCESS)
+ for access in aia.value:
+ if access.access_method._name == "caIssuers":
+ uri = access.access_location._value
+ with urlopen(uri) as response:
+ der_cert = response.read()
+ pem_cert = ssl.DER_cert_to_PEM_cert(der_cert)
+ cert = x509.load_pem_x509_certificate(pem_cert.encode(), default_backend())
+ cert_chain.append(pem_cert)
+ break
+ except: # noqa: E722
+ break
+
+ cert_chain_str = ""
+ for cert in cert_chain:
+ cert_chain_str += cert + "\n"
+ return cert_chain_str
diff --git a/press/utils/email_validator.py b/press/utils/email_validator.py
new file mode 100644
index 0000000000..5cc03fb7e1
--- /dev/null
+++ b/press/utils/email_validator.py
@@ -0,0 +1,170 @@
+"""
+Customized function of validate-email package
+
+RFC 2822 - style email validation for Python
+
+(c) 2012 Syrus Akbary