Skip to content

Commit

Permalink
Doc - Update database schema documentation
Browse files Browse the repository at this point in the history
  • Loading branch information
@mdouchin
mdouchin committed Jun 15, 2021
1 parent c8c1e8b commit be45576
Show file tree
Hide file tree
Showing 8 changed files with 347 additions and 575 deletions.
474 changes: 237 additions & 237 deletions docs/database/columns.html
View file

Large diffs are not rendered by default.

227 changes: 96 additions & 131 deletions docs/database/gis.lizsync.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -763,6 +763,8 @@ want to log row values.
Note that the user name logged is the login role for the session. The audit trigger
cannot obtain the active role because it is reset by the SECURITY DEFINER invocation
of the audit trigger its self.
LizSync has added its own sync_data column to store the needed information for synchronisation purpose.
]]></comment>
<definition language="PLPGSQL"><![CDATA[
DECLARE
Expand Down Expand Up @@ -796,6 +798,8 @@ BEGIN
substring(TG_OP,1,1), -- action
NULL, NULL, -- row_data, changed_fields
'f', -- statement_only
-- LizSync specific data field
jsonb_build_object(
'origin', current_setting('lizsync.server_from', true),
'replayed_by',
Expand All @@ -807,8 +811,10 @@ BEGIN
current_setting('lizsync.sync_id', true)
)
ELSE jsonb_build_object()
END
END,
-- when the clone has created/updated/deleted the data in the clone database
'action_tstamp_tx',
Cast(current_setting('lizsync.clone_action_tstamp_tx', true) AS TIMESTAMP WITH TIME ZONE)
)
);
Expand Down Expand Up @@ -1097,10 +1103,18 @@ DECLARE
p_sync_id uuid;
rec record;
p_counter integer;
final_counter integer;
p_temporary_table text;
dblink_connection_name text;
dblink_msg text;
current_status boolean;
event_ids_on_error bigint[];
BEGIN
-- Default values
final_counter = 0;
current_status = True;
event_ids_on_error = array[]::bigint[];
-- Get the total number of logs to replay
SELECT count(*) AS nb
FROM temp_clone_audit
Expand All @@ -1110,17 +1124,25 @@ BEGIN
-- If there are some logs, process them
IF p_counter > 0 THEN
-- Get central server id
SELECT server_id::text INTO p_central_id
FROM central_lizsync.server_metadata
LIMIT 1;
-- RAISE NOTICE 'central id %', p_central_id;
BEGIN
-- Get central server id
SELECT server_id::text INTO p_central_id
FROM central_lizsync.server_metadata
LIMIT 1;
-- RAISE NOTICE 'central id %', p_central_id;
-- Get clone server id
SELECT server_id::text INTO p_clone_id
FROM lizsync.server_metadata
LIMIT 1;
-- RAISE NOTICE 'clone id %', p_clone_id;
-- Get clone server id
SELECT server_id::text INTO p_clone_id
FROM lizsync.server_metadata
LIMIT 1;
-- RAISE NOTICE 'clone id %', p_clone_id;
EXCEPTION WHEN OTHERS THEN
RAISE NOTICE 'LizSync: % %', SQLSTATE, SQLERRM;
-- set status
current_status = False;
RETURN QUERY SELECT -1::integer;
END;
-- Create dblink connection
dblink_connection_name = (md5(((random())::text || (clock_timestamp())::text)))::text;
Expand All @@ -1134,46 +1156,40 @@ BEGIN
-- Add a new item in the central history table
-- p_server_from = clone
-- p_server_to = central
sqltemplate = format(
'SELECT lizsync.insert_history_item(
''%1$s''::text, ''%2$s''::text,
NULL, NULL, NULL,
''partial'', ''pending''
) AS sync_id
',
p_clone_id,
p_central_id
);
SELECT sync_id FROM dblink(
dblink_connection_name,
sqltemplate
) AS foo(sync_id uuid)
INTO p_sync_id;
-- RAISE NOTICE 'sync id %', p_sync_id;
BEGIN
sqltemplate = format(
'SELECT lizsync.insert_history_item(
''%1$s''::text, ''%2$s''::text,
NULL, NULL, NULL,
''partial'', ''pending''
) AS sync_id
',
p_clone_id,
p_central_id
);
SELECT sync_id FROM dblink(
dblink_connection_name,
sqltemplate
) AS foo(sync_id uuid)
INTO p_sync_id;
-- RAISE NOTICE 'sync id %', p_sync_id;
EXCEPTION WHEN OTHERS THEN
RAISE NOTICE 'LizSync: % %', SQLSTATE, SQLERRM;
-- set status
current_status = False;
RETURN QUERY SELECT -1::integer;
END;
-- Replay SQL queries in central db
-- The session variables are used by the central server audit function
-- to fill the sync_data field
-- do not break line in sqlsession to avoid bugs
sqlsession = format('SET SESSION "lizsync.server_from" = ''%1$s''; SET SESSION "lizsync.server_to" = ''%2$s''; SET SESSION "lizsync.sync_id" = ''%3$s'';',
sqlsession = format(
'SET SESSION "lizsync.server_from" = ''%1$s''; SET SESSION "lizsync.server_to" = ''%2$s''; SET SESSION "lizsync.sync_id" = ''%3$s'';',
p_clone_id,
p_central_id,
p_sync_id
);
-- Store SQL query to update central logs afterward with original log timestamp
p_temporary_table = 'temp_' || md5(random()::text || clock_timestamp()::text);
sqlupdatelogs = '
CREATE TEMPORARY TABLE ' || p_temporary_table || ' (
action_tstamp_tx TIMESTAMP WITH TIME ZONE,
sync_id uuid,
client_query_hash text,
action_type text,
ident text
) ON COMMIT DROP;
';
-- Loop through logs and replay action
-- We need to query one by one to be able
-- to update the sync_data->action_tstamp_tx afterwards
Expand All @@ -1185,63 +1201,44 @@ BEGIN
ORDER BY tid
LOOP
-- Run the query in the central database via dblink
sqltemplate = sqlsession || trim(rec.action) || ';';
SELECT dblink_exec(
dblink_connection_name,
sqltemplate
)
INTO dblink_msg;
-- Concatenate action in sqlupdatelogs
-- We need it to add the clone actions timestamp in the central log table
-- since we find the central logs rows to update with a WHERE including action SQL ie sqltemplate
-- we use a hash on sqltemplate to avoid too big SQL
sqltemplate = trim(quote_literal(sqltemplate), '''');
sqlupdatelogs = concat(
sqlupdatelogs,
format('
INSERT INTO ' || p_temporary_table || '
(action_tstamp_tx, sync_id, client_query_hash, action_type, ident)
VALUES (
Cast(''%1$s'' AS TIMESTAMP WITH TIME ZONE),
''%2$s''::uuid,
''%3$s'',
''%4$s'',
''%5$s''
);
',
rec.action_tstamp_tx,
p_sync_id,
md5(sqltemplate)::text,
rec.action_type,
rec.ident
)
-- We add a session variable with the action timestamp in the clone
-- This will let us compare clone and central timestamp when UPDATE conflicts appear
sqltemplate = concat(
sqlsession,
format(
'SET SESSION "lizsync.clone_action_tstamp_tx" = ''%1$s'';',
rec.action_tstamp_tx
),
trim(rec.action) || ';'
);
END LOOP;
BEGIN
SELECT dblink_exec(
dblink_connection_name,
sqltemplate
)
INTO dblink_msg;
-- Add the necessary call the to security definer function
-- making possible from the clone to update the central lizsync.logged_actions
-- with the key action_tstamp_tx
sqlupdatelogs = concat(
sqlupdatelogs,
'SELECT lizsync.update_central_logs_add_clone_action_timestamps(' || quote_literal(p_temporary_table) || ');'
);
-- increase counter
final_counter = final_counter + 1;
EXCEPTION WHEN OTHERS THEN
RAISE NOTICE 'LizSync: % %', SQLSTATE, SQLERRM;
-- set status
current_status = False;
-- Update central lizsync.logged_actions
PERFORM * FROM dblink(
dblink_connection_name,
sqlupdatelogs
) AS foo(update_status boolean)
;
-- add event id to keep because on error
event_ids_on_error = event_ids_on_error || rec.event_id;
CONTINUE;
END;
END LOOP;
-- Update central history table item
sqltemplate = format(
'SELECT lizsync.update_history_item(''%1$s''::uuid, ''done'', NULL)',
p_sync_id
)
;
PERFORM * FROM dblink(
dblink_connection_name,
sqltemplate
Expand All @@ -1254,14 +1251,19 @@ BEGIN
END IF;
-- Remove successfully replayed logs from clone audit table
DELETE FROM lizsync.logged_actions
WHERE NOT (event_id = ANY (event_ids_on_error))
;
-- Remove logs from clone audit table
TRUNCATE lizsync.logged_actions
RESTART IDENTITY;
-- Set sequence to the minimum required
PERFORM setval(
'"lizsync"."logged_actions_event_id_seq"',
(SELECT max(event_id) FROM lizsync.logged_actions)
);
-- Return
RETURN QUERY
SELECT p_counter;
RETURN QUERY SELECT final_counter;
END;
]]></definition>
</routine>
Expand Down Expand Up @@ -1547,43 +1549,6 @@ BEGIN
p_number_conflicts
;
END;
]]></definition>
</routine>
<routine dataAccess="MODIFIES" deterministic="false" name="update_central_logs_add_clone_action_timestamps" returnType="boolean" securityType="DEFINER" type="FUNCTION">
<comment><![CDATA[Update all logs created by the central database after the clone has replayed its local logs in the central database. It is necessary to update the action_tstamp_tx key of lizsync.logged_actions sync_data column. The SECURITY DEFINER allows the clone to update the protected lizsync.logged_actions table. DO NOT USE MANUALLY.]]></comment>
<definition language="PLPGSQL"><![CDATA[
DECLARE
sqltemplate text;
p_central_id text;
BEGIN
-- Get central server id
SELECT server_id::text INTO p_central_id
FROM lizsync.server_metadata
LIMIT 1;
-- Use the data stored in the temporary table given to update the lizsync.logged_actions
sqltemplate = '
UPDATE lizsync.logged_actions
SET sync_data = sync_data || jsonb_build_object(
''action_tstamp_tx'',
Cast(t.action_tstamp_tx AS TIMESTAMP WITH TIME ZONE)
)
FROM ' || quote_ident(temporary_table_name) || ' AS t
WHERE True
-- same synchronisation id
AND sync_data->''replayed_by''->>' || quote_literal(p_central_id) || ' = t.sync_id::text
-- same query (we compare hash)
AND md5(client_query)::text = t.client_query_hash
-- same action type
AND action = t.action_type
-- same table
AND concat(schema_name, ''.'', table_name) = t.ident
';
EXECUTE sqltemplate;
-- Return
RETURN True;
END;
]]></definition>
</routine>
<routine dataAccess="MODIFIES" deterministic="false" name="update_central_logs_add_clone_id" returnType="boolean" securityType="DEFINER" type="FUNCTION">
Expand Down
16 changes: 8 additions & 8 deletions docs/database/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ <h1>Tables</h1><br />
<div class="col-md-12">
<div class="callout callout-info">
<h4>SchemaSpy Analysis of gis</h4>
<p>Generated on Thu May 27 09:24 GMT 2021</p>
<p>Generated on Tue Jun 15 15:23 GMT 2021</p>
</div>
</div>
</div>
Expand Down Expand Up @@ -154,7 +154,7 @@ <h4>SchemaSpy Analysis of gis</h4>
<span class="info-box-icon bg-navy"><i class="fa fa-file-code-o" aria-hidden="true"></i></span>
<div class="info-box-content">
<span class="info-box-text">Routines</span>
<span class="info-box-number">23</span>
<span class="info-box-number">22</span>
</div>
<!-- /.info-box-content -->
</div>
Expand Down Expand Up @@ -209,31 +209,31 @@ <h3 class="box-title">Tables</h3>
<td class="comment detail" style="display: table-cell;"><p>Database structure metadata used for migration by QGIS plugin</p></td>
</tr>
<tr class="tbl even" valign="top">
<td class="detail"><a href="tables/server_metadata.html">server_metadata</a></td>
<td class="detail"><a href="tables/synchronized_tables.html">synchronized_tables</a></td>
<td class="detail" align="right">0</td>
<td class="detail" align="right">0</td>
<td class="detail" align="right">2</td>
<td class="detail" align="right">0</td>
<td class="detail" align="right">Table</td>
<td class="comment detail" style="display: table-cell;"></td>
<td class="comment detail" style="display: table-cell;"><p>List of tables to synchronise per clone server id. This list works as a white list. Only listed tables will be synchronised for each server ids.</p></td>
</tr>
<tr class="tbl even" valign="top">
<td class="detail"><a href="tables/synchronized_tables.html">synchronized_tables</a></td>
<td class="detail"><a href="tables/logged_relations.html">logged_relations</a></td>
<td class="detail" align="right">0</td>
<td class="detail" align="right">0</td>
<td class="detail" align="right">2</td>
<td class="detail" align="right">0</td>
<td class="detail" align="right">Table</td>
<td class="comment detail" style="display: table-cell;"><p>List of tables to synchronise per clone server id. This list works as a white list. Only listed tables will be synchronised for each server ids.</p></td>
<td class="comment detail" style="display: table-cell;"><p>Table used to store unique identifier columns for table or views, so that events can be replayed</p></td>
</tr>
<tr class="tbl even" valign="top">
<td class="detail"><a href="tables/logged_relations.html">logged_relations</a></td>
<td class="detail"><a href="tables/server_metadata.html">server_metadata</a></td>
<td class="detail" align="right">0</td>
<td class="detail" align="right">0</td>
<td class="detail" align="right">2</td>
<td class="detail" align="right">0</td>
<td class="detail" align="right">Table</td>
<td class="comment detail" style="display: table-cell;"><p>Table used to store unique identifier columns for table or views, so that events can be replayed</p></td>
<td class="comment detail" style="display: table-cell;"></td>
</tr>
<tr class="tbl even" valign="top">
<td class="detail"><a href="tables/logged_actions.html">logged_actions</a></td>
Expand Down
4 changes: 2 additions & 2 deletions docs/database/info-html.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
date=2021-05-27 09:25:00+0000
os=Linux 5.8.0-53-generic
date=2021-06-15 15:23:04+0000
os=Linux 5.8.0-55-generic
schemaspy-version=6.1.0
schemaspy-build=6.1.0.41 2019-09-17 21:52:57
diagramImplementation=Graphviz dot 2.38
Loading

0 comments on commit be45576

Please sign in to comment.