From d717be43fee8cee9430674daa617813db7f120e9 Mon Sep 17 00:00:00 2001
From: Jan Mach <jan.mach@cesnet.cz>
Date: Tue, 2 Nov 2021 15:45:57 +0100
Subject: [PATCH] Further improvements in devsnapshot script.
(Redmine issue: #7041)
---
Makefile | 37 ++++++++++++++++++++++++-------------
conf/scripts/devsnapshot.sh | 14 +++++++-------
2 files changed, 31 insertions(+), 20 deletions(-)
diff --git a/Makefile b/Makefile
index 44ca05b3..c809385b 100644
--- a/Makefile
+++ b/Makefile
@@ -490,24 +490,35 @@ data-fetch-dbsnapshot: FORCE
@echo "\n$(GREEN)*** Fetching latest production database dump ***$(NC)\n"
@rm -rf ./data/var
@mkdir -p ./data
- @scp mentat-alt:/var/mentat/production-dump/mentat_psqldb_latest.tar.gz ./data/
- @cd ./data/ && tar -xzvf ./mentat_psqldb_latest.tar.gz
- @rm -f ./data/mentat_psqldb_latest.tar.gz
- @mv ./data/var/mentat/tmp/mentat_* ./data/mentat_psqldb_latest
+ @time scp mentat-alt:/var/mentat/devsnapshots/devsnapshot-latest.tar.gz ./data/
+ @ls -alh mentat-alt:/var/mentat/devsnapshots/devsnapshot-latest.tar.gz ./data/
+ @cd ./data/ && tar -xzvf ./devsnapshot-latest.tar.gz
+ @rm -f ./data/devsnapshot-latest.tar.gz
+ @mv ./data/var/mentat/devsnapshots/devsnapshot ./data/devsnapshot
@rm -rf ./data/var
@echo ""
data-import-dbsnapshot: FORCE
@echo "\n$(GREEN)*** Importing latest production database dump ***$(NC)\n"
- @sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/mentat_psqldb_latest/main
- @echo ""
- @sudo -u postgres psql mentat_main -c "select count(*) as user_count, min(createtime) as from_date, max(createtime) as to_date from users;"
- @sudo -u postgres psql mentat_main -c "select count(*) as group_count, min(createtime) as from_date, max(createtime) as to_date from groups;"
- @sudo -u postgres psql mentat_main -c "select count(*) as filter_count, min(createtime) as from_date, max(createtime) as to_date from filters;"
- @sudo -u postgres psql mentat_main -c "select count(*) as network_count, min(createtime) as from_date, max(createtime) as to_date from networks;"
- @sudo -u postgres psql mentat_main -c "select count(*) as report_count, min(createtime) as from_date, max(createtime) as to_date from reports_events;"
- @sudo -u postgres psql mentat_main -c "select count(*) as statistics_count, min(createtime) as from_date, max(createtime) as to_date from statistics_events;"
- @sudo -u postgres psql mentat_main -c "select count(*) as changelog_count, min(createtime) as from_date, max(createtime) as to_date from changelogs_items;"
+
+ @sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/devsnapshot/main
+ @sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/devsnapshot/chunks/reports_events.dat
+ @sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/devsnapshot/chunks/statistics_events.dat
+ @sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/devsnapshot/chunks/changelogs_items.dat
+
+ @sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_events /vagrant/data/devsnapshot/events/events.dat
+ @sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_events /vagrant/data/devsnapshot/events/events_json.dat
+ @sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_events /vagrant/data/devsnapshot/events/events_thresholded.dat
+ @sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_events /vagrant/data/devsnapshot/events/thresholds.dat
+
+ @echo ""
+ @sudo -u postgres psql mentat_main -e -c "select count(*) as user_count, min(createtime) as from_date, max(createtime) as to_date from users;"
+ @sudo -u postgres psql mentat_main -e -c "select count(*) as group_count, min(createtime) as from_date, max(createtime) as to_date from groups;"
+ @sudo -u postgres psql mentat_main -e -c "select count(*) as filter_count, min(createtime) as from_date, max(createtime) as to_date from filters;"
+ @sudo -u postgres psql mentat_main -e -c "select count(*) as network_count, min(createtime) as from_date, max(createtime) as to_date from networks;"
+ @sudo -u postgres psql mentat_main -e -c "select count(*) as report_count, min(createtime) as from_date, max(createtime) as to_date from reports_events;"
+ @sudo -u postgres psql mentat_main -e -c "select count(*) as statistics_count, min(createtime) as from_date, max(createtime) as to_date from statistics_events;"
+ @sudo -u postgres psql mentat_main -e -c "select count(*) as changelog_count, min(createtime) as from_date, max(createtime) as to_date from changelogs_items;"
@echo ""
#-------------------------------------------------------------------------------
diff --git a/conf/scripts/devsnapshot.sh b/conf/scripts/devsnapshot.sh
index 468cc0b5..a9bb1628 100755
--- a/conf/scripts/devsnapshot.sh
+++ b/conf/scripts/devsnapshot.sh
@@ -15,18 +15,19 @@ source /etc/mentat/scripts/lib.sh
print_title '<BEGIN> CREATING DEVELOPMENT SNAPSHOT'
-BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/main
BACKUP_DATE_FROM=`date --date="8 days ago" --iso-8601=seconds`
+BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/main
mkdir -p $BACKUP_DIR
chown mentat:mentat $BACKUP_DIR
cd $BACKUP_DIR
print_subtitle 'Dumping main tables from metadata database'
sudo -u mentat pg_dump --clean --create --if-exists --format=d --jobs=3 --dbname=mentat_main --username=mentat --no-password --exclude-table=reports_events --exclude-table=statistics_events --exclude-table=changelogs_items --file=$BACKUP_DIR
+BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/chunks
+mkdir -p $BACKUP_DIR
for table_name in reports_events statistics_events changelogs_items
do
print_subtitle "Dumping chunk of table ${table_name} from metadata database"
sudo -u mentat psql mentat_main -c "copy (select * from ${table_name} where createtime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/${table_name}.dat
- sudo -u mentat gzip ${BACKUP_DIR}/${table_name}.dat
done
BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/events
@@ -36,19 +37,15 @@ cd $BACKUP_DIR
print_subtitle "Dumping chunk of table events from event database"
sudo -u mentat psql mentat_events -c "copy (select * from events where storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/events.dat
-sudo -u mentat gzip ${BACKUP_DIR}/events.dat
print_subtitle "Dumping chunk of table events_json from event database"
sudo -u mentat psql mentat_events -c "copy (select events_json.* from events_json join events on events_json.id = events.id where events.storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/events_json.dat
-sudo -u mentat gzip ${BACKUP_DIR}/events_json.dat
print_subtitle "Dumping chunk of table events_thresholded from event database"
sudo -u mentat psql mentat_events -c "copy (select * from events_thresholded join events on events_thresholded.eventid = events.id where events.storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/events_thresholded.dat
-sudo -u mentat gzip ${BACKUP_DIR}/events_thresholded.dat
print_subtitle "Dumping chunk of table thresholds from event database"
sudo -u mentat psql mentat_events -c "copy (select * from thresholds where thresholdtime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/thresholds.dat
-sudo -u mentat gzip ${BACKUP_DIR}/thresholds.dat
cd /var/mentat
SNAPSHOT_FILE_NAME="/var/mentat/devsnapshots/devsnapshot-${BACKUP_DATE_FROM}.tar.gz"
@@ -58,6 +55,9 @@ rm -f "${LATEST_SNAPSHOT_FILE_NAME}"
ln -s "${SNAPSHOT_FILE_NAME}" "${LATEST_SNAPSHOT_FILE_NAME}"
rm -rf /var/mentat/devsnapshots/devsnapshot/
-print_subtitle "Created DB snapshot file: ${SNAPSHOT_FILE_NAME} as ${LATEST_SNAPSHOT_FILE_NAME}"
+print_subtitle "Created DB snapshot file:"
+ls -alh ${SNAPSHOT_FILE_NAME}
+
+find /var/mentat/devsnapshots/ -type f -mtime +7 -name '*.tar.gz' -delete
print_title '<DONE> CREATING DEVELOPMENT SNAPSHOT'
--
GitLab