Further improvements in devsnapshot script.

(Redmine issue: #7041)

Makefile

@@ -490,24 +490,35 @@ data-fetch-dbsnapshot: FORCE
 	@echo "\n$(GREEN)*** Fetching latest production database dump ***$(NC)\n"
 	@rm -rf ./data/var
 	@mkdir -p ./data
-	@scp mentat-alt:/var/mentat/production-dump/mentat_psqldb_latest.tar.gz ./data/
-	@cd ./data/ && tar -xzvf ./mentat_psqldb_latest.tar.gz
-	@rm -f ./data/mentat_psqldb_latest.tar.gz
-	@mv ./data/var/mentat/tmp/mentat_* ./data/mentat_psqldb_latest
+	@time scp mentat-alt:/var/mentat/devsnapshots/devsnapshot-latest.tar.gz ./data/
+	@ls -alh mentat-alt:/var/mentat/devsnapshots/devsnapshot-latest.tar.gz ./data/
+	@cd ./data/ && tar -xzvf ./devsnapshot-latest.tar.gz
+	@rm -f ./data/devsnapshot-latest.tar.gz
+	@mv ./data/var/mentat/devsnapshots/devsnapshot ./data/devsnapshot
 	@rm -rf ./data/var
 	@echo ""
 
 data-import-dbsnapshot: FORCE
 	@echo "\n$(GREEN)*** Importing latest production database dump ***$(NC)\n"
-	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/mentat_psqldb_latest/main
-	@echo ""
-	@sudo -u postgres psql mentat_main -c "select count(*) as user_count, min(createtime) as from_date, max(createtime) as to_date from users;"
-	@sudo -u postgres psql mentat_main -c "select count(*) as group_count, min(createtime) as from_date, max(createtime) as to_date from groups;"
-	@sudo -u postgres psql mentat_main -c "select count(*) as filter_count, min(createtime) as from_date, max(createtime) as to_date from filters;"
-	@sudo -u postgres psql mentat_main -c "select count(*) as network_count, min(createtime) as from_date, max(createtime) as to_date from networks;"
-	@sudo -u postgres psql mentat_main -c "select count(*) as report_count, min(createtime) as from_date, max(createtime) as to_date from reports_events;"
-	@sudo -u postgres psql mentat_main -c "select count(*) as statistics_count, min(createtime) as from_date, max(createtime) as to_date from statistics_events;"
-	@sudo -u postgres psql mentat_main -c "select count(*) as changelog_count, min(createtime) as from_date, max(createtime) as to_date from changelogs_items;"
+
+	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/devsnapshot/main
+	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/devsnapshot/chunks/reports_events.dat
+	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/devsnapshot/chunks/statistics_events.dat
+	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/devsnapshot/chunks/changelogs_items.dat
+
+	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_events /vagrant/data/devsnapshot/events/events.dat
+	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_events /vagrant/data/devsnapshot/events/events_json.dat
+	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_events /vagrant/data/devsnapshot/events/events_thresholded.dat
+	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_events /vagrant/data/devsnapshot/events/thresholds.dat
+
+	@echo ""
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as user_count, min(createtime) as from_date, max(createtime) as to_date from users;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as group_count, min(createtime) as from_date, max(createtime) as to_date from groups;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as filter_count, min(createtime) as from_date, max(createtime) as to_date from filters;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as network_count, min(createtime) as from_date, max(createtime) as to_date from networks;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as report_count, min(createtime) as from_date, max(createtime) as to_date from reports_events;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as statistics_count, min(createtime) as from_date, max(createtime) as to_date from statistics_events;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as changelog_count, min(createtime) as from_date, max(createtime) as to_date from changelogs_items;"
 	@echo ""
 
 #-------------------------------------------------------------------------------

conf/scripts/devsnapshot.sh

@@ -15,18 +15,19 @@ source /etc/mentat/scripts/lib.sh
 
 print_title '<BEGIN> CREATING DEVELOPMENT SNAPSHOT'
 
-BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/main
 BACKUP_DATE_FROM=`date --date="8 days ago" --iso-8601=seconds`
+BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/main
 mkdir -p $BACKUP_DIR
 chown mentat:mentat $BACKUP_DIR
 cd $BACKUP_DIR
 print_subtitle 'Dumping main tables from metadata database'
 sudo -u mentat pg_dump --clean --create --if-exists --format=d --jobs=3 --dbname=mentat_main --username=mentat --no-password --exclude-table=reports_events --exclude-table=statistics_events --exclude-table=changelogs_items --file=$BACKUP_DIR
+BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/chunks
+mkdir -p $BACKUP_DIR
 for table_name in reports_events statistics_events changelogs_items
 do
    print_subtitle "Dumping chunk of table ${table_name} from metadata database"
    sudo -u mentat psql mentat_main -c "copy (select * from ${table_name} where createtime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/${table_name}.dat
-   sudo -u mentat gzip ${BACKUP_DIR}/${table_name}.dat
 done
 
 BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/events
@@ -36,19 +37,15 @@ cd $BACKUP_DIR
 
 print_subtitle "Dumping chunk of table events from event database"
 sudo -u mentat psql mentat_events -c "copy (select * from events where storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/events.dat
-sudo -u mentat gzip ${BACKUP_DIR}/events.dat
 
 print_subtitle "Dumping chunk of table events_json from event database"
 sudo -u mentat psql mentat_events -c "copy (select events_json.* from events_json join events on events_json.id = events.id where events.storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/events_json.dat
-sudo -u mentat gzip ${BACKUP_DIR}/events_json.dat
 
 print_subtitle "Dumping chunk of table events_thresholded from event database"
 sudo -u mentat psql mentat_events -c "copy (select * from events_thresholded join events on events_thresholded.eventid = events.id where events.storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/events_thresholded.dat
-sudo -u mentat gzip ${BACKUP_DIR}/events_thresholded.dat
 
 print_subtitle "Dumping chunk of table thresholds from event database"
 sudo -u mentat psql mentat_events -c "copy (select * from thresholds where thresholdtime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/thresholds.dat
-sudo -u mentat gzip ${BACKUP_DIR}/thresholds.dat
 
 cd /var/mentat
 SNAPSHOT_FILE_NAME="/var/mentat/devsnapshots/devsnapshot-${BACKUP_DATE_FROM}.tar.gz"
@@ -58,6 +55,9 @@ rm -f "${LATEST_SNAPSHOT_FILE_NAME}"
 ln -s "${SNAPSHOT_FILE_NAME}" "${LATEST_SNAPSHOT_FILE_NAME}"
 rm -rf /var/mentat/devsnapshots/devsnapshot/
 
-print_subtitle "Created DB snapshot file: ${SNAPSHOT_FILE_NAME} as ${LATEST_SNAPSHOT_FILE_NAME}"
+print_subtitle "Created DB snapshot file:"
+ls -alh ${SNAPSHOT_FILE_NAME}
+
+find /var/mentat/devsnapshots/ -type f -mtime +7 -name '*.tar.gz' -delete
 
 print_title '<DONE> CREATING DEVELOPMENT SNAPSHOT'