Fixes in devsnapshot dump and import logic, after consultation with Radko.

Untested.

(Redmine issue: #7041)

Makefile

@@ -502,12 +502,12 @@ data-import-dbsnapshot: FORCE
 	@echo "\n$(GREEN)*** Importing latest production database dump ***$(NC)\n"
 
 	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/devsnapshot/main
-	@sudo -u postgres psql mentat_main -c "COPY FROM stdin;" < /vagrant/data/devsnapshot/chunks/reports_events.dat
-	@sudo -u postgres psql mentat_main -c "COPY FROM stdin;" < /vagrant/data/devsnapshot/chunks/statistics_events.dat
-	@sudo -u postgres psql mentat_main -c "COPY FROM stdin;" < /vagrant/data/devsnapshot/chunks/changelogs_items.dat
+	@sudo -u postgres psql mentat_main -c "COPY reports_events ($(cat /vagrant/data/devsnapshot/main/reports_events.cols)) FROM stdin;" < /vagrant/data/devsnapshot/chunks/reports_events.dat
+	@sudo -u postgres psql mentat_main -c "COPY statistics_events ($(cat /vagrant/data/devsnapshot/main/statistics_events.cols)) FROM stdin;" < /vagrant/data/devsnapshot/chunks/statistics_events.dat
+	@sudo -u postgres psql mentat_main -c "COPY changelogs_items ($(cat /vagrant/data/devsnapshot/main/changelogs_items.cols)) FROM stdin;" < /vagrant/data/devsnapshot/chunks/changelogs_items.dat
 
-	@sudo -u postgres psql mentat_events -c "COPY FROM stdin;" < /vagrant/data/devsnapshot/events/events.dat
-	@sudo -u postgres psql mentat_events -c "COPY FROM stdin;" < /vagrant/data/devsnapshot/events/events_json.dat
+	@sudo -u postgres psql mentat_events -c "COPY events ($(cat /vagrant/data/devsnapshot/events/events.cols)) FROM stdin;" < /vagrant/data/devsnapshot/events/events.dat
+	@sudo -u postgres psql mentat_events -c "COPY events_json ($(cat /vagrant/data/devsnapshot/events/events_json.cols)) FROM stdin;" < /vagrant/data/devsnapshot/events/events_json.dat
 
 	@echo ""
 	@sudo -u postgres psql mentat_main -e -c "select count(*) as user_count, min(createtime) as from_date, max(createtime) as to_date from users;"

conf/scripts/devsnapshot.sh

@@ -27,7 +27,8 @@ mkdir -p $BACKUP_DIR
 for table_name in reports_events statistics_events changelogs_items
 do
    print_subtitle "Dumping chunk of table ${table_name} from metadata database"
-   sudo -u mentat psql mentat_main -c "copy (select * from ${table_name} where createtime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/${table_name}.dat
+   sudo -u mentat psql mentat_main -c "COPY (SELECT * FROM ${table_name} WHERE createtime >= '${BACKUP_DATE_FROM}'::timestamptz) TO stdout" > ${BACKUP_DIR}/${table_name}.dat
+   sudo -u mentat psql mentat_main -c "COPY (SELECT string_agg(column_name, ',') FROM information_schema.columns WHERE table_name = '${table_name}' ORDER BY ordinal_position) TO stdout" > ${BACKUP_DIR}/${table_name}.cols
 done
 
 BACKUP_DIR=/var/mentat/devsnapshots/devsnapshot/events
@@ -36,16 +37,12 @@ chown mentat:mentat $BACKUP_DIR
 cd $BACKUP_DIR
 
 print_subtitle "Dumping chunk of table events from event database"
-sudo -u mentat psql mentat_events -c "copy (select * from events where storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/events.dat
+sudo -u mentat psql mentat_events -c "COPY (SELECT * FROM events WHERE storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) TO stdout" > ${BACKUP_DIR}/events.dat
+sudo -u mentat psql mentat_events -c "COPY (SELECT string_agg(column_name, ',') FROM information_schema.columns WHERE table_name = 'events' ORDER BY ordinal_position) TO stdout" > ${BACKUP_DIR}/events.cols
 
 print_subtitle "Dumping chunk of table events_json from event database"
-sudo -u mentat psql mentat_events -c "copy (select events_json.* from events_json join events on events_json.id = events.id where events.storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/events_json.dat
-
-print_subtitle "Dumping chunk of table events_thresholded from event database"
-sudo -u mentat psql mentat_events -c "copy (select * from events_thresholded join events on events_thresholded.eventid = events.id where events.storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/events_thresholded.dat
-
-print_subtitle "Dumping chunk of table thresholds from event database"
-sudo -u mentat psql mentat_events -c "copy (select * from thresholds where thresholdtime >= '${BACKUP_DATE_FROM}'::timestamptz) to stdout" > ${BACKUP_DIR}/thresholds.dat
+sudo -u mentat psql mentat_events -c "COPY (SELECT events_json.* FROM events_json JOIN events ON events_json.id = events.id WHERE events.storagetime >= '${BACKUP_DATE_FROM}'::timestamptz) TO stdout" > ${BACKUP_DIR}/events_json.dat
+sudo -u mentat psql mentat_events -c "COPY (SELECT string_agg(column_name, ',') FROM information_schema.columns WHERE table_name = 'events_json' ORDER BY ordinal_position) TO stdout" > ${BACKUP_DIR}/events_json.cols
 
 print_subtitle "Size before packaging:"
 du -csh /var/mentat/devsnapshots/devsnapshot
@@ -64,3 +61,9 @@ ls -alh ${SNAPSHOT_FILE_NAME}
 find /var/mentat/devsnapshots/ -type f -mtime +7 -name '*.tar.gz' -delete
 
 print_title '<DONE> CREATING DEVELOPMENT SNAPSHOT'
+
+
+sudo -u mentat psql mentat_main -c "COPY (SELECT string_agg(column_name, ',') FROM information_schema.columns WHERE table_name = '${table_name}' ORDER BY ordinal_position) TO stdout" > ${BACKUP_DIR}/${table_name}.cols
+sudo -u mentat psql mentat_main -c "COPY (SELECT * FROM ${table_name} WHERE createtime >= '${BACKUP_DATE_FROM}'::timestamptz) TO stdout" > ${BACKUP_DIR}/${table_name}.dat
+
+sudo -u postgres psql mentat_main -c "COPY ${table_name} ($(cat ${BACKUP_DIR}/${table_name}.cols)) FROM stdin;" < ${BACKUP_DIR}/${table_name}.dat