-
Jan Mach authored
The database migrations for IDEA event database *mentat_events* are now implemented. I have moved this issue to the next release, because there is code already merged in the current development branch that depended performing the migrations on target system. It was a choice of either removing that commit, or going forward with implementation. I have decided to implement it immediately, because this issue is blocking many more features. The implementation is again based on Alembic tool, which is already being used for migrations on mentat_main metadata database. It is however separatelly configured instance, merging both instances into single one would require considerably more amount of research and work. I have also updated accordingly manual pages regarding installation and upgrading. Very important bit of information was stamping the database with latest migration revision after clean installation. (Redmine issue: #4230)
Jan Mach authoredThe database migrations for IDEA event database *mentat_events* are now implemented. I have moved this issue to the next release, because there is code already merged in the current development branch that depended performing the migrations on target system. It was a choice of either removing that commit, or going forward with implementation. I have decided to implement it immediately, because this issue is blocking many more features. The implementation is again based on Alembic tool, which is already being used for migrations on mentat_main metadata database. It is however separatelly configured instance, merging both instances into single one would require considerably more amount of research and work. I have also updated accordingly manual pages regarding installation and upgrading. Very important bit of information was stamping the database with latest migration revision after clean installation. (Redmine issue: #4230)
Gruntfile.js 26.89 KiB
module.exports = function(grunt) {
// ---------------------------------------------------------------------
// Command line options.
var distribution = grunt.option('distribution') || 'development';
var buildnumber = grunt.option('buildnumber') || '0';
function cbk_dir_size(err, stdout, stderr, cb) {
grunt.config.set('template.deb-control.options.data.package_size', stdout.trim());
cb();
}
function version() {
if (distribution == 'production') {
return '<%= pkg.version %>'
}
else {
return '<%= pkg.version %>.' + buildnumber;
}
}
function deb_package_filename() {
return '<%= pkg.name %>_' + version() + '_<%= meta.architecture %>.deb';
}
function deb_package_latest_filename() {
return '<%= pkg.name %>_latest_<%= meta.architecture %>.deb';
}
function minify_json_file(src, tgt) {
src_content = grunt.file.readJSON(src);
tgt = tgt.replace(/\.json$/, '.min.json')
grunt.file.write(
tgt,
JSON.stringify(src_content)
);
console.log("Minified JSON file '" + src + "' to '" + tgt + "'");
}
// Project configuration.
grunt.initConfig({
pkg: grunt.file.readJSON('package.json'),
meta: grunt.file.readJSON('gruntconf.json'),
// Project paths to important directories.
project_paths: {
'deploy_dir': 'deploy/mentat/',
'package_dir': 'deploy/mentat/package/',
'control_dir': 'deploy/mentat/ctrl/',
'archive_dir': 'deploy/mentat/archive/',
'web_dir': 'lib/hawat/',
'web_static_dir': 'lib/hawat/blueprints/design/static/'
},
// Paths related to DEB packages.
paths_deb: {
'bin_dir': 'usr/local/bin/',
'etc_dir': 'etc/mentat/',
'lib_dir': 'usr/lib/python3/dist-packages/',
'scr_dir': 'etc/mentat/scripts/',
'man_dir': 'usr/share/man/'
},
// ---------------------------------------------------------------------
// Cleanup various locations.
clean: {
// Cleanup package directory
build: {
src: [
"<%= project_paths.package_dir %>" ]
},
// Cleanup web UI directory
webui: {
src: [
"<%= project_paths.web_static_dir %>vendor"
]
}
},
// ---------------------------------------------------------------------
// Running shell commands.
shell: {
// Install Yarn managed packages (web interface frontend packages).
yarn_install: {
command: 'yarn install'
},
// Upgrade Yarn managed packages (web interface frontend packages).
yarn_upgrade: {
command: 'yarn upgrade'
},
// Clean precompiled Python modules.
pyclean: {
command: 'find lib/ -name *.pyc -delete'
},
// Compile language dictionaries.
pybabel_hawat: {
command: 'make hpybabel-compile'
},
pybabel_mentat: {
command: 'make mpybabel-compile'
},
// Calculate directory size, will be used in DEB package metadata.
dir_size: {
command: '/usr/bin/du -k -s <%= project_paths.package_dir %> | /usr/bin/cut -f 1',
options: {
callback: cbk_dir_size
}
},
// Archive previously built DEB packages from build directory.
deb_archive: {
command: 'pkgs=`find <%= project_paths.deploy_dir %> -maxdepth 1 -name *.deb` && if [ -n "$pkgs" ]; then mv -f <%= project_paths.deploy_dir %>*.deb <%= project_paths.archive_dir %>; fi;'
},
// Build DEB package.
deb_build: {
command: '/usr/bin/fakeroot /usr/bin/dpkg-deb --build <%= project_paths.package_dir %> <%= project_paths.deploy_dir %>' + deb_package_filename()
},
// Duplicate package from deb_package_filename() to deb_package_latest_filename().
deb_duplicate: {
command: 'cp -f <%= project_paths.deploy_dir %>' + deb_package_filename() + ' <%= project_paths.deploy_dir %>' + deb_package_latest_filename()
},
// SCP deployment - upload built DEB packages to repository server.
deb_scp_upload_repo: {
command: '/usr/bin/scp <%= project_paths.deploy_dir %>' + deb_package_filename() + ' <%= meta.pkg_server %>:<%= meta.pkg_location %>/stable'
},
// SCP deployment - upload built DEB packages to development server.
deb_scp_upload_dev: {
command: '/usr/bin/scp <%= project_paths.deploy_dir %>' + deb_package_filename() + ' <%= meta.dev_server %>:<%= meta.dev_location %>'
},
// SCP deployment - upload built DEB packages to demo server.
deb_scp_upload_demo: {
command: '/usr/bin/scp <%= project_paths.deploy_dir %>' + deb_package_filename() + ' <%= meta.demo_server %>:<%= meta.demo_location %>'
},
// Refresh DEB repository metadata on repository server.
repo_refresh: {
command: '/usr/bin/ssh mach@<%= meta.pkg_server %> apt-update mentat'
},
// Install latest package via APT on production server - step 1.
ssh_apt_update_prod: {
command: '/usr/bin/ssh root@<%= meta.dep_server %> aptitude update' },
// Install latest package via APT on production server - step 2.
ssh_apt_install_prod: {
command: '/usr/bin/ssh root@<%= meta.dep_server %> aptitude install mentat-ng'
},
// Install latest package via DPKG on local system.
dpkg_install_local: {
command: 'sudo dpkg -i <%= project_paths.deploy_dir %>' + deb_package_filename()
},
// Install latest package via DPKG on development server.
dpkg_install_dev: {
command: '/usr/bin/ssh root@<%= meta.dev_server %> dpkg -i <%= meta.dev_location %>/' + deb_package_filename()
},
// Install latest package via DPKG on demo server.
dpkg_install_demo: {
command: '/usr/bin/ssh root@<%= meta.demo_server %> dpkg -i <%= meta.demo_location %>/' + deb_package_filename()
},
},
// ---------------------------------------------------------------------
// Fill in certain template files.
template: {
'deb-control': {
'options': {
'data': {
'package_name': '<%= pkg.name %>',
'package_version': version(),
'package_size': '0',
'architecture': '<%= meta.architecture %>'
}
},
'files': {
'<%= project_paths.package_dir %>DEBIAN/control': ['<%= project_paths.control_dir %>control.tmpl']
}
}
},
// ---------------------------------------------------------------------
// Copy certain files to appropriate locations.
copy: {
// Copy components for DEB package.
deb: {
files: [
// ----- Copy binaries to appropriate package location.
{
expand: true,
flatten: true,
cwd: 'bin/',
src: './mentat-*.py',
dest: '<%= project_paths.package_dir %><%= paths_deb.bin_dir %>'
},
// ----- Copy WSGI scripts to appropriate package location.
{
expand: true,
flatten: true,
cwd: 'bin/',
src: './*.wsgi',
dest: '<%= project_paths.package_dir %><%= paths_deb.bin_dir %>'
},
// ----- Copy libraries to appropriate package location.
{
expand: true,
cwd: 'lib/',
src: './**',
dest: '<%= project_paths.package_dir %><%= paths_deb.lib_dir %>'
},
// ----- Copy Debian package meta files to appropriate package location.
{
src: '<%= project_paths.control_dir %>conffiles',
dest: '<%= project_paths.package_dir %>DEBIAN/conffiles' },
{
src: '<%= project_paths.control_dir %>copyright',
dest: '<%= project_paths.package_dir %>DEBIAN/copyright'
},
{
src: '<%= project_paths.control_dir %>postinst',
dest: '<%= project_paths.package_dir %>DEBIAN/postinst'
},
// ----- Copy scripts to appropriate package location.
{
expand: true,
cwd: 'scripts/',
src: './**',
dest: '<%= project_paths.package_dir %><%= paths_deb.scr_dir %>'
},
// ----- Copy configurations to appropriate package location.
{
expand: true,
cwd: 'conf/',
src: './**',
dest: '<%= project_paths.package_dir %><%= paths_deb.etc_dir %>'
},
// ----- Copy migrations to appropriate package location.
{
expand: true,
cwd: 'migrations/',
src: './**',
dest: '<%= project_paths.package_dir %><%= paths_deb.etc_dir %>/migrations'
},
{
expand: true,
cwd: 'migrations-events/',
src: './**',
dest: '<%= project_paths.package_dir %><%= paths_deb.etc_dir %>/migrations-events'
},
// ----- Copy additional files to appropriate package locations.
{
expand: true,
flatten: true,
cwd: 'etc/bash_completion.d/',
src: './mentat',
dest: '<%= project_paths.package_dir %>etc/bash_completion.d'
},
{
expand: true,
flatten: true,
cwd: 'etc/default/',
src: './mentat',
dest: '<%= project_paths.package_dir %>etc/default'
},
{
expand: true,
flatten: true,
cwd: 'etc/init.d/',
src: './mentat',
dest: '<%= project_paths.package_dir %>etc/init.d'
},
{
expand: true,
flatten: true,
cwd: 'etc/systemd/',
src: './mentat.service',
dest: '<%= project_paths.package_dir %>lib/systemd/system'
}
]
},
// Copy components for web user interface.
webui: {
files: [ // ----- bootstrap
{
expand: true,
flatten: true,
cwd: 'node_modules/bootstrap/dist/css/',
src: './*',
dest: '<%= project_paths.web_static_dir %>vendor/bootstrap/css/'
},
{
expand: true,
flatten: true,
cwd: 'node_modules/bootstrap/dist/fonts/',
src: './*',
dest: '<%= project_paths.web_static_dir %>vendor/bootstrap/fonts/'
},
{
expand: true,
flatten: true,
cwd: 'node_modules/bootstrap/dist/js/',
src: './*',
dest: '<%= project_paths.web_static_dir %>vendor/bootstrap/js/'
},
// ----- CLDRJS.
{
expand: true,
cwd: 'node_modules/cldrjs/dist/',
src: './cldr.js',
dest: '<%= project_paths.web_static_dir %>vendor/cldrjs'
},
{
expand: true,
cwd: 'node_modules/cldrjs/dist/cldr/',
src: './*.js',
dest: '<%= project_paths.web_static_dir %>vendor/cldrjs/cldr'
},
// ----- CLDR.
{
expand: true,
cwd: 'node_modules/cldr-data/main/cs/',
src: './*.json',
dest: '<%= project_paths.web_static_dir %>vendor/cldr/main/cs'
},
{
expand: true,
cwd: 'node_modules/cldr-data/main/en/',
src: './*.json',
dest: '<%= project_paths.web_static_dir %>vendor/cldr/main/en'
},
{
expand: true,
cwd: 'node_modules/cldr-data/supplemental/',
src: './*.json',
dest: '<%= project_paths.web_static_dir %>vendor/cldr/supplemental'
},
{
expand: true,
cwd: 'node_modules/iana-tz-data/',
src: './iana-tz-data.json',
dest: '<%= project_paths.web_static_dir %>vendor/iana-tz-data'
},
// ----- Globalize.
{
expand: true,
cwd: 'node_modules/globalize/dist/',
src: './globalize.js',
dest: '<%= project_paths.web_static_dir %>vendor/globalize'
},
{
expand: true,
cwd: 'node_modules/globalize/dist/globalize/', src: './*.js',
dest: '<%= project_paths.web_static_dir %>vendor/globalize/globalize'
},
// ----- D3
{
expand: true,
flatten: true,
cwd: 'node_modules/d3/',
src: './d3*.js',
dest: '<%= project_paths.web_static_dir %>vendor/d3/js/'
},
// ----- NVD3
{
expand: true,
flatten: true,
cwd: 'node_modules/nvd3/build/',
src: './*.js*',
dest: '<%= project_paths.web_static_dir %>vendor/nvd3/js/'
},
{
expand: true,
flatten: true,
cwd: 'node_modules/nvd3/build/',
src: './*.css*',
dest: '<%= project_paths.web_static_dir %>vendor/nvd3/css/'
},
// ----- moment
{
expand: true,
flatten: true,
cwd: 'node_modules/moment/min/',
src: './*',
dest: '<%= project_paths.web_static_dir %>vendor/moment/js/'
},
// ----- bootstrap-select
{
expand: true,
flatten: true,
cwd: 'node_modules/bootstrap-select/dist/css/',
src: './*',
dest: '<%= project_paths.web_static_dir %>vendor/bootstrap-select/css/'
},
{
expand: true,
flatten: true,
cwd: 'node_modules/bootstrap-select/dist/js/',
src: './*',
dest: '<%= project_paths.web_static_dir %>vendor/bootstrap-select/js/'
},
// ----- eonasdan-bootstrap-datetimepicker
{
expand: true,
flatten: true,
cwd: 'node_modules/eonasdan-bootstrap-datetimepicker/build/css/',
src: './*',
dest: '<%= project_paths.web_static_dir %>vendor/bootstrap-datetimepicker/css/'
},
{
expand: true,
flatten: true,
cwd: 'node_modules/eonasdan-bootstrap-datetimepicker/build/js/',
src: './*',
dest: '<%= project_paths.web_static_dir %>vendor/bootstrap-datetimepicker/js/'
},
// ----- font-awesome
{
expand: true,
flatten: true,
cwd: 'node_modules/@fortawesome/fontawesome-free/css/',
src: './**', dest: '<%= project_paths.web_static_dir %>vendor/font-awesome/css/'
},
{
expand: true,
flatten: true,
cwd: 'node_modules/@fortawesome/fontawesome-free/webfonts/',
src: './**',
dest: '<%= project_paths.web_static_dir %>vendor/font-awesome/webfonts/'
},
// ----- jquery
{
expand: true,
flatten: true,
cwd: 'node_modules/jquery/dist/',
src: './jquery.min.js',
dest: '<%= project_paths.web_static_dir %>vendor/jquery/js/'
},
// ----- DataTables.net
{
expand: true,
flatten: true,
cwd: 'node_modules/datatables.net/js/',
src: './*.js',
dest: '<%= project_paths.web_static_dir %>vendor/datatables/js/'
},
{
expand: true,
flatten: true,
cwd: 'node_modules/datatables.net-plugins/',
src: [
'./integration/bootstrap/3/dataTables.bootstrap*.js'
],
dest: '<%= project_paths.web_static_dir %>vendor/datatables/js/'
},
{
expand: true,
flatten: true,
cwd: 'node_modules/datatables.net-plugins/',
src: [
'./integration/bootstrap/3/dataTables.bootstrap.css',
'./integration/font-awesome/dataTables.fontAwesome.css'
],
dest: '<%= project_paths.web_static_dir %>vendor/datatables/css/'
},
{
expand: true,
flatten: true,
cwd: 'node_modules/datatables.net-plugins/i18n/',
src: [
'./*.lang'
],
dest: '<%= project_paths.web_static_dir %>vendor/datatables/i18n/'
},
]
}
},
// ---------------------------------------------------------------------
// Strip comments from source files
comments: {
// Strip comments from datatables localization files, otherwise they
// are not valid JSON files and table localization does not work.
dti18n: {
options: {
singleline: true,
multiline: true
},
src: [ '<%= project_paths.web_static_dir %>vendor/datatables/i18n/*.lang']
},
},
// ---------------------------------------------------------------------
// Make sure all files have necessary file permissions.
chmod: {
options: {
mode: '755'
},
deb_control: {
src: ['<%= project_paths.package_dir %>DEBIAN/postinst']
},
deb_init: {
src: ['<%= project_paths.package_dir %>etc/init.d/*']
},
deb_bin: {
src: ['<%= project_paths.package_dir %><%= paths_deb.bin_dir %>/*']
},
deb_scr: {
src: ['<%= project_paths.package_dir %><%= paths_deb.scr_dir %>/*']
}
}
});
// ---------------------------------------------------------------------
// Load grunt modules.
require('load-grunt-tasks')(grunt, { scope: 'devDependencies' });
require('time-grunt')(grunt);
// Subtask for displaying current version number.
grunt.registerTask('check-version', function() {
console.log("Current package version: " + grunt.template.process(version()));
});
// Subtask for minification of CLDR JSON files.
grunt.registerTask('minify-cldrs', function() {
// Minify IANA timezone metadata.
[
[
'node_modules/iana-tz-data/iana-tz-data.json',
'<%= project_paths.web_dir %>templates/iana-tz-data/iana-tz-data.json'
],
[
'node_modules/iana-tz-data/iana-tz-data.json',
'<%= project_paths.web_static_dir %>vendor/iana-tz-data/iana-tz-data.json'
]
].forEach(function(item) {
src = grunt.template.process(item[0]);
tgt = grunt.template.process(item[1]);
minify_json_file(src, tgt);
});
// Minify CLDR metadata.
[
{
expand: true,
cwd: 'node_modules/cldr-data/main/cs/',
src: './*.json',
dest: '<%= project_paths.web_static_dir %>vendor/cldr/main/cs'
},
{
expand: true,
cwd: 'node_modules/cldr-data/main/en/',
src: './*.json',
dest: '<%= project_paths.web_static_dir %>vendor/cldr/main/en'
},
{
expand: true,
cwd: 'node_modules/cldr-data/supplemental/',
src: './*.json',
dest: '<%= project_paths.web_static_dir %>vendor/cldr/supplemental' }
].forEach(function(item) {
file_mapping = grunt.file.expandMapping(item.src, item.dest, item);
file_mapping.forEach(function(subitem) {
src = grunt.template.process(subitem.src[0]);
tgt = grunt.template.process(subitem.dest);
minify_json_file(src, tgt);
});
});
});
// ---------------------------------------------------------------------
// Setup custom task(s).
// ---------------------------------------------------------------------
grunt.registerTask('deb-build',
'(RUN) Build Debian packages for Mentat-ng system.',
['check-version', 'shell:pyclean', 'clean:build', 'shell:deb_archive', 'webui', 'copy:deb', 'chmod', 'shell:dir_size', 'template:deb-control', 'shell:deb_build', 'clean:build']
);
grunt.registerTask('deb-buildbot',
'(RUN) Build Debian packages for Mentat-ng system by Buildbot automation system.',
['check-version', 'shell:pyclean', 'clean:build', 'shell:deb_archive', 'shell:pybabel_mentat', 'webui', 'copy:deb', 'chmod', 'shell:dir_size', 'template:deb-control', 'shell:deb_build', 'clean:build']
);
grunt.registerTask('deb-scp-upload',
'(RUN) Upload latest Debian packages to repository server, development server and demo server.',
['shell:deb_scp_upload_repo', 'shell:repo_refresh', 'shell:deb_scp_upload_dev', 'shell:deb_scp_upload_demo']
);
grunt.registerTask('deb-ssh-install',
'(RUN) Install latest Debian packages on local system and development and demo servers.',
['shell:dpkg_install_local', 'shell:dpkg_install_dev', 'shell:dpkg_install_demo']
);
grunt.registerTask('deb-deploy',
'(RUN) Build Debian packages for Mentat-ng system, upload them to repository, development and demo servers and install them.',
['deb-build', 'deb-scp-upload', 'deb-ssh-install']
);
grunt.registerTask('deb-deploy-local',
'(RUN) Build Debian packages for Mentat-ng system and install them on local system.',
['deb-build', 'shell:dpkg_install_local']
);
grunt.registerTask('deb-deploy-dev',
'(RUN) Build Debian packages for Mentat-ng system and upload and install them on development server.',
['deb-build', 'shell:deb_scp_upload_dev', 'shell:dpkg_install_dev']
);
grunt.registerTask('deb-deploy-demo',
'(RUN) Build Debian packages for Mentat-ng system and upload and install them on demo server.',
['deb-build', 'shell:deb_scp_upload_demo', 'shell:dpkg_install_demo']
);
grunt.registerTask('update',
'(RUN) Update internal NPM packages for build suite.',
['shell:yarn_install', 'shell:yarn_upgrade']
);
grunt.registerTask('webui',
'(RUN) Build and install web user interface dependencies.',
['shell:yarn_install', 'shell:pybabel_hawat', 'clean:webui', 'copy:webui', 'comments', 'minify-cldrs']
);
grunt.registerTask('default',
'(RUN) Alias for deb-build, only build Debian packages.',
['deb-build']
);
};