From 6629c0b512818642d36d3e21e221f3d6a0c0a3c1 Mon Sep 17 00:00:00 2001
From: Alexander Rose <alex.rose@rcsb.org>
Date: Fri, 23 Feb 2018 20:25:34 -0800
Subject: [PATCH] improved schema generation and moved to apps folder
---
.../schema-generator/generate-typescript.ts} | 4 +-
.../schema-from-mmcif-dic.ts | 44 +++++++++++++------
.../schema-generator/util}/cif-dic.ts | 3 +-
.../schema-generator/util}/generate.ts | 17 ++++---
.../schema-generator/util}/json-schema.ts | 2 +-
.../schema-generator/util}/validate.ts | 2 +-
.../schema-generator}/validate-schema.ts | 4 +-
7 files changed, 49 insertions(+), 27 deletions(-)
rename src/{scripts/generate-schema.ts => apps/schema-generator/generate-typescript.ts} (84%)
rename src/{scripts => apps/schema-generator}/schema-from-mmcif-dic.ts (77%)
rename src/{scripts/schema-generation => apps/schema-generator/util}/cif-dic.ts (98%)
rename src/{scripts/schema-generation => apps/schema-generator/util}/generate.ts (76%)
rename src/{scripts/schema-generation => apps/schema-generator/util}/json-schema.ts (94%)
rename src/{scripts/schema-generation => apps/schema-generator/util}/validate.ts (96%)
rename src/{scripts => apps/schema-generator}/validate-schema.ts (80%)
diff --git a/src/scripts/generate-schema.ts b/src/apps/schema-generator/generate-typescript.ts
similarity index 84%
rename from src/scripts/generate-schema.ts
rename to src/apps/schema-generator/generate-typescript.ts
index df5381d4e..651beadda 100644
--- a/src/scripts/generate-schema.ts
+++ b/src/apps/schema-generator/generate-typescript.ts
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
@@ -7,7 +7,7 @@
import * as argparse from 'argparse'
import * as fs from 'fs'
-import { generate } from './schema-generation/generate'
+import { generate } from './util/generate'
function generateSchema (name: string, path: string) {
const str = fs.readFileSync(path, 'utf8')
diff --git a/src/scripts/schema-from-mmcif-dic.ts b/src/apps/schema-generator/schema-from-mmcif-dic.ts
similarity index 77%
rename from src/scripts/schema-from-mmcif-dic.ts
rename to src/apps/schema-generator/schema-from-mmcif-dic.ts
index d35b57a52..5cf4c58cf 100644
--- a/src/scripts/schema-from-mmcif-dic.ts
+++ b/src/apps/schema-generator/schema-from-mmcif-dic.ts
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
@@ -11,9 +11,9 @@ import fetch from 'node-fetch'
import Csv from 'mol-io/reader/csv/parser'
import CIF from 'mol-io/reader/cif'
-import { generateSchema } from './schema-generation/cif-dic'
-import { generate } from './schema-generation/generate'
-import { Filter, mergeFilters } from './schema-generation/json-schema'
+import { generateSchema } from './util/cif-dic'
+import { generate } from './util/generate'
+import { Filter, mergeFilters } from './util/json-schema'
async function runGenerateSchema(name: string, fieldNamesPath?: string, minCount = 0, typescript = false, out?: string) {
await ensureMmcifDicAvailable()
@@ -21,12 +21,21 @@ async function runGenerateSchema(name: string, fieldNamesPath?: string, minCount
const parsed = await comp();
if (parsed.isError) throw parsed
- let filter = await getUsageCountsFilter(minCount)
- // console.log(util.inspect(filter, {showHidden: false, depth: 3}))
- if (fieldNamesPath) {
- filter = mergeFilters(filter, await getFieldNamesFilter(fieldNamesPath))
+ console.log(fieldNamesPath, minCount)
+
+ let filter: Filter | undefined
+ if (minCount && fieldNamesPath) {
+ filter = mergeFilters(
+ await getUsageCountsFilter(minCount),
+ await getFieldNamesFilter(fieldNamesPath)
+ )
+ } else if (minCount) {
+ filter = await getUsageCountsFilter(minCount)
+ } else if (fieldNamesPath) {
+ console.log('MOIN')
+ filter = await getFieldNamesFilter(fieldNamesPath)
}
- // console.log(util.inspect(filter, {showHidden: false, depth: 3}))
+
const schema = generateSchema(parsed.result.blocks[0])
const output = typescript ? generate(name, schema, filter) : JSON.stringify(schema, undefined, 4)
@@ -49,10 +58,12 @@ async function getFieldNamesFilter(fieldNamesPath: string): Promise<Filter> {
const filter: Filter = {}
fieldNames.forEach((name, i) => {
- const [ category, field ] = name.substr(1).split('.')
+ const [ category, field ] = name.split('.')
+ console.log(category, field)
if (!filter[ category ]) filter[ category ] = {}
filter[ category ][ field ] = true
})
+ console.log(filter)
return filter
}
@@ -82,18 +93,22 @@ async function ensureMmcifDicAvailable() {
if (FORCE_MMCIF_DOWNLOAD || !fs.existsSync(MMCIF_DIC_PATH)) {
console.log('downloading mmcif dic...')
const data = await fetch(MMCIF_DIC_URL)
+ if (!fs.existsSync(MMCIF_DIC_DIR)){
+ fs.mkdirSync(MMCIF_DIC_DIR);
+ }
fs.writeFileSync(MMCIF_DIC_PATH, await data.text())
console.log('done downloading mmcif dic')
}
}
const MMCIF_USAGE_COUNTS_PATH = './data/mmcif-usage-counts.txt'
-const MMCIF_DIC_PATH = './build/dics/mmcif_pdbx_v50.dic'
+const MMCIF_DIC_DIR = './build/dics'
+const MMCIF_DIC_PATH = `${MMCIF_DIC_DIR}/mmcif_pdbx_v50.dic`
const MMCIF_DIC_URL = 'http://mmcif.wwpdb.org/dictionaries/ascii/mmcif_pdbx_v50.dic'
const parser = new argparse.ArgumentParser({
addHelp: true,
- description: 'Create schema from mmcif dictionary'
+ description: 'Create schema from mmcif dictionary (v50, downloaded from wwPDB)'
});
parser.addArgument([ '--name', '-n' ], {
defaultValue: 'mmCIF',
@@ -107,11 +122,12 @@ parser.addArgument([ '--typescript', '-ts' ], {
help: 'Output schema as TypeScript instead of as JSON'
});
parser.addArgument([ '--minFieldUsageCount', '-mc' ], {
- defaultValue: 1,
+ defaultValue: 0,
+ type: parseInt,
help: 'Minimum mmcif field usage counts'
});
parser.addArgument([ '--fieldNamesPath', '-fn' ], {
- defaultValue: 1,
+ defaultValue: '',
help: 'Field names to include'
});
parser.addArgument([ '--forceMmcifDicDownload', '-f' ], {
diff --git a/src/scripts/schema-generation/cif-dic.ts b/src/apps/schema-generator/util/cif-dic.ts
similarity index 98%
rename from src/scripts/schema-generation/cif-dic.ts
rename to src/apps/schema-generator/util/cif-dic.ts
index 7537f3e98..87626e2eb 100644
--- a/src/scripts/schema-generation/cif-dic.ts
+++ b/src/apps/schema-generator/util/cif-dic.ts
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
@@ -48,6 +48,7 @@ export function getFieldType (type: string, values?: string[]): Column {
case 'point_symmetry':
case 'id_list':
case '4x3_matrix':
+ case '3x4_matrices':
case 'point_group':
case 'point_group_helical':
case 'boolean':
diff --git a/src/scripts/schema-generation/generate.ts b/src/apps/schema-generator/util/generate.ts
similarity index 76%
rename from src/scripts/schema-generation/generate.ts
rename to src/apps/schema-generator/util/generate.ts
index c829bf037..22364bd63 100644
--- a/src/scripts/schema-generation/generate.ts
+++ b/src/apps/schema-generator/util/generate.ts
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
@@ -7,13 +7,13 @@
import { validate } from './validate'
import { Database, getTypeAndArgs, Filter } from './json-schema'
-function header (name: string, importDatabasePath = 'mol-base/collections/database') {
+function header (name: string, importDatabasePath = 'mol-data/db') {
return `/**
- * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* Code-generated '${name}' schema file
*
- * @author mol-star package (src/scripts/schema-generation/generate)
+ * @author mol-star package (src/apps/schema-generator/generate)
*/
import { Database, Column } from '${importDatabasePath}'
@@ -48,6 +48,11 @@ const value: { [k: string]: (...args: any[]) => string } = {
}
}
+const reSafePropertyName = /^[a-zA-Z_$][0-9a-zA-Z_$]*$/
+function safePropertyString(name: string) {
+ return name.match(reSafePropertyName) ? name : `'${name}'`
+}
+
export function generate (name: string, schema: Database, fields?: Filter, importDatabasePath?: string) {
const validationResult = validate(schema)
if (validationResult !== true) {
@@ -59,7 +64,7 @@ export function generate (name: string, schema: Database, fields?: Filter, impor
codeLines.push(`export const ${name}_Schema = {`)
Object.keys(schema).forEach(table => {
if (fields && !fields[ table ]) return
- codeLines.push(`\t'${table}': {`)
+ codeLines.push(`\t${safePropertyString(table)}: {`)
const columns = schema[ table ]
Object.keys(columns).forEach(columnName => {
if (fields && !fields[ table ][ columnName ]) return
@@ -71,7 +76,7 @@ export function generate (name: string, schema: Database, fields?: Filter, impor
} else {
typeDef = fieldType
}
- codeLines.push(`\t\t'${columnName}': ${typeDef},`)
+ codeLines.push(`\t\t${safePropertyString(columnName)}: ${typeDef},`)
})
codeLines.push('\t},')
})
diff --git a/src/scripts/schema-generation/json-schema.ts b/src/apps/schema-generator/util/json-schema.ts
similarity index 94%
rename from src/scripts/schema-generation/json-schema.ts
rename to src/apps/schema-generator/util/json-schema.ts
index 30cf9c2fa..45c662d93 100644
--- a/src/scripts/schema-generation/json-schema.ts
+++ b/src/apps/schema-generator/util/json-schema.ts
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
diff --git a/src/scripts/schema-generation/validate.ts b/src/apps/schema-generator/util/validate.ts
similarity index 96%
rename from src/scripts/schema-generation/validate.ts
rename to src/apps/schema-generator/util/validate.ts
index 591cc21ee..08355031d 100644
--- a/src/scripts/schema-generation/validate.ts
+++ b/src/apps/schema-generator/util/validate.ts
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
diff --git a/src/scripts/validate-schema.ts b/src/apps/schema-generator/validate-schema.ts
similarity index 80%
rename from src/scripts/validate-schema.ts
rename to src/apps/schema-generator/validate-schema.ts
index 7dfd27055..698090c99 100644
--- a/src/scripts/validate-schema.ts
+++ b/src/apps/schema-generator/validate-schema.ts
@@ -1,5 +1,5 @@
/**
- * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2017-2018 mol* contributors, licensed under MIT, See LICENSE file for more info.
*
* @author Alexander Rose <alexander.rose@weirdbyte.de>
*/
@@ -7,7 +7,7 @@
import * as argparse from 'argparse'
import * as fs from 'fs'
-import { validate } from './schema-generation/validate'
+import { validate } from './util/validate'
function runValidateSchema (path: string) {
const str = fs.readFileSync(path, 'utf8')
--
GitLab