Commit d4592990 authored by Tamlyn Rhodes's avatar Tamlyn Rhodes

feat: switch to PostgreSQL

BREAKING CHANGE: All data is now persisted in a PostgreSQL database instead of PouchDB
BREAKING CHANGE: Database server must be running and have an existing database before running `pubsweet setupdb` (Docker config provided)
`pubsweet start` runs `npm start` script if found and falls back to `pubsweet server`
`pubsweet server` starts the PubSweet server (like the old `pubsweet start`)
`pubsweet-server` model API is unchanged
parent ce5599d0
Pipeline #4822 passed with stages
in 8 minutes and 27 seconds
node_modules
coverage
*.log
coverage/
# Created by https://www.gitignore.io/api/osx,linux
......@@ -48,5 +49,3 @@ Temporary Items
.apdisk
# End of https://www.gitignore.io/api/osx,linux
node_modules/
......@@ -52,10 +52,21 @@ test:
image: $IMAGE_ORG/$IMAGE_NAME:$CI_COMMIT_SHA
stage: test
variables:
# don't clone repo as image already has it
GIT_STRATEGY: none
# setup data for postgres image
POSTGRES_USER: test
POSTGRES_PASSWORD: pw
# connection details for tests
PGUSER: test
PGPASSWORD: pw
NODE_ENV: test
services:
- postgres
script:
- cd ${HOME}
- NODE_ENV=test npm run test
# specify host here else it confuses the linked postgres image
- PGHOST=postgres npm run test
# if tests pass we will push latest, labelled with current commit hash
push:latest:
......
This diff is collapsed.
#!/usr/bin/env node
require('../cli/server')().catch(require('../src/error-exit'))
......@@ -8,7 +8,8 @@ program
.command('new', 'create and set up a new pubsweet app')
.command('setupdb', 'generate a database for a pubsweet app')
.command('build', 'build static assets for a pubsweet app')
.command('start', 'build static assets and start a pubsweet app')
.command('start', 'start pubsweet server and backing services')
.command('server', 'build static assets and start a pubsweet app')
.command('add', 'add one or more components to a pubsweet app')
.command('remove', 'remove one or more components from a pubsweet app')
.command('adduser', 'add a user to the database for a pubsweet app')
......
......@@ -4,6 +4,7 @@ const program = require('commander')
const fs = require('fs-extra')
const { spawnSync } = require('child_process')
const path = require('path')
const crypto = require('crypto')
const { STARTER_REPO_URL } = require('../src/constants')
const readCommand = async argsOverride => {
......@@ -28,7 +29,7 @@ const readCommand = async argsOverride => {
const overWrite = appPath => {
if (!fs.statSync(appPath).isDirectory()) {
throw new Error(appPath, 'exists as a file. Will not overwrite.')
throw new Error(`${appPath} exists as a file. Will not overwrite.`)
}
logger.info(`Overwriting ${appPath} due to --clobber flag`)
fs.removeSync(appPath)
......@@ -55,5 +56,11 @@ module.exports = async argsOverride => {
stdio: 'inherit',
})
// generate secret
const configFilePath = path.join(appPath, 'config', `local.json`)
const secret = crypto.randomBytes(64).toString('hex')
fs.writeJsonSync(configFilePath, { secret })
logger.info(`Added secret to ${configFilePath} under pubsweet-server.secret`)
logger.info('Finished generating initial app')
}
const forever = require('forever-monitor')
const _ = require('lodash')
const config = require('config')
const logger = require('@pubsweet/logger')
const path = require('path')
const program = require('commander')
const { dbExists, setupDb } = require('@pubsweet/db-manager')
const { ordinalize } = require('inflection')
const readCommand = async argsOverride => {
program
.option('--reduxlog-off', 'Switch off Redux logger')
.description(
'Build assets and start the app with forever (not recommended for production).',
)
return program.parse(argsOverride || process.argv)
}
module.exports = async argsOverride => {
const commandOpts = await readCommand(argsOverride)
logger.info('Starting PubSweet app')
if (!await dbExists()) {
if (config.had('dbManager')) {
await setupDb(config.get('dbManager'))
} else {
throw new Error(
'Setup database with "pubsweet setupdb" before starting app',
)
}
}
const executable = path.join(__dirname, '..', 'src', 'startup', 'start.js')
const defaultOpts = {
silent: false,
watch: true,
// By default we'll restart the app when config is edited
watchDirectory: path.resolve('config'),
// watchIgnorePatterns: ["./client-config.js"] // perhaps
max: 10,
env: _.clone(process.env),
}
const configOpts = config.has('forever') ? config.get('forever') : {}
const overrideOpts = {
env: {
REDUXLOG_OFF: commandOpts.reduxlogOff,
},
}
const finalOpts = _.merge(defaultOpts, configOpts, overrideOpts)
const child = forever.start(executable, finalOpts)
child.on('start', () => {
logger.info(`App started.`)
logger.info(
'The app will be kept running, even if errors occur, until you stop it.',
)
logger.info('To stop the app use ctrl-C')
})
child.on('stop', proc => {
logger.info(`App stopped (${proc})`)
})
child.on('restart', () => {
logger.warn(ordinalize(`Restarting app for ${child.times} time`))
})
child.on('exit:code', code => {
logger.error(`App exited with code ${code}`)
})
child.on('error', err => {
logger.error(err.stack)
throw err
})
return child
}
const program = require('commander')
const properties = require('../src/schemas/').db
const properties = require('../src/schemas').db
const { setupDb } = require('@pubsweet/db-manager')
const db = require('pubsweet-server/src/db')
const config = require('config')
const _ = require('lodash')
const { forEach, merge } = require('lodash')
const runPrompt = require('../src/run-prompt')
const readCommand = async argsOverride => {
......@@ -10,7 +11,7 @@ const readCommand = async argsOverride => {
'Setup a database for a PubSweet app. Run from your project root',
)
_.forEach(properties, (value, key) => {
forEach(properties, (value, key) => {
if (value.type === 'boolean') {
program.option(`--${key}`, value.description)
} else {
......@@ -23,11 +24,14 @@ const readCommand = async argsOverride => {
module.exports = async argsOverride => {
const commandOpts = await readCommand(argsOverride)
commandOpts.clobber = !!commandOpts.clobber // Always interpret absence of option as clobber = false
const configOpts = config.has('dbManager') ? config.get('dbManager') : {}
const promptOverride = _.merge(configOpts, commandOpts)
const promptOverride = merge(configOpts, commandOpts)
promptOverride.clobber = !!promptOverride.clobber // Always interpret absence of option as clobber = false
const finalOpts = await runPrompt({ properties, override: promptOverride })
return setupDb(finalOpts)
await setupDb(finalOpts)
// drain pool to avoid 10 second delay before command exits
db.end()
}
const forever = require('forever-monitor')
const _ = require('lodash')
const config = require('config')
const logger = require('@pubsweet/logger')
const path = require('path')
const program = require('commander')
const { dbExists } = require('@pubsweet/db-manager')
const { ordinalize } = require('inflection')
const readCommand = async argsOverride => {
program
.option('--reduxlog-off', 'Switch off Redux logger')
.description(
'Build assets and start the app with forever (not recommended for production).',
)
return program.parse(argsOverride || process.argv)
}
module.exports = async argsOverride => {
const commandOpts = await readCommand(argsOverride)
const { spawn } = require('child_process')
const requireRelative = require('require-relative')
const serverCommand = require('./server')
module.exports = async () => {
logger.info('Starting PubSweet app')
if (!await dbExists()) {
throw new Error(
'Create database with "pubsweet setupdb" before starting app',
)
}
const executable = path.join(__dirname, '..', 'src', 'startup', 'start.js')
const defaultOpts = {
silent: false,
watch: true,
// By default we'll restart the app when config is edited
watchDirectory: path.resolve('config'),
// watchIgnorePatterns: ["./client-config.js"] // perhaps
max: 10,
env: _.clone(process.env),
}
const configOpts = config.has('forever') ? config.get('forever') : {}
const overrideOpts = {
env: {
REDUXLOG_OFF: commandOpts.reduxlogOff,
},
}
const finalOpts = _.merge(defaultOpts, configOpts, overrideOpts)
const child = forever.start(executable, finalOpts)
child.on('start', () => {
logger.info(`App started.`)
const pkg = requireRelative('./package.json')
if (
pkg.scripts &&
pkg.scripts.start &&
pkg.scripts.start !== 'pubsweet start'
) {
logger.info('Using "start" script from app.')
spawn('npm', ['start'], { stdio: 'inherit' })
} else {
logger.info(
'The app will be kept running, even if errors occur, until you stop it.',
'No "start" script defined in app. Falling back to "pubsweet server" behavior.',
)
logger.info('To stop the app use ctrl-C')
})
child.on('stop', proc => {
logger.info(`App stopped (${proc})`)
})
child.on('restart', () => {
logger.warn(ordinalize(`Restarting app for ${child.times} time`))
})
child.on('exit:code', code => {
logger.error(`App exited with code ${code}`)
})
child.on('error', err => {
logger.error(err.stack)
throw err
})
return child
await serverCommand()
}
}
module.exports = {}
module.exports = {
'pubsweet-server': {
db: {
database: global.__testDbName || 'test',
},
secret: 'not very secret',
},
}
......@@ -4,6 +4,7 @@
"description": "Pubsweet command-line interface, app generator and manager",
"bin": "./bin/pubsweet.js",
"scripts": {
"test": "jest",
"vuln-test": "nsp check --output checkstyle"
},
"keywords": ["pubsweet", "cli"],
......@@ -14,6 +15,7 @@
"@pubsweet/logger": "^0.2.2",
"bluebird": "^3.5.0",
"colors": "^1.1.2",
"crypto": "^1.0.1",
"commander": "^2.9.0",
"express": "^4.15.3",
"forever-monitor": "^1.7.0",
......@@ -35,6 +37,7 @@
"@pubsweet/starter":
"git+https://gitlab.coko.foundation/pubsweet/pubsweet-starter.git",
"jest": "^22.1.4",
"jest-environment-db": "^0.1.0",
"nsp": "^2.8.1"
},
"jest": {
......@@ -43,9 +46,9 @@
"collectCoverage": true,
"collectCoverageFrom": ["src/*.js", "cli/*.js"],
"modulePaths": ["<rootDir>/node_modules"],
"testEnvironment": "node",
"testEnvironment": "jest-environment-db",
"unmockedModulePathPatterns": ["/src/models"],
"setupTestFrameworkScriptFile": "<rootDir>/test/jest-setup.js",
"setupTestFrameworkScriptFile": "<rootDir>/test/helpers/jest-setup.js",
"verbose": true
}
}
jest.mock('child_process', () => ({ spawnSync: jest.fn() }))
jest.mock('@pubsweet/db-manager', () => ({ addUser: jest.fn() }))
const { getMockArgv } = require('../helpers/')
const runAddUser = require('../../cli/adduser')
const addUserSpy = require('@pubsweet/db-manager').addUser
const user = {
username: 'anotheruser',
email: 'bar@example.com',
password: '12345',
admin: true,
}
describe('adduser', () => {
it('calls dbManager.addUser with correct arguments', async () => {
await runAddUser(getMockArgv({ options: user }))
expect(addUserSpy.mock.calls[0][0]).toEqual(user)
})
})
jest.mock('child_process', () => ({ spawnSync: jest.fn() }))
jest.mock('fs-extra', () => {
const fs = require.requireActual('fs-extra')
fs.removeSync = jest.fn(fs.removeSync)
return fs
})
const path = require('path')
const fs = require('fs-extra')
const { getMockArgv } = require('../helpers/')
const runNew = require('../../cli/new')
const spawnSpy = require('child_process').spawnSync
const removeSpy = fs.removeSync
const appName = 'testapp'
const appPath = path.join(process.cwd(), appName)
describe('new', () => {
it('spawns git and yarn child processes with correct arguments', async () => {
await runNew(getMockArgv({ args: appName }))
const { calls } = spawnSpy.mock
expect(calls).toHaveLength(2)
expect(calls[0][1][2]).toBe(appName)
expect(calls[1][2].cwd).toBe(appPath)
})
it('will not overwrite dir without clobber passed', async () => {
fs.ensureDirSync(path.join(appPath, 'block-write'))
await runNew(getMockArgv({ args: appName }))
const { calls } = removeSpy.mock
expect(calls).toHaveLength(0)
const notOverwritten = fs.existsSync(appPath)
expect(notOverwritten).toBeTruthy()
require.requireActual('fs-extra').removeSync(appPath)
})
it('will overwrite dir with clobber passed', async () => {
fs.ensureDirSync(path.join(appPath, 'block-write'))
await runNew(getMockArgv({ args: appName, options: { clobber: true } }))
const { calls } = removeSpy.mock
expect(calls[0][0]).toBe(appPath)
require.requireActual('fs-extra').removeSync(appPath)
})
})
jest.mock('child_process', () => ({ spawnSync: jest.fn() }))
jest.mock('@pubsweet/db-manager', () => ({ setupDb: jest.fn() }))
const { getMockArgv } = require('../helpers/')
const runSetupDb = require('../../cli/setupdb')
const setupDbSpy = require('@pubsweet/db-manager').setupDb
const dbOpts = {
username: 'anotheruser',
email: 'bar@example.com',
password: '12345',
}
describe('setupdb', () => {
it('calls dbManager.setupDb with correct arguments', async () => {
await runSetupDb(getMockArgv({ options: dbOpts }))
const expected = Object.assign({}, dbOpts, { clobber: false })
expect(setupDbSpy.mock.calls[0][0]).toEqual(expected)
})
})
jest.mock('webpack', () => {})
jest.mock(
require('path').resolve(
'webpack',
`webpack.${require('config').util.getEnv('NODE_ENV')}.config.js`,
),
() => {},
{ virtual: true },
)
jest.mock(require('path').resolve('config', 'components.json'), () => [], {
virtual: true,
})
jest.mock('forever-monitor', () => ({
start: jest.fn(() => ({ on: jest.fn() })),
}))
jest.mock('require-relative', () => required => {
if (required === 'webpack') {
const compiler = {
run: cb => cb(null, {}),
}
return () => compiler
}
return app => require('bluebird').resolve({ on: jest.fn(), app })
})
const { getMockArgv } = require('../helpers/')
const Promise = require('bluebird')
const config = require('config')
config['pubsweet-server'] = { dbPath: __dirname, adapter: 'leveldb' }
const runStart = require('../../cli/start')
const start = require('../../src/startup/start.js')
describe('start', () => {
let server
afterAll(async () => {
const closeServer = Promise.promisify(server.close, { context: server })
await closeServer()
})
it('throws an error if no database found', async () => {
await expect(runStart(getMockArgv(''))).rejects.toHaveProperty(
'message',
`Create database with "pubsweet setupdb" before starting app`,
)
})
it('calls startServer with an express app', async () => {
server = await start()
expect(server.app).toHaveProperty('mountpath', '/')
})
})
const { spawnSync } = require('child_process')
const { spawn } = require('child_process')
const { spawn, spawnSync } = require('child_process')
const path = require('path')
const reduce = require('lodash/fp/reduce').convert({ cap: false })
......
......@@ -8,16 +8,12 @@ const fetch = require('isomorphic-fetch')
const os = require('os')
const appName = `pubsweet-test-${Math.floor(Math.random() * 99999)}`
const dbName = 'test_db'
const tempDir = os.tmpdir()
const appPath = path.join(tempDir, appName)
const dbDir = path.join(appPath, 'api', 'db')
const dbPath = path.join(dbDir, dbName)
const nodeConfig = {
'pubsweet-server': {
dbPath,
adapter: 'leveldb',
db: { database: global.__testDbName },
},
// TODO: Remove this once version of server that handles
// undefined app validations is released.
......@@ -33,7 +29,7 @@ const nodeConfig = {
},
}
const dbOptions = {
const defaultUser = {
username: 'someuser',
email: 'user@test.com',
password: '12345678',
......@@ -43,13 +39,7 @@ const dbOptions = {
/* They perform a full installation cycle, including multiple yarn commands */
describe('CLI: integration test', () => {
beforeAll(() => {
fs.ensureDirSync(tempDir)
})
afterAll(() => {
fs.removeSync(appPath)
})
afterAll(() => fs.removeSync(appPath))
describe('new', () => {
it('will not overwrite non-empty dir', () => {
......@@ -57,7 +47,6 @@ describe('CLI: integration test', () => {
const { stderr } = runCommandSync({
args: `new ${appName}`,
cwd: tempDir,
stdio: 'pipe',
})
expect(stderr).toContain(
`destination path '${appName}' already exists and is not an empty directory`,
......@@ -65,9 +54,12 @@ describe('CLI: integration test', () => {
fs.removeSync(appPath)
})
it('runs git clone <appname> and yarn install', () => {
it('clones repo, installs dependencies and generates secret', () => {
runCommandSync({ args: `new ${appName}`, cwd: tempDir, stdio: 'inherit' })
expect(fs.existsSync(path.join(appPath, 'node_modules'))).toBe(true)
expect(fs.readdirSync(appPath)).toContain('node_modules')
expect(fs.readdirSync(path.join(appPath, 'config'))).toContain(
'local.json',
)
})
})
......@@ -108,12 +100,10 @@ describe('CLI: integration test', () => {
})
describe('setupdb', () => {
it('creates a new database', () => {
fs.ensureDirSync(dbDir)
it('creates tables', () => {
const { stdout, stderr } = runCommandSync({
args: 'setupdb',
options: dbOptions,
options: defaultUser,
stdio: 'pipe',
cwd: appPath,
nodeConfig,
......@@ -121,8 +111,6 @@ describe('CLI: integration test', () => {
console.log(stdout, stderr)
expect(stdout).toContain('Finished')
expect(fs.existsSync(path.join(dbPath, 'CURRENT'))).toBe(true)
fs.removeSync(dbDir)
})
})
......@@ -142,26 +130,16 @@ describe('CLI: integration test', () => {
})
})
describe('start', () => {
describe('server', () => {
it('starts an app', done => {
fs.ensureDirSync(dbDir)
runCommandSync({
args: 'setupdb',
options: dbOptions,
stdio: 'inherit',
cwd: appPath,
nodeConfig,
})
const app = runCommandAsync({
args: 'start',
args: 'server',
cwd: appPath,
stdio: 'pipe',
nodeConfig,
})
app.stderr.on('data', async data => {
app.stderr.on('data', data => {
console.log('stderr:', data.toString())
})
......
module.exports = {
'pubsweet-server': {
db: {
// temporary database name set by jest-environment-db
database: global.__testDbName || 'test',
},
ignoreTerminatedConnectionError: true,
secret: 'test',
dbPath: 'dummy value for validation',
adapter: 'memory',
logger: {
error: () => false,
warn: () => false,
......
......@@ -4,6 +4,7 @@
"description": "Provides database management utilities to Pubsweet apps.",
"main": "src/index.js",
"scripts": {
"test": "jest",
"vuln-test": "nsp check --output checkstyle"
},
"repository": {
......@@ -20,13 +21,14 @@
"collectCoverage": true,
"collectCoverageFrom": ["src/**/*.js"],
"setupTestFrameworkScriptFile": "<rootDir>/test/jest-setup.js",
"testEnvironment": "node",
"testEnvironment": "jest-environment-db",
"verbose": true
},
"author": "Samuel Galson",
"license": "MIT",
"devDependencies": {
"jest": "^22.1.4",
"jest-environment-db": "^0.1.0",
"nsp": "^2.8.0"
},
"dependencies": {
......@@ -34,7 +36,7 @@
"fs-extra": "^4.0.2",
"isomorphic-fetch": "^2.2.1",
"joi": "^13.1.0",
"pouchdb": "^6.3.4",
"pg": "^7.4.1",
"pubsweet-server": "^1.1.1"
}
}
const logger = require('@pubsweet/logger')
const Collection = require('pubsweet-server/src/models/Collection')
const User = require('pubsweet-server/src/models/User')
module.exports = async collectionData => {
logger.info('Creating collection')
const Collection = require('pubsweet-server/src/models/Collection')
const User = require('pubsweet-server/src/models/User')
const collection = new Collection(collectionData)
const collection = await new Collection(collectionData).save()
const [user] = await User.all()
if (user) collection.setOwners([user.id])
await collection.save()
......