diff --git a/.cz-config.js b/.cz-config.js index 8bb18634163bd5f21079697a82568b7036ee5017..8af9b961242d8f16994e766ee2cbbedc91f2fc46 100644 --- a/.cz-config.js +++ b/.cz-config.js @@ -1,6 +1,6 @@ const { commitizen } = require('@coko/lint') commitizen.skipQuestions = ['body', 'footer'] // do NOT skip 'breaking' -commitizen.scopes = ['server', 'middleware', 'models', 'db-manager', '*'] +commitizen.scopes = ['server', 'middleware', 'models', 'db manager', 'cli', '*'] module.exports = commitizen diff --git a/package.json b/package.json index 3eb4bee9fd3ef46e954bc61ef716be67200014fa..3eec276dbc2e6be5947bee67a1cbf3c0e51fecf8 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "bcrypt": "5.1.1", "body-parser": "^1.19.0", "command-exists": "^1.2.9", - "commander": "^2.20.0", + "commander": "^12.0.0", "config": "^3.3.2", "cookie-parser": "^1.4.5", "cors": "^2.8.5", diff --git a/src/__tests__/apiFileUpload.test.js b/src/__tests__/apiFileUpload.test.js index 7edcedd137eb64f9995aa8bbae5f262b543f1fe3..63bad139e20f950a4b1281f5c748aff5d8054ed6 100644 --- a/src/__tests__/apiFileUpload.test.js +++ b/src/__tests__/apiFileUpload.test.js @@ -1,6 +1,6 @@ const fs = require('fs') const path = require('path') -const migrate = require('../dbManager/migrate') +const { migrate } = require('../dbManager/migrate') const { User } = require('../models') const api = require('./helpers/api') diff --git a/src/cli/coko-server-migrate.js b/src/cli/coko-server-migrate.js deleted file mode 100644 index 8509d0b75005008f93bb56e82bda159f3cafca16..0000000000000000000000000000000000000000 --- a/src/cli/coko-server-migrate.js +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env node - -const program = require('commander') - -const migrate = require('../dbManager/migrate') -const logger = require('../logger') - -const commandArguments = process.argv -const options = program.parse(commandArguments) - -migrate(options) - .then(() => { - process.exit(0) - }) - .catch(e => { - logger.error(e) - process.exit(1) - }) diff --git a/src/cli/coko-server.js b/src/cli/coko-server.js index 7f5557e4d25303d0165fcecc26bb713d371074ad..666a4285590054f3871355ce6d859dd76cb84d6c 100755 --- a/src/cli/coko-server.js +++ b/src/cli/coko-server.js @@ -1,15 +1,103 @@ #!/usr/bin/env node -/* eslint-disable no-underscore-dangle */ +const { program } = require('commander') -const program = require('commander') const pkg = require('../../package.json') +const logger = require('../logger') +const { migrate, rollback, pending, executed } = require('../dbManager/migrate') + +const migrateCommand = program + .command('migrate') + .description('Run or roll back migrations') + .showHelpAfterError() + +migrateCommand + .command('up') + .option('-s, --step <number>', 'How many migrations to run') + .option( + '-l, --skip-last <number>', + 'Run all except for the last <number> migrations. If used, the --step option is discarded.', + ) + .description('Run migrations') + .alias('run') + .action(async options => { + try { + const optionsToPass = {} + + if (options.skipLast) { + optionsToPass.skipLast = parseInt(options.skipLast, 10) + } + + if (options.step) { + optionsToPass.step = parseInt(options.step, 10) + } + + await migrate(optionsToPass) + process.exit(0) + } catch (e) { + logger.error(e) + process.exit(1) + } + }) + +migrateCommand + .command('down') + .option('-s, --step <number>', 'How many migrations to roll back', 1) + .option( + '-l, --last-successful-run', + 'Roll back to the last time migrate completed successfully. If used, the --step option is discarded.', + ) + .description('Roll back migrations') + .alias('rollback') + .action(async options => { + const optionsToPass = {} + const lastSuccessfulRun = options.lastSuccessfulRun === true + const step = parseInt(options.step, 10) + + if (!lastSuccessfulRun) { + if (step > 1) optionsToPass.step = step + } else { + optionsToPass.lastSuccessfulRun = true + } + + try { + await rollback(optionsToPass) + process.exit(0) + } catch (e) { + logger.error(e) + process.exit(1) + } + }) + +migrateCommand + .command('pending') + .description('Display pending migrations') + .action(async () => { + try { + await pending() + process.exit(0) + } catch (e) { + logger.error(e) + process.exit(1) + } + }) + +migrateCommand + .command('executed') + .description('Display executed migrations') + .action(async () => { + try { + await executed() + process.exit(0) + } catch (e) { + logger.error(e) + process.exit(1) + } + }) program - .version(pkg.version) - .command('migrate', 'run pending database migrations') + .name('coko-server') + .version(pkg.version, '-v, --version') + .description("Coko server's cli tool") + .showHelpAfterError() .parse(process.argv) - -if (!program.commands.map(cmd => cmd._name).includes(program.args[0])) { - program.help() -} diff --git a/src/dbManager/__tests__/migrate.test.js b/src/dbManager/__tests__/migrate.test.js index 7067bc36bb8c2f0978730a2ed5954028cff368f4..d2c2d9623e96e9e8e23670c09b979bebe616148c 100644 --- a/src/dbManager/__tests__/migrate.test.js +++ b/src/dbManager/__tests__/migrate.test.js @@ -7,7 +7,7 @@ Object.assign(config, { }, }) -const migrate = require('../migrate') +const { migrate } = require('../migrate') describe('Migrate', () => { it('throws an error when a broken migration runs', async () => { diff --git a/src/dbManager/createTables.js b/src/dbManager/createTables.js index 03b180fb65b587e7be03e79945a2f404077ac0fe..e00fcb3129585250ad46eb0751835964b05ce08f 100644 --- a/src/dbManager/createTables.js +++ b/src/dbManager/createTables.js @@ -1,6 +1,6 @@ const logger = require('../logger') const db = require('./db') -const migrate = require('./migrate') +const { migrate } = require('./migrate') const createTables = async clobber => { const { rows } = await db.raw(` diff --git a/src/dbManager/migrate.js b/src/dbManager/migrate.js index 5d6e8b726b51e441e1ff69f477d2185699c2e851..a8beab1f0b0af15f3375ea2ff695b91e005921af 100644 --- a/src/dbManager/migrate.js +++ b/src/dbManager/migrate.js @@ -8,7 +8,17 @@ const isFunction = require('lodash/isFunction') const logger = require('../logger') const db = require('./db') +const { migrations, meta } = require('./migrateDbHelpers') +const MigrateOptionIntegrityError = require('../errors/migrate/MigrateOptionIntegrityError') +const MigrateSkipLimitError = require('../errors/migrate/MigrateSkipLimitError') +const MigrationResolverRulesError = require('../errors/migrate/MigrationResolverRulesError') +const RollbackLimitError = require('../errors/migrate/RollbackLimitError') +const RollbackUnavailableError = require('../errors/migrate/RollbackUnavailableError') + +const META_ID = '1715865523-create-coko-server-meta.js' + +// #region umzug const resolveRelative = m => require.resolve(m, { paths: [process.cwd()] }) const tryRequireRelative = componentPath => { @@ -69,22 +79,12 @@ const getGlobPattern = () => { } const customStorage = { - async logMigration(migration) { - await db.raw('INSERT INTO migrations (id) VALUES (?)', [migration.name]) - }, - - async unlogMigration(migration) { - await db.raw('DELETE FROM migrations WHERE id = ?', [migration.name]) - }, + logMigration: async migration => migrations.logMigration(migration.name), + unlogMigration: async migration => migrations.unlogMigration(migration.name), - async executed() { - await db.raw( - `CREATE TABLE IF NOT EXISTS migrations ( - id TEXT PRIMARY KEY, - run_at TIMESTAMPTZ DEFAULT current_timestamp - )`, - ) - const { rows } = await db.raw('SELECT id FROM migrations') + executed: async () => { + await migrations.createTable() + const rows = await migrations.getRows() return rows.map(row => row.id) }, } @@ -121,26 +121,18 @@ const customResolver = (params, threshold) => { const isSql = extname(filePath) === '.sql' const isPastThreshold = isMigrationAfterThreshold(name, threshold) - class MigrationResolverRulesError extends Error { - constructor(message) { - super( - `Starting with coko server v4: ${message}. This error occured in ${name}.`, - ) - this.name = 'MigrationResolverRulesError' - } - } - if (!doesMigrationFilenameStartWithUnixTimestamp(name)) { throw new MigrationResolverRulesError( `Migration files must start with a unix timestamp larger than 1000000000, followed by a dash (-)`, + name, ) } if (isPastThreshold) { if (isSql) { - // TO DO -- migration error? throw new MigrationResolverRulesError( `Migration files must be js files. Use knex.raw if you need to write sql code`, + name, ) } } @@ -162,6 +154,7 @@ const customResolver = (params, threshold) => { if (!migration.down || !isFunction(migration.down)) { throw new MigrationResolverRulesError( `All migrations need to define a down function so that the migration can be rolled back`, + name, ) } } @@ -181,7 +174,6 @@ const getUmzug = threshold => { glob: globPattern, resolve: params => customResolver(params, threshold), }, - context: { knex: db }, storage: customStorage, logger, }) @@ -198,13 +190,12 @@ const getUmzug = threshold => { return umzug } +// #endregion umzug -const getMetaCreated = async () => { - const tableExists = await db.schema.hasTable('coko_server_meta') - if (!tableExists) return null - - const { rows } = await db.raw(`SELECT created FROM coko_server_meta`) - const data = rows[0] // this table always has one row only +// #region helpers +const getMetaCreatedAsUnixTimestamp = async () => { + if (!(await meta.exists())) return null + const data = meta.getData() const createdDateAsUnixTimestamp = Math.floor( new Date(data.created).getTime() / 1000, @@ -213,21 +204,24 @@ const getMetaCreated = async () => { return createdDateAsUnixTimestamp } -const updateLastSuccessfulMigrateCheckpoint = async () => { - logger.info('Migrate: Updating last successful migration checkpoint') +const updateCheckpoint = async () => { + if (!(await meta.exists())) { + logger.info( + 'Migrate: Coko server meta table does not exist! Not updating last successful migrate checkpoint', + ) + return + } - const lastMigrationRow = await db('migrations') - .select('id') - .orderBy('runAt', 'desc') - .first() + logger.info('Migrate: Last successful migrate checkpoint: updating') - await db('coko_server_meta').update({ - lastSuccessfulMigrateCheckpoint: lastMigrationRow.id, - }) + const lastMigration = await migrations.getLastMigration() + await meta.setCheckpoint(lastMigration) - logger.info('Migrate: Last successful migration checkpoint updated') + logger.info('Migrate: Last successful migrate checkpoint: updated') } +// #endregion helpers +// #region commands /** * After installing v4, some rules will apply for migrations, but only for new * migrations, so that developers don't have to rewrite all existing migrations. @@ -237,13 +231,148 @@ const updateLastSuccessfulMigrateCheckpoint = async () => { * coko server v4). */ const migrate = async options => { - const threshold = await getMetaCreated() + const threshold = await getMetaCreatedAsUnixTimestamp() const umzug = getUmzug(threshold) - await umzug.up(options) + const { skipLast, ...otherOptions } = options + + if (skipLast || Number.isNaN(skipLast)) { + if (!Number.isInteger(skipLast) || skipLast <= 0) { + throw new MigrateOptionIntegrityError( + 'Skip value must be a positive integer.', + ) + } + + const pending = await umzug.pending() + + if (pending.length === 0) { + throw new MigrateSkipLimitError('There are no pending migrations.') + } + + if (skipLast === pending.length) { + throw new MigrateSkipLimitError( + 'Skip value equals number of pending migrations. There is nothing to migrate.', + ) + } + + if (skipLast > pending.length) { + throw new MigrateSkipLimitError( + 'Skip value exceeds number of pending migrations.', + pending.length - 1, + ) + } + + const runTo = pending[pending.length - 1 - skipLast].name + await umzug.up({ to: runTo }) + } else { + await umzug.up(otherOptions) + } + logger.info('Migrate: All migrations ran successfully!') + await updateCheckpoint() +} + +const rollback = async options => { + if (!(await meta.exists())) throw new RollbackUnavailableError() + + const migrationRows = await migrations.getRows() + const metaPosition = migrationRows.findIndex(item => item.id === META_ID) + const metaIsLast = metaPosition === migrationRows.length - 1 + + if (metaIsLast) { + throw new RollbackLimitError('No migrations have run after the upgrade.', { + metaLimit: true, + }) + } + + const downOptions = {} + const checkpoint = await meta.getCheckpoint() + + if (!options.lastSuccessfulRun) { + const maximum = migrationRows.length - 1 - metaPosition + const stepTooFar = (options.step || 1) > maximum + + if (stepTooFar) { + throw new RollbackLimitError( + `Maximum steps value for the current state of the migration table is ${maximum}.`, + { metaLimit: true }, + ) + } + + if (options.step && options.step > 1) downOptions.step = options.step + } else { + const checkpointPosition = migrationRows.findIndex( + item => item.id === checkpoint, + ) + + const checkpointTooFar = checkpointPosition <= metaPosition + + if (checkpointTooFar) { + throw new RollbackLimitError( + `Check that the checkpoint in the coko_server_meta table in your database is a migration that ran after ${META_ID}`, + { metaLimit: true }, + ) + } + + /** + * The 'to' option is inclusive, ie. it will revert all migrations, + * INCLUDING the one specified. We want to roll back up to, but not + * including the specified migration. So we find the one right after. + */ + if (migrationRows.length - 1 === checkpointPosition) { + throw new RollbackLimitError( + 'No migrations have completed successfully since the last checkpoint. There is nothing to revert.', + ) + } + + const revertTo = migrationRows[checkpointPosition + 1].id - await updateLastSuccessfulMigrateCheckpoint() + downOptions.to = revertTo + } + + // If we don't clear the checkpoint, we get a reference error, as the checkpoint + // is a foreign key to the migrations id column + await meta.clearCheckpoint() + + try { + const umzug = getUmzug() + await umzug.down(downOptions) + logger.info('Migrate: Migration rollback successful!') + } catch (e) { + logger.error(e) + + // Restore original cleared checkpoint + if (checkpoint) await meta.setCheckpoint(checkpoint) + + throw e + } + + await updateCheckpoint() +} + +const pending = async () => { + const umzug = getUmzug() + const pendingMigrations = await umzug.pending() + + if (pendingMigrations.length === 0) { + logger.info('Migrate: There are no pending migrations.') + } else { + logger.info(`Migrate: Pending migrations:`) + logger.info(pendingMigrations) + } +} + +const executed = async () => { + const umzug = getUmzug() + const executedMigrations = await umzug.executed() + + if (executedMigrations.length === 0) { + logger.info('Migrate: There are no executed migrations.') + } else { + logger.info(`Migrate: Executed migrations:`) + logger.info(executedMigrations) + } } +// #endregion commmands -module.exports = migrate +module.exports = { migrate, rollback, pending, executed } diff --git a/src/dbManager/migrateDbHelpers.js b/src/dbManager/migrateDbHelpers.js new file mode 100644 index 0000000000000000000000000000000000000000..cfb0c4681484e5d1ccac731f3d7b0bc54f9a04e3 --- /dev/null +++ b/src/dbManager/migrateDbHelpers.js @@ -0,0 +1,60 @@ +const db = require('./db') + +const MIGRATIONS_TABLE = 'migrations' +const META_TABLE = 'coko_server_meta' + +const migrations = { + createTable: async () => + db.raw(` + CREATE TABLE IF NOT EXISTS ${MIGRATIONS_TABLE} ( + id TEXT PRIMARY KEY, + run_at TIMESTAMPTZ DEFAULT current_timestamp + ) + `), + + getLastMigration: async () => { + const row = await db(MIGRATIONS_TABLE) + .select('id') + .orderBy('runAt', 'desc') + .first() + + return row.id + }, + + getRows: async () => db(MIGRATIONS_TABLE).orderBy('runAt', 'asc'), + + logMigration: async migrationName => + db.raw(`INSERT INTO ${MIGRATIONS_TABLE} (id) VALUES (?)`, [migrationName]), + + unlogMigration: async migrationName => + db.raw(`DELETE FROM ${MIGRATIONS_TABLE} WHERE id = ?`, [migrationName]), +} + +const meta = { + clearCheckpoint: async () => + db(META_TABLE).update({ + lastSuccessfulMigrateCheckpoint: null, + }), + + exists: async () => db.schema.hasTable(META_TABLE), + + getCheckpoint: async () => { + const row = await db(META_TABLE) + .select('lastSuccessfulMigrateCheckpoint') + .first() + + return row.lastSuccessfulMigrateCheckpoint + }, + + getData: async () => { + const rows = await db(META_TABLE) + return rows[0] // this table always has one row only + }, + + setCheckpoint: async value => + db(META_TABLE).update({ + lastSuccessfulMigrateCheckpoint: value, + }), +} + +module.exports = { migrations, meta } diff --git a/src/errors/migrate/MigrateOptionIntegrityError.js b/src/errors/migrate/MigrateOptionIntegrityError.js new file mode 100644 index 0000000000000000000000000000000000000000..b8e8fdc5936bd8fe51416402671d1b6ed4d9ab73 --- /dev/null +++ b/src/errors/migrate/MigrateOptionIntegrityError.js @@ -0,0 +1,10 @@ +class MigrateOptionIntegrityError extends Error { + constructor(message, max) { + super(message) + + this.message = message + this.name = 'MigrateOptionIntegrityError' + } +} + +module.exports = MigrateOptionIntegrityError diff --git a/src/errors/migrate/MigrateSkipLimitError.js b/src/errors/migrate/MigrateSkipLimitError.js new file mode 100644 index 0000000000000000000000000000000000000000..f24d9b4ea299a596ff57bc86095c92964e3798b7 --- /dev/null +++ b/src/errors/migrate/MigrateSkipLimitError.js @@ -0,0 +1,15 @@ +class MigrateSkipLimitError extends Error { + constructor(message, max) { + super(message) + + if (max) { + this.message = `${message} Maximum value for skip with current pending migrations is ${max}.` + } else { + this.message = message + } + + this.name = 'MigrateSkipLimitError' + } +} + +module.exports = MigrateSkipLimitError diff --git a/src/errors/migrate/MigrationResolverRulesError.js b/src/errors/migrate/MigrationResolverRulesError.js new file mode 100644 index 0000000000000000000000000000000000000000..8216d1c1d45e53b19c0af82d0fdf6475da13349f --- /dev/null +++ b/src/errors/migrate/MigrationResolverRulesError.js @@ -0,0 +1,10 @@ +class MigrationResolverRulesError extends Error { + constructor(message, name) { + super(message) + + this.message = `Starting with coko server v4: ${message}. This error occured in ${name}.` + this.name = 'MigrationResolverRulesError' + } +} + +module.exports = MigrationResolverRulesError diff --git a/src/errors/migrate/RollbackLimitError.js b/src/errors/migrate/RollbackLimitError.js new file mode 100644 index 0000000000000000000000000000000000000000..15b233142f27c50634afd6938beb7dc94ac3d977 --- /dev/null +++ b/src/errors/migrate/RollbackLimitError.js @@ -0,0 +1,20 @@ +const ROLLBACK_LIMIT_MESSAGE = + 'Rollbacks can only go as far as the point where the coko server v4 upgrade occurred.' + +class RollbackLimitError extends Error { + constructor(message, options = {}) { + super(message) + + const { metaLimit } = options + + if (metaLimit) { + this.message = `${ROLLBACK_LIMIT_MESSAGE} ${message}` + } else { + this.message = message + } + + this.name = 'RollbackLimitError' + } +} + +module.exports = RollbackLimitError diff --git a/src/errors/migrate/RollbackUnavailableError.js b/src/errors/migrate/RollbackUnavailableError.js new file mode 100644 index 0000000000000000000000000000000000000000..5ef53c1ebe07bc96e6f5d84327e396d3e833bc0d --- /dev/null +++ b/src/errors/migrate/RollbackUnavailableError.js @@ -0,0 +1,10 @@ +class RollbackUnavailableError extends Error { + constructor(message) { + super(message) + + this.message = `'Coko server meta table does not exist! Rollbacks only work starting coko server v4, which creates that table.'` + this.name = 'RollbackUnavailableError' + } +} + +module.exports = RollbackUnavailableError diff --git a/src/index.js b/src/index.js index 5662d81b9c0eb2945bbdc85235117733622f5893..3e75070b5da460d2b5b07f75fb76913753553930 100644 --- a/src/index.js +++ b/src/index.js @@ -5,7 +5,7 @@ const { send: sendEmail } = require('./services/sendEmail') const logger = require('./logger') const db = require('./dbManager/db') -const migrate = require('./dbManager/migrate') +const { migrate } = require('./dbManager/migrate') const createTables = require('./dbManager/createTables') const pubsubManager = require('./graphql/pubsub') const authentication = require('./authentication') diff --git a/src/models/__tests__/_setup.js b/src/models/__tests__/_setup.js index 9bb53399ecc96905241a2f8e898d46fb1125dc0e..99a2675efa8a69aad7bcde7c8dd41ecb81119d97 100644 --- a/src/models/__tests__/_setup.js +++ b/src/models/__tests__/_setup.js @@ -1,4 +1,4 @@ -const migrate = require('../../dbManager/migrate') +const { migrate } = require('../../dbManager/migrate') // Ideally, instead of running a single worker, we should be spinning up // one db per worker, so that the tests run in parallel without interfering diff --git a/src/startServer.js b/src/startServer.js index 39eb2d5f0cff806b5c9c8f6e455a7935bc7c80e0..43d60eccfc20249082601e0bf0310038b9fe7317 100644 --- a/src/startServer.js +++ b/src/startServer.js @@ -7,7 +7,7 @@ const path = require('path') const isFunction = require('lodash/isFunction') const logger = require('./logger') -const migrate = require('./dbManager/migrate') +const { migrate } = require('./dbManager/migrate') const seedGlobalTeams = require('./startup/seedGlobalTeams') let server diff --git a/yarn.lock b/yarn.lock index 0e4c62468d360858159d63dcbb44445a107f99bb..c22c770870765ce17943316ebaca60cddfbb1649 100644 --- a/yarn.lock +++ b/yarn.lock @@ -655,7 +655,7 @@ __metadata: bcrypt: "npm:5.1.1" body-parser: "npm:^1.19.0" command-exists: "npm:^1.2.9" - commander: "npm:^2.20.0" + commander: "npm:^12.0.0" config: "npm:^3.3.2" cookie-parser: "npm:^1.4.5" cors: "npm:^2.8.5" @@ -3755,7 +3755,14 @@ __metadata: languageName: node linkType: hard -"commander@npm:^2.20.0, commander@npm:^2.20.3": +"commander@npm:^12.0.0": + version: 12.0.0 + resolution: "commander@npm:12.0.0" + checksum: 10c0/e51cac1d1d0aa1f76581981d2256a9249497e08f5a370bf63b0dfc7e76a647fc8cbc3ddd507928f2bdca6c514c83834e87e2687ace2fe2fc7cc7e631bf80f83d + languageName: node + linkType: hard + +"commander@npm:^2.20.3": version: 2.20.3 resolution: "commander@npm:2.20.3" checksum: 10c0/74c781a5248c2402a0a3e966a0a2bba3c054aad144f5c023364be83265e796b20565aa9feff624132ff629aa64e16999fa40a743c10c12f7c61e96a794b99288