diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34673fe5d8940544b6e614c0767701375fe5f5e1..95eb8708f163e3986bff41a46aec700312bb3a45 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -84,7 +84,7 @@ deploy: - apk --no-cache add --update python python-dev py-pip - pip install ecs-deploy # Deploy - - ecs deploy ${CI_CLUSTER_NAME} ${CI_SERVICE_NAME} --region ${AWS_REGION} --timeout 600 -e ${CI_CONTAINER_NAME} AWS_S3_ACCESS_KEY $AWS_S3_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_S3_SECRET_KEY $AWS_S3_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_S3_REGION $AWS_S3_REGION -e ${CI_CONTAINER_NAME} AWS_S3_BUCKET $AWS_S3_BUCKET -e ${CI_CONTAINER_NAME} AWS_SES_SECRET_KEY $AWS_SES_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_SES_ACCESS_KEY $AWS_SES_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_SES_REGION $AWS_SES_REGION -e ${CI_CONTAINER_NAME} EMAIL_SENDER $EMAIL_SENDER -e ${CI_CONTAINER_NAME} secret $SECRET -e ${CI_CONTAINER_NAME} DATABASE $DATABASE -e ${CI_CONTAINER_NAME} DB_USER $DB_USER -e ${CI_CONTAINER_NAME} DB_PASS $DB_PASS -e ${CI_CONTAINER_NAME} DB_HOST $DB_HOST -e ${CI_CONTAINER_NAME} ORCID_CLIENT_ID $ORCID_CLIENT_ID -e ${CI_CONTAINER_NAME} ORCID_CLIENT_SECRET $ORCID_CLIENT_SECRET -e ${CI_CONTAINER_NAME} CLIENT_BASE_URL $CLIENT_BASE_URL + - ecs deploy ${CI_CLUSTER_NAME} ${CI_SERVICE_NAME} --region ${AWS_REGION} --timeout 600 -e ${CI_CONTAINER_NAME} AWS_S3_ACCESS_KEY $AWS_S3_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_S3_SECRET_KEY $AWS_S3_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_S3_REGION $AWS_S3_REGION -e ${CI_CONTAINER_NAME} AWS_S3_BUCKET $AWS_S3_BUCKET -e ${CI_CONTAINER_NAME} AWS_SES_SECRET_KEY $AWS_SES_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_SES_ACCESS_KEY $AWS_SES_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_SES_REGION $AWS_SES_REGION -e ${CI_CONTAINER_NAME} EMAIL_SENDER $EMAIL_SENDER -e ${CI_CONTAINER_NAME} secret $SECRET -e ${CI_CONTAINER_NAME} DATABASE $DATABASE -e ${CI_CONTAINER_NAME} DB_USER $DB_USER -e ${CI_CONTAINER_NAME} DB_PASS $DB_PASS -e ${CI_CONTAINER_NAME} DB_HOST $DB_HOST -e ${CI_CONTAINER_NAME} ORCID_CLIENT_ID $ORCID_CLIENT_ID -e ${CI_CONTAINER_NAME} ORCID_CLIENT_SECRET $ORCID_CLIENT_SECRET -e ${CI_CONTAINER_NAME} CLIENT_BASE_URL $CLIENT_BASE_URL -e ${CI_CONTAINER_NAME} FTP_USERNAME $FTP_USERNAME -e ${CI_CONTAINER_NAME} FTP_PASSWORD $FTP_PASSWORD -e ${CI_CONTAINER_NAME} FTP_HOST $FTP_HOST environment: name: qa url: $CI_ALB_URL @@ -101,25 +101,11 @@ aws-qa: - apk --no-cache add --update python python-dev py-pip - pip install ecs-deploy # Deploy - - ecs deploy ${CI_CLUSTER_NAME} ${CI_SERVICE_NAME} --region ${AWS_REGION} --timeout 600 -e ${CI_CONTAINER_NAME} AWS_S3_ACCESS_KEY $AWS_S3_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_S3_SECRET_KEY $AWS_S3_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_S3_REGION $AWS_S3_REGION -e ${CI_CONTAINER_NAME} AWS_S3_BUCKET $AWS_S3_BUCKET -e ${CI_CONTAINER_NAME} AWS_SES_SECRET_KEY $AWS_SES_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_SES_ACCESS_KEY $AWS_SES_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_SES_REGION $AWS_SES_REGION -e ${CI_CONTAINER_NAME} EMAIL_SENDER $EMAIL_SENDER -e ${CI_CONTAINER_NAME} secret $SECRET -e ${CI_CONTAINER_NAME} DATABASE $DATABASE -e ${CI_CONTAINER_NAME} DB_USER $DB_USER -e ${CI_CONTAINER_NAME} DB_PASS $DB_PASS -e ${CI_CONTAINER_NAME} DB_HOST $DB_HOST -e ${CI_CONTAINER_NAME} ORCID_CLIENT_ID $ORCID_CLIENT_ID -e ${CI_CONTAINER_NAME} ORCID_CLIENT_SECRET $ORCID_CLIENT_SECRET -e ${CI_CONTAINER_NAME} CLIENT_BASE_URL $CLIENT_BASE_URL + - ecs deploy ${CI_CLUSTER_NAME} ${CI_SERVICE_NAME} --region ${AWS_REGION} --timeout 600 -e ${CI_CONTAINER_NAME} AWS_S3_ACCESS_KEY $AWS_S3_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_S3_SECRET_KEY $AWS_S3_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_S3_REGION $AWS_S3_REGION -e ${CI_CONTAINER_NAME} AWS_S3_BUCKET $AWS_S3_BUCKET -e ${CI_CONTAINER_NAME} AWS_SES_SECRET_KEY $AWS_SES_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_SES_ACCESS_KEY $AWS_SES_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_SES_REGION $AWS_SES_REGION -e ${CI_CONTAINER_NAME} EMAIL_SENDER $EMAIL_SENDER -e ${CI_CONTAINER_NAME} secret $SECRET -e ${CI_CONTAINER_NAME} DATABASE $DATABASE -e ${CI_CONTAINER_NAME} DB_USER $DB_USER -e ${CI_CONTAINER_NAME} DB_PASS $DB_PASS -e ${CI_CONTAINER_NAME} DB_HOST $DB_HOST -e ${CI_CONTAINER_NAME} ORCID_CLIENT_ID $ORCID_CLIENT_ID -e ${CI_CONTAINER_NAME} ORCID_CLIENT_SECRET $ORCID_CLIENT_SECRET -e ${CI_CONTAINER_NAME} CLIENT_BASE_URL $CLIENT_BASE_URL -e ${CI_CONTAINER_NAME} FTP_USERNAME $FTP_USERNAME -e ${CI_CONTAINER_NAME} FTP_PASSWORD $FTP_PASSWORD -e ${CI_CONTAINER_NAME} FTP_HOST $FTP_HOST environment: name: qa url: $CI_ALB_URL -now.sh: - image: $CI_ECR_URL:latest - stage: deploy - when: manual - script: - - npm i -g --unsafe-perm now - - cd ${HOME}/now - - echo "FROM ${CI_ECR_URL}:latest" >> Dockerfile - - now --public --docker --token $NOW_TOKEN -e AWS_S3_ACCESS_KEY=$AWS_S3_ACCESS_KEY -e AWS_S3_SECRET_KEY=$AWS_S3_SECRET_KEY -e AWS_S3_REGION=$AWS_S3_REGION -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_SES_SECRET_KEY=$AWS_SES_SECRET_KEY -e AWS_SES_ACCESS_KEY=$AWS_SES_ACCESS_KEY -e AWS_SES_REGION=$AWS_SES_REGION -e EMAIL_SENDER=$EMAIL_SENDER -e secret=$SECRET -e DATABASE=$DATABASE -e DB_USER=$DB_USER -e DB_PASS=$DB_PASS -e DB_HOST=$DB_HOST - - now alias $NOW_URL xpub-faraday-qa --token $NOW_TOKEN - environment: - name: qa - url: https://xpub-faraday-qa.now.sh/ - rollback: stage: rollback when: on_failure @@ -136,7 +122,7 @@ rollback: - aws s3 cp s3://${CI_REV_BUCKET}/${CI_SERVICE_NAME} ./ - REV=`cat ./${CI_SERVICE_NAME}` - echo rev is $REV - - ecs deploy --region ${AWS_REGION} ${CLUSTER_NAME} ${CI_SERVICE_NAME} --task ${REV} -e ${CI_CONTAINER_NAME} AWS_S3_ACCESS_KEY $AWS_S3_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_S3_SECRET_KEY $AWS_S3_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_S3_REGION $AWS_S3_REGION -e ${CI_CONTAINER_NAME} AWS_S3_BUCKET $AWS_S3_BUCKET -e ${CI_CONTAINER_NAME} AWS_SES_SECRET_KEY $AWS_SES_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_SES_ACCESS_KEY $AWS_SES_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_SES_REGION $AWS_SES_REGION -e ${CI_CONTAINER_NAME} EMAIL_SENDER $EMAIL_SENDER -e ${CI_CONTAINER_NAME} secret $SECRET -e ${CI_CONTAINER_NAME} DATABASE $DATABASE -e ${CI_CONTAINER_NAME} DB_USER $DB_USER -e ${CI_CONTAINER_NAME} DB_PASS $DB_PASS -e ${CI_CONTAINER_NAME} DB_HOST $DB_HOST -e ${CI_CONTAINER_NAME} ORCID_CLIENT_ID $ORCID_CLIENT_ID -e ${CI_CONTAINER_NAME} ORCID_CLIENT_SECRET $ORCID_CLIENT_SECRET -e ${CI_CONTAINER_NAME} CLIENT_BASE_URL $CLIENT_BASE_URL + - ecs deploy --region ${AWS_REGION} ${CLUSTER_NAME} ${CI_SERVICE_NAME} --task ${REV} -e ${CI_CONTAINER_NAME} AWS_S3_ACCESS_KEY $AWS_S3_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_S3_SECRET_KEY $AWS_S3_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_S3_REGION $AWS_S3_REGION -e ${CI_CONTAINER_NAME} AWS_S3_BUCKET $AWS_S3_BUCKET -e ${CI_CONTAINER_NAME} AWS_SES_SECRET_KEY $AWS_SES_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_SES_ACCESS_KEY $AWS_SES_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_SES_REGION $AWS_SES_REGION -e ${CI_CONTAINER_NAME} EMAIL_SENDER $EMAIL_SENDER -e ${CI_CONTAINER_NAME} secret $SECRET -e ${CI_CONTAINER_NAME} DATABASE $DATABASE -e ${CI_CONTAINER_NAME} DB_USER $DB_USER -e ${CI_CONTAINER_NAME} DB_PASS $DB_PASS -e ${CI_CONTAINER_NAME} DB_HOST $DB_HOST -e ${CI_CONTAINER_NAME} ORCID_CLIENT_ID $ORCID_CLIENT_ID -e ${CI_CONTAINER_NAME} ORCID_CLIENT_SECRET $ORCID_CLIENT_SECRET -e ${CI_CONTAINER_NAME} CLIENT_BASE_URL $CLIENT_BASE_URL -e ${CI_CONTAINER_NAME} FTP_USERNAME $FTP_USERNAME -e ${CI_CONTAINER_NAME} FTP_PASSWORD $FTP_PASSWORD -e ${CI_CONTAINER_NAME} FTP_HOST $FTP_HOST environment: name: qa url: $CI_ALB_URL @@ -157,7 +143,7 @@ rollback-qa: - aws s3 cp s3://${CI_REV_BUCKET}/${CI_SERVICE_NAME} ./ - REV=`cat ./${CI_SERVICE_NAME}` - echo rev is $REV - - ecs deploy --region ${AWS_REGION} ${CLUSTER_NAME} ${CI_SERVICE_NAME} --task ${REV} -e ${CI_CONTAINER_NAME} AWS_S3_ACCESS_KEY $AWS_S3_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_S3_SECRET_KEY $AWS_S3_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_S3_REGION $AWS_S3_REGION -e ${CI_CONTAINER_NAME} AWS_S3_BUCKET $AWS_S3_BUCKET -e ${CI_CONTAINER_NAME} AWS_SES_SECRET_KEY $AWS_SES_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_SES_ACCESS_KEY $AWS_SES_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_SES_REGION $AWS_SES_REGION -e ${CI_CONTAINER_NAME} EMAIL_SENDER $EMAIL_SENDER -e ${CI_CONTAINER_NAME} secret $SECRET -e ${CI_CONTAINER_NAME} DATABASE $DATABASE -e ${CI_CONTAINER_NAME} DB_USER $DB_USER -e ${CI_CONTAINER_NAME} DB_PASS $DB_PASS -e ${CI_CONTAINER_NAME} DB_HOST $DB_HOST -e ${CI_CONTAINER_NAME} ORCID_CLIENT_ID $ORCID_CLIENT_ID -e ${CI_CONTAINER_NAME} ORCID_CLIENT_SECRET $ORCID_CLIENT_SECRET -e ${CI_CONTAINER_NAME} CLIENT_BASE_URL $CLIENT_BASE_URL + - ecs deploy --region ${AWS_REGION} ${CLUSTER_NAME} ${CI_SERVICE_NAME} --task ${REV} -e ${CI_CONTAINER_NAME} AWS_S3_ACCESS_KEY $AWS_S3_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_S3_SECRET_KEY $AWS_S3_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_S3_REGION $AWS_S3_REGION -e ${CI_CONTAINER_NAME} AWS_S3_BUCKET $AWS_S3_BUCKET -e ${CI_CONTAINER_NAME} AWS_SES_SECRET_KEY $AWS_SES_SECRET_KEY -e ${CI_CONTAINER_NAME} AWS_SES_ACCESS_KEY $AWS_SES_ACCESS_KEY -e ${CI_CONTAINER_NAME} AWS_SES_REGION $AWS_SES_REGION -e ${CI_CONTAINER_NAME} EMAIL_SENDER $EMAIL_SENDER -e ${CI_CONTAINER_NAME} secret $SECRET -e ${CI_CONTAINER_NAME} DATABASE $DATABASE -e ${CI_CONTAINER_NAME} DB_USER $DB_USER -e ${CI_CONTAINER_NAME} DB_PASS $DB_PASS -e ${CI_CONTAINER_NAME} DB_HOST $DB_HOST -e ${CI_CONTAINER_NAME} ORCID_CLIENT_ID $ORCID_CLIENT_ID -e ${CI_CONTAINER_NAME} ORCID_CLIENT_SECRET $ORCID_CLIENT_SECRET -e ${CI_CONTAINER_NAME} CLIENT_BASE_URL $CLIENT_BASE_URL -e ${CI_CONTAINER_NAME} FTP_USERNAME $FTP_USERNAME -e ${CI_CONTAINER_NAME} FTP_PASSWORD $FTP_PASSWORD -e ${CI_CONTAINER_NAME} FTP_HOST $FTP_HOST environment: name: qa url: $CI_ALB_URL \ No newline at end of file diff --git a/packages/component-manuscript-manager/src/routes/fragments/post.js b/packages/component-manuscript-manager/src/routes/fragments/post.js index b54399b0cc006f469863bbfabaaecb0c8bab3150..97ad2389943da68a469015f21cf483f465cdd1ff 100644 --- a/packages/component-manuscript-manager/src/routes/fragments/post.js +++ b/packages/component-manuscript-manager/src/routes/fragments/post.js @@ -1,3 +1,5 @@ +const config = require('config') +const { get } = require('lodash') const { Email, Fragment, @@ -5,6 +7,10 @@ const { authsome: authsomeHelper, } = require('pubsweet-component-helper-service') +const s3Config = get(config, 'pubsweet-component-aws-s3', {}) +const mtsConfig = get(config, 'mts-service', {}) +const MTSService = require('pubsweet-component-mts-package') + module.exports = models => async (req, res) => { const { collectionId, fragmentId } = req.params let collection, fragment @@ -51,6 +57,18 @@ module.exports = models => async (req, res) => { collection.status = 'submitted' collection.save() + const { journal, xmlParser, ftp } = mtsConfig + const MTS = new MTSService(journal, xmlParser, s3Config, ftp) + const packageFragment = { + ...fragment, + metadata: { + ...fragment.metadata, + customId: collection.customId, + }, + } + + await MTS.sendPackage(packageFragment) + return res.status(200).json(fragment) } catch (e) { const notFoundError = await services.handleNotFoundError(e, 'Item') diff --git a/packages/component-manuscript-manager/src/tests/fragments/post.test.js b/packages/component-manuscript-manager/src/tests/fragments/post.test.js index 07b6ff7a80ec6024f7e9126551c001eeeb32fad9..18fbd3612a51604c0dcec8d45f172eba0f197819 100644 --- a/packages/component-manuscript-manager/src/tests/fragments/post.test.js +++ b/packages/component-manuscript-manager/src/tests/fragments/post.test.js @@ -10,6 +10,7 @@ jest.mock('pubsweet-component-mail-service', () => ({ sendNotificationEmail: jest.fn(), sendSimpleEmail: jest.fn(), })) +jest.mock('pubsweet-component-mts-package') const reqBody = {} const path = '../routes/fragments/post' diff --git a/packages/component-mts-package/config/default.js b/packages/component-mts-package/config/default.js index 64fad090508eac085fabc26f4ca3cf026a028ecf..bf1f239eb09ce13400de74acb78501622867fb33 100644 --- a/packages/component-mts-package/config/default.js +++ b/packages/component-mts-package/config/default.js @@ -16,7 +16,26 @@ const defaultConfig = { prefix: 'RESEARCH-F-', } +const defaultS3Config = { + secretAccessKey: process.env.AWS_S3_SECRET_KEY, + accessKeyId: process.env.AWS_S3_ACCESS_KEY, + region: process.env.AWS_S3_REGION, + bucket: process.env.AWS_S3_BUCKET, +} + +const defaultFTPConfig = { + user: 'dlpuser@dlptest.com', + password: '3D6XZV9MKdhM5fF', + host: 'ftp.dlptest.com', + port: 21, + localRoot: `../files`, + remoteRoot: '/', + exclude: ['*.js'], +} + module.exports = { defaultConfig, defaultParseXmlOptions, + defaultS3Config, + defaultFTPConfig, } diff --git a/packages/component-mts-package/package.json b/packages/component-mts-package/package.json index 2fe71737554a827b88fa8fde15d58d2313632cad..20b9f9c86d65c196ad5da70e266fa8bbf45b9e28 100644 --- a/packages/component-mts-package/package.json +++ b/packages/component-mts-package/package.json @@ -33,6 +33,9 @@ "access": "public" }, "dependencies": { + "archiver": "^2.1.1", + "aws-sdk": "^2.185.0", + "ftp-deploy": "^2.1.2", "lodash": "^4.17.10", "xml-js": "^1.6.7" } diff --git a/packages/component-mts-package/src/MTS.js b/packages/component-mts-package/src/MTS.js index 7449255e045e9a9fba0da7ce92f1ac154b2f1888..b7504983db641427edc3b6566fb93a5c13781ed3 100644 --- a/packages/component-mts-package/src/MTS.js +++ b/packages/component-mts-package/src/MTS.js @@ -1,22 +1,40 @@ -const fs = require('fs') const convert = require('xml-js') -const { set, get } = require('lodash') -const logger = require('@pubsweet/logger') +const { set, get, reduce } = require('lodash') +const PackageManager = require('./PackageManager') const mts = require('./mts-json-template') -const { defaultConfig, defaultParseXmlOptions } = require('../config/default') +const { + defaultConfig, + defaultParseXmlOptions, + defaultS3Config, + defaultFTPConfig, +} = require('../config/default') class MTS { - constructor(config = defaultConfig, options = defaultParseXmlOptions) { + constructor( + config = defaultConfig, + options = defaultParseXmlOptions, + s3Config = defaultS3Config, + ftpConfig = defaultFTPConfig, + ) { this.config = config this.options = options this.jsonTemplate = mts.getJsonTemplate(config) + this.s3Config = s3Config + this.ftpConfig = ftpConfig } createFileName(id = Date.now()) { return `${this.config.prefix}${id}` } + parseHtml(content = '') { + if (/<\/?[^>]*>/.test(content)) { + return convert.xml2js(content, this.options) + } + return content + } + convertToXML(json = {}) { const content = convert.json2xml(json, this.options) const customId = get( @@ -31,19 +49,10 @@ class MTS { } } - async createXMLFile(json = {}, defaultPath = '../files/') { - const { content, name } = this.convertToXML(json) - await fs.writeFile(`${defaultPath}${name}`, content, err => { - if (err) return logger.error(err) - logger.info('Created XML file') - }) - } - setMetadata(metadata, jsonTemplate) { const fileName = this.createFileName(metadata.customId) const titleGroup = { - 'article-title': - convert.xml2js(metadata.title, this.options) || 'Untitled', + 'article-title': this.parseHtml(metadata.title), } const articleId = [ { @@ -79,11 +88,7 @@ class MTS { 'article.front.article-meta.article-categories', articleType, ) - set( - jsonTemplate, - 'article.front.article-meta.abstract', - convert.xml2js(metadata.abstract, this.options) || '', - ) + set(jsonTemplate, 'article.front.article-meta.abstract', metadata.abstract) return jsonTemplate } @@ -96,10 +101,10 @@ class MTS { 'date-type': 'received', }, day: { - _text: date.getDay(), + _text: date.getDate(), }, month: { - _text: date.getMonth(), + _text: date.getMonth() + 1, }, year: { _text: date.getFullYear(), @@ -111,20 +116,29 @@ class MTS { return jsonTemplate } - static setFigures({ supplementary = [] }, jsonTemplate) { - const figs = supplementary.map((f, i) => ({ - label: { - _text: `Figure ${i + 1}`, - }, - graphic: { - _attributes: { - 'xlink:href': f.name, - 'xmlns:xlink': 'http://www.w3.org/1999/xlink', - }, + static setFiles(files, jsonTemplate) { + const jsonFiles = reduce( + files, + (result, value, key) => { + value.map(v => + result.push({ + item_type: { + _text: key, + }, + item_description: { + _text: key, + }, + item_name: { + _text: v.name, + }, + }), + ) + + return result }, - })) - - set(jsonTemplate, 'article.body.fig', figs) + [], + ) + set(jsonTemplate, 'article.front.files.file', jsonFiles) return jsonTemplate } @@ -165,8 +179,9 @@ class MTS { _attributes: { id: `aff${i + 1}`, }, - country: { - _text: a.affiliation, + country: {}, + 'addr-line': { + _text: a.affiliation || '', }, })) @@ -183,6 +198,7 @@ class MTS { composeJson(fragment = {}) { const { authors = [], + files = [], metadata = { title: 'untitled', abstract: '' }, submitted = new Date(), } = fragment @@ -192,17 +208,30 @@ class MTS { ...this.setMetadata(metadata, this.jsonTemplate), ...this.constructor.setContributors(authors, this.jsonTemplate), ...this.constructor.setHistory(submitted, this.jsonTemplate), - // ...this.constructor.setFigures(files, this.jsonTemplate), + ...this.constructor.setFiles(files, this.jsonTemplate), } } - createXMLFileFromFragment(fragment = {}, defaultPath) { - return this.createXMLFile(this.composeJson(fragment), defaultPath) - } - convertFragmentToXML(fragment = {}) { return this.convertToXML(this.composeJson(fragment)) } + + sendPackage(fragment = {}) { + const xmlFile = this.convertFragmentToXML(fragment) + + return PackageManager.createFilesPackage(this.s3Config)({ + fragment, + xmlFile, + }).then(() => { + const manuscriptName = get(xmlFile, 'name', '').replace('.xml', '') + const filename = `${manuscriptName}.zip` + return PackageManager.uploadFiles({ + filename, + s3Config: this.s3Config, + config: this.ftpConfig, + }) + }) + } } module.exports = MTS diff --git a/packages/component-mts-package/src/PackageManager.js b/packages/component-mts-package/src/PackageManager.js new file mode 100644 index 0000000000000000000000000000000000000000..43d9386c5037e78176ee3a41116fdaf071dfe366 --- /dev/null +++ b/packages/component-mts-package/src/PackageManager.js @@ -0,0 +1,140 @@ +const fs = require('fs') +const AWS = require('aws-sdk') +const { get } = require('lodash') +const { promisify } = require('util') +const FtpDeploy = require('ftp-deploy') +const nodeArchiver = require('archiver') +const logger = require('@pubsweet/logger') + +const filterByType = (fileTypes = []) => ({ Metadata: { filetype } }) => + fileTypes.length > 0 ? fileTypes.includes(filetype) : true + +const createFilesPackage = (s3Config, archiver = nodeArchiver) => { + AWS.config.update({ + secretAccessKey: s3Config.secretAccessKey, + accessKeyId: s3Config.accessKeyId, + region: s3Config.region, + }) + const s3 = new AWS.S3() + const asyncGetObject = promisify(s3.getObject.bind(s3)) + const asyncListObjects = promisify(s3.listObjects.bind(s3)) + + return async ({ fragment, fileTypes, xmlFile }) => { + const { id } = fragment + const manuscriptName = get(xmlFile, 'name', '').replace('.xml', '') + try { + const params = { + Bucket: s3Config.bucket, + Prefix: `${id}`, + } + const s3Items = await asyncListObjects(params) + if (s3Items) { + const s3Files = await Promise.all( + s3Items.Contents.map(content => + asyncGetObject({ + Bucket: s3Config.bucket, + Key: content.Key, + }), + ), + ) + + if (s3Files) { + const packageOutput = fs.createWriteStream(`${manuscriptName}.zip`) + const archive = archiver('zip') + + archive.pipe(packageOutput) + archive.append(xmlFile.content, { name: xmlFile.name }) + + s3Files.filter(filterByType(fileTypes)).forEach(f => { + const name = get(f, 'Metadata.filename', f.ETag) + archive.append(f.Body, { + name, + }) + }) + + archive.on('error', err => { + throw err + }) + archive.on('end', err => { + if (err) throw err + }) + archive.finalize() + } + } else { + logger.error('Failed to create package') + throw new Error('Failed to create a package') + } + } catch (err) { + logger.error(err.message) + return err.message + } + } +} + +const readFile = filename => + new Promise((resolve, reject) => { + fs.readFile(filename, (err, data) => { + if (err) reject(err) + resolve(data) + }) + }) + +const fileError = filename => err => { + logger.error(err) + deleteFile(filename) + throw err +} + +const uploadFiles = async ({ filename, s3Config, config }) => { + const data = await readFile(filename) + AWS.config.update({ + secretAccessKey: s3Config.secretAccessKey, + accessKeyId: s3Config.accessKeyId, + region: s3Config.region, + }) + const s3 = new AWS.S3() + + const params = { + Bucket: s3Config.bucket, + Body: data, + Key: `mts/${filename}`, + } + + const asyncUploadS3 = promisify(s3.upload.bind(s3)) + + return asyncUploadS3(params) + .then(() => { + logger.info(`Successfully uploaded ${filename} to S3`) + return uploadFTP({ filename, config }) + }) + .then(() => { + logger.info(`Successfully uploaded ${filename} to FTP`) + deleteFile(filename) + }) + .catch(fileError(filename)) +} + +const deleteFile = filename => { + fs.access(filename, fs.constants.F_OK, err => { + if (!err) { + fs.unlink(filename, err => { + if (err) throw err + logger.info(`Deleted ${filename}`) + }) + } + }) +} + +const uploadFTP = ({ filename, config }) => { + const ftpDeploy = new FtpDeploy() + const configs = { + ...config, + include: [filename], + } + return ftpDeploy.deploy(configs) +} + +module.exports = { + createFilesPackage, + uploadFiles, +} diff --git a/packages/component-mts-package/src/mts-json-template.js b/packages/component-mts-package/src/mts-json-template.js index ee0c091bba0b57fe0584b00ecf687b9535c07a3a..330744aeddb2936f92af2eee0f4368212acc97aa 100644 --- a/packages/component-mts-package/src/mts-json-template.js +++ b/packages/component-mts-package/src/mts-json-template.js @@ -142,6 +142,9 @@ const getJsonTemplate = (config = {}) => ({ }, 'funding-group': {}, }, + files: { + file: [], + }, }, }, }) diff --git a/packages/component-mts-package/src/output.xml b/packages/component-mts-package/src/output.xml deleted file mode 100644 index ea3de542f7fd341e9b5aa6f742ba3ec2b0525b6d..0000000000000000000000000000000000000000 --- a/packages/component-mts-package/src/output.xml +++ /dev/null @@ -1,68 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!DOCTYPE article SYSTEM "JATS-archivearticle1-mathml3.dtd"> -<article dtd-version="1.1d1" article-type="Research Article"> - <front> - <journal-meta> - <journal-id journal-id-type="publisher">research</journal-id> - <journal-id journal-id-type="email">faraday@hindawi.com</journal-id> - <journal-title-group> - <journal-title>Bioinorganic Chemistry and Applications</journal-title> - </journal-title-group> - <issn pub-type="ppub">2474-7394</issn> - <issn pub-type="epub"></issn> - </journal-meta> - <article-meta> - <article-id pub-id-type="publisher-id">RESEARCH-F-7654321</article-id> - <article-id pub-id-type="manuscript">RESEARCH-F-7654321</article-id> - <article-categories> - <subj-group subj-group-type="Article Type"> - <subject>clinical-study</subject> - </subj-group> - </article-categories> - <title-group> - <article-title> - <p>Harry Kane</p> - </article-title> - </title-group> - <contrib-group> - <contrib contrib-type="author" corresp="yes"> - <role content-type="1"></role> - <name> - <surname>Author</surname> - <given-names>Manuscriptunson</given-names> - <prefix>mr</prefix> - </name> - <email>hindawi+auth@thinslices.com</email> - <xref ref-type="aff" rid="aff1"></xref> - </contrib> - <contrib contrib-type="author" corresp="no"> - <role content-type="1"></role> - <name> - <surname>Gareth 4thplace</surname> - <given-names>Southgate</given-names> - <prefix>Dr.</prefix> - </name> - <email>hindawi+coauthor@thinslices.com</email> - <xref ref-type="aff" rid="aff2"></xref> - </contrib> - <aff id="aff1"> - <country>Hindawi</country> - </aff> - <aff id="aff2"> - <country>UK</country> - </aff> - </contrib-group> - <history> - <date date-type="received"> - <day>5</day> - <month>6</month> - <year>2018</year> - </date> - </history> - <abstract> - <p>Golden boot, golden boy.</p> - </abstract> - <funding-group></funding-group> - </article-meta> - </front> -</article> \ No newline at end of file diff --git a/packages/component-mts-package/tests/MTS.test.js b/packages/component-mts-package/tests/MTS.test.js index 0dc22cadabd7f1060b4936d9833d556e40408a9c..6655fbe07f3602b353cb89be4371bde679d4ddff 100644 --- a/packages/component-mts-package/tests/MTS.test.js +++ b/packages/component-mts-package/tests/MTS.test.js @@ -8,10 +8,6 @@ jest.mock('xml-js', () => ({ json2xml: jest.fn(), xml2js: jest.fn(), })) -jest.mock('fs', () => ({ - json2xml: jest.fn(), - xml2js: jest.fn(), -})) describe('MTS integration', () => { let MTS diff --git a/packages/component-wizard/src/components/SubmissionWizard.js b/packages/component-wizard/src/components/SubmissionWizard.js index d425016fb8dee8861acb58139b102d03a7355526..162de41426cb1e4d6ab014aeac2c8e9f6c632031 100644 --- a/packages/component-wizard/src/components/SubmissionWizard.js +++ b/packages/component-wizard/src/components/SubmissionWizard.js @@ -27,7 +27,12 @@ import { import { Steps, FormItems } from 'pubsweet-components-faraday/src/components' import { wizardSteps } from './' -import { autosaveRequest } from '../redux/autosave' +import { + getAutosave, + autosaveRequest, + autosaveSuccess, + autosaveFailure, +} from '../redux/autosave' import AutosaveIndicator from './AutosaveIndicator' import { submitManuscript } from '../redux/conversion' import { onChange, onSubmit, setInitialValues, validate } from './utils' @@ -83,6 +88,14 @@ const NewWizard = ({ ) // #endregion +const ModalWrapper = compose( + connect(state => ({ + fetching: get(getAutosave(state), 'isFetching', false), + })), +)(({ fetching, ...rest }) => ( + <SubmissionModal {...rest} isFetching={fetching} /> +)) + // #region export export default compose( ConnectPage(({ match }) => [ @@ -103,6 +116,8 @@ export default compose( }), { autosaveRequest, + autosaveSuccess, + autosaveFailure, submitManuscript, updateFragment: actions.updateFragment, }, @@ -125,7 +140,7 @@ export default compose( hasConflicts: get(formValues, 'conflicts.hasConflicts', 'no') === 'yes', })), withModal(() => ({ - modalComponent: SubmissionModal, + modalComponent: ModalWrapper, })), reduxForm({ form: 'submission', diff --git a/packages/component-wizard/src/components/utils.js b/packages/component-wizard/src/components/utils.js index 052f731d68102b32a44da1253efafbc90888e15e..eb3f2317884c860d5524a81a1e28fc7d9566fb1b 100644 --- a/packages/component-wizard/src/components/utils.js +++ b/packages/component-wizard/src/components/utils.js @@ -1,4 +1,3 @@ -import { SubmissionError } from 'redux-form' import { set, get, @@ -83,6 +82,10 @@ export const onSubmit = ( nextStep, showModal, hideModal, + setModalError, + autosaveRequest, + autosaveSuccess, + autosaveFailure, submitManuscript, version: { id: fragmentId }, project: { id: collectionId, customId }, @@ -96,20 +99,22 @@ export const onSubmit = ( 'By submitting the manuscript you agree to the following statements:', confirmText: 'AGREE & SUBMIT', cancelText: 'BACK TO SUBMISSION', - onConfirm: () => + onConfirm: () => { + dispatch(autosaveRequest()) submitManuscript(collectionId, fragmentId) .then(r => { + dispatch(autosaveSuccess()) history.push('/confirmation-page', { customId, version: fragmentId, project: collectionId, }) }) - .catch(err => { - if (err.validationErrors) { - throw new SubmissionError() - } - }), + .catch(e => { + dispatch(autosaveFailure(e)) + setModalError('Something went wrong.') + }) + }, onCancel: hideModal, }) } diff --git a/packages/xpub-faraday/config/default.js b/packages/xpub-faraday/config/default.js index 22d48a7d70e81b486f47841d157e79f137882789..d7958d81f5985b02d32c1b026e20856fcb543f1d 100644 --- a/packages/xpub-faraday/config/default.js +++ b/packages/xpub-faraday/config/default.js @@ -71,6 +71,17 @@ module.exports = { bucket: process.env.AWS_S3_BUCKET, validations: path.resolve(__dirname, 'upload-validations.js'), }, + 'mts-service': { + ftp: { + user: process.env.FTP_USERNAME, + password: process.env.FTP_PASSWORD, + host: process.env.FTP_HOST, + port: 21, + localRoot: `./`, + remoteRoot: '/', + exclude: ['*.js'], + }, + }, 'invite-reset-password': { url: process.env.PUBSWEET_INVITE_PASSWORD_RESET_URL || '/invite', }, diff --git a/yarn.lock b/yarn.lock index 580f88dd02b7311bd99001b4a7271ef4be751fd8..965f00ccb37aa4b27d827dd63a31af4e9d7b5447 100644 --- a/yarn.lock +++ b/yarn.lock @@ -85,6 +85,13 @@ lodash "^4.2.0" to-fast-properties "^2.0.0" +"@icetee/ftp@^0.3.15": + version "0.3.15" + resolved "https://registry.yarnpkg.com/@icetee/ftp/-/ftp-0.3.15.tgz#d32efd91ab7585f0a3b6cbed9ceffe2763b04ec6" + dependencies: + readable-stream "1.1.x" + xregexp "2.0.0" + "@pubsweet/component-aws-s3@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@pubsweet/component-aws-s3/-/component-aws-s3-1.1.2.tgz#ef7c6c7f22a19ce6f547412b73ab8de3fc81c3ee" @@ -1633,6 +1640,10 @@ block-stream@*: dependencies: inherits "~2.0.0" +bluebird@2.x: + version "2.11.0" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-2.11.0.tgz#534b9033c022c9579c56ba3b3e5a5caafbb650e1" + bluebird@^3.4.7, bluebird@^3.5.0, bluebird@^3.5.1: version "3.5.1" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" @@ -4083,6 +4094,16 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: mkdirp ">=0.5 0" rimraf "2" +ftp-deploy@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ftp-deploy/-/ftp-deploy-2.1.2.tgz#d8b4c6619f563eff189fa64ee1f5c8159ca15960" + dependencies: + bluebird "^3.5.1" + minimatch "3.0.4" + promise-ftp "^1.3.5" + read "^1.0.7" + upath "^1.0.5" + function-bind@^1.0.2, function-bind@^1.1.0, function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -6253,7 +6274,7 @@ minimalistic-crypto-utils@^1.0.0, minimalistic-crypto-utils@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a" -minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4, minimatch@~3.0.2: +minimatch@3.0.4, minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: @@ -7534,6 +7555,18 @@ progress@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.0.tgz#8a1be366bf8fc23db2bd23f10c6fe920b4389d1f" +promise-ftp-common@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/promise-ftp-common/-/promise-ftp-common-1.1.5.tgz#b4f8082a74035647703506763edb14230d9865da" + +promise-ftp@^1.3.5: + version "1.3.5" + resolved "https://registry.yarnpkg.com/promise-ftp/-/promise-ftp-1.3.5.tgz#ecfa4a5e5b779a6bfdd4dd3096957b58286f5104" + dependencies: + "@icetee/ftp" "^0.3.15" + bluebird "2.x" + promise-ftp-common "^1.1.5" + promise-inflight@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3" @@ -8241,7 +8274,7 @@ read-pkg@^3.0.0: normalize-package-data "^2.3.2" path-type "^3.0.0" -read@1.0.x: +read@1.0.x, read@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/read/-/read-1.0.7.tgz#b3da19bd052431a97671d44a42634adf710b40c4" dependencies: @@ -9876,6 +9909,10 @@ unzip-response@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/unzip-response/-/unzip-response-2.0.1.tgz#d2f0f737d16b0615e72a6935ed04214572d56f97" +upath@^1.0.5: + version "1.1.0" + resolved "https://registry.yarnpkg.com/upath/-/upath-1.1.0.tgz#35256597e46a581db4793d0ce47fa9aebfc9fabd" + upper-case@^1.1.1: version "1.1.3" resolved "https://registry.yarnpkg.com/upper-case/-/upper-case-1.1.3.tgz#f6b4501c2ec4cdd26ba78be7222961de77621598" @@ -10414,6 +10451,10 @@ xpub-validators@^0.0.3: dependencies: striptags "^3.1.0" +xregexp@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/xregexp/-/xregexp-2.0.0.tgz#52a63e56ca0b84a7f3a5f3d61872f126ad7a5943" + xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"