Commit d155ae9d authored by Audrey Hamelers's avatar Audrey Hamelers

Merge branch 'shared-data-model' into 'dev'

Shared data model

See merge request !53
parents c025d5c2 5867fb5b
Pipeline #11307 failed with stages
in 12 seconds
......@@ -8,7 +8,7 @@ import { TextField, Icon } from '@pubsweet/ui'
import { LoadingIcon, Select } from '../ui'
import { FileLightbox } from '../preview-files'
import { SubmissionTypes, ManuscriptTypes } from './uploadtypes'
import { GET_MANUSCRIPT } from '../submission-wizard/operations'
import { GET_MANUSCRIPT } from '../operations'
import {
DELETE_FILE_MUTATION,
UPDATE_FILE_MUTATION,
......
......@@ -9,7 +9,7 @@ import { LoadingIcon, Notification } from '../ui'
import { FileTypes, ManuscriptTypes, XMLTypes } from './uploadtypes'
import UploadFileListItem from './UploadFileListItem'
import { UPLOAD_MUTATION } from './operations'
import { GET_MANUSCRIPT } from '../submission-wizard/operations'
import { GET_MANUSCRIPT } from '../operations'
import { fileSort } from '../preview-files'
const UploadDiv = styled.div`
......
......@@ -16,6 +16,7 @@
"./server/xmlValidation",
"./server/db",
"./server/annotator",
"./server/cron",
"./server/xpub-server",
"./server/xpub-model/entities/user",
"./server/xpub-model/entities/file",
......
......@@ -20,4 +20,36 @@ module.exports = {
freq: 120000,
},
users: [],
ncbiFileTypes: [
{
value: 'supplement',
numeric: 6,
},
{
value: 'PMC',
numeric: 8,
},
{
value: 'IMGsmall',
numeric: 9,
},
{
value: 'IMGview',
numeric: 10,
},
{
value: 'IMGprint',
numeric: 11,
},
{
value: 'supplement_tag',
numeric: 21,
},
{
value: 'pdf4load',
numeric: 33,
},
],
ncbiInitialState: 'ncbi-ready',
ncbiSentState: 'ncbi-sent',
}
......@@ -20,4 +20,36 @@ module.exports = {
freq: 120000,
},
users: [],
ncbiFileTypes: [
{
value: 'supplement',
numeric: 6,
},
{
value: 'PMC',
numeric: 8,
},
{
value: 'IMGsmall',
numeric: 9,
},
{
value: 'IMGview',
numeric: 10,
},
{
value: 'IMGprint',
numeric: 11,
},
{
value: 'supplement_tag',
numeric: 21,
},
{
value: 'pdf4load',
numeric: 33,
},
],
ncbiInitialState: 'ncbi-ready',
ncbiSentState: 'ncbi-sent',
}
......@@ -93,7 +93,10 @@
"xpub-selectors": "^0.1.0",
"xpub-theme": "^0.0.7",
"xslt4node": "^0.3.2",
"yup": "^0.26.6"
"yup": "^0.26.6",
"md5-file": "^4.0.0",
"ftp": "^0.3.10",
"node-cron": "^2.0.3"
},
"resolutions": {
"graphql": "^0.13.0"
......
......@@ -80,6 +80,7 @@ CREATE TABLE manuscript (
"meta,unmatched_journal" TEXT,
pdf_deposit_id TEXT,
pdf_deposit_state TEXT,
ncbi_state TEXT,
updated_by UUID NOT NULL,
claimed_by TEXT
);
......
*/1 * * * * node server/ncbi-integration/test.js
module.exports = {
backend: () => require('./app'),
}
module.exports = {
backend: () => require('./jobs'),
}
const cron = require('node-cron')
const toNcbi = require('../ncbi-integration/toNcbi')
module.exports = jobs => {
cron.schedule('*/1 * * * *', () => {
toNcbi()
})
}
module.exports = {
backend: () => require('./toNcbi'),
}
const logger = require('@pubsweet/logger')
const path = require('path')
const { exec } = require('child_process')
const http = require('http')
const fs = require('fs')
const os = require('os')
const config = require('config')
const md5File = require('md5-file')
const Client = require('ftp')
const pubsweetServer = config.get('pubsweet-server.baseUrl')
// const ftpLocation = '/home/mselim/test'
const ncbiFileTypes = config.get('ncbiFileTypes')
const ManuscriptManager = require('../xpub-model/entities/manuscript')
module.exports = toNcbi => {
ManuscriptManager.findNcbiReady().then(manuscripts => {
manuscripts.forEach(manuscript => {
logger.info(`Sending package ${manuscript.id} to ncbi FTP location.`)
createTempDir(manuscript).then(tmpPath =>
getFiles(tmpPath, manuscript).then(() =>
getFunderInfo(manuscript).then(funders =>
createManifest(tmpPath, manuscript, funders).then(() =>
compress(tmpPath, manuscript).then(destination => {
send(destination, manuscript).then(() => {
tidyUp(tmpPath)
updateManuscriptNcbiStatus(manuscript)
})
}),
),
),
),
)
})
})
}
process.on('uncaughtException', err => {
logger.info(err)
})
function createTempDir(manuscript) {
return new Promise((resolve, reject) => {
// noinspection JSCheckFunctionSignatures
const directory = `${os.tmpdir()}/${getFilesPrefix(manuscript)}`
// if temp directory exists, remove its contents:
fs.access(directory, fs.constants.F_OK, err => {
if (!err) {
tidyUp(directory)
}
fs.mkdir(directory, err => {
if (err) {
reject(err)
}
resolve(directory)
})
})
})
}
function getFunderInfo(manuscript) {
return new Promise((resolve, reject) => {
http
.get(
'http://www.ebi.ac.uk/europepmc/GristAPI/rest/api/get/funders',
res => {
let data = ''
res.on('data', chunk => {
data += chunk
})
res.on('end', () => {
resolve(JSON.parse(data))
})
},
)
.on('error', err => {
// Handle errors
reject(err)
})
})
}
function getFiles(tmpPath, manuscript) {
const promises = [] // array of promises
// list of files needed by ncbi
const files = getNcbiFiles(manuscript)
files.forEach(file => {
const promise = new Promise((resolve, reject) => {
const fileOnDisk = fs.createWriteStream(`${tmpPath}/${file.filename}`)
http
.get(`${pubsweetServer}${file.url}`, response => {
response.pipe(fileOnDisk)
resolve(true)
})
.on('error', err => {
// Handle errors
reject(err)
})
})
promises.push(promise)
})
return Promise.all(promises)
}
function createManifest(tmpPath, manuscript, funders) {
const { funderInfoList } = funders
const pmcId = manuscript['meta,articleIds'].find(
id => id.pubIdType === 'pmcid',
)
? manuscript['meta,articleIds'].find(id => id.pubIdType === 'pmcid').id
: 0
const pmId = manuscript['meta,articleIds'].find(id => id.pubIdType === 'pmid')
? manuscript['meta,articleIds'].find(id => id.pubIdType === 'pmid').id
: 0
const createdDate = new Date(manuscript.created).toISOString().split('T')[0]
const publishDate = new Date(
manuscript['meta,publicationDates'].find(date => date.type === 'ppub').date,
)
.toISOString()
.split('T')[0]
const hasPubPdf = manuscript.files.find(file => file.type === 'pdf4load')
? '1'
: '0'
const journalNlmId = manuscript.journal['meta,nlmuniqueid']
const grant = funderInfoList.find(
funder => funder.name === manuscript['meta,fundingGroup'][0].fundingSource,
)
const granteName = grant.grantPrefix.includes('_')
? `${grant.grantPrefix}${manuscript['meta,fundingGroup'][0].awardId}`
: grant.grantPrefix
const files = getNcbiFiles(manuscript)
let text = `<?xml version="1.0"?>\n <!DOCTYPE pnihms-xdata PUBLIC "-//PNIHMS-EXCHANGE//DTD pNIHMS Exchange DTD//EN" "pnihms_exchange.dtd"> \n
<pnihms-xdata mid="${getFilesPrefix(
manuscript,
)}" pubmed-id="${pmId}" pmc-id="${pmcId}" create-date="${createdDate}" publish-date="${publishDate}" pub-pdf="${hasPubPdf}" nlm-journal-id="${journalNlmId}"> \n
<grants> \n
<grant name="${granteName}" authority="WT" /> \n
</grants> \n
<blobs> \n`
files.forEach(file => {
const { numeric } = ncbiFileTypes.find(type => type.value === file.type)
const fileMd5 = getFileMd5(`${tmpPath}/${file.filename}`)
text += ` <blob md5="${fileMd5}" name="${
file.filename
}" type="${numeric}" /> \n`
})
text += ` </blobs> \n
</pnihms-xdata>`
return new Promise((resolve, reject) => {
fs.writeFile(`${tmpPath}/${getFilesPrefix(manuscript)}.mxml`, text, err => {
if (err) {
reject(err)
}
resolve(tmpPath)
})
})
}
function compress(tmpPath, manuscript) {
return new Promise((resolve, reject) => {
const currentDate = new Date()
// const date = currentDate.toISOString().split("T")[0]
const timestamp = currentDate
.toISOString()
.replace(/-/g, '_')
.replace('T', '-')
.replace(/:/g, '_')
.split('.')[0]
const dest = `${tmpPath}/${getFilesPrefix(manuscript)}.${timestamp}.ld.zip`
const cmd = `cd ${path.dirname(tmpPath)} && mkdir ${getFilesPrefix(
manuscript,
)}.${timestamp}.ld && cp ${path.basename(tmpPath)}/* ${getFilesPrefix(
manuscript,
)}.${timestamp}.ld && zip -r ${dest} ${getFilesPrefix(
manuscript,
)}.${timestamp}.ld/`
exec(cmd, err => {
if (err) {
// node couldn't execute the command
throw new Error('cannot compress files')
}
global.destination = dest
resolve(dest)
})
})
}
function send(dest, manuscript) {
return new Promise((resolve, reject) => {
const c = new Client()
c.on('ready', () => {
c.put(
dest,
`/${config.get('ncbi-ftp')['receive-folder']}/${path.basename(dest)}`,
err => {
if (err) {
reject(err)
}
c.end()
resolve(true)
},
)
})
const { host, user, password } = config.get('ncbi-ftp')
c.connect({ host, user, password })
})
}
process.on('unhandledRejection', (reason, promise) => {
logger.error(`Unhandled Rejection at: ${reason.stack || reason}`)
tidyUp()
})
function tidyUp(tmpPath) {
try {
const cmd = `rm -rf ${tmpPath}*`
return new Promise((resolve, reject) => {
exec(cmd, (err, stdout, stderr) => {
if (err) {
// node couldn't execute the command
reject(err)
}
// logger.info(`Finished process`)
resolve(true)
})
})
} catch (e) {
logger.error('Error while tidying up after another error!')
}
}
function getFilesPrefix(manuscript) {
return manuscript.id.toLowerCase()
}
function getNcbiFiles(manuscript) {
return manuscript.files.filter(file =>
ncbiFileTypes.find(type => type.value === file.type),
)
}
function getFileMd5(filePath) {
return md5File.sync(filePath)
}
function updateManuscriptNcbiStatus(manuscript) {
ManuscriptManager.update(
{
id: manuscript.id,
status: config.get('ncbiSentState'),
ncbiState: 'sent',
},
manuscript.updatedBy,
).then(() =>
logger.info(`Created package ${manuscript.id} to ncbi FTP location.`),
)
}
......@@ -30,6 +30,7 @@ class Manuscript extends EpmcBaseModel {
decision: { type: ['string', 'null'] },
pdfDepositId: { type: 'uuid' },
pdfDepositState: { type: ['string', 'null'] },
ncbiState: { type: ['string', 'null'] },
'meta,title': { type: ['string', 'null'] },
'meta,articleType': { type: ['string', 'null'] },
'meta,articleIds': {
......@@ -435,6 +436,15 @@ class Manuscript extends EpmcBaseModel {
return manuscripts
}
static async findNcbiReady() {
const manuscripts = Manuscript.query()
.where('status', config.get('ncbiInitialState'))
.whereNull('ncbiState')
.whereNull('deleted')
.eager('[journal, files]')
logger.debug('manuscripts: ', manuscripts)
return manuscripts
}
static async insert(manuscript) {
const { id } = await Manuscript.query().insertGraph(manuscript)
const manuscriptInserted = await Manuscript.query()
......
......@@ -25,6 +25,9 @@ const dManuscriptUpdate = (data, userId) => {
if (data.status) {
properties.status = data.status
}
if (data.ncbiState) {
properties.ncbiState = data.ncbiState
}
if (data.hasOwnProperty('claimedBy')) {
properties.claimedBy = data.claimedBy
}
......
......@@ -140,7 +140,7 @@ const Manuscript = {
},
findByDepositStates: ManuscriptAccess.selectByPdfDepositStates,
findByDepositStatesNull: ManuscriptAccess.selectByPdfDepositStatesNull,
findNcbiReady: ManuscriptAccess.findNcbiReady,
delete: async (id, userId) => {
try {
const manuscript = await ManuscriptAccess.selectById(id)
......
......@@ -5133,6 +5133,14 @@ fstream@^1.0.0, fstream@^1.0.2:
mkdirp ">=0.5 0"
rimraf "2"
ftp@^0.3.10:
version "0.3.10"
resolved "https://registry.yarnpkg.com/ftp/-/ftp-0.3.10.tgz#9197d861ad8142f3e63d5a83bfe4c59f7330885d"
integrity sha1-kZfYYa2BQvPmPVqDv+TFn3MwiF0=
dependencies:
readable-stream "1.1.x"
xregexp "2.0.0"
function-bind@^1.0.2, function-bind@^1.1.0, function-bind@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
......@@ -7711,6 +7719,11 @@ mathml-tag-names@^2.0.1:
resolved "https://registry.yarnpkg.com/mathml-tag-names/-/mathml-tag-names-2.1.0.tgz#490b70e062ee24636536e3d9481e333733d00f2c"
integrity sha512-3Zs9P/0zzwTob2pdgT0CHZuMbnSUSp8MB1bddfm+HDmnFWHGT4jvEZRf+2RuPoa+cjdn/z25SEt5gFTqdhvJAg==
md5-file@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/md5-file/-/md5-file-4.0.0.tgz#f3f7ba1e2dd1144d5bf1de698d0e5f44a4409584"
integrity sha512-UC0qFwyAjn4YdPpKaDNw6gNxRf7Mcx7jC1UGCY4boCzgvU2Aoc1mOGzTtrjjLKhM5ivsnhoKpQVxKPp+1j1qwg==
md5.js@^1.3.4:
version "1.3.5"
resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f"
......@@ -8195,6 +8208,14 @@ nocache@2.0.0:
resolved "https://registry.yarnpkg.com/nocache/-/nocache-2.0.0.tgz#202b48021a0c4cbde2df80de15a17443c8b43980"
integrity sha1-ICtIAhoMTL3i34DeFaF0Q8i0OYA=
node-cron@^2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/node-cron/-/node-cron-2.0.3.tgz#b9649784d0d6c00758410eef22fa54a10e3f602d"
integrity sha512-eJI+QitXlwcgiZwNNSRbqsjeZMp5shyajMR81RZCqeW0ZDEj4zU9tpd4nTh/1JsBiKbF8d08FCewiipDmVIYjg==
dependencies:
opencollective-postinstall "^2.0.0"
tz-offset "0.0.1"
node-ensure@^0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7"
......@@ -8668,6 +8689,11 @@ onetime@^2.0.0, onetime@^2.0.1:
dependencies:
mimic-fn "^1.0.0"
opencollective-postinstall@^2.0.0:
version "2.0.2"
resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz#5657f1bede69b6e33a45939b061eb53d3c6c3a89"
integrity sha512-pVOEP16TrAO2/fjej1IdOyupJY8KDUM1CvsaScRbw6oddvpQoOfGk4ywha0HKKVAD6RkW4x6Q+tNBwhf3Bgpuw==
optimism@^0.6.6:
version "0.6.8"
resolved "https://registry.yarnpkg.com/optimism/-/optimism-0.6.8.tgz#0780b546da8cd0a72e5207e0c3706c990c8673a6"
......@@ -12940,6 +12966,11 @@ typescript@^2.2.2:
resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.9.2.tgz#1cbf61d05d6b96269244eb6a3bce4bd914e0f00c"
integrity sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w==
tz-offset@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/tz-offset/-/tz-offset-0.0.1.tgz#fef920257024d3583ed9072a767721a18bdb8a76"
integrity sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ==
ua-parser-js@^0.7.18:
version "0.7.19"
resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.19.tgz#94151be4c0a7fb1d001af7022fdaca4642659e4b"
......@@ -13897,6 +13928,11 @@ xpub-validators@^0.0.6:
dependencies:
striptags "^3.1.0"
xregexp@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/xregexp/-/xregexp-2.0.0.tgz#52a63e56ca0b84a7f3a5f3d61872f126ad7a5943"
integrity sha1-UqY+VsoLhKfzpfPWGHLxJq16WUM=
xslt4node@^0.3.2:
version "0.3.2"
resolved "https://registry.yarnpkg.com/xslt4node/-/xslt4node-0.3.2.tgz#a54063ac9d03a9318a363a7bec2a8ed6de28b610"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment