Commit 39d3086f authored by Nikos Marinos's avatar Nikos Marinos

perform local seeding of journals

parent 12ea454e
select json_agg(t) from (select date_revised "dateRevised", journal_title "journalTitle", "meta,publisher_name" "meta,publisherName", "meta,issn", "meta,nlmta", "meta,nlmuniqueid", "meta,pubmed_status" "meta,pubmedStatus", "meta,pmc_status" "meta,pmcStatus", "meta,first_year" "meta,firstYear", "meta,end_year" "meta,endYear" from journal) t \g journals.json
This diff is collapsed.
......@@ -3,10 +3,15 @@
const logger = require('@pubsweet/logger')
const Identity = require('../server/xpub-model/entities/identity/data-access')
const User = require('../server/xpub-model/entities/user/data-access')
const Journal = require('../server/xpub-model/entities/journal/data-access')
const config = require('config')
const journals = require('../config/journals.json')
async function seed() {
try {
// const journals = config.get('journals')
await Journal.upsertMulti(journals)
const users = config.get('users')
let counter = 0
/* eslint-disable no-await-in-loop */
......@@ -34,10 +39,14 @@ async function seed() {
}
})
} catch (e) {
logger.warn('Could not load any ftpUsers')
logger.warn('Could not load any seeds', e)
}
logger.info('Seeding complete.')
}
seed()
;(async () => {
const beforeUpdate = Date.now()
await seed()
logger.info(`Seeding was finished in ${Date.now() - beforeUpdate} ms`)
})()
const { Model } = require('objection')
const { Model, transaction } = require('objection')
const uuid = require('uuid')
const EpmcBaseModel = require('../epmc-base-model')
const { rowToEntity, entityToRow, buildQuery, runQuery } = require('../util')
......@@ -102,77 +102,65 @@ class Journal extends EpmcBaseModel {
return row.id
}
/*
function insertJournals() {
const Journal = require('../xpub-model/entities/journal/data-access')
const journals = [
{
journalTitle: 'title1',
'meta,publisherName': 'publisher11',
'meta,issn': [
{id: '1234', type:'issn'},
{id: '4567', type:'essn'}
],
'meta,nlmta': 'dont know what this is',
'meta,nlmuniqueid': 'xxxx',
'meta,pmc_status': true,
'meta,pubmed_status': false,
},
{
journalTitle: 'title2',
'meta,publisherName': 'publisher22',
'meta,issn': [
{id: '1234', type:'issn'},
{id: '4567', type:'essn'}
],
'meta,nlmta': 'dont know what this is',
'meta,nlmuniqueid': 'yyyy',
'meta,pmc_status': true,
'meta,pubmed_status': false,
},
]
Journal.upsertMulti(journals)
}
insertJournals()
*/
static async upsertMulti(journals) {
const promises = [] // array of promises
const util = require('util')
journals.forEach(journal => {
const insert = knex('journal')
.insert(journal)
.toString()
const update = knex('journal')
.update(journal)
.whereRaw(
`journal."meta,nlmuniqueid" = '${
journal['meta,nlmuniqueid']
}' AND journal.date_revised < '${journal.dateRevised}'`,
let promises = [] // array of promises
let trx
try {
trx = await transaction.start(knex)
let counter = 0
const util = require('util')
journals.forEach(journal => {
const insert = knex('journal')
.insert(journal)
.toString()
const update = knex('journal')
.update(journal)
.whereRaw(
`journal."meta,nlmuniqueid" = '${
journal['meta,nlmuniqueid']
}' AND journal.date_revised < '${journal.dateRevised}'`,
)
const query = util.format(
'%s ON CONFLICT ("meta,nlmuniqueid") DO UPDATE SET %s',
insert.toString(),
update.toString().replace(/^update\s.*\sset\s/i, ''),
'RETURNING id',
)
counter += 1
promises.push(
knex
.raw(query)
.transacting(trx)
.then(row => {
if (row.rowCount > 0) return row.rows[0].id
}),
)
const query = util.format(
'%s ON CONFLICT ("meta,nlmuniqueid") DO UPDATE SET %s',
insert.toString(),
update.toString().replace(/^update\s.*\sset\s/i, ''),
'RETURNING id',
)
promises.push(
knex.raw(query).then(row => {
if (row.rowCount > 0) return row.rows[0].id
}),
)
})
return Promise.all(promises).then(arr => {
const updatedIds = arr.filter(el => el !== undefined)
if (updatedIds.length > 0) {
logger.info(`These records were updated: ${updatedIds.join(',')}`)
if (counter % 1000 === 0) {
Promise.all(promises).then(arr => {
trx.commit()
const updatedIds = arr.filter(el => el !== undefined)
if (updatedIds.length > 0) {
}
promises = []
})
}
})
return Promise.all(promises).then(() => {
trx.commit()
logger.info(`${counter} records were updated.`)
})
} catch (error) {
if (trx) {
await trx.rollback()
}
})
logger.error('No journals were inserted')
logger.error(error)
throw error
}
}
static update(journal) {
......
......@@ -25,10 +25,4 @@ done
echo "ERROR: xPub is not running"
exit 1
exit 1
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment