Allow different recipes to be called via the INK component

The existing INK components (frontend and backend) were built to handle conversion of DOCX files to HTML; this change allows any INK recipe to be called in the same way.

• Adds a "recipe" parameter (fixes #54)
• Adds an "outputFile" parameter (so the backend knows which file to download at the end of the process)
• Refactor to improve promise handling

The changes to the backend component are backwards-compatible (tested in Editoria) - finding the recipe id works as follows:

1. Read the "recipe" parameter.
2. If there's a matching key (e.g. 'editoria-typescript') in config['pubsweet-component-ink-backend']['recipes'] use that value as the recipe ID. This is the ideal situation, where the app configuration maps keys used in the frontend to recipe IDs in the configured INK instance.
3. If there's no key, use "Editoria Typescript" as the default (for backwards compatibility). Fetch the list of recipes from the INK API and find the recipe with title that matches this key (this is how it used to work, but is inefficient as it has to fetch and filter that list for every conversion).

packages/InkBackend/InkBackend.js

-const config = require('config')
-const rp = require('request-promise-native')
-const Busboy = require('busboy')
-const temp = require('temp').track()
 const fs = require('fs')
 const path = require('path')
-const promiseRetry = require('promise-retry')
 const logger = require('@pubsweet/logger')
+const Busboy = require('busboy')
+const config = require('config')
+const rp = require('request-promise-native')
+const temp = require('temp')
+
+// rp.debug = true
+
+const inkConfig = config.get('pubsweet-component-ink-backend')
 
-let inkConfig = config.get('pubsweet-component-ink-backend')
-let inkEndpoint = inkConfig.inkEndpoint
-let email = inkConfig.email
-let password = inkConfig.password
-let maxRetries = inkConfig.maxRetries || 60
+// Generate the absolute URL
+const inkUrl = path => inkConfig.inkEndpoint + 'api/' + path
 
-let authRequest = {
-  uri: inkEndpoint + '/api/auth/sign_in',
+// Sign in
+const authorize = () => rp({
   method: 'POST',
-  body: {
-    email: email,
-    password: password
+  uri: inkUrl('auth/sign_in'),
+  formData: {
+    email: inkConfig.email,
+    password: inkConfig.password
   },
-  json: true,
   headers: {
     'Accept': 'application/vnd.ink.1'
   },
   resolveWithFullResponse: true
-}
+}).then(res => ({
+  'client': res.headers['client'],
+  'access-token': res.headers['access-token']
+}))
 
-// Get an access token
-let getAuth = () => {
-  return rp(authRequest).then(response => {
-    return {
-      accessToken: response.headers['access-token'],
-      client: response.headers['client']
-    }
-  }).catch(
-    err => {
-      logger.error('INK API LOGIN FAILURE:', err)
-      throw err
-    }
-  )
-}
+// Upload file to INK and execute the recipe
+const upload = (recipeId, inputFile, auth) => rp({
+  method: 'POST',
+  uri: inkUrl('recipes/' + recipeId + '/execute'),
+  headers: {
+    uid: inkConfig.email,
+    ...auth
+  },
+  formData: {
+    input_files: [inputFile]
+  },
+  json: true,
+  timeout: 60 * 60 * 1000 // 3600 seconds
+})
 
-let defaultRecipeId = null
-let recipeListUrl = inkEndpoint + '/api/recipes'
+// Download the output file (keep trying if there's a 404 response, until it's ready)
+const download = async (chain, auth, outputFileName) => {
+  const manifest = chain.input_file_manifest
 
-const getRecipeId = () => {
-  if (defaultRecipeId) return Promise.resolve(defaultRecipeId)
+  if (manifest.length === 0) {
+    throw new Error('The INK server gave a malformed response (no input files in the process chain)')
+  }
 
-  const listRequest = auth => ({
-    method: 'GET',
-    uri: recipeListUrl,
-    headers: {
-      'uid': email,
-      'access-token': auth.accessToken,
-      'client': auth.client
-    }
-  })
+  const interval = inkConfig.interval || 1000 // try once per second
 
-  return getAuth().then(
-    auth => rp(listRequest(auth))
-  ).then(
-    response => Promise.resolve(JSON.parse(response))
-  ).then(
-    response => {
-      const defaultRecipe = response.recipes.find(
-        recipe => recipe.name === 'Editoria Typescript' // XSweet recipe
-      )
-      if (!defaultRecipe) throw new Error('could not get default recipe from INK')
-      defaultRecipeId = defaultRecipe.id
-      return Promise.resolve(defaultRecipeId)
-    }
-  )
-}
+  const maxRetries = inkConfig.maxRetries || 300 // retry for up to 5 minutes
 
-getRecipeId()
-
-const inkRecipeUrl = () => getRecipeId().then(
-  recipeId => Promise.resolve(inkEndpoint + '/api/recipes/' + recipeId + '/execute')
-)
-
-const healthCheckRequest = auth => inkRecipeUrl().then(
-  recipeUrl => {
-    const opts = {
-      uri: recipeUrl,
-      method: 'OPTIONS',
-      headers: {
-        'Access-Control-Request-Method': 'POST',
-        'Access-Control-Request-Headers': 'access-token, client, expiry, token-type, uid',
-        'Origin': 'http://ink.coko.foundation',
-        'access-token': auth.accessToken,
-        'client': auth.client
-      }
-    }
-    return Promise.resolve(opts)
-  }
-)
-
-const uploadRequest = (data, auth) => inkRecipeUrl().then(
-  recipeUrl => {
-    const opts = {
-      uri: recipeUrl,
-      method: 'POST',
-      headers: {
-        'uid': email,
-        'access-token': auth.accessToken,
-        'client': auth.client
-      },
-      formData: {
-        input_files: [data]
-      }
-    }
-    return Promise.resolve(opts)
-  }
-)
+  const uri = inkUrl('process_chains/' + chain.id + '/download_output_file')
 
-// Upload file to INK and execute the recipe
-const uploadToInk = data => auth => uploadRequest(
-  data, auth
-).then(
-  rp
-).then(
-  response => Promise.resolve([auth, JSON.parse(response)])
-)
-
-// Check if INK is alive and well
-const checkInk = auth => healthCheckRequest(auth).then(
-  rp
-).then(
-  response => {
-    return Promise.resolve(auth)
+  const qs = {
+    relative_path: outputFileName || path.basename(manifest[0].path, '.docx') + '.html'
   }
-).catch(
-  err => {
-    throw err
+
+  const headers = {
+    uid: inkConfig.email,
+    ...auth
   }
-)
-
-const retryFor30SecondsUntil200 = (uri, auth) => {
-  const downloadRequest = {
-    method: 'GET',
-    uri: uri,
-    headers: {
-      'uid': email,
-      'access-token': auth.accessToken,
-      'client': auth.client
+
+  for (let i = 0; i < maxRetries; i++) {
+    // delay
+    await new Promise(resolve => setTimeout(resolve, interval))
+
+    const response = await rp({
+      uri,
+      qs,
+      headers,
+      simple: false,
+      resolveWithFullResponse: true
+    }).catch(error => {
+      logger.error('Error downloading from INK:', error.message)
+      throw error
+    })
+
+    // a successful request: return the data
+    if (response.statusCode === 200) {
+      return response.body
+    }
+
+    // not a 404 response - stop trying
+    if (response.statusCode !== 404) {
+      break
     }
   }
 
-  return promiseRetry(
-    (retry, number) => {
-      return rp(downloadRequest).catch(retry)
-    },
-    { retries: maxRetries, factor: 1, minTimeout: 3000 }
-  )
+  throw new Error('Unable to download the output from INK')
 }
 
-const downloadUrl = (chainId, relPath) => inkEndpoint +
-  '/api/process_chains/' +
-  chainId +
-  '/download_output_file?relative_path=' +
-  relPath +
-  '.html'
+const findRecipeId = (recipeKey = 'Editoria Typescript', auth) => rp({
+  method: 'GET',
+  uri: inkUrl('recipes'),
+  headers: {
+    uid: inkConfig.email,
+    ...auth
+  },
+  json: true
+}).then(data => {
+  const recipe = data.recipes.find(recipe => recipe.name === recipeKey)
 
-const downloadFromInk = ([auth, response]) => {
-  if (response.process_chain.input_file_manifest.length === 0) {
-    throw new Error('The INK server gave a malformed response (no input files in the process chain)')
-  }
-  const relPath = path.basename(response.process_chain.input_file_manifest[0].path, '.docx')
-  const url = downloadUrl(response.process_chain.id, relPath)
-  return retryFor30SecondsUntil200(url, auth)
+  return recipe ? recipe.id : null
+})
+
+const process = async (inputFile, options) => {
+  const auth = await authorize().catch(err => {
+    logger.error('INK API LOGIN FAILURE:', err.message)
+    throw err
+  })
+
+  // either use the recipe id from the configuration or search for it by name
+  const recipeId = inkConfig.recipes[options.recipe] || await findRecipeId(options.recipe, auth)
+  if (!recipeId) throw new Error('Unknown recipe')
+
+  const response = await upload(recipeId, inputFile, auth).catch(err => {
+    logger.error('INK API UPLOAD FAILURE:', err.message)
+    throw err
+  })
+
+  return download(response.process_chain, auth, options.outputFileName)
 }
 
-var InkBackend = function (app) {
+const InkBackend = function (app) {
+  // TODO: authentication on this route
   app.use('/api/ink', (req, res, next) => {
-    var fileStream = new Busboy({ headers: req.headers })
-
-    const handleErr = err => {
-      logger.error('ERROR CONVERTING WITH INK', err)
-      next(err)
-    }
+    const fileStream = new Busboy({ headers: req.headers })
 
     fileStream.on('file', (fieldname, file, filename, encoding, contentType) => {
-      var stream = temp.createWriteStream()
-      file.pipe(stream)
-
-      file.on('end', () => {
-        stream.end()
+      const stream = temp.createWriteStream()
 
-        var fileOpts = {
+      stream.on('finish', () => {
+        const inputFile = {
           value: fs.createReadStream(stream.path),
-          options: {
-            filename: filename,
-            contentType: contentType
-          }
+          options: { filename, contentType }
         }
 
-        getAuth().then(
-          checkInk
-        ).then(
-          uploadToInk(fileOpts)
-        ).then(
-          downloadFromInk
-        ).then(
-          response => res.send(response)
-        ).catch(handleErr)
+        process(inputFile, req.query).then(converted => {
+          res.json({ converted })
+
+          // clean up temp file
+          fs.unlink(stream.path, () => {
+            logger.info('Deleted temporary file', stream.path)
+          })
+        }).catch(err => {
+          logger.error('ERROR CONVERTING WITH INK:', err.message)
+          next(err)
+        })
+      })
+
+      file.pipe(stream)
+
+      file.on('end', () => {
+        stream.end()
       })
     })
 
-    fileStream.on('error', handleErr)
+    fileStream.on('error', err => {
+      logger.error(err)
+      next(err)
+    })
 
     req.pipe(fileStream)
   })