Commit f61b3ba4 authored by charlie-ablett's avatar charlie-ablett
Browse files

Integrate arbitrary step execution with step-at-a-time execution.

parent 6cdddedb
......@@ -29,8 +29,8 @@ gem 'pg', '~> 0.15'
################### basic steps #############################
# gem 'ink_step', path: "~/Projects/coko/ink-step"
gem 'ink_step', git: 'https://gitlab.coko.foundation/INK/ink-step.git'
gem 'ink_step', path: "~/RubymineProjects/coko/ink-step"
# gem 'ink_step', git: 'https://gitlab.coko.foundation/INK/ink-step.git'
# Load StepGemfile
if File.exists?(File.join(File.dirname(__FILE__), "StepGemfile"))
......
......@@ -6,15 +6,6 @@ GIT
capistrano (~> 3.1)
sshkit (~> 1.3)
GIT
remote: https://gitlab.coko.foundation/INK/ink-step.git
revision: 30a166cc2b813cf143b59c5c11552f9dd62636d2
specs:
ink_step (1.3.3)
awesome_print
httparty
rubyzip
GIT
remote: https://gitlab.coko.foundation/INK/inkstep_coko_conversion.git
revision: 6cb33e4f6f15c30cb5467a96d3afc69557801067
......@@ -33,6 +24,14 @@ GIT
specs:
inkstep_development (1.0.5)
PATH
remote: ../ink-step
specs:
ink_step (1.3.3)
awesome_print
httparty
rubyzip
GEM
remote: https://rubygems.org/
specs:
......
......@@ -6,6 +6,7 @@ module Api
before_action :authenticate!
before_action :authorise_account!, only: [:show, :download_input_file, :download_input_zip, :download_output_file, :download_output_zip]
before_action :check_processing_finished!, only: [:download_output_file, :download_output_zip]
respond_to :json
......@@ -16,7 +17,7 @@ module Api
def create
new_single_step_execution[:execution_parameters] = params[:single_step_execution][:execution_parameters]
new_single_step_execution.save!
new_single_step_execution.start_execution!(code: params[:single_step_execution][:code])
new_single_step_execution.start_execution!(input_files: input_file_param, code: params[:single_step_execution][:code])
render json: new_single_step_execution
rescue => e
ap e.message
......@@ -28,13 +29,9 @@ module Api
render json: current_entity.account.single_step_executions.as_json
end
def zip_path
"/tmp/standalone_#{id}_input.zip"
end
def download_input_file
ap "Downloading #{params[:relative_path]}..."
file_path = assemble_file_path(location: single_step_execution.input_files_directory, relative_path: params[:relative_path])
ap "Downloading #{relative_path_param}..."
file_path = assemble_file_path(location: single_step_execution.input_files_directory, relative_path: relative_path_param)
send_file(file_path,
:disposition => 'attachment',
......@@ -51,22 +48,14 @@ module Api
end
def download_output_file
ap "Downloading #{params[:relative_path]} from #{single_step_execution.working_directory}..."
unless single_step_execution.finished?
render_not_found_error("Not finished processing yet")
return
end
file_path = assemble_file_path(location: single_step_execution.working_directory, relative_path: params[:relative_path])
ap "Downloading #{relative_path_param} from #{single_step_execution.working_directory}..."
file_path = assemble_file_path(location: single_step_execution.working_directory, relative_path: relative_path_param)
send_file(file_path,
:disposition => 'attachment',
:url_based_filename => true)
end
def download_output_zip
unless single_step_execution.finished?
render_not_found_error("Not finished processing yet")
return
end
zip_path = single_step_execution.assemble_output_file_zip
ap "Assembled #{zip_path}"
......@@ -77,10 +66,18 @@ module Api
private
def input_file_param
params.require(:single_step_execution).require(:input_file_list)
end
def single_step_execution_params
params.require(:single_step_execution).permit(:description, :step_class_name, :execution_parameters, :input_file_list)
end
def relative_path_param
params.require(:relative_path)
end
def single_step_execution
@single_step_execution ||= current_entity.account.single_step_executions.find(params[:id])
end
......@@ -94,10 +91,18 @@ module Api
end
def authorise_account!
if single_step_execution.account != current_entity.account || !current_entity.account.admin?
if single_step_execution.account.id != current_entity.account.id && !current_entity.account.admin?
e = ExecutionErrors::NotAuthorisedError.new("This is not accessible to you.")
render_error(e)
end
rescue => e
render_error(e)
end
def check_processing_finished!
unless single_step_execution.finished?
render_not_found_error("Not finished processing yet")
end
end
end
end
......
......@@ -64,4 +64,18 @@ module DirectoryMethods
path.fnmatch?(File.join(parent_directory,'**'))
end
def write_input_files(input_files)
# Input file is received by the controller like so
#<ActionDispatch::Http::UploadedFile:0x007f2ad1984fd8
# @tempfile=#<Tempfile:/tmp/RackMultipart20161211-6674-1d8irke.docx>,
# @original_filename="a_very_nice_document.docx",
# @content_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
# @headers="Content-Disposition: form-data; name=\"input_file\"; filename=\"a_very_nice_document.docx\"\r\nContent-Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document\r\n">
input_files.each do |uploaded_file|
target_file = File.join(input_files_directory, uploaded_file.original_filename)
FileUtils.cp(uploaded_file.tempfile, target_file)
end
end
end
\ No newline at end of file
......@@ -6,12 +6,12 @@ module Execution
include DirectoryMethods
include ExecutionErrors
attr_accessor :step_array, :process_steps, :chain_file_location, :chain_id, :recipe_id, :cumulative_file_manifest
attr_accessor :step_array, :process_steps, :base_file_location, :chain_id, :recipe_id, :cumulative_file_manifest
def initialize(process_steps_in_order:, chain_file_location:, process_chain:)
def initialize(process_steps_in_order:, base_file_location:, process_chain:)
@step_array = []
@process_steps = process_steps_in_order
@chain_file_location = chain_file_location
@base_file_location = base_file_location
@process_chain = process_chain
@recipe_id = process_chain.recipe_id
@chain_id = @process_chain.id
......@@ -37,7 +37,7 @@ module Execution
def execute_process_steps
@cumulative_file_manifest = {}
@process_steps.each do |process_step|
behaviour_step = process_step.step_class.new(chain_file_location: chain_file_location,
behaviour_step = process_step.step_class.new(base_file_location: base_file_location,
position: process_step.position)
@step_array << behaviour_step
trigger_step_started_event(behaviour_step)
......
......@@ -6,7 +6,7 @@ module Execution
include DirectoryMethods
include ExecutionErrors
attr_accessor :working_directory, :klass
attr_accessor :working_directory, :klass, :standalone_execution
def initialize(klass:, single_step_execution:)
# load behaviour class
......@@ -33,7 +33,7 @@ module Execution
end
def execute_standalone_step
behaviour_step = klass.new(chain_file_location: working_directory, position: 1)
behaviour_step = klass.new(base_file_location: working_directory, position: 1)
trigger_step_started_event(behaviour_step)
begin
behaviour_step.combined_parameters = @standalone_execution.execution_parameters
......@@ -42,8 +42,8 @@ module Execution
# log(e.message)
# log(e.backtrace)
ensure
process_step.map_results(behaviour_step: behaviour_step)
trigger_standalone_execution_finished_event(behaviour_step, process_step)
standalone_execution.map_results(behaviour_step: behaviour_step)
trigger_standalone_execution_finished_event(behaviour_step, standalone_execution)
end
end
......@@ -53,15 +53,15 @@ module Execution
data: { position: behaviour_step.position })
end
def trigger_standalone_execution_finished_event(behaviour_step, process_step)
def trigger_standalone_execution_finished_event(behaviour_step, standalone_step)
trigger_event(channels: standalone_execution_channel(@standalone_execution.account_id),
event: standalone_execution_finished_event(@standalone_execution.account_id),
data: { position: behaviour_step.position,
successful: behaviour_step.successful,
notes: behaviour_step.notes,
execution_errors: behaviour_step.errors,
process_log_location: process_step.process_log_file_name,
output_file_manifest: process_step.output_file_manifest })
process_log_location: standalone_step.process_log_file_name,
output_file_manifest: standalone_step.output_file_manifest })
end
end
end
......
......@@ -65,20 +65,6 @@ class ProcessChain < ApplicationRecord
process_steps.sort_by(&:position)
end
def write_input_files(input_files)
# Input file is received by the controller like so
#<ActionDispatch::Http::UploadedFile:0x007f2ad1984fd8
# @tempfile=#<Tempfile:/tmp/RackMultipart20161211-6674-1d8irke.docx>,
# @original_filename="a_very_nice_document.docx",
# @content_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
# @headers="Content-Disposition: form-data; name=\"input_file\"; filename=\"a_very_nice_document.docx\"\r\nContent-Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document\r\n">
input_files.each do |uploaded_file|
target_file = File.join(input_files_directory, uploaded_file.original_filename)
FileUtils.cp(uploaded_file.tempfile, target_file)
end
end
def initialize_directories
create_directory_if_needed(Constants::FILE_LOCATION)
create_directory_if_needed(working_directory)
......
......@@ -30,9 +30,10 @@ class SingleStepExecution < ApplicationRecord
File.join(Constants::FILE_LOCATION, "single_step_executions", slug)
end
def start_execution!(code:, callback_url: nil)
def start_execution!(input_files:, code:, callback_url: nil)
initialize_directories
write_code_to_file(code: code)
write_input_files(input_files)
StandaloneExecutionWorker.perform_async(self.id, callback_url)
end
......@@ -82,14 +83,30 @@ class SingleStepExecution < ApplicationRecord
File.join(working_directory, Constants::OUTPUT_FILE_DIRECTORY_NAME)
end
def input_zip_path
"/tmp/standalone_#{id}_input.zip"
end
def output_zip_path
"/tmp/standalone_#{id}_output.zip"
end
def assemble_output_file_zip
zip_path = "/tmp/step_#{id}_output.zip"
Dir.chdir(working_directory) do
unless File.exists?(zip_path)
`zip -r "#{zip_path}" *`
unless File.exists?(output_zip_path)
`zip -r "#{output_zip_path}" *`
end
end
output_zip_path
end
def assemble_input_file_zip
Dir.chdir(working_directory) do
unless File.exists?(input_zip_path)
`zip -r "#{input_zip_path}" *`
end
end
zip_path
input_zip_path
end
def process_log_file_name
......@@ -99,7 +116,7 @@ class SingleStepExecution < ApplicationRecord
def map_results(behaviour_step:)
self.execution_errors = [behaviour_step.errors].flatten.map{|line| line.gsub(working_directory, "$process_step_working_directory")}
self.notes = [behaviour_step.notes].flatten.map{|line| line.gsub(working_directory, "$process_step_working_directory")}
self.executed_at = behaviour_step.executed_at
self.executed_at = behaviour_step.started_at
self.finished_at = behaviour_step.finished_at
self.successful = behaviour_step.successful
self.output_file_list = behaviour_step.semantically_tagged_manifest
......
......@@ -12,7 +12,7 @@ class ExecutionWorker
sleep(5) unless Rails.env.test? # icky hack to solve race condition - not final solution
@process_chain = ProcessChain.includes(:process_steps).find(chain_id)
runner = Execution::RecipeExecutionRunner.new(process_steps_in_order: process_steps_in_order, chain_file_location: @process_chain.working_directory, process_chain: @process_chain)
runner = Execution::RecipeExecutionRunner.new(process_steps_in_order: process_steps_in_order, base_file_location: @process_chain.working_directory, process_chain: @process_chain)
begin
runner.run!
rescue => e
......
......@@ -7,6 +7,7 @@ Sidekiq::Testing.inline!
describe Api::V1::SingleStepExecutionsController, type: :controller do
let!(:account) { create(:account, password: "password", password_confirmation: "password") }
let!(:other_account) { create(:account, password: "password", password_confirmation: "password") }
let!(:text_file) { File.new('spec/fixtures/files/plaintext.txt', 'r') }
let!(:single_step_execution) {
create(:single_step_execution, account: account, step_class_name: "InkStep::Base")
......@@ -37,7 +38,7 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
description: description,
step_class_name: step_class_name,
execution_parameters: execution_parameters,
input_files: html_file,
input_file_list: [html_file],
code: klass_file
}
}
......@@ -64,7 +65,7 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
context 'if the code supplied is blank' do
before do
single_step_execution_params[:code] = File.read(Rails.root.join('spec/fixtures/files/standalone/blank_file.rb'))
single_step_execution_params[:single_step_execution][:code] = File.read(Rails.root.join('spec/fixtures/files/standalone/blank_file.rb'))
end
specify do
......@@ -79,14 +80,13 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
context 'if the code supplied does not contain a class' do
before do
single_step_execution_params[:code] = File.read(Rails.root.join('spec/fixtures/files/standalone/valid_not_a_class.rb'))
single_step_execution_params[:single_step_execution][:code] = File.read(Rails.root.join('spec/fixtures/files/standalone/valid_not_a_class.rb'))
end
specify do
request_with_auth(account.new_jwt) do
perform_create_request(single_step_execution_params)
end
expect(response.status).to eq 422
expect(body_as_json['errors']).to eq ["Mismatch! You provided InkStep::AwesomeClass and the file defined something else"]
end
......@@ -94,7 +94,7 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
context 'if the code supplied is not valid syntax' do
before do
single_step_execution_params[:code] = File.read(Rails.root.join('spec/fixtures/files/standalone/rubbish_class'))
single_step_execution_params[:single_step_execution][:code] = File.read(Rails.root.join('spec/fixtures/files/standalone/rubbish_class'))
end
specify do
......@@ -145,14 +145,24 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
}
context 'if a valid token is supplied' do
context 'and the chain belongs to that account' do
context 'and the execution belongs to that account' do
it 'serves the file successfully' do
request_with_auth(account.new_jwt) do
perform_download_input_zip_request(download_params)
end
expect(response.status).to eq 200
expect(response.stream.to_path).to eq "/tmp/chain_#{single_step_execution.id}_input.zip"
expect(response.stream.to_path).to eq single_step_execution.input_zip_path
end
end
context 'and the execution does not belong to that account' do
it 'does not allow access' do
request_with_auth(other_account.new_jwt) do
perform_download_input_zip_request(download_params)
end
expect(response.status).to eq 404
end
end
end
......@@ -168,18 +178,19 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
before do
create_directory_if_needed(single_step_execution.working_directory)
copy_fixture_file('some_text.txt', single_step_execution.working_directory)
single_step_execution.update_attribute(:finished_at, 3.minutes.ago)
end
context 'if a valid token is supplied' do
context 'and the chain belongs to that account' do
context 'and exeuction is finished' do
context 'and the execution belongs to that account' do
context 'and execuction is finished' do
it 'serves the file successfully' do
request_with_auth(account.new_jwt) do
perform_download_output_zip_request(download_params)
end
expect(response.status).to eq 200
expect(response.stream.to_path).to eq "/tmp/step_#{single_step_execution.last_step.id}_output.zip"
expect(response.stream.to_path).to eq single_step_execution.output_zip_path
end
end
context 'and the execution is not finished' do
......@@ -196,6 +207,16 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
end
end
end
context 'and the execution does not belong to that account' do
it 'does not allow access' do
request_with_auth(other_account.new_jwt) do
perform_download_input_zip_request(download_params)
end
expect(response.status).to eq 404
end
end
end
end
......@@ -204,6 +225,7 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
before do
create_directory_if_needed(single_step_execution.send(:working_directory))
copy_fixture_file("plaintext.txt", single_step_execution.send(:working_directory))
single_step_execution.update_attribute(:finished_at, 3.minutes.ago)
end
context 'when there is an output file' do
......@@ -214,7 +236,18 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
}
}
context 'and the execution does not belong to that account' do
it 'does not allow access' do
request_with_auth(other_account.new_jwt) do
perform_download_input_zip_request(download_params)
end
expect(response.status).to eq 404
end
end
context 'when execution is finished' do
it 'downloads the file' do
request_with_auth(account.new_jwt) do
perform_download_output_file_request(download_params)
......@@ -223,6 +256,7 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
expect(response.status).to eq 200
end
end
context 'and the execution is not finished' do
before do
single_step_execution.update_attribute(:finished_at, nil)
......@@ -248,7 +282,7 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
it 'fails' do
request_with_auth(account.new_jwt) do
expect{perform_download_output_file_request(download_params)}.to raise_error("Please provide a relative file path")
expect{perform_download_output_file_request(download_params)}.to raise_error("param is missing or the value is empty: relative_path")
end
end
end
......@@ -313,7 +347,6 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
end
end
end
end
describe "GET download_input_file" do
......@@ -331,6 +364,16 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
perform_download_input_file_request(download_params)
end
end
context 'and the execution does not belong to that account' do
it 'does not allow access' do
request_with_auth(other_account.new_jwt) do
perform_download_input_zip_request(download_params)
end
expect(response.status).to eq 404
end
end
end
context 'if no filename is supplied' do
......@@ -343,7 +386,7 @@ describe Api::V1::SingleStepExecutionsController, type: :controller do
it 'fails' do
request_with_auth(account.new_jwt) do
expect{perform_download_input_file_request(download_params)}.to raise_error("Please provide a relative file path")
expect{perform_download_input_file_request(download_params)}.to raise_error("param is missing or the value is empty: relative_path")
end
end
end
......
......@@ -4,5 +4,13 @@ module InkStep
def perform_step
# The goggles! They do nothing!
end
def version
"3.1415926"
end
def description
"An exciting test"
end
end
end
\ No newline at end of file
......@@ -18,7 +18,7 @@ describe Execution::RecipeExecutionRunner do
describe '#run!' do
context 'for a successful execution' do
context 'if there are no steps' do
subject { Execution::RecipeExecutionRunner.new(process_steps_in_order: [], chain_file_location: chain.working_directory, process_chain: chain) }
subject { Execution::RecipeExecutionRunner.new(process_steps_in_order: [], base_file_location: chain.working_directory, process_chain: chain) }
it 'returns nil - no change was made' do
expect{subject.run!}.to raise_error(ExecutionErrors::EmptyChainError, error_message)
......@@ -28,7 +28,7 @@ describe Execution::RecipeExecutionRunner do
context 'if there is 1 step' do
let!(:step1) { create(:process_step, process_chain: chain, position: 1, step_class_name: base_step_class.to_s) }
subject { Execution::RecipeExecutionRunner.new(process_steps_in_order: [step1], chain_file_location: chain.working_directory, process_chain: chain) }
subject { Execution::RecipeExecutionRunner.new(process_steps_in_order: [step1], base_file_location: chain.working_directory, process_chain: chain) }
before { chain.reload }
......@@ -45,7 +45,7 @@ describe Execution::RecipeExecutionRunner do
let!(:step2) { create(:process_step, process_chain: chain, position: 2, step_class_name: rot_thirteen_step_class.to_s) }
let!(:step3) { create(:process_step, process_chain: chain, position: 3, step_class_name: base_step_class.to_s) }
let(:steps) { [step1, step2, step3] }
subject { Execution::RecipeExecutionRunner.new(process_steps_in_order: steps, chain_file_location: chain.working_directory, process_chain: chain) }
subject { Execution::RecipeExecutionRunner.new(process_steps_in_order: steps, base_file_location: chain.working_directory, process_chain: chain) }
before { chain.reload }
......@@ -61,7 +61,7 @@ describe Execution::RecipeExecutionRunner do
context 'if there is a failure' do
let(:steps) { [step1] }
subject { Execution::RecipeExecutionRunner.new(process_steps_in_order: steps, chain_file_location: chain.working_directory, process_chain: chain) }
subject { Execution::RecipeExecutionRunner.new(process_steps_in_order: steps, base_file_location: chain.working_directory, process_chain: chain) }
before do
allow_any_instance_of(base_step_class).to receive(:perform_step) { raise "Oh noes! Error!" }
......
......@@ -2,10 +2,10 @@ require 'rails_helper'
describe base_step_class do
let(:chain_file_location) { File.join("tmp", "ink_api_files", Time.now.to_i.to_s) }
subject(:base_step) { base_step_class.new(chain_file_location: chain_file_location, position: 1) }
let(:base_file_location) { File.join("tmp", "ink_api_files", Time.now.to_i.to_s) }
subject(:base_step) { base_step_class.new(base_file_location: base_file_location, position: 1) }
let(:input_file) { Rails.root.join('spec/fixtures/files/some_text.html') }
let(:input_directory) { File.join(chain_file_location, Constants::INPUT_FILE_DIRECTORY_NAME) }
let(:input_directory) { File.join(base_file_location, Constants::INPUT_FILE_DIRECTORY_NAME) }
before do
create_directory_if_needed(input_directory)
......
......@@ -52,7 +52,7 @@ def unfavourite_recipe(version, data = {}.to_json)
get :unfavourite, params: data, headers: {'Content-Type' => "application/json", 'Accept' => "application/vnd.ink.#{version}" }
end
# process_chain / process_stepcontroller requests
# process_chain / process_step / single_step_execution_controller requests
def download_input_file(version, data = {}.to_json)
get :download_input_file, params: data, headers: {'Content-Type' => "application/json", 'Accept' => "application/vnd.ink.#{version}" }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment