Commit 7f01e022 authored by charlie-ablett's avatar charlie-ablett

Steps clean up after themselves, improve zip/unzip, store previous manifest for comparison

parent 52ba57ae
PATH
remote: .
specs:
ink_step (1.1.0)
ink_step (1.1.1)
awesome_print
httparty
rubyzip
......
......@@ -9,20 +9,23 @@ module InkStep
include Mixins::HelperMethods
include Mixins::ShellMethods
attr_accessor :next_step, :errors, :status_code, :process_step, :started_at, :finished_at,
attr_accessor :errors, :status_code, :process_step, :started_at, :finished_at,
:position, :chain_file_location, :successful, :notes, :process_log,
:combined_parameters
:combined_parameters, :input_file_manifest, :output_file_manifest
INPUT_FILE_DIRECTORY_NAME = "input_files"
OUTPUT_FILE_DIRECTORY_NAME = "output_files"
def initialize(chain_file_location:, position:, next_step: nil)
def initialize(chain_file_location:, position:, incoming_file_manifest:)
@position = position
@next_step = next_step
@chain_file_location = chain_file_location
@input_file_manifest = incoming_file_manifest
@successful = nil
@errors = []
@notes = []
ap "incoming manifest"
ap @input_file_manifest
end
def execute(options: {})
......@@ -31,9 +34,12 @@ module InkStep
check_parameters(combined_parameters)
create_directory_if_needed(working_directory)
get_input_files
input_file_manifest = assemble_manifest(directory: working_directory)
log_as_step "Using version #{version}"
log_as_step "Execution parameters:"
log_as_step combined_parameters
log_as_step "Previous step manifest:"
log_as_step input_file_manifest
perform_step(options: combined_parameters)
rescue => e
raise_and_log_error(e)
......@@ -145,5 +151,71 @@ module InkStep
def success!
@successful = true
end
private
def write_manifest
semantically_tagged_manifest(working_directory, input_file_manifest)
end
def semantically_tagged_manifest(working_directory, manifest_at_start)
file_manifest_at_end = assemble_manifest(directory: working_directory)
file_manifest_at_end.each do |file_hash|
start_file_hash = find_start_file(file_hash, manifest_at_start)
if start_file_hash.nil?
if file_hash[:path] == "basic_doc.html"
ap "NEW FILE #{file_hash[:path]}"
end
file_hash[:tag] = :new
elsif file_hash[:checksum] == start_file_hash[:checksum]
if file_hash[:path] == "basic_doc.html"
ap "IDENTICAL FILE #{file_hash[:path]}"
end
file_hash[:tag] = :identical
else
file_hash[:tag] = :modified
end
end
end
def assemble_manifest(directory:)
files = recursive_file_list(directory)
files.inject([]) do |result, file_relative_path|
file_info_hash = {}
file_info_hash[:path] = file_relative_path
absolute_file_path = File.join(directory, file_relative_path)
file_info_hash[:size] = file_size_for_humans(absolute_file_path)
file_info_hash[:checksum] = Digest::MD5.hexdigest(File.read(absolute_file_path))
result << file_info_hash
result
end
end
def recursive_file_list(directory_path)
Dir.chdir(directory_path) do
files = Dir["**/*"] # Dir["**/*.*"]
files_only = []
files.each do |file|
full_path = File.join(directory_path, file)
files_only << file if File.file?(full_path)
end
files_only
end
end
def file_size_for_humans(path)
file_size_in_bytes = File.size(path)
if file_size_in_bytes > 299000
"#{(file_size_in_bytes / 1000000.0).round(1)} MB"
elsif file_size_in_bytes > 1000
"#{(file_size_in_bytes / 1000.0).round(1)} kB"
else
"#{file_size_in_bytes} bytes"
end
end
def find_start_file(current_file_hash, manifest_at_start)
manifest_at_start.collect{|f| return f if f[:path] == current_file_hash[:path]}.first
end
end
end
\ No newline at end of file
module InkStep::Mixins
# From rubyzip readme, accessed 24 July 2017
# This is a simple example which uses rubyzip to
# recursively generate a zip file from the contents of
# a specified directory. The directory itself is not
# included in the archive, rather just its contents.
#
# Usage:
# directory_to_zip = "/tmp/input"
# output_file = "/tmp/out.zip"
# zf = ZipFileGenerator.new(directory_to_zip, output_file)
# zf.write()
class ZipFileGenerator
# Initialize with the directory to zip and the location of the output archive.
def initialize(input_dir, output_file)
@input_dir = input_dir
@output_file = output_file
end
# Zip the input directory.
def write
entries = Dir.entries(@input_dir) - %w(. ..)
::Zip::File.open(@output_file, ::Zip::File::CREATE) do |io|
write_entries entries, '', io
end
end
private
# A helper method to make the recursion work.
def write_entries(entries, path, io)
entries.each do |e|
zip_file_path = path == '' ? e : File.join(path, e)
disk_file_path = File.join(@input_dir, zip_file_path)
puts "Deflating #{disk_file_path}"
if File.directory? disk_file_path
recursively_deflate_directory(disk_file_path, io, zip_file_path)
else
put_into_archive(disk_file_path, io, zip_file_path)
end
end
end
def recursively_deflate_directory(disk_file_path, io, zip_file_path)
io.mkdir zip_file_path
subdir = Dir.entries(disk_file_path) - %w(. ..)
write_entries subdir, zip_file_path, io
end
def put_into_archive(disk_file_path, io, zip_file_path)
io.get_output_stream(zip_file_path) do |f|
f.write(File.open(disk_file_path, 'rb').read)
end
end
end
end
\ No newline at end of file
require 'ink_step/base'
require 'ink_step/mixins/shell_methods'
module InkStep::UtilitySteps
class ModifiedFileCollectorStep < ::InkStep::Base
def perform_step(options: {})
discard_identical_files
success!
end
def self.description
"Keeps all new and modified files from the previous step, discards any unchanged files"
end
def version
InkStep::VERSION
end
protected
def discard_identical_files
identical_files = input_file_manifest.select{|file_hash| file_hash[:tag] == :identical}
identical_files.map do |file_hash|
log_as_step("Removing #{file_hash[:path]}")
File.delete(File.join(working_directory, file_hash[:path]))
end
end
end
end
\ No newline at end of file
......@@ -7,6 +7,10 @@ module InkStep::UtilitySteps
def perform_step(options: {})
source_file_path = find_source_file(regex: /\.#{parameter(options, :zip_file_extension)}$/)
unzip_file(File.join(working_directory, source_file_path))
# cleanup - destroy original archive
File.delete(File.join(working_directory, source_file_path))
success!
end
......
......@@ -5,8 +5,10 @@ module InkStep::UtilitySteps
class ZipStep < ::InkStep::Base
def perform_step(options: {})
# source_file_path = find_source_file(regex: /\.#{parameter(options, :zip_file_name)}$/)
# unzip_file(File.join(working_directory, source_file_path))
@options = options
create_archive
# cleanup
delete_other_files
success!
end
......@@ -20,26 +22,49 @@ module InkStep::UtilitySteps
def required_parameters
# e.g. [:foo, :bar]
[:zip_file_name]
[:archive_file_name]
end
def accepted_parameters
# e.g. {foo: "For setting the grobblegronx measure", bar: "Can be X, Y or Z"}
{zip_file_name: "The name of the archive file (.zip will be added)"}
{archive_file_name: "The name of the archive file (including .zip)"}
end
def default_parameter_values
# e.g. {foo: 1, bar: nil}
{zip_file_name: "archive.zip"}
{archive_file_name: "archive.zip"}
end
protected
def zip_file(directory, archive_file_name)
log_as_step "Zipping archive..."
def archive_file_name
@archive_file_name ||= parameter(@options, :archive_file_name)
end
def delete_other_files
Dir.chdir working_directory
Dir.glob("**/*").each do |f|
next if f == archive_file_name
if File.file?(f)
log_as_step("Deleting file #{f}")
File.delete(f)
elsif File.directory?(f)
log_as_step("Deleting directory #{f}")
FileUtils.remove_dir(f)
end
end
end
def create_archive
log_as_step "Zipping archive #{archive_file_name}..."
zipper = ::InkStep::Mixins::ZipGenerator.new(directory, archive_file_name)
zipper.write
Zip::File.open(archive_file_name, Zip::File::CREATE) do |zipfile|
Dir.chdir working_directory
Dir.glob("**/*").reject {|fn| File.directory?(fn) }.each do |file|
log_as_step "Adding #{file}"
zipfile.add(file.sub(working_directory + '/', ''), file)
end
end
end
end
end
\ No newline at end of file
......@@ -3,5 +3,9 @@ require 'ink_step/base'
module InkStep
class ValidationStep < Base
def self.description
"A base validator."
end
end
end
\ No newline at end of file
......@@ -25,7 +25,7 @@ module InkStep
end
def self.description
"A validator that returns the result you tell it to."
"A demo validator that returns the result you tell it to."
end
end
end
......
module InkStep
VERSION = '1.1.0'
VERSION = '1.1.1'
end
\ No newline at end of file
......@@ -10,7 +10,7 @@ describe InkStep::Base do
let!(:chain_file_location) { temp_directory }
let!(:input_file_path) { File.join(chain_file_location, InkStep::Base::INPUT_FILE_DIRECTORY_NAME) }
let!(:subject) { InkStep::Base.new(chain_file_location: chain_file_location, position: 1) }
let!(:subject) { InkStep::Base.new(chain_file_location: chain_file_location, position: 1, incoming_file_manifest: [path: target_file_name]) }
let!(:working_directory) { subject.send(:working_directory) }
before do
......
......@@ -7,7 +7,7 @@ describe "#perform_step" do
let(:target_file) { File.join(Dir.pwd, "spec", "fixtures", "files", target_file_name) }
let!(:input_file_path) { File.join(temp_directory, InkStep::Base::INPUT_FILE_DIRECTORY_NAME) }
subject { InkStep::UtilitySteps::UnzipStep.new(chain_file_location: temp_directory, position: 1) }
subject { InkStep::UtilitySteps::UnzipStep.new(chain_file_location: temp_directory, position: 1, incoming_file_manifest: [path: target_file_name]) }
before do
create_directory_if_needed(subject.working_directory)
......
......@@ -8,7 +8,7 @@ describe "Testing validators" do
let(:target_file) { File.join(Dir.pwd, "spec", "fixtures", "files", target_file_name) }
let!(:input_file_path) { File.join(temp_directory, InkStep::Base::INPUT_FILE_DIRECTORY_NAME) }
subject { InkStep::ValidationSteps::ArbitraryValidator.new(chain_file_location: temp_directory, position: 1) }
subject { InkStep::ValidationSteps::ArbitraryValidator.new(chain_file_location: temp_directory, position: 1, incoming_file_manifest: [path: target_file_name]) }
before do
create_directory_if_needed(temp_directory)
......
......@@ -8,7 +8,7 @@ describe "#perform_step" do
let(:target_file) { File.join(Dir.pwd, "spec", "fixtures", "files", target_file_name) }
let!(:input_file_path) { File.join(temp_directory, InkStep::Base::INPUT_FILE_DIRECTORY_NAME) }
subject { InkStep::UtilitySteps::ZipStep.new(chain_file_location: temp_directory, position: 1) }
subject { InkStep::UtilitySteps::ZipStep.new(chain_file_location: temp_directory, position: 1, incoming_file_manifest: [path: target_file_name]) }
before do
create_directory_if_needed(subject.working_directory)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment