exercism/analyzer-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support all platform operations over zmq

Browse Source
This commit is contained in:
Charles Care
2019-09-10 14:29:00 +01:00
parent 61561aadfa
commit 2331da3289
19 changed files with 166 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
Gemfile
View File

@@ -4,9 +4,10 @@ gem "activesupport"
gem "mandate"
gem "propono"
gem "rugged"
gem 'aws-sdk-ecr'
gem "aws-sdk-s3"
gem "aws-sdk-ecr"
gem 'simplecov', require: false, group: :test
gem "simplecov", require: false, group: :test
gem "ffi-rzmq"

8
Gemfile.lock
View File

@@ -16,6 +16,13 @@ GEM
aws-sdk-ecr (1.20.0)
aws-sdk-core (~> 3, >= 3.61.1)
aws-sigv4 (~> 1.1)
aws-sdk-kms (1.24.0)
aws-sdk-core (~> 3, >= 3.61.1)
aws-sigv4 (~> 1.1)
aws-sdk-s3 (1.46.0)
aws-sdk-core (~> 3, >= 3.61.1)
aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.1)
aws-sdk-sns (1.19.0)
aws-sdk-core (~> 3, >= 3.61.1)
aws-sigv4 (~> 1.1)
@@ -60,6 +67,7 @@ PLATFORMS
DEPENDENCIES
activesupport
aws-sdk-ecr
aws-sdk-s3
ffi-rzmq
mandate
minitest

11
bin/build_analyzer
View File

@@ -1,11 +0,0 @@
#!/usr/bin/env ruby
require "bundler/setup"
$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
require "pipeline"
Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
Pipeline.build_analyzer(ARGV[0])
# Pipeline.build_analyzer("ruby")
# Pipeline.release("ruby")
# Pipeline.analyzer("ruby")

41
bin/builderd Executable file → Normal file
View File

@@ -2,42 +2,7 @@
require "bundler/setup"
$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
require 'ffi-rzmq'
require "pipeline"
class PipelineRpcServer
attr_reader :context, :socket
def initialize
@context = ZMQ::Context.new(1)
@socket = context.socket(ZMQ::REP)
socket.bind("tcp://*:5555")
end
def run
require "pipeline"
Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
loop do
request = ''
socket.recv_string(request)
puts "Received request. Data: #{request.inspect}"
if request.start_with? "build-analyzer_"
_, arg = request.split("_")
result = Pipeline.build_analyzer(arg)
socket.send_string(result.to_json)
elsif request.start_with? "release-analyzer_"
_, arg = request.split("_")
result = Pipeline.release(arg)
socket.send_string(result.to_json)
else
socket.send_string("done")
end
end
puts msg
socket.send_string(msg)
end
end
PipelineRpcServer.new.run
Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
Pipeline.daemon

37
bin/client.rb Normal file
View File

@@ -0,0 +1,37 @@
require 'ffi-rzmq'
require 'json'
require 'yaml'
require 'securerandom'
class PipelineClient
attr_reader :context, :socket
def initialize
@context = ZMQ::Context.new(1)
@socket = context.socket(ZMQ::REQ)
socket.setsockopt(ZMQ::LINGER, 0)
socket.connect("tcp://localhost:5555")
end
def send_msg(msg)
socket.send_string(msg)
response = ''
rc = socket.recv_string(response)
parsed = JSON.parse(response)
parsed
end
def build_analyzer(track_slug)
send_msg("build-analyzer_#{track_slug}")
end
def release_latest(track_slug)
send_msg("release-analyzer_#{track_slug}")
end
def analyze(track_slug, exercise_slug, solution_slug, iteration_folder)
send_msg("analyze_#{track_slug}|#{exercise_slug}|#{solution_slug}|#{iteration_folder}")
end
end

47
bin/invoke.rb
View File

@@ -1,44 +1,13 @@
#!/usr/bin/env ruby
require 'ffi-rzmq'
require 'json'
class PipelineClient
attr_reader :context, :socket
def initialize
@context = ZMQ::Context.new(1)
@socket = context.socket(ZMQ::REQ)
socket.setsockopt(ZMQ::LINGER, 0)
socket.connect("tcp://localhost:5555")
end
def send_msg(msg)
socket.send_string(msg)
response = ''
rc = socket.recv_string(response)
parsed = JSON.parse(response)
parsed
end
def build_analyzer(track_slug)
send_msg("build-analyzer_#{track_slug}")
end
def release_latest(track_slug)
send_msg("release-analyzer_#{track_slug}")
end
end
require_relative "./client"
pipeline = PipelineClient.new
r = pipeline.analyze("ruby", "two-fer", "soln-42", "s3://exercism-dev/iterations/fff07700-e1c3-402d-8937-823aeefb159f")
r["logs"].each do |log_line|
puts "+ #{log_line["cmd"]}"
puts log_line["stdout"]
puts log_line["stderr"]
end
# result = pipeline.build_analyzer("ruby")
# result["logs"].each do |log_line|
# puts "+ #{log_line["cmd"]}"
# puts log_line["stdout"]
# puts log_line["stderr"]
# end
pipeline.release_latest("ruby")
puts r["result"]

8
bin/release_analyzer
View File

@@ -1,8 +0,0 @@
#!/usr/bin/env ruby
require "bundler/setup"
$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
require "pipeline"
Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
Pipeline.release(ARGV[0])

3
bin/test
View File

@@ -1,3 +0,0 @@
#!/bin/bash
bundle exec rake test

1
foo.rb Normal file
View File

@@ -0,0 +1 @@
hello

31
lib/pipeline.rb
View File

@@ -4,8 +4,10 @@ require "active_support"
require 'securerandom'
require 'rugged'
require 'aws-sdk-ecr'
require 'aws-sdk-s3'
require 'yaml'
require 'json'
require 'ffi-rzmq'
module Pipeline
@@ -21,13 +23,17 @@ module Pipeline
@config
end
def self.daemon
server = Pipeline::RpcServer.new
server.listen
end
def self.build_analyzer(track_slug)
repo = Pipeline::AnalyzerRepo.for_track(track_slug)
latest_tag = repo.tags.keys.last
if (latest_tag.nil?)
latest_tag = "master"
end
puts latest_tag
Pipeline::Build::AnalyzerBuild.(latest_tag, track_slug)
end
@@ -41,31 +47,32 @@ module Pipeline
environment.release_analyzer(language_slug)
end
def self.analyzer(language_slug)
def self.analyze!(language_slug, exercise_slug, solution_slug)
env_base = "/tmp/analyzer-env/1e9c733fd7502974c2a3fdd85da9c844"
environment = Runtime::RuntimeEnvironment.new(env_base)
analysis_run = environment.new_analysis(language_slug, "two-fer", 42)
analysis_run = environment.new_analysis(language_slug, exercise_slug, solution_slug)
analysis_run.prepare_iteration do |iteration_folder|
File.write("#{iteration_folder}/two_fer.rb", 'puts "hello"')
yield(iteration_folder)
end
begin
analysis_run.analyze!
rescue => e
puts e
ensure
puts "---"
puts analysis_run.stdout
puts "==="
puts analysis_run.stderr
puts "---"
puts analysis_run.success?
puts analysis_run.exit_status
puts analysis_run.result
# puts "---"
# puts analysis_run.stdout
# puts "==="
# puts analysis_run.stderr
# puts "---"
# puts analysis_run.success?
# puts analysis_run.exit_status
# puts analysis_run.result
puts "DONE"
end
end
end
require "pipeline/rpc_server"
require "pipeline/analyzer_repo"
require "pipeline/validation/check_invokable"
require "pipeline/validation/check_environment_invariants"

2
lib/pipeline/analyzer_repo.rb
View File

@@ -1,5 +1,3 @@
require 'pp'
class Pipeline::AnalyzerRepo
BASE_DIR = ENV.fetch("ANALYZER_REPO_BASE_DIR", "./tmp/repos")

3
lib/pipeline/build/analyzer_build.rb
View File

@@ -12,7 +12,6 @@ module Pipeline::Build
build
validate
publish
puts "DONE"
{
track: track_slug,
image: image_name,
@@ -34,7 +33,6 @@ module Pipeline::Build
def build
@image_tag = Pipeline::Build::BuildImage.(build_tag, image_name, repo, img)
puts ">>>> #{image_tag}"
end
def validate
@@ -51,7 +49,6 @@ module Pipeline::Build
end
memoize
def repo
Pipeline::AnalyzerRepo.for_track(track_slug)
end

55
lib/pipeline/rpc_server.rb Executable file
View File

@@ -0,0 +1,55 @@
class Pipeline::RpcServer
attr_reader :context, :socket
def initialize
@context = ZMQ::Context.new(1)
@socket = context.socket(ZMQ::REP)
socket.bind("tcp://*:5555")
end
def listen
loop do
request = ''
socket.recv_string(request)
puts "Received request. Data: #{request.inspect}"
if request.start_with? "build-analyzer_"
_, arg = request.split("_")
result = Pipeline.build_analyzer(arg)
socket.send_string(result.to_json)
elsif request.start_with? "release-analyzer_"
_, arg = request.split("_")
result = Pipeline.release(arg)
socket.send_string(result.to_json)
elsif request.start_with? "analyze_"
_, arg = request.split("_", 2)
track, exercise_slug, solution_slug, location = arg.split("|")
result = Pipeline.analyze!(track, exercise_slug, solution_slug) do |iteration_folder|
location_uri = URI(location)
bucket = location_uri.host
path = location_uri.path[1..]
s3 = Aws::S3::Client.new(region: 'eu-west-1')
params = {
bucket: bucket,
prefix: "#{path}/",
}
resp = s3.list_objects(params)
resp.contents.each do |item|
key = item[:key]
filename = File.basename(key)
s3.get_object({
bucket: bucket,
key: key,
response_target: "#{iteration_folder}/#{filename}"
})
end
end
socket.send_string(result.to_json)
else
socket.send_string("done")
end
end
socket.send_string(msg)
end
end

22
lib/pipeline/runtime/analysis_run.rb
View File

@@ -2,7 +2,8 @@ module Pipeline::Runtime
class AnalysisRun
attr_reader :track_dir, :exercise_slug, :runs_dir, :solution_dir,
:iteration_folder, :tmp_folder, :current_dir
:iteration_folder, :tmp_folder, :current_dir, :img,
:runc
def initialize(track_dir, exercise_slug, solution_slug)
@track_dir = track_dir
@@ -12,6 +13,9 @@ module Pipeline::Runtime
@solution_dir = "#{runs_dir}/iteration_#{Time.now.to_i}-#{solution_slug}-#{SecureRandom.hex}"
@iteration_folder = "#{solution_dir}/iteration"
@tmp_folder = "#{solution_dir}/tmp"
@logs = Pipeline::Util::LogCollector.new
@img = Pipeline::Util::ImgWrapper.new(@logs)
@runc = Pipeline::Util::RuncWrapper.new(@logs)
end
def prepare_iteration
@@ -27,6 +31,14 @@ module Pipeline::Runtime
def analyze!
container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, solution_dir)
@result = container_driver.run_analyzer_for(exercise_slug)
{
exercise_slug: exercise_slug,
solution_dir: solution_dir,
rootfs_source: rootfs_source,
result: result,
invocation: @result.report,
logs: @logs.inspect
}
end
def result
@@ -60,14 +72,6 @@ module Pipeline::Runtime
end
end
def img
@img ||= Pipeline::Util::ImgWrapper.new
end
def runc
@runc ||= Pipeline::Util::RuncWrapper.new
end
def rootfs_source
@rootfs_source ||= begin
release_folder = File.readlink(current_dir)

42
lib/pipeline/runtime/runtime_environment.rb
View File

@@ -27,8 +27,6 @@ module Pipeline::Runtime
container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, release_dir)
# container_driver.prepare_workdir
# container_driver.unpack_image("track_slug:master")
ecr = Aws::ECR::Client.new(region: 'eu-west-1')
authorization_token = ecr.get_authorization_token.authorization_data[0].authorization_token
plain = Base64.decode64(authorization_token)
@@ -64,45 +62,5 @@ module Pipeline::Runtime
AnalysisRun.new(track_dir, exercise_slug, solution_slug)
end
# def prepare_analysis(track_slug, solution_id)
# track_dir = "#{env_base}/#{track_slug}"
# runs_dir = "#{track_dir}/runs"
# current_dir = "#{track_dir}/current"
# solution_dir = "#{runs_dir}/iteration_#{Time.now.to_i}-#{solution_id}-#{SecureRandom.hex}"
#
# iteration_folder = "#{solution_dir}/iteration"
# tmp_folder = "#{solution_dir}/tmp"
#
#
# FileUtils.mkdir_p iteration_folder
# FileUtils.mkdir_p tmp_folder
# solution_dir
# end
#
# def run_analysis(track_slug, solution_dir, exercise_slug)
# track_dir = "#{env_base}/#{track_slug}"
# runs_dir = "#{track_dir}/runs"
# current_dir = "#{track_dir}/current"
# img = Pipeline::Util::ImgWrapper.new
# runc = Pipeline::Util::RuncWrapper.new
# configurator = Pipeline::Util::RuncConfigurator.new
# configurator.seed_from_env
#
# rootfs_source = "#{File.readlink(current_dir)}/rootfs"
# configurator.rootfs = rootfs_source
#
# configurator.setup_for_terminal_access
# File.write("#{solution_dir}/terminal_config.json", configurator.build.to_json)
#
# container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, solution_dir)
# container_driver.run_analyzer_for("two-fer")
# end
def analyze_solution(track_slug, solution_id, exercise_slug)
iteration_folder = prepare_analysis
yield iteration_folder
run_analysis(iteration_folder, exercise_slug)
end
end
end

9
lib/pipeline/util/external_command.rb
View File

@@ -42,5 +42,14 @@ module Pipeline::Util
def timeout=(timeout)
@timeout = timeout
end
def report
{
cmd: cmd_string,
success: success?,
stdout: stdout,
stderr: stderr
}
end
end
end

6
lib/pipeline/util/img_wrapper.rb
View File

@@ -3,11 +3,11 @@ module Pipeline::Util
attr_accessor :binary_path, :state_location, :suppress_output, :logs
def initialize
def initialize(logs)
@binary_path = File.expand_path "./opt/img"
@state_location = "/tmp/state-img"
@suppress_output = false
@logs = Pipeline::Util::LogCollector.new
@logs = logs || Pipeline::Util::LogCollector.new
end
def build(local_tag)
@@ -61,7 +61,7 @@ module Pipeline::Util
run_cmd = ExternalCommand.new(cmd)
run_cmd.call
logs << run_cmd
raise "Failed #{cmd}" unless run_cmd.success?
raise "Failed #{cmd}" unless run_cmd.success?
end
end

9
lib/pipeline/util/log_collector.rb
View File

@@ -5,13 +5,8 @@ module Pipeline::Util
@logs = []
end
def <<(external_result)
@logs << {
cmd: external_result.cmd_string,
success: external_result.success?,
stdout: external_result.stdout,
stderr: external_result.stderr
}
def <<(external_command)
@logs << external_command.report
end
def inspect

6
lib/pipeline/util/runc_wrapper.rb
View File

@@ -2,10 +2,11 @@ module Pipeline::Util
class RuncWrapper
attr_accessor :binary_path, :suppress_output, :memory_limit
def initialize
def initialize(logs)
@binary_path = File.expand_path "./opt/runc"
@suppress_output = false
@memory_limit = 3000000
@logs = logs || Pipeline::Util::LogCollector.new
end
def run(container_folder)
@@ -17,10 +18,13 @@ module Pipeline::Util
kill_cmd = ExternalCommand.new("#{binary_path} --root root-state kill #{container_id} KILL")
Dir.chdir(container_folder) do
puts "HERE: #{@logs}" + @logs.class.to_s
begin
run_cmd.call
@logs << run_cmd
ensure
kill_cmd.call
@logs << kill_cmd
end
end