From 2331da3289467b72df72538819cdf4dd9e3a2565 Mon Sep 17 00:00:00 2001 From: Charles Care Date: Tue, 10 Sep 2019 14:29:00 +0100 Subject: [PATCH] Support all platform operations over zmq --- Gemfile | 5 +- Gemfile.lock | 8 +++ bin/build_analyzer | 11 ----- bin/builderd | 41 ++------------- bin/client.rb | 37 ++++++++++++++ bin/invoke.rb | 47 +++--------------- bin/release_analyzer | 8 --- bin/test | 3 -- foo.rb | 1 + lib/pipeline.rb | 31 +++++++----- lib/pipeline/analyzer_repo.rb | 2 - lib/pipeline/build/analyzer_build.rb | 3 -- lib/pipeline/rpc_server.rb | 55 +++++++++++++++++++++ lib/pipeline/runtime/analysis_run.rb | 22 +++++---- lib/pipeline/runtime/runtime_environment.rb | 42 ---------------- lib/pipeline/util/external_command.rb | 9 ++++ lib/pipeline/util/img_wrapper.rb | 6 +-- lib/pipeline/util/log_collector.rb | 9 +--- lib/pipeline/util/runc_wrapper.rb | 6 ++- 19 files changed, 166 insertions(+), 180 deletions(-) delete mode 100755 bin/build_analyzer mode change 100755 => 100644 bin/builderd create mode 100644 bin/client.rb delete mode 100644 bin/release_analyzer delete mode 100755 bin/test create mode 100644 foo.rb create mode 100755 lib/pipeline/rpc_server.rb diff --git a/Gemfile b/Gemfile index bd6a04e..8afe7b9 100644 --- a/Gemfile +++ b/Gemfile @@ -4,9 +4,10 @@ gem "activesupport" gem "mandate" gem "propono" gem "rugged" -gem 'aws-sdk-ecr' +gem "aws-sdk-s3" +gem "aws-sdk-ecr" -gem 'simplecov', require: false, group: :test +gem "simplecov", require: false, group: :test gem "ffi-rzmq" diff --git a/Gemfile.lock b/Gemfile.lock index fd89365..c235d82 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,6 +16,13 @@ GEM aws-sdk-ecr (1.20.0) aws-sdk-core (~> 3, >= 3.61.1) aws-sigv4 (~> 1.1) + aws-sdk-kms (1.24.0) + aws-sdk-core (~> 3, >= 3.61.1) + aws-sigv4 (~> 1.1) + aws-sdk-s3 (1.46.0) + aws-sdk-core (~> 3, >= 3.61.1) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.1) aws-sdk-sns (1.19.0) aws-sdk-core (~> 3, >= 3.61.1) aws-sigv4 (~> 1.1) @@ -60,6 +67,7 @@ PLATFORMS DEPENDENCIES activesupport aws-sdk-ecr + aws-sdk-s3 ffi-rzmq mandate minitest diff --git a/bin/build_analyzer b/bin/build_analyzer deleted file mode 100755 index 2e3968d..0000000 --- a/bin/build_analyzer +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env ruby -require "bundler/setup" -$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) - -require "pipeline" -Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__)) -Pipeline.build_analyzer(ARGV[0]) - -# Pipeline.build_analyzer("ruby") -# Pipeline.release("ruby") -# Pipeline.analyzer("ruby") diff --git a/bin/builderd b/bin/builderd old mode 100755 new mode 100644 index 1addd11..4be9d11 --- a/bin/builderd +++ b/bin/builderd @@ -2,42 +2,7 @@ require "bundler/setup" $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) -require 'ffi-rzmq' +require "pipeline" -class PipelineRpcServer - - attr_reader :context, :socket - - def initialize - @context = ZMQ::Context.new(1) - @socket = context.socket(ZMQ::REP) - socket.bind("tcp://*:5555") - end - - def run - require "pipeline" - Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__)) - - loop do - request = '' - socket.recv_string(request) - puts "Received request. Data: #{request.inspect}" - if request.start_with? "build-analyzer_" - _, arg = request.split("_") - result = Pipeline.build_analyzer(arg) - socket.send_string(result.to_json) - elsif request.start_with? "release-analyzer_" - _, arg = request.split("_") - result = Pipeline.release(arg) - socket.send_string(result.to_json) - else - socket.send_string("done") - end - end - puts msg - socket.send_string(msg) - end - -end - -PipelineRpcServer.new.run +Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__)) +Pipeline.daemon diff --git a/bin/client.rb b/bin/client.rb new file mode 100644 index 0000000..410a0d0 --- /dev/null +++ b/bin/client.rb @@ -0,0 +1,37 @@ +require 'ffi-rzmq' +require 'json' +require 'yaml' +require 'securerandom' + +class PipelineClient + + attr_reader :context, :socket + + def initialize + @context = ZMQ::Context.new(1) + @socket = context.socket(ZMQ::REQ) + socket.setsockopt(ZMQ::LINGER, 0) + socket.connect("tcp://localhost:5555") + end + + def send_msg(msg) + socket.send_string(msg) + response = '' + rc = socket.recv_string(response) + parsed = JSON.parse(response) + parsed + end + + def build_analyzer(track_slug) + send_msg("build-analyzer_#{track_slug}") + end + + def release_latest(track_slug) + send_msg("release-analyzer_#{track_slug}") + end + + def analyze(track_slug, exercise_slug, solution_slug, iteration_folder) + send_msg("analyze_#{track_slug}|#{exercise_slug}|#{solution_slug}|#{iteration_folder}") + end + +end diff --git a/bin/invoke.rb b/bin/invoke.rb index 3684eef..ec8c66b 100644 --- a/bin/invoke.rb +++ b/bin/invoke.rb @@ -1,44 +1,13 @@ #!/usr/bin/env ruby -require 'ffi-rzmq' -require 'json' - -class PipelineClient - - attr_reader :context, :socket - - def initialize - @context = ZMQ::Context.new(1) - @socket = context.socket(ZMQ::REQ) - socket.setsockopt(ZMQ::LINGER, 0) - socket.connect("tcp://localhost:5555") - end - - def send_msg(msg) - socket.send_string(msg) - response = '' - rc = socket.recv_string(response) - parsed = JSON.parse(response) - parsed - end - - def build_analyzer(track_slug) - send_msg("build-analyzer_#{track_slug}") - end - - def release_latest(track_slug) - send_msg("release-analyzer_#{track_slug}") - end - -end +require_relative "./client" pipeline = PipelineClient.new +r = pipeline.analyze("ruby", "two-fer", "soln-42", "s3://exercism-dev/iterations/fff07700-e1c3-402d-8937-823aeefb159f") +r["logs"].each do |log_line| + puts "+ #{log_line["cmd"]}" + puts log_line["stdout"] + puts log_line["stderr"] +end -# result = pipeline.build_analyzer("ruby") -# result["logs"].each do |log_line| -# puts "+ #{log_line["cmd"]}" -# puts log_line["stdout"] -# puts log_line["stderr"] -# end - -pipeline.release_latest("ruby") +puts r["result"] diff --git a/bin/release_analyzer b/bin/release_analyzer deleted file mode 100644 index 1a70562..0000000 --- a/bin/release_analyzer +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env ruby -require "bundler/setup" -$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) - -require "pipeline" -Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__)) - -Pipeline.release(ARGV[0]) diff --git a/bin/test b/bin/test deleted file mode 100755 index c8c30e2..0000000 --- a/bin/test +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -bundle exec rake test diff --git a/foo.rb b/foo.rb new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/foo.rb @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/lib/pipeline.rb b/lib/pipeline.rb index e7e4e28..808b72e 100644 --- a/lib/pipeline.rb +++ b/lib/pipeline.rb @@ -4,8 +4,10 @@ require "active_support" require 'securerandom' require 'rugged' require 'aws-sdk-ecr' +require 'aws-sdk-s3' require 'yaml' require 'json' +require 'ffi-rzmq' module Pipeline @@ -21,13 +23,17 @@ module Pipeline @config end + def self.daemon + server = Pipeline::RpcServer.new + server.listen + end + def self.build_analyzer(track_slug) repo = Pipeline::AnalyzerRepo.for_track(track_slug) latest_tag = repo.tags.keys.last if (latest_tag.nil?) latest_tag = "master" end - puts latest_tag Pipeline::Build::AnalyzerBuild.(latest_tag, track_slug) end @@ -41,31 +47,32 @@ module Pipeline environment.release_analyzer(language_slug) end - def self.analyzer(language_slug) + def self.analyze!(language_slug, exercise_slug, solution_slug) env_base = "/tmp/analyzer-env/1e9c733fd7502974c2a3fdd85da9c844" environment = Runtime::RuntimeEnvironment.new(env_base) - analysis_run = environment.new_analysis(language_slug, "two-fer", 42) + analysis_run = environment.new_analysis(language_slug, exercise_slug, solution_slug) analysis_run.prepare_iteration do |iteration_folder| - File.write("#{iteration_folder}/two_fer.rb", 'puts "hello"') + yield(iteration_folder) end begin analysis_run.analyze! rescue => e puts e ensure - puts "---" - puts analysis_run.stdout - puts "===" - puts analysis_run.stderr - puts "---" - puts analysis_run.success? - puts analysis_run.exit_status - puts analysis_run.result + # puts "---" + # puts analysis_run.stdout + # puts "===" + # puts analysis_run.stderr + # puts "---" + # puts analysis_run.success? + # puts analysis_run.exit_status + # puts analysis_run.result puts "DONE" end end end +require "pipeline/rpc_server" require "pipeline/analyzer_repo" require "pipeline/validation/check_invokable" require "pipeline/validation/check_environment_invariants" diff --git a/lib/pipeline/analyzer_repo.rb b/lib/pipeline/analyzer_repo.rb index 18eb994..b0990e1 100644 --- a/lib/pipeline/analyzer_repo.rb +++ b/lib/pipeline/analyzer_repo.rb @@ -1,5 +1,3 @@ -require 'pp' - class Pipeline::AnalyzerRepo BASE_DIR = ENV.fetch("ANALYZER_REPO_BASE_DIR", "./tmp/repos") diff --git a/lib/pipeline/build/analyzer_build.rb b/lib/pipeline/build/analyzer_build.rb index 38b48e1..e1ad25a 100644 --- a/lib/pipeline/build/analyzer_build.rb +++ b/lib/pipeline/build/analyzer_build.rb @@ -12,7 +12,6 @@ module Pipeline::Build build validate publish - puts "DONE" { track: track_slug, image: image_name, @@ -34,7 +33,6 @@ module Pipeline::Build def build @image_tag = Pipeline::Build::BuildImage.(build_tag, image_name, repo, img) - puts ">>>> #{image_tag}" end def validate @@ -51,7 +49,6 @@ module Pipeline::Build end memoize - def repo Pipeline::AnalyzerRepo.for_track(track_slug) end diff --git a/lib/pipeline/rpc_server.rb b/lib/pipeline/rpc_server.rb new file mode 100755 index 0000000..567f8c9 --- /dev/null +++ b/lib/pipeline/rpc_server.rb @@ -0,0 +1,55 @@ +class Pipeline::RpcServer + + attr_reader :context, :socket + + def initialize + @context = ZMQ::Context.new(1) + @socket = context.socket(ZMQ::REP) + socket.bind("tcp://*:5555") + end + + def listen + loop do + request = '' + socket.recv_string(request) + puts "Received request. Data: #{request.inspect}" + if request.start_with? "build-analyzer_" + _, arg = request.split("_") + result = Pipeline.build_analyzer(arg) + socket.send_string(result.to_json) + elsif request.start_with? "release-analyzer_" + _, arg = request.split("_") + result = Pipeline.release(arg) + socket.send_string(result.to_json) + elsif request.start_with? "analyze_" + _, arg = request.split("_", 2) + track, exercise_slug, solution_slug, location = arg.split("|") + result = Pipeline.analyze!(track, exercise_slug, solution_slug) do |iteration_folder| + location_uri = URI(location) + bucket = location_uri.host + path = location_uri.path[1..] + s3 = Aws::S3::Client.new(region: 'eu-west-1') + params = { + bucket: bucket, + prefix: "#{path}/", + } + resp = s3.list_objects(params) + resp.contents.each do |item| + key = item[:key] + filename = File.basename(key) + s3.get_object({ + bucket: bucket, + key: key, + response_target: "#{iteration_folder}/#{filename}" + }) + end + end + socket.send_string(result.to_json) + else + socket.send_string("done") + end + end + socket.send_string(msg) + end + +end diff --git a/lib/pipeline/runtime/analysis_run.rb b/lib/pipeline/runtime/analysis_run.rb index 52d267f..75afa03 100644 --- a/lib/pipeline/runtime/analysis_run.rb +++ b/lib/pipeline/runtime/analysis_run.rb @@ -2,7 +2,8 @@ module Pipeline::Runtime class AnalysisRun attr_reader :track_dir, :exercise_slug, :runs_dir, :solution_dir, - :iteration_folder, :tmp_folder, :current_dir + :iteration_folder, :tmp_folder, :current_dir, :img, + :runc def initialize(track_dir, exercise_slug, solution_slug) @track_dir = track_dir @@ -12,6 +13,9 @@ module Pipeline::Runtime @solution_dir = "#{runs_dir}/iteration_#{Time.now.to_i}-#{solution_slug}-#{SecureRandom.hex}" @iteration_folder = "#{solution_dir}/iteration" @tmp_folder = "#{solution_dir}/tmp" + @logs = Pipeline::Util::LogCollector.new + @img = Pipeline::Util::ImgWrapper.new(@logs) + @runc = Pipeline::Util::RuncWrapper.new(@logs) end def prepare_iteration @@ -27,6 +31,14 @@ module Pipeline::Runtime def analyze! container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, solution_dir) @result = container_driver.run_analyzer_for(exercise_slug) + { + exercise_slug: exercise_slug, + solution_dir: solution_dir, + rootfs_source: rootfs_source, + result: result, + invocation: @result.report, + logs: @logs.inspect + } end def result @@ -60,14 +72,6 @@ module Pipeline::Runtime end end - def img - @img ||= Pipeline::Util::ImgWrapper.new - end - - def runc - @runc ||= Pipeline::Util::RuncWrapper.new - end - def rootfs_source @rootfs_source ||= begin release_folder = File.readlink(current_dir) diff --git a/lib/pipeline/runtime/runtime_environment.rb b/lib/pipeline/runtime/runtime_environment.rb index 4381f06..9710039 100644 --- a/lib/pipeline/runtime/runtime_environment.rb +++ b/lib/pipeline/runtime/runtime_environment.rb @@ -27,8 +27,6 @@ module Pipeline::Runtime container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, release_dir) - # container_driver.prepare_workdir - # container_driver.unpack_image("track_slug:master") ecr = Aws::ECR::Client.new(region: 'eu-west-1') authorization_token = ecr.get_authorization_token.authorization_data[0].authorization_token plain = Base64.decode64(authorization_token) @@ -64,45 +62,5 @@ module Pipeline::Runtime AnalysisRun.new(track_dir, exercise_slug, solution_slug) end - # def prepare_analysis(track_slug, solution_id) - # track_dir = "#{env_base}/#{track_slug}" - # runs_dir = "#{track_dir}/runs" - # current_dir = "#{track_dir}/current" - # solution_dir = "#{runs_dir}/iteration_#{Time.now.to_i}-#{solution_id}-#{SecureRandom.hex}" - # - # iteration_folder = "#{solution_dir}/iteration" - # tmp_folder = "#{solution_dir}/tmp" - # - # - # FileUtils.mkdir_p iteration_folder - # FileUtils.mkdir_p tmp_folder - # solution_dir - # end - # - # def run_analysis(track_slug, solution_dir, exercise_slug) - # track_dir = "#{env_base}/#{track_slug}" - # runs_dir = "#{track_dir}/runs" - # current_dir = "#{track_dir}/current" - # img = Pipeline::Util::ImgWrapper.new - # runc = Pipeline::Util::RuncWrapper.new - # configurator = Pipeline::Util::RuncConfigurator.new - # configurator.seed_from_env - # - # rootfs_source = "#{File.readlink(current_dir)}/rootfs" - # configurator.rootfs = rootfs_source - # - # configurator.setup_for_terminal_access - # File.write("#{solution_dir}/terminal_config.json", configurator.build.to_json) - # - # container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, solution_dir) - # container_driver.run_analyzer_for("two-fer") - # end - - def analyze_solution(track_slug, solution_id, exercise_slug) - iteration_folder = prepare_analysis - yield iteration_folder - run_analysis(iteration_folder, exercise_slug) - end - end end diff --git a/lib/pipeline/util/external_command.rb b/lib/pipeline/util/external_command.rb index 93e5c24..246f623 100644 --- a/lib/pipeline/util/external_command.rb +++ b/lib/pipeline/util/external_command.rb @@ -42,5 +42,14 @@ module Pipeline::Util def timeout=(timeout) @timeout = timeout end + + def report + { + cmd: cmd_string, + success: success?, + stdout: stdout, + stderr: stderr + } + end end end diff --git a/lib/pipeline/util/img_wrapper.rb b/lib/pipeline/util/img_wrapper.rb index d1739a6..6ff54f1 100644 --- a/lib/pipeline/util/img_wrapper.rb +++ b/lib/pipeline/util/img_wrapper.rb @@ -3,11 +3,11 @@ module Pipeline::Util attr_accessor :binary_path, :state_location, :suppress_output, :logs - def initialize + def initialize(logs) @binary_path = File.expand_path "./opt/img" @state_location = "/tmp/state-img" @suppress_output = false - @logs = Pipeline::Util::LogCollector.new + @logs = logs || Pipeline::Util::LogCollector.new end def build(local_tag) @@ -61,7 +61,7 @@ module Pipeline::Util run_cmd = ExternalCommand.new(cmd) run_cmd.call logs << run_cmd - raise "Failed #{cmd}" unless run_cmd.success? + raise "Failed #{cmd}" unless run_cmd.success? end end diff --git a/lib/pipeline/util/log_collector.rb b/lib/pipeline/util/log_collector.rb index cfdc1c3..0e82506 100644 --- a/lib/pipeline/util/log_collector.rb +++ b/lib/pipeline/util/log_collector.rb @@ -5,13 +5,8 @@ module Pipeline::Util @logs = [] end - def <<(external_result) - @logs << { - cmd: external_result.cmd_string, - success: external_result.success?, - stdout: external_result.stdout, - stderr: external_result.stderr - } + def <<(external_command) + @logs << external_command.report end def inspect diff --git a/lib/pipeline/util/runc_wrapper.rb b/lib/pipeline/util/runc_wrapper.rb index 168ae88..9cf0536 100644 --- a/lib/pipeline/util/runc_wrapper.rb +++ b/lib/pipeline/util/runc_wrapper.rb @@ -2,10 +2,11 @@ module Pipeline::Util class RuncWrapper attr_accessor :binary_path, :suppress_output, :memory_limit - def initialize + def initialize(logs) @binary_path = File.expand_path "./opt/runc" @suppress_output = false @memory_limit = 3000000 + @logs = logs || Pipeline::Util::LogCollector.new end def run(container_folder) @@ -17,10 +18,13 @@ module Pipeline::Util kill_cmd = ExternalCommand.new("#{binary_path} --root root-state kill #{container_id} KILL") Dir.chdir(container_folder) do + puts "HERE: #{@logs}" + @logs.class.to_s begin run_cmd.call + @logs << run_cmd ensure kill_cmd.call + @logs << kill_cmd end end