Support all platform operations over zmq

Gemfile

@@ -4,9 +4,10 @@ gem "activesupport"
 gem "mandate"
 gem "propono"
 gem "rugged"
-gem 'aws-sdk-ecr'
+gem "aws-sdk-s3"
+gem "aws-sdk-ecr"
 
-gem 'simplecov', require: false, group: :test
+gem "simplecov", require: false, group: :test
 
 gem "ffi-rzmq"
 

Gemfile.lock

@@ -16,6 +16,13 @@ GEM
     aws-sdk-ecr (1.20.0)
       aws-sdk-core (~> 3, >= 3.61.1)
       aws-sigv4 (~> 1.1)
+    aws-sdk-kms (1.24.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.46.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.1)
     aws-sdk-sns (1.19.0)
       aws-sdk-core (~> 3, >= 3.61.1)
       aws-sigv4 (~> 1.1)
@@ -60,6 +67,7 @@ PLATFORMS
 DEPENDENCIES
   activesupport
   aws-sdk-ecr
+  aws-sdk-s3
   ffi-rzmq
   mandate
   minitest

bin/build_analyzer

@@ -1,11 +0,0 @@
-#!/usr/bin/env ruby
-require "bundler/setup"
-$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
-
-require "pipeline"
-Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
-Pipeline.build_analyzer(ARGV[0])
-
-# Pipeline.build_analyzer("ruby")
-# Pipeline.release("ruby")
-# Pipeline.analyzer("ruby")

bin/builderd

@@ -2,42 +2,7 @@
 require "bundler/setup"
 $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
 
-require 'ffi-rzmq'
+require "pipeline"
 
-class PipelineRpcServer
+Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
-
+Pipeline.daemon
-  attr_reader :context, :socket
-
-  def initialize
-    @context = ZMQ::Context.new(1)
-    @socket = context.socket(ZMQ::REP)
-    socket.bind("tcp://*:5555")
-  end
-
-  def run
-    require "pipeline"
-    Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
-
-    loop do
-      request = ''
-      socket.recv_string(request)
-      puts "Received request. Data: #{request.inspect}"
-      if request.start_with? "build-analyzer_"
-        _, arg = request.split("_")
-        result = Pipeline.build_analyzer(arg)
-        socket.send_string(result.to_json)
-      elsif request.start_with? "release-analyzer_"
-        _, arg = request.split("_")
-        result = Pipeline.release(arg)
-        socket.send_string(result.to_json)
-      else
-        socket.send_string("done")
-      end
-    end
-    puts msg
-    socket.send_string(msg)
-  end
-
-end
-
-PipelineRpcServer.new.run

bin/client.rb

@@ -0,0 +1,37 @@
+require 'ffi-rzmq'
+require 'json'
+require 'yaml'
+require 'securerandom'
+
+class PipelineClient
+
+  attr_reader :context, :socket
+
+  def initialize
+    @context = ZMQ::Context.new(1)
+    @socket = context.socket(ZMQ::REQ)
+    socket.setsockopt(ZMQ::LINGER, 0)
+    socket.connect("tcp://localhost:5555")
+  end
+
+  def send_msg(msg)
+    socket.send_string(msg)
+    response = ''
+    rc = socket.recv_string(response)
+    parsed = JSON.parse(response)
+    parsed
+  end
+
+  def build_analyzer(track_slug)
+    send_msg("build-analyzer_#{track_slug}")
+  end
+
+  def release_latest(track_slug)
+    send_msg("release-analyzer_#{track_slug}")
+  end
+
+  def analyze(track_slug, exercise_slug, solution_slug, iteration_folder)
+    send_msg("analyze_#{track_slug}|#{exercise_slug}|#{solution_slug}|#{iteration_folder}")
+  end
+
+end

bin/invoke.rb

@@ -1,44 +1,13 @@
 #!/usr/bin/env ruby
 
-require 'ffi-rzmq'
+require_relative "./client"
-require 'json'
-
-class PipelineClient
-
-  attr_reader :context, :socket
-
-  def initialize
-    @context = ZMQ::Context.new(1)
-    @socket = context.socket(ZMQ::REQ)
-    socket.setsockopt(ZMQ::LINGER, 0)
-    socket.connect("tcp://localhost:5555")
-  end
-
-  def send_msg(msg)
-    socket.send_string(msg)
-    response = ''
-    rc = socket.recv_string(response)
-    parsed = JSON.parse(response)
-    parsed
-  end
-
-  def build_analyzer(track_slug)
-    send_msg("build-analyzer_#{track_slug}")
-  end
-
-  def release_latest(track_slug)
-    send_msg("release-analyzer_#{track_slug}")
-  end
-
-end
 
 pipeline = PipelineClient.new
+r = pipeline.analyze("ruby", "two-fer", "soln-42", "s3://exercism-dev/iterations/fff07700-e1c3-402d-8937-823aeefb159f")
+r["logs"].each do |log_line|
+  puts "+ #{log_line["cmd"]}"
+  puts log_line["stdout"]
+  puts log_line["stderr"]
+end
 
-# result = pipeline.build_analyzer("ruby")
+puts r["result"]
-# result["logs"].each do |log_line|
-#   puts "+ #{log_line["cmd"]}"
-#   puts log_line["stdout"]
-#   puts log_line["stderr"]
-# end
-
-pipeline.release_latest("ruby")

bin/release_analyzer

@@ -1,8 +0,0 @@
-#!/usr/bin/env ruby
-require "bundler/setup"
-$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
-
-require "pipeline"
-Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
-
-Pipeline.release(ARGV[0])

bin/test

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-bundle exec rake test

foo.rb

@@ -0,0 +1 @@
+hello

lib/pipeline.rb

@@ -4,8 +4,10 @@ require "active_support"
 require 'securerandom'
 require 'rugged'
 require 'aws-sdk-ecr'
+require 'aws-sdk-s3'
 require 'yaml'
 require 'json'
+require 'ffi-rzmq'
 
 module Pipeline
 
@@ -21,13 +23,17 @@ module Pipeline
     @config
   end
 
+  def self.daemon
+    server = Pipeline::RpcServer.new
+    server.listen
+  end
+
   def self.build_analyzer(track_slug)
     repo = Pipeline::AnalyzerRepo.for_track(track_slug)
     latest_tag = repo.tags.keys.last
     if (latest_tag.nil?)
       latest_tag = "master"
     end
-    puts latest_tag
     Pipeline::Build::AnalyzerBuild.(latest_tag, track_slug)
   end
 
@@ -41,31 +47,32 @@ module Pipeline
     environment.release_analyzer(language_slug)
   end
 
-  def self.analyzer(language_slug)
+  def self.analyze!(language_slug, exercise_slug, solution_slug)
     env_base = "/tmp/analyzer-env/1e9c733fd7502974c2a3fdd85da9c844"
     environment = Runtime::RuntimeEnvironment.new(env_base)
-    analysis_run = environment.new_analysis(language_slug, "two-fer", 42)
+    analysis_run = environment.new_analysis(language_slug, exercise_slug, solution_slug)
     analysis_run.prepare_iteration do |iteration_folder|
-      File.write("#{iteration_folder}/two_fer.rb", 'puts "hello"')
+      yield(iteration_folder)
     end
     begin
       analysis_run.analyze!
     rescue => e
       puts e
     ensure
-      puts "---"
+      # puts "---"
-      puts analysis_run.stdout
+      # puts analysis_run.stdout
-      puts "==="
+      # puts "==="
-      puts analysis_run.stderr
+      # puts analysis_run.stderr
-      puts "---"
+      # puts "---"
-      puts analysis_run.success?
+      # puts analysis_run.success?
-      puts analysis_run.exit_status
+      # puts analysis_run.exit_status
-      puts analysis_run.result
+      # puts analysis_run.result
       puts "DONE"
     end
   end
 end
 
+require "pipeline/rpc_server"
 require "pipeline/analyzer_repo"
 require "pipeline/validation/check_invokable"
 require "pipeline/validation/check_environment_invariants"

lib/pipeline/analyzer_repo.rb

@@ -1,5 +1,3 @@
-require 'pp'
-
 class Pipeline::AnalyzerRepo
 
   BASE_DIR = ENV.fetch("ANALYZER_REPO_BASE_DIR", "./tmp/repos")

lib/pipeline/build/analyzer_build.rb

@@ -12,7 +12,6 @@ module Pipeline::Build
       build
       validate
       publish
-      puts "DONE"
       {
         track: track_slug,
         image: image_name,
@@ -34,7 +33,6 @@ module Pipeline::Build
 
     def build
       @image_tag = Pipeline::Build::BuildImage.(build_tag, image_name, repo, img)
-      puts ">>>> #{image_tag}"
     end
 
     def validate
@@ -51,7 +49,6 @@ module Pipeline::Build
     end
 
     memoize
-
     def repo
       Pipeline::AnalyzerRepo.for_track(track_slug)
     end

lib/pipeline/rpc_server.rb

@@ -0,0 +1,55 @@
+class Pipeline::RpcServer
+
+  attr_reader :context, :socket
+
+  def initialize
+    @context = ZMQ::Context.new(1)
+    @socket = context.socket(ZMQ::REP)
+    socket.bind("tcp://*:5555")
+  end
+
+  def listen
+    loop do
+      request = ''
+      socket.recv_string(request)
+      puts "Received request. Data: #{request.inspect}"
+      if request.start_with? "build-analyzer_"
+        _, arg = request.split("_")
+        result = Pipeline.build_analyzer(arg)
+        socket.send_string(result.to_json)
+      elsif request.start_with? "release-analyzer_"
+        _, arg = request.split("_")
+        result = Pipeline.release(arg)
+        socket.send_string(result.to_json)
+      elsif request.start_with? "analyze_"
+        _, arg = request.split("_", 2)
+        track, exercise_slug, solution_slug, location = arg.split("|")
+        result = Pipeline.analyze!(track, exercise_slug, solution_slug) do |iteration_folder|
+          location_uri = URI(location)
+          bucket = location_uri.host
+          path = location_uri.path[1..]
+          s3 = Aws::S3::Client.new(region: 'eu-west-1')
+          params = {
+            bucket: bucket,
+            prefix: "#{path}/",
+          }
+          resp = s3.list_objects(params)
+          resp.contents.each do |item|
+            key = item[:key]
+            filename = File.basename(key)
+            s3.get_object({
+              bucket: bucket,
+              key: key,
+              response_target: "#{iteration_folder}/#{filename}"
+            })
+          end
+        end
+        socket.send_string(result.to_json)
+      else
+        socket.send_string("done")
+      end
+    end
+    socket.send_string(msg)
+  end
+
+end

lib/pipeline/runtime/analysis_run.rb

@@ -2,7 +2,8 @@ module Pipeline::Runtime
   class AnalysisRun
 
     attr_reader :track_dir, :exercise_slug, :runs_dir, :solution_dir,
-                :iteration_folder, :tmp_folder, :current_dir
+                :iteration_folder, :tmp_folder, :current_dir, :img,
+                :runc
 
     def initialize(track_dir, exercise_slug, solution_slug)
       @track_dir = track_dir
@@ -12,6 +13,9 @@ module Pipeline::Runtime
       @solution_dir = "#{runs_dir}/iteration_#{Time.now.to_i}-#{solution_slug}-#{SecureRandom.hex}"
       @iteration_folder = "#{solution_dir}/iteration"
       @tmp_folder = "#{solution_dir}/tmp"
+      @logs = Pipeline::Util::LogCollector.new
+      @img  = Pipeline::Util::ImgWrapper.new(@logs)
+      @runc = Pipeline::Util::RuncWrapper.new(@logs)
     end
 
     def prepare_iteration
@@ -27,6 +31,14 @@ module Pipeline::Runtime
     def analyze!
       container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, solution_dir)
       @result = container_driver.run_analyzer_for(exercise_slug)
+      {
+        exercise_slug: exercise_slug,
+        solution_dir: solution_dir,
+        rootfs_source: rootfs_source,
+        result: result,
+        invocation: @result.report,
+        logs: @logs.inspect
+      }
     end
 
     def result
@@ -60,14 +72,6 @@ module Pipeline::Runtime
       end
     end
 
-    def img
-      @img ||= Pipeline::Util::ImgWrapper.new
-    end
-
-    def runc
-      @runc ||= Pipeline::Util::RuncWrapper.new
-    end
-
     def rootfs_source
       @rootfs_source ||= begin
         release_folder = File.readlink(current_dir)

lib/pipeline/runtime/runtime_environment.rb

@@ -27,8 +27,6 @@ module Pipeline::Runtime
 
       container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, release_dir)
 
-      # container_driver.prepare_workdir
-      # container_driver.unpack_image("track_slug:master")
       ecr = Aws::ECR::Client.new(region: 'eu-west-1')
       authorization_token = ecr.get_authorization_token.authorization_data[0].authorization_token
       plain = Base64.decode64(authorization_token)
@@ -64,45 +62,5 @@ module Pipeline::Runtime
       AnalysisRun.new(track_dir, exercise_slug, solution_slug)
     end
 
-    # def prepare_analysis(track_slug, solution_id)
-    #   track_dir = "#{env_base}/#{track_slug}"
-    #   runs_dir = "#{track_dir}/runs"
-    #   current_dir = "#{track_dir}/current"
-    #   solution_dir = "#{runs_dir}/iteration_#{Time.now.to_i}-#{solution_id}-#{SecureRandom.hex}"
-    #
-    #   iteration_folder = "#{solution_dir}/iteration"
-    #   tmp_folder = "#{solution_dir}/tmp"
-    #
-    #
-    #   FileUtils.mkdir_p iteration_folder
-    #   FileUtils.mkdir_p tmp_folder
-    #   solution_dir
-    # end
-    #
-    # def run_analysis(track_slug, solution_dir, exercise_slug)
-    #   track_dir = "#{env_base}/#{track_slug}"
-    #   runs_dir = "#{track_dir}/runs"
-    #   current_dir = "#{track_dir}/current"
-    #   img = Pipeline::Util::ImgWrapper.new
-    #   runc = Pipeline::Util::RuncWrapper.new
-    #   configurator = Pipeline::Util::RuncConfigurator.new
-    #   configurator.seed_from_env
-    #
-    #   rootfs_source = "#{File.readlink(current_dir)}/rootfs"
-    #   configurator.rootfs = rootfs_source
-    #
-    #   configurator.setup_for_terminal_access
-    #   File.write("#{solution_dir}/terminal_config.json", configurator.build.to_json)
-    #
-    #   container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, solution_dir)
-    #   container_driver.run_analyzer_for("two-fer")
-    # end
-
-    def analyze_solution(track_slug, solution_id, exercise_slug)
-      iteration_folder = prepare_analysis
-      yield iteration_folder
-      run_analysis(iteration_folder, exercise_slug)
-    end
-
   end
 end

lib/pipeline/util/external_command.rb

@@ -42,5 +42,14 @@ module Pipeline::Util
     def timeout=(timeout)
       @timeout = timeout
     end
+
+    def report
+      {
+        cmd: cmd_string,
+        success: success?,
+        stdout: stdout,
+        stderr: stderr
+      }
+    end
   end
 end

lib/pipeline/util/img_wrapper.rb

@@ -3,11 +3,11 @@ module Pipeline::Util
 
     attr_accessor :binary_path, :state_location, :suppress_output, :logs
 
-    def initialize
+    def initialize(logs)
       @binary_path = File.expand_path "./opt/img"
       @state_location = "/tmp/state-img"
       @suppress_output = false
-      @logs = Pipeline::Util::LogCollector.new
+      @logs = logs || Pipeline::Util::LogCollector.new
     end
 
     def build(local_tag)

lib/pipeline/util/log_collector.rb

@@ -5,13 +5,8 @@ module Pipeline::Util
       @logs = []
     end
 
-    def <<(external_result)
+    def <<(external_command)
-      @logs << {
+      @logs << external_command.report
-        cmd: external_result.cmd_string,
-        success: external_result.success?,
-        stdout: external_result.stdout,
-        stderr:  external_result.stderr
-      }
     end
 
     def inspect

lib/pipeline/util/runc_wrapper.rb

@@ -2,10 +2,11 @@ module Pipeline::Util
   class RuncWrapper
     attr_accessor :binary_path, :suppress_output, :memory_limit
 
-    def initialize
+    def initialize(logs)
       @binary_path = File.expand_path "./opt/runc"
       @suppress_output = false
       @memory_limit = 3000000
+      @logs = logs || Pipeline::Util::LogCollector.new
     end
 
     def run(container_folder)
@@ -17,10 +18,13 @@ module Pipeline::Util
       kill_cmd = ExternalCommand.new("#{binary_path} --root root-state kill #{container_id} KILL")
 
       Dir.chdir(container_folder) do
+        puts "HERE: #{@logs}" + @logs.class.to_s
         begin
           run_cmd.call
+          @logs << run_cmd
         ensure
           kill_cmd.call
+          @logs << kill_cmd
         end
       end