Add RPC daemon

Gemfile

@@ -8,6 +8,8 @@ gem 'aws-sdk-ecr'
 
 gem 'simplecov', require: false, group: :test
 
+gem "ffi-rzmq"
+
 group :development, :test do
   # gem "bundler"
   gem "rake"

Gemfile.lock

@@ -26,6 +26,11 @@ GEM
       aws-eventstream (~> 1.0, >= 1.0.2)
     concurrent-ruby (1.1.5)
     docile (1.3.2)
+    ffi (1.11.1)
+    ffi-rzmq (2.0.7)
+      ffi-rzmq-core (>= 1.0.7)
+    ffi-rzmq-core (1.0.7)
+      ffi
     i18n (1.6.0)
       concurrent-ruby (~> 1.0)
     jmespath (1.4.0)
@@ -55,6 +60,7 @@ PLATFORMS
 DEPENDENCIES
   activesupport
   aws-sdk-ecr
+  ffi-rzmq
   mandate
   minitest
   mocha

bin/builderd

@@ -1,3 +1,43 @@
 #!/usr/bin/env ruby
+require "bundler/setup"
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
 
-puts "OK"
+require 'ffi-rzmq'
+
+class PipelineRpcServer
+
+  attr_reader :context, :socket
+
+  def initialize
+    @context = ZMQ::Context.new(1)
+    @socket = context.socket(ZMQ::REP)
+    socket.bind("tcp://*:5555")
+  end
+
+  def run
+    require "pipeline"
+    Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
+
+    loop do
+      request = ''
+      socket.recv_string(request)
+      puts "Received request. Data: #{request.inspect}"
+      if request.start_with? "build-analyzer_"
+        _, arg = request.split("_")
+        result = Pipeline.build_analyzer(arg)
+        socket.send_string(result.to_json)
+      elsif request.start_with? "release-analyzer_"
+        _, arg = request.split("_")
+        result = Pipeline.release(arg)
+        socket.send_string(result.to_json)
+      else
+        socket.send_string("done")
+      end
+    end
+    puts msg
+    socket.send_string(msg)
+  end
+
+end
+
+PipelineRpcServer.new.run

bin/invoke.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require 'ffi-rzmq'
+require 'json'
+
+class PipelineClient
+
+  attr_reader :context, :socket
+
+  def initialize
+    @context = ZMQ::Context.new(1)
+    @socket = context.socket(ZMQ::REQ)
+    socket.setsockopt(ZMQ::LINGER, 0)
+    socket.connect("tcp://localhost:5555")
+  end
+
+  def send_msg(msg)
+    socket.send_string(msg)
+    response = ''
+    rc = socket.recv_string(response)
+    parsed = JSON.parse(response)
+    parsed
+  end
+
+  def build_analyzer(track_slug)
+    send_msg("build-analyzer_#{track_slug}")
+  end
+
+  def release_latest(track_slug)
+    send_msg("release-analyzer_#{track_slug}")
+  end
+
+end
+
+pipeline = PipelineClient.new
+
+# result = pipeline.build_analyzer("ruby")
+# result["logs"].each do |log_line|
+#   puts "+ #{log_line["cmd"]}"
+#   puts log_line["stdout"]
+#   puts log_line["stderr"]
+# end
+
+pipeline.release_latest("ruby")

lib/pipeline.rb

@@ -5,6 +5,7 @@ require 'securerandom'
 require 'rugged'
 require 'aws-sdk-ecr'
 require 'yaml'
+require 'json'
 
 module Pipeline
 
@@ -76,6 +77,7 @@ require "pipeline/util/runc_configurator"
 require "pipeline/util/img_wrapper"
 require "pipeline/util/runc_wrapper"
 require "pipeline/util/external_command"
+require "pipeline/util/log_collector"
 require "pipeline/build/build_image"
 require "pipeline/build/publish_image"
 require "pipeline/build/analyzer_build"

lib/pipeline/build/analyzer_build.rb

@@ -12,6 +12,15 @@ module Pipeline::Build
       build
       validate
       publish
+      puts "DONE"
+      {
+        track: track_slug,
+        image: image_name,
+        image_tag: image_tag,
+        git_sha: target_sha,
+        git_tag: build_tag,
+        logs: img.logs.inspect
+      }
     end
 
     def setup_utilities
@@ -25,6 +34,7 @@ module Pipeline::Build
 
     def build
       @image_tag = Pipeline::Build::BuildImage.(build_tag, image_name, repo, img)
+      puts ">>>> #{image_tag}"
     end
 
     def validate

lib/pipeline/build/build_image.rb

@@ -10,6 +10,7 @@ module Pipeline::Build
       repo.fetch!
       checkout
       build
+      local_tag
     end
 
     def checkout
@@ -20,7 +21,6 @@ module Pipeline::Build
       Dir.chdir(repo.workdir) do
         img.build(local_tag)
       end
-      local_tag
     end
 
     def local_tag

lib/pipeline/util/container_driver.rb

@@ -16,6 +16,7 @@ module Pipeline::Util
     end
 
     def unpack_image(build_tag)
+      puts "unpack #{build_tag}"
       Dir.chdir(workdir) do
         img.unpack(build_tag)
       end

lib/pipeline/util/external_command.rb

@@ -1,3 +1,5 @@
+require 'open3'
+
 module Pipeline::Util
   class ExternalCommand
 

lib/pipeline/util/img_wrapper.rb

@@ -1,12 +1,13 @@
 module Pipeline::Util
   class ImgWrapper
 
-    attr_accessor :binary_path, :state_location, :suppress_output
+    attr_accessor :binary_path, :state_location, :suppress_output, :logs
 
     def initialize
       @binary_path = File.expand_path "./opt/img"
       @state_location = "/tmp/state-img"
       @suppress_output = false
+      @logs = Pipeline::Util::LogCollector.new
     end
 
     def build(local_tag)
@@ -57,8 +58,10 @@ module Pipeline::Util
     def exec_cmd(cmd)
       puts "> #{cmd}" unless suppress_output
       puts "------------------------------------------------------------" unless suppress_output
-      success = system({}, cmd)
-      raise "Failed #{cmd}" unless success
+      run_cmd = ExternalCommand.new(cmd)
+      run_cmd.call
+      logs << run_cmd
+      raise "Failed #{cmd}" unless run_cmd.success?      
     end
 
   end

lib/pipeline/util/log_collector.rb

@@ -0,0 +1,22 @@
+module Pipeline::Util
+  class LogCollector
+
+    def initialize
+      @logs = []
+    end
+
+    def <<(external_result)
+      @logs << {
+        cmd: external_result.cmd_string,
+        success: external_result.success?,
+        stdout: external_result.stdout,
+        stderr:  external_result.stderr
+      }
+    end
+
+    def inspect
+      @logs
+    end
+
+  end
+end

lib/pipeline/util/runc_wrapper.rb

@@ -1,6 +1,3 @@
-require 'open3'
-
-
 module Pipeline::Util
   class RuncWrapper
     attr_accessor :binary_path, :suppress_output, :memory_limit