Building in a worker

bin/builder

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+require "bundler/setup"
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+
+require "pipeline"
+
+Pipeline::Cmd::BuilderDaemon.(ARGV)

lib/pipeline/analyzer_repo.rb

@@ -18,13 +18,25 @@ class Pipeline::AnalyzerRepo
     repo.fetch('origin')
   end
 
+  def valid_commit?(reference)
+    return false unless reference.match? /^[0-9a-f]{40}$/
+    repo.exists?(reference)
+  end
+
   def checkout(ref)
     if tags[ref]
       oid = tags[ref]
       repo.checkout(oid)
       return oid
+    elsif valid_commit?(ref)
+      repo.checkout(ref)
+      repo.reset(ref, :hard)
+      return ref
     else
+      puts "checkout #{ref}"
       ref_pointer = repo.checkout(ref)
+      puts "repo #{repo_dir}"
+      puts ref_pointer
       return ref_pointer.target.target.oid
     end
   end

lib/pipeline/build/analyzer_build.rb

@@ -5,6 +5,10 @@ module Pipeline::Build
       suffix = "-dev" unless ENV["env"] == "production"
       "#{track_slug}-analyzer#{suffix}"
     end
-    
+
+    def validate
+      Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}")
+    end
+
   end
 end

lib/pipeline/build/container_build.rb

@@ -7,8 +7,10 @@ module Pipeline::Build
     initialize_with :build_tag, :track_slug, :repo, :container_repo
 
     def call
+      puts "Setting up utilities"
       setup_utilities
       setup_remote_repo
+      fetch_code
       check_tag_exists
       if already_built?
         puts "already_built"
@@ -34,6 +36,10 @@ module Pipeline::Build
       }
     end
 
+    def fetch_code
+      repo.fetch!
+    end
+
     def setup_utilities
       @logs = Pipeline::Util::LogCollector.new
       @img = Pipeline::Util::ImgWrapper.new(@logs)
@@ -45,6 +51,7 @@ module Pipeline::Build
 
     def check_tag_exists
       return if build_tag == "master"
+      return if repo.valid_commit?(build_tag)
       raise "Build tag does not exist" unless repo.tags[build_tag]
     end
 
@@ -55,7 +62,6 @@ module Pipeline::Build
       puts "current: #{@container_repo.git_shas}"
       puts "repo: #{repo}"
       current_tags = @container_repo.git_shas
-      repo.fetch!
       target_sha = repo.checkout(build_tag)
       puts target_sha
       current_tags.include? target_sha
@@ -67,7 +73,6 @@ module Pipeline::Build
     end
 
     def validate
-      Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}")
     end
 
     def publish

lib/pipeline/build/representer_build.rb

@@ -0,0 +1,15 @@
+module Pipeline::Build
+  class RepresenterBuild < ContainerBuild
+
+    def image_name
+      suffix = "-dev" unless ENV["env"] == "production"
+      "#{track_slug}-representer#{suffix}"
+    end
+
+    def validate
+      # No validation implemented for this yet
+      # Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}")
+    end
+
+  end
+end

lib/pipeline/cmd/builder_daemon.rb

@@ -0,0 +1,65 @@
+require "docopt"
+
+module Pipeline::Cmd
+
+  class BuilderDaemon
+
+    SPEC = <<-DOCOPT
+    Exercism builder.
+
+    Usage:
+      #{__FILE__} listen <channel_address> <work_folder> [--dryrun]
+      #{__FILE__} -h | --help
+      #{__FILE__} --version
+
+    DOCOPT
+
+    include Mandate
+
+    initialize_with :argv
+
+    def call
+      puts "*** Exercism Worker ***"
+
+      begin
+        daemon.bootstrap
+        exit 0 if dryrun?
+
+        daemon.configure
+        exit 0 if configure?
+
+        daemon.listen
+      rescue Pipeline::Rpc::Worker::DaemonRestartException
+        puts "Restarting Daemon"
+        retry
+      end
+    end
+
+    def options
+      @options ||= begin
+        Docopt::docopt(SPEC, argv: argv)
+      rescue Docopt::Exit => e
+        puts e.message
+        exit 1
+      end
+    end
+
+    def dryrun?
+      options["--dryrun"] == true
+    end
+
+    def configure?
+      options["configure"] == true
+    end
+
+    def daemon
+      @daemon ||= begin
+        worker_identity = options["<identity>"]
+        channel_address = options["<channel_address>"]
+        env_base = options["<work_folder>"]
+        Pipeline::Rpc::Worker::Daemon.new(worker_identity, channel_address, env_base)
+      end
+    end
+
+  end
+end

lib/pipeline/container_repo.rb

@@ -68,6 +68,7 @@ class Pipeline::ContainerRepo
     tags = []
     images.image_ids.each do |image|
       tag = image.image_tag
+      next if tag.nil?
       # Only return git-based shas
       if tag.start_with?("sha-")
         tag = tag.gsub(/sha-/, "")

lib/pipeline/rpc/front_end_request.rb

@@ -46,6 +46,11 @@ module Pipeline::Rpc
       (Time.now.to_f * 1000)
     end
 
+    def default_timeout
+      return 300 if parsed_msg["action"] == "build_container"
+      5
+    end
+
     def send_reply(msg)
       @end = current_timestamp
       @duration_milliseconds = @end - @start

lib/pipeline/rpc/request_register.rb

@@ -6,7 +6,8 @@ module Pipeline::Rpc
     end
 
     def register(req)
-      timeout_at = Time.now.to_i + 5
+      timeout_seconds = req.default_timeout
+      timeout_at = Time.now.to_i + timeout_seconds
       @in_flight[req.raw_address] = {timeout: timeout_at, req: req}
     end
 

lib/pipeline/rpc/router.rb

@@ -8,6 +8,7 @@ module Pipeline::Rpc
       @response_port = 5556
       @notification_port = 5557
       @front_end_port = 5555
+      # @builder_port = 5557
 
       @zmq_context = zmq_context
 
@@ -38,6 +39,10 @@ module Pipeline::Rpc
         end
       end
 
+      # @builder_channel = {
+      #   "*" => WorkChannel.new(zmq_context, "tcp://*:#{@builder_port}")
+      # }
+
       @container_versions = {}
       config["workers"].each do |worker_class, worker_config|
         worker_class = worker_class.to_sym
@@ -97,6 +102,8 @@ module Pipeline::Rpc
           handle_with_worker(:test_runners, req)
         elsif action == "represent"
           handle_with_worker(:representers, req)
+        elsif action == "build_container"
+          handle_with_worker(:builders, req)
         elsif action == "restart_workers"
           force_worker_restart!
           req.send_result({ message: "Request accepted" })
@@ -130,7 +137,7 @@ module Pipeline::Rpc
     end
 
     def handle_with_worker(worker_class, req)
-      channel = @backend_channels[worker_class]
+      channel = select_channel(worker_class)
       if channel.nil?
         req.send_error({ status: :worker_class_unknown })
       else
@@ -138,15 +145,20 @@ module Pipeline::Rpc
       end
     end
 
+    def select_channel(worker_class)
+      # return @builder_channel if worker_class == :builders
+      @backend_channels[worker_class]
+    end
+
     def select_backend_and_forward(req, channel)
       track_slug = req.parsed_msg["track_slug"]
       backend = channel[track_slug]
-      if backend.worker_available?
+      if backend && backend.worker_available?
         forward(backend, req)
         return
       end
       backend = channel["*"]
-      if backend.worker_available?
+      if backend && backend.worker_available?
         forward(backend, req)
       else
         req.send_error({ status: :worker_unavailable })
@@ -182,11 +194,7 @@ module Pipeline::Rpc
       topics = req.parsed_msg["topics"] || ["*"]
       workqueue_addresses = []
 
-      puts channel
-      puts @backend_channels.keys
-      puts "------"
       channel_entry = @backend_channels[channel]
-      puts channel_entry.keys
       topics.each do |topic|
         next unless channel_entry.has_key?(topic)
         port = channel_entry[topic].port

lib/pipeline/rpc/worker/build_container_action.rb

@@ -0,0 +1,44 @@
+module Pipeline::Rpc::Worker
+
+  class BuildContainerAction < WorkerAction
+
+    attr_reader :reader, :return_address
+
+    def initialize(request, return_address)
+      @request = request
+      @return_address = return_address
+    end
+
+    def invoke
+      track_slug = request["track_slug"]
+      channel = request["channel"]
+      build_tag = request["git_reference"]
+      puts "Building #{build_tag}"
+      credentials = parse_credentials(request["context"])
+      container_repo = Pipeline::Runtime::RuntimeEnvironment.container_repo(channel, track_slug, credentials)
+      repo = Pipeline::Runtime::RuntimeEnvironment.source_repo(channel, track_slug)
+
+      result = case channel
+      when "static_analyzers"
+        Pipeline::Build::AnalyzerBuild.(build_tag, track_slug, repo, container_repo)
+      when "test_runners"
+        Pipeline::Build::TestRunnerBuild.(build_tag, track_slug, repo, container_repo)
+      when "representers"
+        Pipeline::Build::RepresenterBuild.(build_tag, track_slug, repo, container_repo)
+      else
+        raise "Unknown channel: #{channel}"
+      end
+      response = {return_address: return_address}
+
+      if @error
+        response[:msg_type] = :error_response
+        response.merge(@error)
+      else
+        response[:msg_type] = :response
+        response[:return_address] = return_address
+        response.merge(result)
+      end
+    end
+
+  end
+end

lib/pipeline/rpc/worker/work_socket_wrapper.rb

@@ -24,6 +24,8 @@ module Pipeline::Rpc::Worker
         RepresentAction.new(request, return_address)
       elsif action == "test_solution"
         TestRunnerAction.new(request, return_address)
+      elsif action == "build_container"
+        BuildContainerAction.new(request, return_address)
       else
         puts "HERE ELSE: #{request}"
       end

lib/pipeline/runtime/runtime_environment.rb

@@ -2,19 +2,34 @@ module Pipeline::Runtime
   class RuntimeEnvironment
 
     def self.container_repo(channel, language_slug, credentials)
+      suffix = "-dev" unless ENV["env"] == "production"
       container_slug = case channel
       when "static_analyzers"
-        "#{language_slug}-analyzer"
+        "#{language_slug}-analyzer#{suffix}"
       when "test_runners"
-        "#{language_slug}-test-runner"
+        "#{language_slug}-test-runner#{suffix}"
       when "representers"
-        "#{language_slug}-representer"
+        "#{language_slug}-representer#{suffix}"
       else
         raise "Unknown channel: #{channel}"
       end
       Pipeline::ContainerRepo.instance_for(container_slug, credentials)
     end
 
+    def self.source_repo(channel, language_slug)
+      suffix = case channel
+      when "static_analyzers"
+        "analyzer"
+      when "test_runners"
+        "test-runner"
+      when "representers"
+        "representer"
+      else
+        raise "Unknown channel: #{channel}"
+      end
+      Pipeline::AnalyzerRepo.for_track(language_slug, suffix)
+    end
+
     attr_reader :env_base
 
     def initialize(env_base)