diff --git a/bin/builder b/bin/builder new file mode 100755 index 0000000..7527fba --- /dev/null +++ b/bin/builder @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby +require "bundler/setup" +$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) + +require "pipeline" + +Pipeline::Cmd::BuilderDaemon.(ARGV) diff --git a/lib/pipeline/analyzer_repo.rb b/lib/pipeline/analyzer_repo.rb index 6267f87..dd3d7a1 100644 --- a/lib/pipeline/analyzer_repo.rb +++ b/lib/pipeline/analyzer_repo.rb @@ -18,13 +18,25 @@ class Pipeline::AnalyzerRepo repo.fetch('origin') end + def valid_commit?(reference) + return false unless reference.match? /^[0-9a-f]{40}$/ + repo.exists?(reference) + end + def checkout(ref) if tags[ref] oid = tags[ref] repo.checkout(oid) return oid + elsif valid_commit?(ref) + repo.checkout(ref) + repo.reset(ref, :hard) + return ref else + puts "checkout #{ref}" ref_pointer = repo.checkout(ref) + puts "repo #{repo_dir}" + puts ref_pointer return ref_pointer.target.target.oid end end diff --git a/lib/pipeline/build/analyzer_build.rb b/lib/pipeline/build/analyzer_build.rb index d46fc17..c51968d 100644 --- a/lib/pipeline/build/analyzer_build.rb +++ b/lib/pipeline/build/analyzer_build.rb @@ -5,6 +5,10 @@ module Pipeline::Build suffix = "-dev" unless ENV["env"] == "production" "#{track_slug}-analyzer#{suffix}" end - + + def validate + Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}") + end + end end diff --git a/lib/pipeline/build/container_build.rb b/lib/pipeline/build/container_build.rb index bf3280a..495f558 100644 --- a/lib/pipeline/build/container_build.rb +++ b/lib/pipeline/build/container_build.rb @@ -7,8 +7,10 @@ module Pipeline::Build initialize_with :build_tag, :track_slug, :repo, :container_repo def call + puts "Setting up utilities" setup_utilities setup_remote_repo + fetch_code check_tag_exists if already_built? puts "already_built" @@ -34,6 +36,10 @@ module Pipeline::Build } end + def fetch_code + repo.fetch! + end + def setup_utilities @logs = Pipeline::Util::LogCollector.new @img = Pipeline::Util::ImgWrapper.new(@logs) @@ -45,6 +51,7 @@ module Pipeline::Build def check_tag_exists return if build_tag == "master" + return if repo.valid_commit?(build_tag) raise "Build tag does not exist" unless repo.tags[build_tag] end @@ -55,7 +62,6 @@ module Pipeline::Build puts "current: #{@container_repo.git_shas}" puts "repo: #{repo}" current_tags = @container_repo.git_shas - repo.fetch! target_sha = repo.checkout(build_tag) puts target_sha current_tags.include? target_sha @@ -67,7 +73,6 @@ module Pipeline::Build end def validate - Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}") end def publish diff --git a/lib/pipeline/build/representer_build.rb b/lib/pipeline/build/representer_build.rb new file mode 100644 index 0000000..dea0692 --- /dev/null +++ b/lib/pipeline/build/representer_build.rb @@ -0,0 +1,15 @@ +module Pipeline::Build + class RepresenterBuild < ContainerBuild + + def image_name + suffix = "-dev" unless ENV["env"] == "production" + "#{track_slug}-representer#{suffix}" + end + + def validate + # No validation implemented for this yet + # Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}") + end + + end +end diff --git a/lib/pipeline/cmd/builder_daemon.rb b/lib/pipeline/cmd/builder_daemon.rb new file mode 100644 index 0000000..621243f --- /dev/null +++ b/lib/pipeline/cmd/builder_daemon.rb @@ -0,0 +1,65 @@ +require "docopt" + +module Pipeline::Cmd + + class BuilderDaemon + + SPEC = <<-DOCOPT + Exercism builder. + + Usage: + #{__FILE__} listen [--dryrun] + #{__FILE__} -h | --help + #{__FILE__} --version + + DOCOPT + + include Mandate + + initialize_with :argv + + def call + puts "*** Exercism Worker ***" + + begin + daemon.bootstrap + exit 0 if dryrun? + + daemon.configure + exit 0 if configure? + + daemon.listen + rescue Pipeline::Rpc::Worker::DaemonRestartException + puts "Restarting Daemon" + retry + end + end + + def options + @options ||= begin + Docopt::docopt(SPEC, argv: argv) + rescue Docopt::Exit => e + puts e.message + exit 1 + end + end + + def dryrun? + options["--dryrun"] == true + end + + def configure? + options["configure"] == true + end + + def daemon + @daemon ||= begin + worker_identity = options[""] + channel_address = options[""] + env_base = options[""] + Pipeline::Rpc::Worker::Daemon.new(worker_identity, channel_address, env_base) + end + end + + end +end diff --git a/lib/pipeline/container_repo.rb b/lib/pipeline/container_repo.rb index ec6ba21..7a83f18 100644 --- a/lib/pipeline/container_repo.rb +++ b/lib/pipeline/container_repo.rb @@ -68,6 +68,7 @@ class Pipeline::ContainerRepo tags = [] images.image_ids.each do |image| tag = image.image_tag + next if tag.nil? # Only return git-based shas if tag.start_with?("sha-") tag = tag.gsub(/sha-/, "") diff --git a/lib/pipeline/rpc/front_end_request.rb b/lib/pipeline/rpc/front_end_request.rb index d03ae28..e563b27 100644 --- a/lib/pipeline/rpc/front_end_request.rb +++ b/lib/pipeline/rpc/front_end_request.rb @@ -46,6 +46,11 @@ module Pipeline::Rpc (Time.now.to_f * 1000) end + def default_timeout + return 300 if parsed_msg["action"] == "build_container" + 5 + end + def send_reply(msg) @end = current_timestamp @duration_milliseconds = @end - @start diff --git a/lib/pipeline/rpc/request_register.rb b/lib/pipeline/rpc/request_register.rb index efb1daa..3691145 100644 --- a/lib/pipeline/rpc/request_register.rb +++ b/lib/pipeline/rpc/request_register.rb @@ -6,7 +6,8 @@ module Pipeline::Rpc end def register(req) - timeout_at = Time.now.to_i + 5 + timeout_seconds = req.default_timeout + timeout_at = Time.now.to_i + timeout_seconds @in_flight[req.raw_address] = {timeout: timeout_at, req: req} end diff --git a/lib/pipeline/rpc/router.rb b/lib/pipeline/rpc/router.rb index 53f4810..7f3b82d 100644 --- a/lib/pipeline/rpc/router.rb +++ b/lib/pipeline/rpc/router.rb @@ -8,6 +8,7 @@ module Pipeline::Rpc @response_port = 5556 @notification_port = 5557 @front_end_port = 5555 + # @builder_port = 5557 @zmq_context = zmq_context @@ -38,6 +39,10 @@ module Pipeline::Rpc end end + # @builder_channel = { + # "*" => WorkChannel.new(zmq_context, "tcp://*:#{@builder_port}") + # } + @container_versions = {} config["workers"].each do |worker_class, worker_config| worker_class = worker_class.to_sym @@ -97,6 +102,8 @@ module Pipeline::Rpc handle_with_worker(:test_runners, req) elsif action == "represent" handle_with_worker(:representers, req) + elsif action == "build_container" + handle_with_worker(:builders, req) elsif action == "restart_workers" force_worker_restart! req.send_result({ message: "Request accepted" }) @@ -130,7 +137,7 @@ module Pipeline::Rpc end def handle_with_worker(worker_class, req) - channel = @backend_channels[worker_class] + channel = select_channel(worker_class) if channel.nil? req.send_error({ status: :worker_class_unknown }) else @@ -138,15 +145,20 @@ module Pipeline::Rpc end end + def select_channel(worker_class) + # return @builder_channel if worker_class == :builders + @backend_channels[worker_class] + end + def select_backend_and_forward(req, channel) track_slug = req.parsed_msg["track_slug"] backend = channel[track_slug] - if backend.worker_available? + if backend && backend.worker_available? forward(backend, req) return end backend = channel["*"] - if backend.worker_available? + if backend && backend.worker_available? forward(backend, req) else req.send_error({ status: :worker_unavailable }) @@ -182,11 +194,7 @@ module Pipeline::Rpc topics = req.parsed_msg["topics"] || ["*"] workqueue_addresses = [] - puts channel - puts @backend_channels.keys - puts "------" channel_entry = @backend_channels[channel] - puts channel_entry.keys topics.each do |topic| next unless channel_entry.has_key?(topic) port = channel_entry[topic].port diff --git a/lib/pipeline/rpc/worker/build_container_action.rb b/lib/pipeline/rpc/worker/build_container_action.rb new file mode 100644 index 0000000..09d3a54 --- /dev/null +++ b/lib/pipeline/rpc/worker/build_container_action.rb @@ -0,0 +1,44 @@ +module Pipeline::Rpc::Worker + + class BuildContainerAction < WorkerAction + + attr_reader :reader, :return_address + + def initialize(request, return_address) + @request = request + @return_address = return_address + end + + def invoke + track_slug = request["track_slug"] + channel = request["channel"] + build_tag = request["git_reference"] + puts "Building #{build_tag}" + credentials = parse_credentials(request["context"]) + container_repo = Pipeline::Runtime::RuntimeEnvironment.container_repo(channel, track_slug, credentials) + repo = Pipeline::Runtime::RuntimeEnvironment.source_repo(channel, track_slug) + + result = case channel + when "static_analyzers" + Pipeline::Build::AnalyzerBuild.(build_tag, track_slug, repo, container_repo) + when "test_runners" + Pipeline::Build::TestRunnerBuild.(build_tag, track_slug, repo, container_repo) + when "representers" + Pipeline::Build::RepresenterBuild.(build_tag, track_slug, repo, container_repo) + else + raise "Unknown channel: #{channel}" + end + response = {return_address: return_address} + + if @error + response[:msg_type] = :error_response + response.merge(@error) + else + response[:msg_type] = :response + response[:return_address] = return_address + response.merge(result) + end + end + + end +end diff --git a/lib/pipeline/rpc/worker/work_socket_wrapper.rb b/lib/pipeline/rpc/worker/work_socket_wrapper.rb index aa0a86b..31a62aa 100644 --- a/lib/pipeline/rpc/worker/work_socket_wrapper.rb +++ b/lib/pipeline/rpc/worker/work_socket_wrapper.rb @@ -24,6 +24,8 @@ module Pipeline::Rpc::Worker RepresentAction.new(request, return_address) elsif action == "test_solution" TestRunnerAction.new(request, return_address) + elsif action == "build_container" + BuildContainerAction.new(request, return_address) else puts "HERE ELSE: #{request}" end diff --git a/lib/pipeline/runtime/runtime_environment.rb b/lib/pipeline/runtime/runtime_environment.rb index d530073..b81d88c 100644 --- a/lib/pipeline/runtime/runtime_environment.rb +++ b/lib/pipeline/runtime/runtime_environment.rb @@ -2,19 +2,34 @@ module Pipeline::Runtime class RuntimeEnvironment def self.container_repo(channel, language_slug, credentials) + suffix = "-dev" unless ENV["env"] == "production" container_slug = case channel when "static_analyzers" - "#{language_slug}-analyzer" + "#{language_slug}-analyzer#{suffix}" when "test_runners" - "#{language_slug}-test-runner" + "#{language_slug}-test-runner#{suffix}" when "representers" - "#{language_slug}-representer" + "#{language_slug}-representer#{suffix}" else raise "Unknown channel: #{channel}" end Pipeline::ContainerRepo.instance_for(container_slug, credentials) end + def self.source_repo(channel, language_slug) + suffix = case channel + when "static_analyzers" + "analyzer" + when "test_runners" + "test-runner" + when "representers" + "representer" + else + raise "Unknown channel: #{channel}" + end + Pipeline::AnalyzerRepo.for_track(language_slug, suffix) + end + attr_reader :env_base def initialize(env_base)