Building test-runners

bin/builderd

@@ -5,4 +5,6 @@ $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
 require "pipeline"
 
 Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
-Pipeline.daemon
+# Pipeline.daemon
+# Pipeline.build_analyzer "ruby"
+Pipeline.build_test_runner "ruby"

bin/invoke.rb

@@ -8,17 +8,18 @@ pipeline = PipelineClient.new
 lang = ARGV[0] || "ruby"
 
 pipeline.build_test_runner(lang)
-exit
+
+
 # pipeline.release_latest(lang)
 # exit
-r = pipeline.analyze(lang, "two-fer", "soln-42", "s3://exercism-dev/iterations/fff07700-e1c3-402d-8937-823aeefb159f")
+# r = pipeline.analyze(lang, "two-fer", "soln-42", "s3://exercism-dev/iterations/fff07700-e1c3-402d-8937-823aeefb159f")
-puts r
+# puts r
-if r["logs"]
+# if r["logs"]
-  r["logs"].each do |log_line|
+#   r["logs"].each do |log_line|
-    puts "+ #{log_line["cmd"]}"
+#     puts "+ #{log_line["cmd"]}"
-    puts log_line["stdout"]
+#     puts log_line["stdout"]
-    puts log_line["stderr"]
+#     puts log_line["stderr"]
-  end
+#   end
-end
+# end
-
+#
-puts r["result"]
+# puts r["result"]

lib/pipeline.rb

@@ -34,7 +34,7 @@ module Pipeline
     if (latest_tag.nil?)
       latest_tag = "master"
     end
-    Pipeline::Build::AnalyzerBuild.(latest_tag, track_slug)
+    Pipeline::Build::AnalyzerBuild.(latest_tag, track_slug, repo)
   end
 
   def self.build_test_runner(track_slug)
@@ -43,7 +43,7 @@ module Pipeline
     if (latest_tag.nil?)
       latest_tag = "master"
     end
-    Pipeline::Build::AnalyzerBuild.(latest_tag, track_slug)
+    Pipeline::Build::TestRunnerBuild.(latest_tag, track_slug, repo)
   end
 
   def self.release(language_slug)
@@ -82,6 +82,7 @@ end
 
 require "pipeline/rpc_server"
 require "pipeline/analyzer_repo"
+require "pipeline/container_repo"
 require "pipeline/validation/check_invokable"
 require "pipeline/validation/check_environment_invariants"
 require "pipeline/validation/check_fixtures"
@@ -95,6 +96,8 @@ require "pipeline/util/external_command"
 require "pipeline/util/log_collector"
 require "pipeline/build/build_image"
 require "pipeline/build/publish_image"
+require "pipeline/build/container_build"
 require "pipeline/build/analyzer_build"
+require "pipeline/build/test_runner_build"
 require "pipeline/runtime/runtime_environment"
 require "pipeline/runtime/analysis_run"

lib/pipeline/analyzer_repo.rb

@@ -10,7 +10,7 @@ class Pipeline::AnalyzerRepo
   end
 
   def self.test_runner_for_track(track_slug)
-    repo_url = "https://github.com/exercism/#{track_slug}-analyzer"
+    repo_url = "https://github.com/exercism/#{track_slug}-test-runner"
     Pipeline::AnalyzerRepo.new(repo_url)
   end
 

lib/pipeline/build/analyzer_build.rb

@@ -1,56 +1,10 @@
 module Pipeline::Build
-  class AnalyzerBuild
+  class AnalyzerBuild < ContainerBuild
-    include Mandate
-
-    attr_accessor :img, :target_sha, :image_tag
-
-    initialize_with :build_tag, :track_slug
-
-    def call
-      setup_utilities
-      check_tag_exists
-      build
-      validate
-      publish
-      {
-        track: track_slug,
-        image: image_name,
-        image_tag: image_tag,
-        git_sha: target_sha,
-        git_tag: build_tag,
-        logs: img.logs.inspect
-      }
-    end
-
-    def setup_utilities
-      @img = Pipeline::Util::ImgWrapper.new
-    end
-
-    def check_tag_exists
-      return if build_tag == "master"
-      raise "Build tag does not exist" unless repo.tags[build_tag]
-    end
-
-    def build
-      @image_tag = Pipeline::Build::BuildImage.(build_tag, image_name, repo, img)
-    end
-
-    def validate
-      Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}")
-    end
-
-    def publish
-      Pipeline::Build::PublishImage.(img, image_name, image_tag, build_tag)
-    end
 
     def image_name
       suffix = "-dev" unless ENV["env"] == "production"
       "#{track_slug}-analyzer#{suffix}"
     end
-
+    
-    memoize
-    def repo
-      Pipeline::AnalyzerRepo.for_track(track_slug)
-    end
   end
 end

lib/pipeline/build/build_image.rb

@@ -10,15 +10,17 @@ module Pipeline::Build
       repo.fetch!
       checkout
       build
-      local_tag
+      @target_sha
     end
 
     def checkout
-      @target_sha = repo.checkout(build_tag)
+      target_sha = repo.checkout(build_tag)
+      @target_sha = "sha-#{target_sha}"
     end
 
     def build
       Dir.chdir(repo.workdir) do
+        img.reset_hub_login
         img.build(local_tag)
       end
     end

lib/pipeline/build/container_build.rb

@@ -0,0 +1,82 @@
+module Pipeline::Build
+  class ContainerBuild
+    include Mandate
+
+    attr_accessor :img, :local_tag, :image_tag, :container_repo
+
+    initialize_with :build_tag, :track_slug, :repo
+
+    def call
+      setup_utilities
+      setup_remote_repo
+      check_tag_exists
+      if already_built?
+        puts "already_built"
+        return {
+          status: "ignored",
+          message: "Already built",
+          track: track_slug,
+          image: image_name,
+          image_tag: image_tag
+        }
+      end
+      build
+      validate
+      publish
+      {
+        status: "built",
+        message: "Successfully built",
+        track: track_slug,
+        image: image_name,
+        image_tag: image_tag,
+        git_tag: build_tag,
+        logs: img.logs.inspect
+      }
+    end
+
+    def setup_utilities
+      @logs = Pipeline::Util::LogCollector.new
+      @img = Pipeline::Util::ImgWrapper.new(@logs)
+    end
+
+    def setup_remote_repo
+      @container_repo = Pipeline::ContainerRepo.new(image_name)
+      @container_repo.create_if_required
+    end
+
+    def check_tag_exists
+      return if build_tag == "master"
+      raise "Build tag does not exist" unless repo.tags[build_tag]
+    end
+
+    def already_built?
+      puts "Already built?"
+      puts "image_name: #{image_name}"
+      puts "build_tag: #{build_tag}"
+      puts "current: #{@container_repo.git_shas}"
+      puts "repo: #{repo}"
+      current_tags = @container_repo.git_shas
+      repo.fetch!
+      target_sha = repo.checkout(build_tag)
+      puts target_sha
+      current_tags.include? target_sha
+    end
+
+    def build
+      @local_tag = Pipeline::Build::BuildImage.(build_tag, image_name, repo, img)
+      @image_tag = "#{image_name}:#{local_tag}"
+    end
+
+    def validate
+      Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}")
+    end
+
+    def publish
+      Pipeline::Build::PublishImage.(img, container_repo, local_tag, build_tag)
+    end
+
+    def image_name
+      raise "Image not implemented"
+    end
+  end
+end

lib/pipeline/build/publish_image.rb

@@ -2,77 +2,62 @@ module Pipeline::Build
   class PublishImage
     include Mandate
 
-    initialize_with :img, :image_name, :image_tag, :build_tag
+    initialize_with :img, :container_repo, :local_tag, :build_tag
+
+    attr_reader :repository_url
 
     def call
       puts "PUBLISHING #{image_tag}"
       puts "Login to repo"
-      create_repository
       login_to_repository
       tag_build
       push_build
       logout
     end
 
-    def create_repository
-      puts "Checking if repository exists"
-      begin
-        ecr.describe_repositories({
-          repository_names: [image_name]
-        })
-        return
-      rescue Aws::ECR::Errors::RepositoryNotFoundException
-        puts "Repository #{image_name} not found"
-      end
-      puts "Creating repository"
-      ecr.create_repository({
-        repository_name: image_name,
-        image_tag_mutability: "MUTABLE"
-      })
-    end
-
     def login_to_repository
-      authorization_token = ecr.get_authorization_token.authorization_data[0].authorization_token
+      @repository_url = container_repo.repository_url
-      plain = Base64.decode64(authorization_token)
+      user, password = container_repo.create_login_token
-      user,password = plain.split(":")
+      img.login("AWS", password, repository_url)
-      img.login("AWS", password, registry_endpoint)
     end
 
     def logout
-      img.logout(registry_endpoint)
+      img.logout(repository_url)
     end
 
     def tag_build
       img.tag(image_tag, remote_tag)
+      img.tag(image_tag, remote_tag_timestamped)
       img.tag(image_tag, remote_human_tag) unless build_tag.nil?
       img.tag(image_tag, remote_latest_tag)
     end
 
     def push_build
       img.push(remote_tag)
+      img.push(remote_tag_timestamped)
       img.push(remote_human_tag) unless build_tag.nil?
       img.push(remote_latest_tag)
     end
 
+    def image_tag
+      "#{container_repo.image_name}:#{local_tag}"
+    end
+
     def remote_tag
-      "#{registry_endpoint}/#{image_tag}"
+      "#{repository_url}:#{local_tag}"
     end
 
     def remote_human_tag
-      "#{registry_endpoint}/#{image_name}:#{build_tag}"
+      "#{repository_url}:#{build_tag}"
     end
 
     def remote_latest_tag
-      "#{registry_endpoint}/#{image_name}:latest"
+      "#{repository_url}:latest"
-    end
-
-    def registry_endpoint
-      Pipeline.config["registry_endpoint"]
     end
 
     memoize
-    def ecr
+    def remote_tag_timestamped
-      Aws::ECR::Client.new(region: 'eu-west-1')
+      "#{remote_tag}-#{Time.now.to_i}"
     end
 
   end

lib/pipeline/build/test_runner_build.rb

@@ -0,0 +1,15 @@
+module Pipeline::Build
+  class TestRunnerBuild < ContainerBuild
+
+    def image_name
+      suffix = "-dev" unless ENV["env"] == "production"
+      "#{track_slug}-test-runner#{suffix}"
+    end
+
+    def validate
+      # No validation implemented for this yet
+      # Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}")
+    end
+
+  end
+end

lib/pipeline/container_repo.rb

@@ -0,0 +1,61 @@
+class Pipeline::ContainerRepo
+
+  attr_reader :image_name
+
+  def initialize(image_name)
+    @image_name = image_name
+  end
+
+  def create_if_required
+    puts "Checking if repository exists"
+    begin
+      return lookup_repo
+    rescue Aws::ECR::Errors::RepositoryNotFoundException
+      puts "Repository #{image_name} not found"
+    end
+    puts "Creating repository"
+    ecr.create_repository({
+      repository_name: image_name,
+      image_tag_mutability: "MUTABLE"
+    })
+    lookup_repo
+  end
+
+  def lookup_repo
+    repos = ecr.describe_repositories({
+      repository_names: [image_name]
+    })
+    repos.repositories.first
+  end
+
+  def repository_url
+    lookup_repo.repository_uri
+  end
+
+  def create_login_token
+    authorization_token = ecr.get_authorization_token.authorization_data[0].authorization_token
+    plain = Base64.decode64(authorization_token)
+    user,password = plain.split(":")
+  end
+
+  def git_shas
+    images = ecr.list_images({
+      repository_name: image_name
+    })
+    tags = []
+    images.image_ids.each do |image|
+      tag = image.image_tag
+      # Only return git-based shas
+      if tag.start_with?("sha-")
+        tag = tag.gsub(/sha-/, "")
+        tags << tag unless tag.include?("-")
+      end
+    end
+    tags.uniq
+  end
+
+  def ecr
+    @ecr ||= Aws::ECR::Client.new(region: 'eu-west-1')
+  end
+
+end

lib/pipeline/util/img_wrapper.rb

@@ -23,6 +23,10 @@ module Pipeline::Util
       exec_cmd "#{binary_path} login -u #{user} -p \"#{password}\" #{registry_endpoint}"
     end
 
+    def reset_hub_login
+      exec_cmd "#{binary_path} logout"
+    end
+
     def logout(registry_endpoint)
       exec_cmd "#{binary_path} logout #{registry_endpoint}"
     end

lib/pipeline/validation/validate_build.rb

@@ -34,8 +34,9 @@ class Pipeline::Validation::ValidateBuild
 
   memoize
   def container_driver
-    img = Pipeline::Util::ImgWrapper.new
+    @logs = Pipeline::Util::LogCollector.new
-    runc = Pipeline::Util::RuncWrapper.new
+    img = Pipeline::Util::ImgWrapper.new(@logs)
+    runc = Pipeline::Util::RuncWrapper.new(@logs)
     configurator = Pipeline::Util::RuncConfigurator.new
     configurator.seed_from_env
     Pipeline::Util::ContainerDriver.new(runc, img, configurator, workdir)

test/pipeline/build_and_validate_test.rb

@@ -7,8 +7,10 @@ module Pipeline
     def test_build_and_validate_realish_image
       demo_analyzer_repo = "https://github.com/exercism/stub-analyzer.git"
       repo = Pipeline::AnalyzerRepo.new(demo_analyzer_repo)
-      img = Pipeline::Util::ImgWrapper.new
+      logs = Pipeline::Util::LogCollector.new
-      image_tag = Pipeline::Build::BuildImage.("master", "demo", repo, img)
+      img = Pipeline::Util::ImgWrapper.new(logs)
+      local_tag = Pipeline::Build::BuildImage.("master", "demo", repo, img)
+      image_tag = "demo:#{local_tag}"
       Pipeline::Validation::ValidateBuild.(image_tag, "test-fixtures/demo")
     end
 

test/pipeline/validation/check_features_test.rb

@@ -17,7 +17,8 @@ module Pipeline::Validation
       configurator = Pipeline::Util::RuncConfigurator.new
       configurator.seed_from_env
 
-      image_tag = Pipeline::Build::BuildImage.("master", track_slug, repo, img)
+      local_tag = Pipeline::Build::BuildImage.("master", track_slug, repo, img)
+      image_tag = "#{track_slug}:#{local_tag}"
 
       @container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, workdir)
       container_driver.prepare_workdir