exercism/analyzer-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Building test-runners

Browse Source
This commit is contained in:
Charles Care
2019-10-04 14:24:15 +01:00
parent 834c6714fd
commit 832d0cbb7b
14 changed files with 217 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bin/builderd
View File

@@ -5,4 +5,6 @@ $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
require "pipeline"
Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
Pipeline.daemon
# Pipeline.daemon
# Pipeline.build_analyzer "ruby"
Pipeline.build_test_runner "ruby"

25
bin/invoke.rb
View File

@@ -8,17 +8,18 @@ pipeline = PipelineClient.new
lang = ARGV[0] || "ruby"
pipeline.build_test_runner(lang)
exit
# pipeline.release_latest(lang)
# exit
r = pipeline.analyze(lang, "two-fer", "soln-42", "s3://exercism-dev/iterations/fff07700-e1c3-402d-8937-823aeefb159f")
puts r
if r["logs"]
r["logs"].each do |log_line|
puts "+ #{log_line["cmd"]}"
puts log_line["stdout"]
puts log_line["stderr"]
end
end
puts r["result"]
# r = pipeline.analyze(lang, "two-fer", "soln-42", "s3://exercism-dev/iterations/fff07700-e1c3-402d-8937-823aeefb159f")
# puts r
# if r["logs"]
# r["logs"].each do |log_line|
# puts "+ #{log_line["cmd"]}"
# puts log_line["stdout"]
# puts log_line["stderr"]
# end
# end
#
# puts r["result"]

7
lib/pipeline.rb
View File

@@ -34,7 +34,7 @@ module Pipeline
if (latest_tag.nil?)
latest_tag = "master"
end
Pipeline::Build::AnalyzerBuild.(latest_tag, track_slug)
Pipeline::Build::AnalyzerBuild.(latest_tag, track_slug, repo)
end
def self.build_test_runner(track_slug)
@@ -43,7 +43,7 @@ module Pipeline
if (latest_tag.nil?)
latest_tag = "master"
end
Pipeline::Build::AnalyzerBuild.(latest_tag, track_slug)
Pipeline::Build::TestRunnerBuild.(latest_tag, track_slug, repo)
end
def self.release(language_slug)
@@ -82,6 +82,7 @@ end
require "pipeline/rpc_server"
require "pipeline/analyzer_repo"
require "pipeline/container_repo"
require "pipeline/validation/check_invokable"
require "pipeline/validation/check_environment_invariants"
require "pipeline/validation/check_fixtures"
@@ -95,6 +96,8 @@ require "pipeline/util/external_command"
require "pipeline/util/log_collector"
require "pipeline/build/build_image"
require "pipeline/build/publish_image"
require "pipeline/build/container_build"
require "pipeline/build/analyzer_build"
require "pipeline/build/test_runner_build"
require "pipeline/runtime/runtime_environment"
require "pipeline/runtime/analysis_run"

2
lib/pipeline/analyzer_repo.rb
View File

@@ -10,7 +10,7 @@ class Pipeline::AnalyzerRepo
end
def self.test_runner_for_track(track_slug)
repo_url = "https://github.com/exercism/#{track_slug}-analyzer"
repo_url = "https://github.com/exercism/#{track_slug}-test-runner"
Pipeline::AnalyzerRepo.new(repo_url)
end

50
lib/pipeline/build/analyzer_build.rb
View File

@@ -1,56 +1,10 @@
module Pipeline::Build
class AnalyzerBuild
include Mandate
attr_accessor :img, :target_sha, :image_tag
initialize_with :build_tag, :track_slug
def call
setup_utilities
check_tag_exists
build
validate
publish
{
track: track_slug,
image: image_name,
image_tag: image_tag,
git_sha: target_sha,
git_tag: build_tag,
logs: img.logs.inspect
}
end
def setup_utilities
@img = Pipeline::Util::ImgWrapper.new
end
def check_tag_exists
return if build_tag == "master"
raise "Build tag does not exist" unless repo.tags[build_tag]
end
def build
@image_tag = Pipeline::Build::BuildImage.(build_tag, image_name, repo, img)
end
def validate
Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}")
end
def publish
Pipeline::Build::PublishImage.(img, image_name, image_tag, build_tag)
end
class AnalyzerBuild < ContainerBuild
def image_name
suffix = "-dev" unless ENV["env"] == "production"
"#{track_slug}-analyzer#{suffix}"
end
memoize
def repo
Pipeline::AnalyzerRepo.for_track(track_slug)
end
end
end

6
lib/pipeline/build/build_image.rb
View File

@@ -10,15 +10,17 @@ module Pipeline::Build
repo.fetch!
checkout
build
local_tag
@target_sha
end
def checkout
@target_sha = repo.checkout(build_tag)
target_sha = repo.checkout(build_tag)
@target_sha = "sha-#{target_sha}"
end
def build
Dir.chdir(repo.workdir) do
img.reset_hub_login
img.build(local_tag)
end
end

82
lib/pipeline/build/container_build.rb Normal file
View File

@@ -0,0 +1,82 @@
module Pipeline::Build
class ContainerBuild
include Mandate
attr_accessor :img, :local_tag, :image_tag, :container_repo
initialize_with :build_tag, :track_slug, :repo
def call
setup_utilities
setup_remote_repo
check_tag_exists
if already_built?
puts "already_built"
return {
status: "ignored",
message: "Already built",
track: track_slug,
image: image_name,
image_tag: image_tag
}
end
build
validate
publish
{
status: "built",
message: "Successfully built",
track: track_slug,
image: image_name,
image_tag: image_tag,
git_tag: build_tag,
logs: img.logs.inspect
}
end
def setup_utilities
@logs = Pipeline::Util::LogCollector.new
@img = Pipeline::Util::ImgWrapper.new(@logs)
end
def setup_remote_repo
@container_repo = Pipeline::ContainerRepo.new(image_name)
@container_repo.create_if_required
end
def check_tag_exists
return if build_tag == "master"
raise "Build tag does not exist" unless repo.tags[build_tag]
end
def already_built?
puts "Already built?"
puts "image_name: #{image_name}"
puts "build_tag: #{build_tag}"
puts "current: #{@container_repo.git_shas}"
puts "repo: #{repo}"
current_tags = @container_repo.git_shas
repo.fetch!
target_sha = repo.checkout(build_tag)
puts target_sha
current_tags.include? target_sha
end
def build
@local_tag = Pipeline::Build::BuildImage.(build_tag, image_name, repo, img)
@image_tag = "#{image_name}:#{local_tag}"
end
def validate
Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}")
end
def publish
Pipeline::Build::PublishImage.(img, container_repo, local_tag, build_tag)
end
def image_name
raise "Image not implemented"
end
end
end

51
lib/pipeline/build/publish_image.rb
View File

@@ -2,77 +2,62 @@ module Pipeline::Build
class PublishImage
include Mandate
initialize_with :img, :image_name, :image_tag, :build_tag
initialize_with :img, :container_repo, :local_tag, :build_tag
attr_reader :repository_url
def call
puts "PUBLISHING #{image_tag}"
puts "Login to repo"
create_repository
login_to_repository
tag_build
push_build
logout
end
def create_repository
puts "Checking if repository exists"
begin
ecr.describe_repositories({
repository_names: [image_name]
})
return
rescue Aws::ECR::Errors::RepositoryNotFoundException
puts "Repository #{image_name} not found"
end
puts "Creating repository"
ecr.create_repository({
repository_name: image_name,
image_tag_mutability: "MUTABLE"
})
end
def login_to_repository
authorization_token = ecr.get_authorization_token.authorization_data[0].authorization_token
plain = Base64.decode64(authorization_token)
user,password = plain.split(":")
img.login("AWS", password, registry_endpoint)
@repository_url = container_repo.repository_url
user, password = container_repo.create_login_token
img.login("AWS", password, repository_url)
end
def logout
img.logout(registry_endpoint)
img.logout(repository_url)
end
def tag_build
img.tag(image_tag, remote_tag)
img.tag(image_tag, remote_tag_timestamped)
img.tag(image_tag, remote_human_tag) unless build_tag.nil?
img.tag(image_tag, remote_latest_tag)
end
def push_build
img.push(remote_tag)
img.push(remote_tag_timestamped)
img.push(remote_human_tag) unless build_tag.nil?
img.push(remote_latest_tag)
end
def image_tag
"#{container_repo.image_name}:#{local_tag}"
end
def remote_tag
"#{registry_endpoint}/#{image_tag}"
"#{repository_url}:#{local_tag}"
end
def remote_human_tag
"#{registry_endpoint}/#{image_name}:#{build_tag}"
"#{repository_url}:#{build_tag}"
end
def remote_latest_tag
"#{registry_endpoint}/#{image_name}:latest"
end
def registry_endpoint
Pipeline.config["registry_endpoint"]
"#{repository_url}:latest"
end
memoize
def ecr
Aws::ECR::Client.new(region: 'eu-west-1')
def remote_tag_timestamped
"#{remote_tag}-#{Time.now.to_i}"
end
end

15
lib/pipeline/build/test_runner_build.rb Normal file
View File

@@ -0,0 +1,15 @@
module Pipeline::Build
class TestRunnerBuild < ContainerBuild
def image_name
suffix = "-dev" unless ENV["env"] == "production"
"#{track_slug}-test-runner#{suffix}"
end
def validate
# No validation implemented for this yet
# Pipeline::Validation::ValidateBuild.(image_tag, "fixtures/#{track_slug}")
end
end
end

61
lib/pipeline/container_repo.rb Normal file
View File

@@ -0,0 +1,61 @@
class Pipeline::ContainerRepo
attr_reader :image_name
def initialize(image_name)
@image_name = image_name
end
def create_if_required
puts "Checking if repository exists"
begin
return lookup_repo
rescue Aws::ECR::Errors::RepositoryNotFoundException
puts "Repository #{image_name} not found"
end
puts "Creating repository"
ecr.create_repository({
repository_name: image_name,
image_tag_mutability: "MUTABLE"
})
lookup_repo
end
def lookup_repo
repos = ecr.describe_repositories({
repository_names: [image_name]
})
repos.repositories.first
end
def repository_url
lookup_repo.repository_uri
end
def create_login_token
authorization_token = ecr.get_authorization_token.authorization_data[0].authorization_token
plain = Base64.decode64(authorization_token)
user,password = plain.split(":")
end
def git_shas
images = ecr.list_images({
repository_name: image_name
})
tags = []
images.image_ids.each do |image|
tag = image.image_tag
# Only return git-based shas
if tag.start_with?("sha-")
tag = tag.gsub(/sha-/, "")
tags << tag unless tag.include?("-")
end
end
tags.uniq
end
def ecr
@ecr ||= Aws::ECR::Client.new(region: 'eu-west-1')
end
end

4
lib/pipeline/util/img_wrapper.rb
View File

@@ -23,6 +23,10 @@ module Pipeline::Util
exec_cmd "#{binary_path} login -u #{user} -p \"#{password}\" #{registry_endpoint}"
end
def reset_hub_login
exec_cmd "#{binary_path} logout"
end
def logout(registry_endpoint)
exec_cmd "#{binary_path} logout #{registry_endpoint}"
end

5
lib/pipeline/validation/validate_build.rb
View File

@@ -34,8 +34,9 @@ class Pipeline::Validation::ValidateBuild
memoize
def container_driver
img = Pipeline::Util::ImgWrapper.new
runc = Pipeline::Util::RuncWrapper.new
@logs = Pipeline::Util::LogCollector.new
img = Pipeline::Util::ImgWrapper.new(@logs)
runc = Pipeline::Util::RuncWrapper.new(@logs)
configurator = Pipeline::Util::RuncConfigurator.new
configurator.seed_from_env
Pipeline::Util::ContainerDriver.new(runc, img, configurator, workdir)

6
test/pipeline/build_and_validate_test.rb
View File

@@ -7,8 +7,10 @@ module Pipeline
def test_build_and_validate_realish_image
demo_analyzer_repo = "https://github.com/exercism/stub-analyzer.git"
repo = Pipeline::AnalyzerRepo.new(demo_analyzer_repo)
img = Pipeline::Util::ImgWrapper.new
image_tag = Pipeline::Build::BuildImage.("master", "demo", repo, img)
logs = Pipeline::Util::LogCollector.new
img = Pipeline::Util::ImgWrapper.new(logs)
local_tag = Pipeline::Build::BuildImage.("master", "demo", repo, img)
image_tag = "demo:#{local_tag}"
Pipeline::Validation::ValidateBuild.(image_tag, "test-fixtures/demo")
end

3
test/pipeline/validation/check_features_test.rb
View File

@@ -17,7 +17,8 @@ module Pipeline::Validation
configurator = Pipeline::Util::RuncConfigurator.new
configurator.seed_from_env
image_tag = Pipeline::Build::BuildImage.("master", track_slug, repo, img)
local_tag = Pipeline::Build::BuildImage.("master", track_slug, repo, img)
image_tag = "#{track_slug}:#{local_tag}"
@container_driver = Pipeline::Util::ContainerDriver.new(runc, img, configurator, workdir)
container_driver.prepare_workdir