Live updating of container versions

lib/pipeline/build/build_image.rb

@@ -15,7 +15,7 @@ module Pipeline::Build
 
     def checkout
       target_sha = repo.checkout(build_tag)
-      @target_sha = "sha-#{target_sha}"
+      @target_sha = "git-#{target_sha}"
     end
 
     def build

lib/pipeline/build/container_build.rb

@@ -76,7 +76,11 @@ module Pipeline::Build
     end
 
     def publish
-      Pipeline::Build::PublishImage.(img, container_repo, local_tag, build_tag)
+      human_tag = build_tag
+      if repo.valid_commit?(build_tag)
+        human_tag = nil
+      end
+      Pipeline::Build::PublishImage.(img, container_repo, local_tag, human_tag)
     end
 
     def image_name

lib/pipeline/build/publish_image.rb

@@ -55,9 +55,14 @@ module Pipeline::Build
       "#{repository_url}:latest"
     end
 
-    memoize
     def remote_tag_timestamped
-      "#{remote_tag}-#{Time.now.to_i}"
+      tag = local_tag.gsub(/git-/, "build-")
+      "#{repository_url}:#{tag}-#{build_timestamp}"
+    end
+
+    memoize
+    def build_timestamp
+      Time.now.to_i
     end
 
   end

lib/pipeline/cmd/router_daemon.rb

@@ -8,7 +8,7 @@ module Pipeline::Cmd
     Exercism router.
 
     Usage:
-      #{__FILE__} <configuration_file> [--force-worker-restart]
+      #{__FILE__} <configuration_file> [--seed=<seed_configuration] [--force-worker-restart]
       #{__FILE__} -h | --help
       #{__FILE__} --version
 
@@ -40,14 +40,16 @@ module Pipeline::Cmd
     def router
       @router ||= begin
         config_file = options["<configuration_file>"]
+        unless File.file?(config_file)
+          seed = options["--seed"]
+          puts "Seeding config from #{seed}"
+          FileUtils.cp(seed, config_file)
+        end
+        config = Pipeline::Config.new(config_file)
+        config.seed_aws!
+
         context = ZMQ::Context.new
 
-        config = YAML.load(File.read(config_file))
-
-        Aws.config.update({
-           credentials: Aws::Credentials.new(config["aws_access_key_id"], config["aws_secret_access_key"])
-        })
-
         Pipeline::Rpc::Router.new(context, config)
       end
     end

lib/pipeline/config.rb

@@ -0,0 +1,44 @@
+class Pipeline::Config
+
+  attr_reader :config_file
+
+  def initialize(config_file)
+    @config_file = config_file
+  end
+
+  def seed_aws!
+    Aws.config.update({
+       credentials: Aws::Credentials.new(config["aws_access_key_id"], config["aws_secret_access_key"])
+    })
+  end
+
+  def config
+    @config || YAML.load(File.read(config_file))
+  end
+
+  def each_worker(&block)
+    config["workers"].each(&block)
+  end
+
+  def update_container_versions!(worker_class, track_slug, versions)
+    current = config.to_h
+    workers = current["workers"]
+    raise "No worker config" if workers.nil?
+    class_config = workers[worker_class]
+    raise "No worker class config for #{worker_class}" if class_config.nil?
+    track_config = class_config[track_slug]
+    raise "No track config for #{worker_class}:#{track_slug}" if track_config.nil?
+    worker_versions = track_config["worker_versions"]
+    track_config["old_worker_versions"] = worker_versions
+    track_config["worker_versions"] = versions
+    save_config(current)
+  end
+
+  def save_config(updated_config)
+    puts updated_config
+    File.write(config_file, updated_config.to_yaml)
+    @config = YAML.load(File.read(config_file))
+  end
+
+
+end

lib/pipeline/container_repo.rb

@@ -70,8 +70,8 @@ class Pipeline::ContainerRepo
       tag = image.image_tag
       next if tag.nil?
       # Only return git-based shas
-      if tag.start_with?("sha-")
+      if tag.start_with?("git-")
-        tag = tag.gsub(/sha-/, "")
+        tag = tag.gsub(/git-/, "")
         tags << tag unless tag.include?("-")
       end
     end

lib/pipeline/rpc/router.rb

@@ -1,16 +1,16 @@
 module Pipeline::Rpc
 
   class Router
-    attr_reader :zmq_context, :poller, :response_socket, :notification_socket, :container_versions
+    attr_reader :zmq_context, :poller, :response_socket, :notification_socket, :container_versions, :config
 
     def initialize(zmq_context, config)
+      @zmq_context = zmq_context
+      @config = config
+
       @public_hostname = Socket.gethostname
       @response_port = 5556
       @notification_port = 5557
       @front_end_port = 5555
-      # @builder_port = 5557
-
-      @zmq_context = zmq_context
 
       @front_end = FrontEndSocket.new(zmq_context, @front_end_port)
       @response_socket = ResponseSocket.new(zmq_context, @response_port)
@@ -22,7 +22,7 @@ module Pipeline::Rpc
       @in_flight_requests = RequestRegister.new
 
       @backend_channels = {}
-      config["workers"].each do |worker_class, worker_config|
+      config.each_worker do |worker_class, worker_config|
         worker_class = worker_class.to_sym
         backend = @backend_channels[worker_class] = {}
         worker_config.each do |k,v|
@@ -39,21 +39,7 @@ module Pipeline::Rpc
         end
       end
 
-      # @builder_channel = {
+      load_container_versions!
-      #   "*" => WorkChannel.new(zmq_context, "tcp://*:#{@builder_port}")
-      # }
-
-      @container_versions = {}
-      config["workers"].each do |worker_class, worker_config|
-        worker_class = worker_class.to_sym
-        cv = @container_versions[worker_class] = {}
-        worker_config.each do |k,v|
-          if k != "shared_queue"
-            lang_spec = v
-            cv[k] = lang_spec["worker_versions"]
-          end
-        end
-      end
 
       @notification_socket = NotificationSocket.new(zmq_context, @notification_port)
     end
@@ -79,6 +65,20 @@ module Pipeline::Rpc
 
     private
 
+    def load_container_versions!
+      @container_versions = {}
+      config.each_worker do |worker_class, worker_config|
+        worker_class = worker_class.to_sym
+        cv = @container_versions[worker_class] = {}
+        worker_config.each do |k,v|
+          if k != "shared_queue"
+            lang_spec = v
+            cv[k] = lang_spec["worker_versions"]
+          end
+        end
+      end
+    end
+
     def on_service_response(msg)
       if msg.type == "response"
         @in_flight_requests.forward_response(msg)
@@ -112,6 +112,8 @@ module Pipeline::Rpc
           req.send_result({ message: "Request accepted" })
         elsif action == "current_config"
           req.send_result({ container_versions: container_versions })
+        elsif action == "update_container_versions"
+          update_container_versions(req)
         elsif action == "list_available_containers"
           channel = req.parsed_msg["channel"]
           track_slug = req.parsed_msg["track_slug"]
@@ -127,8 +129,6 @@ module Pipeline::Rpc
       end
     end
 
-    private
-
     def to_aws_credentials(raw_credentials)
       key = raw_credentials["access_key_id"]
       secret = raw_credentials["secret_access_key"]
@@ -146,7 +146,6 @@ module Pipeline::Rpc
     end
 
     def select_channel(worker_class)
-      # return @builder_channel if worker_class == :builders
       @backend_channels[worker_class]
     end
 
@@ -211,6 +210,19 @@ module Pipeline::Rpc
       req.send_result(analyzer_spec)
     end
 
+    def update_container_versions(req)
+      channel = req.parsed_msg["channel"]
+      if channel.nil?
+        req.send_error({ msg: "channel unknown" })
+        return
+      end
+      track_slug = req.parsed_msg["track_slug"]
+      versions = req.parsed_msg["versions"]
+      config.update_container_versions!(channel, track_slug, versions)
+      load_container_versions!
+      req.send_result({ container_versions: container_versions })
+    end
+
     def set_temp_credentials(msg)
       msg["credentials"] = temp_credentials
       msg