Implemented a more simplified worker pattern
This commit is contained in:
Charles Care
20
bin/builderd
20
bin/builderd
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env ruby
|
||||
require "bundler/setup"
|
||||
$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
|
||||
|
||||
require "pipeline"
|
||||
|
||||
Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
|
||||
# Pipeline.daemon
|
||||
# Pipeline.build_analyzer "ruby"
|
||||
Pipeline.build_test_runner "ruby"
|
||||
# #!/usr/bin/env ruby
|
||||
# require "bundler/setup"
|
||||
# $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
|
||||
#
|
||||
# require "pipeline"
|
||||
#
|
||||
# Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
|
||||
# # Pipeline.daemon
|
||||
# # Pipeline.build_analyzer "ruby"
|
||||
# Pipeline.build_test_runner "ruby"
|
||||
|
||||
@@ -46,7 +46,7 @@ class PipelineClient
|
||||
end
|
||||
|
||||
def release_latest(track_slug)
|
||||
send_msg("release-analyzer_#{track_slug}", 30)
|
||||
send_msg("release-analyzer_#{track_slug}", 3)
|
||||
end
|
||||
|
||||
def analyze(track_slug, exercise_slug, solution_slug, iteration_folder)
|
||||
|
||||
@@ -6,7 +6,7 @@ pipeline = PipelineClient.new
|
||||
|
||||
# return
|
||||
lang = ARGV[0] || "ruby"
|
||||
|
||||
lang = "ruby"
|
||||
# pipeline.build_test_runner(lang)
|
||||
|
||||
|
||||
|
||||
244
bin/router
244
bin/router
@@ -1,122 +1,122 @@
|
||||
#!/usr/bin/env ruby
|
||||
require "bundler/setup"
|
||||
$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
|
||||
|
||||
require "pipeline"
|
||||
|
||||
context = ZMQ::Context.new
|
||||
front_end_socket = context.socket(ZMQ::ROUTER)
|
||||
front_end_socket.bind('tcp://*:5566')
|
||||
|
||||
back_end_socket = context.socket(ZMQ::DEALER)
|
||||
# back_end_socket = context.socket(ZMQ::ROUTER)
|
||||
@back_end_socket = back_end_socket
|
||||
|
||||
|
||||
status_socket = context.socket(ZMQ::SUB)
|
||||
status_socket.setsockopt(ZMQ::SUBSCRIBE, "")
|
||||
status_socket.bind('tcp://*:5555')
|
||||
|
||||
poller = ZMQ::Poller.new
|
||||
poller.register(back_end_socket, ZMQ::POLLIN)
|
||||
poller.register(front_end_socket, ZMQ::POLLIN)
|
||||
poller.register(status_socket, ZMQ::POLLIN)
|
||||
|
||||
@workers = {}
|
||||
|
||||
def active_workers
|
||||
active = []
|
||||
cut_off = Time.now.to_i - 2000
|
||||
puts "Cut off #{cut_off}"
|
||||
@workers.each do |k, worker|
|
||||
puts "worker: #{worker}"
|
||||
last_seen = worker[:last_seen]
|
||||
puts "last_seen #{last_seen}"
|
||||
active << worker if last_seen > cut_off
|
||||
end
|
||||
active
|
||||
end
|
||||
|
||||
def check_active
|
||||
puts "------------------------------"
|
||||
inactive = []
|
||||
cut_off = Time.now.to_i - 15
|
||||
puts "Cut off #{cut_off}"
|
||||
@workers.each do |k, worker|
|
||||
last_seen = worker[:last_seen]
|
||||
puts "last_seen #{last_seen}. #{last_seen} > #{cut_off} .. #{last_seen > cut_off}"
|
||||
worker[:active] = last_seen > cut_off
|
||||
unless worker[:active]
|
||||
inactive << worker
|
||||
end
|
||||
puts "------------------------------"
|
||||
puts worker
|
||||
puts "------------------------------"
|
||||
end
|
||||
inactive.each do |inactive_worker|
|
||||
puts inactive_worker
|
||||
address = inactive_worker[:status]["address"]
|
||||
puts "Unsub #{address}"
|
||||
@back_end_socket.disconnect(address)
|
||||
end
|
||||
end
|
||||
|
||||
loop do
|
||||
poll_result = poller.poll
|
||||
break if poll_result == -1
|
||||
|
||||
puts "POLL #{poll_result}"
|
||||
|
||||
readables = poller.readables
|
||||
|
||||
puts "readables #{poller.readables.size}"
|
||||
puts "writables #{poller.writables.size}"
|
||||
puts "workers #{@workers.size}"
|
||||
|
||||
continue if readables.empty?
|
||||
|
||||
in_flight = {}
|
||||
|
||||
readables.each do |readable|
|
||||
case readable
|
||||
when status_socket
|
||||
puts "..."
|
||||
msg = ""
|
||||
status_socket.recv_string(msg)
|
||||
status_message = JSON.parse(msg)
|
||||
address = status_message["address"]
|
||||
identity = status_message["identity"]
|
||||
@workers[identity] = { last_seen: Time.now.to_i, status: status_message }
|
||||
back_end_socket.connect(address)
|
||||
puts "STATUS: #{msg}"
|
||||
puts "in_flight: #{in_flight}"
|
||||
check_active
|
||||
when front_end_socket
|
||||
check_active
|
||||
workers = active_workers
|
||||
puts "WORKERS #{workers}"
|
||||
if workers.empty?
|
||||
puts "no workers"
|
||||
msg = []
|
||||
front_end_socket.recv_strings(msg)
|
||||
puts "Address #{msg.first} | #{msg}"
|
||||
reply = [msg.first, "", { status: :failed }.to_json]
|
||||
front_end_socket.send_strings(reply)
|
||||
else
|
||||
msg = []
|
||||
front_end_socket.recv_strings(msg)
|
||||
puts "Address #{msg.first} | #{msg}"
|
||||
in_flight[msg.first] = msg
|
||||
result = back_end_socket.send_strings(msg, 1000)
|
||||
puts result
|
||||
end
|
||||
when back_end_socket
|
||||
msg = []
|
||||
back_end_socket.recv_strings(msg)
|
||||
puts "HERER!!!! #{msg}"
|
||||
front_end_socket.send_strings(msg)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# ZMQ::Device.create(front_end_socket, back_end_socket)
|
||||
# #!/usr/bin/env ruby
|
||||
# require "bundler/setup"
|
||||
# $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
|
||||
#
|
||||
# require "pipeline"
|
||||
#
|
||||
# context = ZMQ::Context.new
|
||||
# front_end_socket = context.socket(ZMQ::ROUTER)
|
||||
# front_end_socket.bind('tcp://*:5566')
|
||||
#
|
||||
# back_end_socket = context.socket(ZMQ::DEALER)
|
||||
# # back_end_socket = context.socket(ZMQ::ROUTER)
|
||||
# @back_end_socket = back_end_socket
|
||||
#
|
||||
#
|
||||
# status_socket = context.socket(ZMQ::SUB)
|
||||
# status_socket.setsockopt(ZMQ::SUBSCRIBE, "")
|
||||
# status_socket.bind('tcp://*:5555')
|
||||
#
|
||||
# poller = ZMQ::Poller.new
|
||||
# poller.register(back_end_socket, ZMQ::POLLIN)
|
||||
# poller.register(front_end_socket, ZMQ::POLLIN)
|
||||
# poller.register(status_socket, ZMQ::POLLIN)
|
||||
#
|
||||
# @workers = {}
|
||||
#
|
||||
# def active_workers
|
||||
# active = []
|
||||
# cut_off = Time.now.to_i - 2000
|
||||
# puts "Cut off #{cut_off}"
|
||||
# @workers.each do |k, worker|
|
||||
# puts "worker: #{worker}"
|
||||
# last_seen = worker[:last_seen]
|
||||
# puts "last_seen #{last_seen}"
|
||||
# active << worker if last_seen > cut_off
|
||||
# end
|
||||
# active
|
||||
# end
|
||||
#
|
||||
# def check_active
|
||||
# puts "------------------------------"
|
||||
# inactive = []
|
||||
# cut_off = Time.now.to_i - 15
|
||||
# puts "Cut off #{cut_off}"
|
||||
# @workers.each do |k, worker|
|
||||
# last_seen = worker[:last_seen]
|
||||
# puts "last_seen #{last_seen}. #{last_seen} > #{cut_off} .. #{last_seen > cut_off}"
|
||||
# worker[:active] = last_seen > cut_off
|
||||
# unless worker[:active]
|
||||
# inactive << worker
|
||||
# end
|
||||
# puts "------------------------------"
|
||||
# puts worker
|
||||
# puts "------------------------------"
|
||||
# end
|
||||
# inactive.each do |inactive_worker|
|
||||
# puts inactive_worker
|
||||
# address = inactive_worker[:status]["address"]
|
||||
# puts "Unsub #{address}"
|
||||
# @back_end_socket.disconnect(address)
|
||||
# end
|
||||
# end
|
||||
#
|
||||
# loop do
|
||||
# poll_result = poller.poll
|
||||
# break if poll_result == -1
|
||||
#
|
||||
# puts "POLL #{poll_result}"
|
||||
#
|
||||
# readables = poller.readables
|
||||
#
|
||||
# puts "readables #{poller.readables.size}"
|
||||
# puts "writables #{poller.writables.size}"
|
||||
# puts "workers #{@workers.size}"
|
||||
#
|
||||
# continue if readables.empty?
|
||||
#
|
||||
# in_flight = {}
|
||||
#
|
||||
# readables.each do |readable|
|
||||
# case readable
|
||||
# when status_socket
|
||||
# puts "..."
|
||||
# msg = ""
|
||||
# status_socket.recv_string(msg)
|
||||
# status_message = JSON.parse(msg)
|
||||
# address = status_message["address"]
|
||||
# identity = status_message["identity"]
|
||||
# @workers[identity] = { last_seen: Time.now.to_i, status: status_message }
|
||||
# back_end_socket.connect(address)
|
||||
# puts "STATUS: #{msg}"
|
||||
# puts "in_flight: #{in_flight}"
|
||||
# check_active
|
||||
# when front_end_socket
|
||||
# check_active
|
||||
# workers = active_workers
|
||||
# puts "WORKERS #{workers}"
|
||||
# if workers.empty?
|
||||
# puts "no workers"
|
||||
# msg = []
|
||||
# front_end_socket.recv_strings(msg)
|
||||
# puts "Address #{msg.first} | #{msg}"
|
||||
# reply = [msg.first, "", { status: :failed }.to_json]
|
||||
# front_end_socket.send_strings(reply)
|
||||
# else
|
||||
# msg = []
|
||||
# front_end_socket.recv_strings(msg)
|
||||
# puts "Address #{msg.first} | #{msg}"
|
||||
# in_flight[msg.first] = msg
|
||||
# result = back_end_socket.send_strings(msg, 1000)
|
||||
# puts result
|
||||
# end
|
||||
# when back_end_socket
|
||||
# msg = []
|
||||
# back_end_socket.recv_strings(msg)
|
||||
# puts "HERER!!!! #{msg}"
|
||||
# front_end_socket.send_strings(msg)
|
||||
# end
|
||||
# end
|
||||
# end
|
||||
#
|
||||
# # ZMQ::Device.create(front_end_socket, back_end_socket)
|
||||
|
||||
139
bin/router2
139
bin/router2
@@ -5,141 +5,10 @@ $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
|
||||
require "pipeline"
|
||||
|
||||
context = ZMQ::Context.new
|
||||
front_end_socket = context.socket(ZMQ::ROUTER)
|
||||
front_end_socket.bind('tcp://*:5566')
|
||||
router = Pipeline::Rpc::Router.new(context)
|
||||
|
||||
back_end_socket = context.socket(ZMQ::PUSH)
|
||||
@back_end_socket = back_end_socket
|
||||
|
||||
|
||||
resp_socket = context.socket(ZMQ::SUB)
|
||||
resp_socket.setsockopt(ZMQ::SUBSCRIBE, "")
|
||||
resp_socket.bind('tcp://*:5556')
|
||||
|
||||
status_socket = context.socket(ZMQ::SUB)
|
||||
status_socket.setsockopt(ZMQ::SUBSCRIBE, "")
|
||||
status_socket.bind('tcp://*:5555')
|
||||
|
||||
poller = ZMQ::Poller.new
|
||||
# poller.register(back_end_socket, ZMQ::POLLIN)
|
||||
poller.register(front_end_socket, ZMQ::POLLIN)
|
||||
poller.register(status_socket, ZMQ::POLLIN)
|
||||
poller.register(resp_socket, ZMQ::POLLIN)
|
||||
|
||||
workers_poller = ZMQ::Poller.new
|
||||
workers_poller.register(back_end_socket, ZMQ::POLLOUT)
|
||||
|
||||
@workers = {}
|
||||
|
||||
def active_workers
|
||||
active = []
|
||||
@workers.each do |k, worker|
|
||||
active << worker if worker[:active]
|
||||
end
|
||||
active
|
||||
Thread.new do
|
||||
router.run_heartbeater
|
||||
end
|
||||
|
||||
def check_active
|
||||
puts "------------------------------"
|
||||
inactive = []
|
||||
cut_off = Time.now.to_i - 15
|
||||
puts "Cut off #{cut_off}"
|
||||
@workers.each do |k, worker|
|
||||
last_seen = worker[:last_seen]
|
||||
puts "last_seen #{last_seen}. #{last_seen} > #{cut_off} .. #{last_seen > cut_off}"
|
||||
worker[:active] = last_seen > cut_off
|
||||
unless worker[:active]
|
||||
inactive << worker
|
||||
end
|
||||
puts "------------------------------"
|
||||
puts worker
|
||||
puts "------------------------------"
|
||||
end
|
||||
inactive.each do |inactive_worker|
|
||||
puts inactive_worker
|
||||
address = inactive_worker[:status]["address"]
|
||||
puts "Unsub #{address}"
|
||||
@back_end_socket.disconnect(address)
|
||||
end
|
||||
end
|
||||
|
||||
loop do
|
||||
poll_result = poller.poll
|
||||
break if poll_result == -1
|
||||
|
||||
puts "POLL #{poll_result}"
|
||||
|
||||
readables = poller.readables
|
||||
|
||||
puts "readables #{poller.readables.size}"
|
||||
puts "writables #{poller.writables.size}"
|
||||
puts "workers #{@workers.size}"
|
||||
|
||||
continue if readables.empty?
|
||||
|
||||
in_flight = {}
|
||||
|
||||
readables.each do |readable|
|
||||
case readable
|
||||
when resp_socket
|
||||
puts "resp_socket"
|
||||
msg = ""
|
||||
resp_socket.recv_string(msg)
|
||||
puts msg
|
||||
puts "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
|
||||
when status_socket
|
||||
puts "..."
|
||||
msg = ""
|
||||
status_socket.recv_string(msg)
|
||||
puts "STATUS MSG: #{msg} "
|
||||
status_message = JSON.parse(msg)
|
||||
type = status_message["msg_type"]
|
||||
puts "STATUS MSG TYPE: #{status_message["msg_type"]} "
|
||||
if type == "status"
|
||||
address = status_message["address"]
|
||||
identity = status_message["identity"]
|
||||
@workers[identity] = { last_seen: Time.now.to_i, status: status_message }
|
||||
back_end_socket.connect(address)
|
||||
puts "in_flight: #{in_flight}"
|
||||
check_active
|
||||
else
|
||||
puts "OTHER"
|
||||
return_address = status_message["return_address"]
|
||||
puts return_address
|
||||
puts return_address.pack("c*")
|
||||
reply = [return_address.pack("c*"), "", msg]
|
||||
front_end_socket.send_strings(reply)
|
||||
end
|
||||
when front_end_socket
|
||||
check_active
|
||||
workers = active_workers
|
||||
puts "WORKERS #{workers}"
|
||||
if workers.empty?
|
||||
puts "no workers"
|
||||
msg = []
|
||||
front_end_socket.recv_strings(msg)
|
||||
puts "Address #{msg.first} | #{msg}"
|
||||
reply = [msg.first, "", { status: :failed }.to_json]
|
||||
front_end_socket.send_strings(reply)
|
||||
else
|
||||
poll_result = workers_poller.poll
|
||||
puts "WORKERS POLL #{poll_result}"
|
||||
msg = []
|
||||
front_end_socket.recv_strings(msg)
|
||||
puts "Address #{msg.first} | #{msg}"
|
||||
in_flight[msg.first] = msg
|
||||
puts "send1"
|
||||
result = back_end_socket.send_strings(msg, ZMQ::DONTWAIT)
|
||||
puts "send2"
|
||||
puts result
|
||||
end
|
||||
when back_end_socket
|
||||
msg = []
|
||||
back_end_socket.recv_strings(msg)
|
||||
puts "HERER!!!! #{msg}"
|
||||
front_end_socket.send_strings(msg)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# ZMQ::Device.create(front_end_socket, back_end_socket)
|
||||
router.run_eventloop
|
||||
|
||||
78
bin/test.rb
78
bin/test.rb
@@ -1,39 +1,39 @@
|
||||
require 'rubygems'
|
||||
require 'ffi-rzmq'
|
||||
|
||||
context = ZMQ::Context.new 2
|
||||
|
||||
req = context.socket(ZMQ::REQ)
|
||||
puts req.bind('ipc://routing.ipc')
|
||||
|
||||
sleep 2
|
||||
|
||||
Thread.new do
|
||||
socket = context.socket(ZMQ::ROUTER)
|
||||
puts socket.setsockopt(ZMQ::IDENTITY, "foobar")
|
||||
puts socket.connect('ipc://routing.ipc')
|
||||
|
||||
sleep 2
|
||||
|
||||
loop do
|
||||
puts "waiting"
|
||||
socket.recv_string(message = '')
|
||||
puts "Received [#{message}]"
|
||||
socket.send_string("OK " + message)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
10.times do |request|
|
||||
ss = "Hello #{request}"
|
||||
puts req.setsockopt(ZMQ::IDENTITY, "baz")
|
||||
req.send_string("foobar", ZMQ::SNDMORE)
|
||||
req.send_string("foobar", ZMQ::SNDMORE)
|
||||
req.send_string("foobar", ZMQ::SNDMORE)
|
||||
req.send_string("foobar", ZMQ::SNDMORE)
|
||||
req.send_string("", ZMQ::SNDMORE)
|
||||
req.send_string(ss)
|
||||
puts "Sending string [#{ss}]"
|
||||
req.recv_string(message = '')
|
||||
puts "Received reply #{request}[#{message}]"
|
||||
end
|
||||
# require 'rubygems'
|
||||
# require 'ffi-rzmq'
|
||||
#
|
||||
# context = ZMQ::Context.new 2
|
||||
#
|
||||
# req = context.socket(ZMQ::REQ)
|
||||
# puts req.bind('ipc://routing.ipc')
|
||||
#
|
||||
# sleep 2
|
||||
#
|
||||
# Thread.new do
|
||||
# socket = context.socket(ZMQ::ROUTER)
|
||||
# puts socket.setsockopt(ZMQ::IDENTITY, "foobar")
|
||||
# puts socket.connect('ipc://routing.ipc')
|
||||
#
|
||||
# sleep 2
|
||||
#
|
||||
# loop do
|
||||
# puts "waiting"
|
||||
# socket.recv_string(message = '')
|
||||
# puts "Received [#{message}]"
|
||||
# socket.send_string("OK " + message)
|
||||
# end
|
||||
#
|
||||
# end
|
||||
#
|
||||
# 10.times do |request|
|
||||
# ss = "Hello #{request}"
|
||||
# puts req.setsockopt(ZMQ::IDENTITY, "baz")
|
||||
# req.send_string("foobar", ZMQ::SNDMORE)
|
||||
# req.send_string("foobar", ZMQ::SNDMORE)
|
||||
# req.send_string("foobar", ZMQ::SNDMORE)
|
||||
# req.send_string("foobar", ZMQ::SNDMORE)
|
||||
# req.send_string("", ZMQ::SNDMORE)
|
||||
# req.send_string(ss)
|
||||
# puts "Sending string [#{ss}]"
|
||||
# req.recv_string(message = '')
|
||||
# puts "Received reply #{request}[#{message}]"
|
||||
# end
|
||||
|
||||
@@ -4,8 +4,8 @@ $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
|
||||
|
||||
require "pipeline"
|
||||
|
||||
env_base = ARGV[0]
|
||||
|
||||
Pipeline.load_config(File.expand_path('../../config/pipeline.yml', __FILE__))
|
||||
# Pipeline.release("ruby")
|
||||
Pipeline.daemon
|
||||
# Pipeline.build_analyzer "ruby"
|
||||
# Pipeline.build_test_runner "ruby"
|
||||
server = Pipeline::Rpc::Server.new(env_base)
|
||||
server.listen
|
||||
|
||||
@@ -23,11 +23,6 @@ module Pipeline
|
||||
@config
|
||||
end
|
||||
|
||||
def self.daemon
|
||||
server = Pipeline::RpcServer.new
|
||||
server.listen
|
||||
end
|
||||
|
||||
def self.build_analyzer(track_slug)
|
||||
repo = Pipeline::AnalyzerRepo.for_track(track_slug)
|
||||
latest_tag = repo.tags.keys.last
|
||||
@@ -80,7 +75,8 @@ module Pipeline
|
||||
end
|
||||
end
|
||||
|
||||
require "pipeline/rpc_server"
|
||||
require "pipeline/rpc/router"
|
||||
require "pipeline/rpc/server"
|
||||
require "pipeline/analyzer_repo"
|
||||
require "pipeline/container_repo"
|
||||
require "pipeline/validation/check_invokable"
|
||||
|
||||
125
lib/pipeline/rpc/router.rb
Normal file
125
lib/pipeline/rpc/router.rb
Normal file
@@ -0,0 +1,125 @@
|
||||
module Pipeline::Rpc
|
||||
class Router
|
||||
attr_reader :context, :front_end_socket, :back_end_socket,
|
||||
:response_socket, :poller, :workers_poller
|
||||
|
||||
def initialize(context)
|
||||
@context = context
|
||||
|
||||
@front_end_socket = context.socket(ZMQ::ROUTER)
|
||||
@front_end_socket.bind('tcp://*:5566')
|
||||
|
||||
@back_end_socket = context.socket(ZMQ::PUSH)
|
||||
@back_end_socket.setsockopt(ZMQ::SNDHWM, 1)
|
||||
@back_end_socket.bind('tcp://*:5577')
|
||||
|
||||
@response_socket = context.socket(ZMQ::SUB)
|
||||
@response_socket.setsockopt(ZMQ::SUBSCRIBE, "")
|
||||
@response_socket.bind('tcp://*:5555')
|
||||
|
||||
@poller = ZMQ::Poller.new
|
||||
@poller.register(@front_end_socket, ZMQ::POLLIN)
|
||||
@poller.register(@response_socket, ZMQ::POLLIN)
|
||||
|
||||
@workers_poller = ZMQ::Poller.new
|
||||
@workers_poller.register(@back_end_socket, ZMQ::POLLOUT)
|
||||
|
||||
@in_flight = {}
|
||||
end
|
||||
|
||||
def run_heartbeater
|
||||
puts "STARTING heartbeat_socket"
|
||||
heartbeat_socket = context.socket(ZMQ::PUB)
|
||||
heartbeat_socket.connect('tcp://127.0.0.1:5555')
|
||||
sleep 2
|
||||
loop do
|
||||
heartbeat_socket.send_string({ msg_type: "heartbeat" }.to_json)
|
||||
puts "ping heartbeat"
|
||||
sleep 10
|
||||
end
|
||||
end
|
||||
|
||||
def run_eventloop
|
||||
loop do
|
||||
poll_result = poller.poll
|
||||
break if poll_result == -1
|
||||
|
||||
readables = poller.readables
|
||||
continue if readables.empty?
|
||||
|
||||
readables.each do |readable|
|
||||
case readable
|
||||
when response_socket
|
||||
incoming_recv
|
||||
when front_end_socket
|
||||
handle_frontend_req
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def incoming_recv
|
||||
puts "..."
|
||||
msg = ""
|
||||
response_socket.recv_string(msg)
|
||||
status_message = JSON.parse(msg)
|
||||
type = status_message["msg_type"]
|
||||
puts "STATUS MSG TYPE: #{status_message["msg_type"]} "
|
||||
if type == "status"
|
||||
# address = status_message["address"]
|
||||
# identity = status_message["identity"]
|
||||
# @workers[identity] = { last_seen: Time.now.to_i, status: status_message }
|
||||
# # back_end_socket.connect(address)
|
||||
# check_active
|
||||
elsif type == "response"
|
||||
# puts "RESP"
|
||||
return_address = status_message["return_address"]
|
||||
# puts return_address
|
||||
# puts return_address.pack("c*")
|
||||
reply = [return_address.pack("c*"), "", msg]
|
||||
front_end_socket.send_strings(reply, ZMQ::DONTWAIT)
|
||||
elsif type == "heartbeat"
|
||||
puts "heartbeat msg"
|
||||
puts "in_flight: #{@in_flight}"
|
||||
timed_out = []
|
||||
now = Time.now.to_i
|
||||
@in_flight.each do |k, v|
|
||||
expiry = v[:timeout]
|
||||
timed_out << k if expiry < now
|
||||
end
|
||||
timed_out.each do |addr|
|
||||
reply = [addr, "", { status: :timeout }.to_json]
|
||||
front_end_socket.send_strings(reply)
|
||||
@in_flight.delete(addr)
|
||||
end
|
||||
else
|
||||
puts "OTHER"
|
||||
end
|
||||
end
|
||||
|
||||
def handle_frontend_req
|
||||
msg = []
|
||||
front_end_socket.recv_strings(msg)
|
||||
puts ">>>> #{msg}"
|
||||
if (msg[2] == "describe_analysers")
|
||||
analyzer_spec = {
|
||||
"ruby" => "master"
|
||||
}
|
||||
reply = [msg.first, "", analyzer_spec.to_json]
|
||||
front_end_socket.send_strings(reply)
|
||||
return
|
||||
end
|
||||
|
||||
poll_result = workers_poller.poll(500)
|
||||
writable = poll_result != -1 && workers_poller.writables.size > 0
|
||||
if !writable
|
||||
reply = [msg.first, "", { status: :failed }.to_json]
|
||||
front_end_socket.send_strings(reply)
|
||||
else
|
||||
@in_flight[msg.first] = {msg: msg, timeout: Time.now.to_i + 5}
|
||||
result = back_end_socket.send_strings(msg, ZMQ::DONTWAIT)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
90
lib/pipeline/rpc/server.rb
Normal file
90
lib/pipeline/rpc/server.rb
Normal file
@@ -0,0 +1,90 @@
|
||||
class Pipeline::Rpc::Server
|
||||
|
||||
attr_reader :context, :incoming, :outgoing, :environment
|
||||
|
||||
def initialize(env_base)
|
||||
@context = ZMQ::Context.new(1)
|
||||
@incoming = context.socket(ZMQ::PULL)
|
||||
@outgoing = context.socket(ZMQ::PUB)
|
||||
@outgoing.connect("tcp://localhost:5555")
|
||||
@environment = Pipeline::Runtime::RuntimeEnvironment.new(env_base)
|
||||
end
|
||||
|
||||
def setup
|
||||
@setup = context.socket(ZMQ::REQ)
|
||||
@setup.setsockopt(ZMQ::LINGER, 0)
|
||||
@setup.connect("tcp://localhost:5566")
|
||||
@setup.send_string("describe_analysers")
|
||||
msg = ""
|
||||
@setup.recv_string(msg)
|
||||
analyzer_spec = JSON.parse(msg)
|
||||
puts analyzer_spec
|
||||
|
||||
environment.prepare
|
||||
|
||||
analyzer_spec.each do |language_slug, version|
|
||||
if environment.released?(language_slug)
|
||||
puts "Already installed #{language_slug}"
|
||||
else
|
||||
puts "Installed #{language_slug}"
|
||||
environment.release_analyzer(language_slug)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def listen
|
||||
setup
|
||||
incoming.connect("tcp://localhost:5577")
|
||||
|
||||
loop do
|
||||
msg = []
|
||||
incoming.recv_strings(msg)
|
||||
puts "Received request. Data: #{msg.inspect}"
|
||||
return_address = msg[0].unpack('c*')
|
||||
puts return_address
|
||||
request = msg[2]
|
||||
if request.start_with? "analyze_"
|
||||
_, arg = request.split("_", 2)
|
||||
track, exercise_slug, solution_slug, location = arg.split("|")
|
||||
result = analyze(track, exercise_slug, solution_slug, location)
|
||||
result["return_address"] = return_address
|
||||
result['msg_type'] = 'response'
|
||||
outgoing.send_string(result.to_json)
|
||||
else
|
||||
puts "HERE ELSE: #{request}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def analyze(language_slug, exercise_slug, solution_slug, location)
|
||||
analysis_run = environment.new_analysis(language_slug, exercise_slug, solution_slug)
|
||||
analysis_run.prepare_iteration do |iteration_folder|
|
||||
location_uri = URI(location)
|
||||
bucket = location_uri.host
|
||||
path = location_uri.path[1..]
|
||||
s3 = Aws::S3::Client.new(region: 'eu-west-1')
|
||||
params = {
|
||||
bucket: bucket,
|
||||
prefix: "#{path}/",
|
||||
}
|
||||
resp = s3.list_objects(params)
|
||||
resp.contents.each do |item|
|
||||
key = item[:key]
|
||||
filename = File.basename(key)
|
||||
s3.get_object({
|
||||
bucket: bucket,
|
||||
key: key,
|
||||
response_target: "#{iteration_folder}/#{filename}"
|
||||
})
|
||||
end
|
||||
end
|
||||
begin
|
||||
analysis_run.analyze!
|
||||
rescue => e
|
||||
puts e
|
||||
ensure
|
||||
puts "DONE"
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
@@ -1,93 +1,100 @@
|
||||
class Pipeline::RpcServer
|
||||
|
||||
attr_reader :context, :socket, :identity
|
||||
|
||||
def initialize
|
||||
@context = ZMQ::Context.new(1)
|
||||
@socket = context.socket(ZMQ::PULL)
|
||||
@outgoing = context.socket(ZMQ::PUB)
|
||||
@outgoing.connect("tcp://localhost:5555")
|
||||
@identity = SecureRandom.uuid
|
||||
end
|
||||
|
||||
def listen
|
||||
hostname = Socket.gethostname
|
||||
# socket.setsockopt(ZMQ::IDENTITY, identity)
|
||||
# socket.setsockopt(ZMQ::ROUTING_ID, identity)
|
||||
# socket.connect("tcp://localhost:5577")
|
||||
port = 5555
|
||||
bind_result = -1
|
||||
until bind_result != -1 || port > 5600
|
||||
port += 1
|
||||
# @identity = "#{port}"
|
||||
bind_result = socket.bind("tcp://*:#{port}")
|
||||
end
|
||||
address = "tcp://#{hostname}:#{port}"
|
||||
|
||||
Thread.new do
|
||||
puts "STARTING"
|
||||
emitter = context.socket(ZMQ::PUB)
|
||||
emitter.connect("tcp://localhost:5555")
|
||||
sleep 2
|
||||
loop do
|
||||
emitter.send_string({ msg_type: "status", address: address, identity: identity}.to_json)
|
||||
puts "Sent"
|
||||
sleep 10
|
||||
end
|
||||
end
|
||||
|
||||
loop do
|
||||
msg = []
|
||||
socket.recv_strings(msg)
|
||||
puts "Received request. Data: #{msg.inspect}"
|
||||
return_address = msg[0].unpack('c*')
|
||||
puts return_address
|
||||
request = msg[2]
|
||||
if request.start_with? "build-analyzer_"
|
||||
_, track = request.split("_")
|
||||
result = Pipeline.build_analyzer(track)
|
||||
socket.send_string(result.to_json)
|
||||
elsif request.start_with? "build-test-runner_"
|
||||
_, track = request.split("_")
|
||||
result = Pipeline.build_test_runner(track)
|
||||
socket.send_string(result.to_json)
|
||||
elsif request.start_with? "release-analyzer_"
|
||||
_, arg = request.split("_")
|
||||
result = Pipeline.release(arg)
|
||||
socket.send_string(result.to_json)
|
||||
elsif request.start_with? "analyze_"
|
||||
_, arg = request.split("_", 2)
|
||||
track, exercise_slug, solution_slug, location = arg.split("|")
|
||||
result = Pipeline.analyze!(track, exercise_slug, solution_slug) do |iteration_folder|
|
||||
location_uri = URI(location)
|
||||
bucket = location_uri.host
|
||||
path = location_uri.path[1..]
|
||||
s3 = Aws::S3::Client.new(region: 'eu-west-1')
|
||||
params = {
|
||||
bucket: bucket,
|
||||
prefix: "#{path}/",
|
||||
}
|
||||
resp = s3.list_objects(params)
|
||||
resp.contents.each do |item|
|
||||
key = item[:key]
|
||||
filename = File.basename(key)
|
||||
s3.get_object({
|
||||
bucket: bucket,
|
||||
key: key,
|
||||
response_target: "#{iteration_folder}/#{filename}"
|
||||
})
|
||||
end
|
||||
end
|
||||
puts "DONE"
|
||||
# socket.send_string(result.to_json)
|
||||
result["return_address"] = return_address
|
||||
@outgoing.send_string(result.to_json)
|
||||
else
|
||||
puts "HERE ELSE: #{request}"
|
||||
# @outgoing.send_string("done")
|
||||
end
|
||||
end
|
||||
# socket.send_string(msg)
|
||||
end
|
||||
|
||||
end
|
||||
# class Pipeline::Rpc::Server
|
||||
#
|
||||
# attr_reader :context, :incoming, :outgoing, :environment
|
||||
#
|
||||
# def initialize(env_base)
|
||||
# @context = ZMQ::Context.new(1)
|
||||
# @incoming = context.socket(ZMQ::PULL)
|
||||
# @outgoing = context.socket(ZMQ::PUB)
|
||||
# @outgoing.connect("tcp://localhost:5555")
|
||||
# @environment = Pipeline::Runtime::RuntimeEnvironment.new(env_base)
|
||||
# end
|
||||
#
|
||||
# def setup
|
||||
# @setup = context.socket(ZMQ::REQ)
|
||||
# @setup.setsockopt(ZMQ::LINGER, 0)
|
||||
# @setup.connect("tcp://localhost:5566")
|
||||
# @setup.send_string("describe_analysers")
|
||||
# msg = ""
|
||||
# @setup.recv_string(msg)
|
||||
# analyzer_spec = JSON.parse(msg)
|
||||
# puts analyzer_spec
|
||||
#
|
||||
# environment.prepare
|
||||
#
|
||||
# analyzer_spec.each do |language_slug, version|
|
||||
# if environment.released?(language_slug)
|
||||
# puts "Already installed #{language_slug}"
|
||||
# else
|
||||
# puts "Installed #{language_slug}"
|
||||
# environment.release_analyzer(language_slug)
|
||||
# end
|
||||
# end
|
||||
# end
|
||||
#
|
||||
# def listen
|
||||
# setup
|
||||
# incoming.connect("tcp://localhost:5577")
|
||||
#
|
||||
# loop do
|
||||
# msg = []
|
||||
# incoming.recv_strings(msg)
|
||||
# puts "Received request. Data: #{msg.inspect}"
|
||||
# return_address = msg[0].unpack('c*')
|
||||
# puts return_address
|
||||
# request = msg[2]
|
||||
# if request.start_with? "analyze_"
|
||||
# _, arg = request.split("_", 2)
|
||||
# track, exercise_slug, solution_slug, location = arg.split("|")
|
||||
# result = analyze(track, exercise_slug, solution_slug) do |iteration_folder|
|
||||
# location_uri = URI(location)
|
||||
# bucket = location_uri.host
|
||||
# path = location_uri.path[1..]
|
||||
# s3 = Aws::S3::Client.new(region: 'eu-west-1')
|
||||
# params = {
|
||||
# bucket: bucket,
|
||||
# prefix: "#{path}/",
|
||||
# }
|
||||
# resp = s3.list_objects(params)
|
||||
# resp.contents.each do |item|
|
||||
# key = item[:key]
|
||||
# filename = File.basename(key)
|
||||
# s3.get_object({
|
||||
# bucket: bucket,
|
||||
# key: key,
|
||||
# response_target: "#{iteration_folder}/#{filename}"
|
||||
# })
|
||||
# end
|
||||
# end
|
||||
# result["return_address"] = return_address
|
||||
# result['msg_type'] = 'response'
|
||||
# outgoing.send_string(result.to_json)
|
||||
# else
|
||||
# puts "HERE ELSE: #{request}"
|
||||
# end
|
||||
# end
|
||||
# end
|
||||
#
|
||||
# def analyze(language_slug, exercise_slug, solution_slug)
|
||||
# analysis_run = environment.new_analysis(language_slug, exercise_slug, solution_slug)
|
||||
# analysis_run.prepare_iteration do |iteration_folder|
|
||||
# yield(iteration_folder)
|
||||
# end
|
||||
# begin
|
||||
# analysis_run.analyze!
|
||||
# rescue => e
|
||||
# puts e
|
||||
# ensure
|
||||
# # puts "---"
|
||||
# # puts analysis_run.stdout
|
||||
# # puts "==="
|
||||
# # puts analysis_run.stderr
|
||||
# # puts "---"
|
||||
# # puts analysis_run.success?
|
||||
# # puts analysis_run.exit_status
|
||||
# # puts analysis_run.result
|
||||
# puts "DONE"
|
||||
# end
|
||||
# end
|
||||
#
|
||||
# end
|
||||
|
||||
@@ -11,6 +11,12 @@ module Pipeline::Runtime
|
||||
FileUtils.mkdir_p env_base
|
||||
end
|
||||
|
||||
def released?(track_slug)
|
||||
track_dir = "#{env_base}/#{track_slug}"
|
||||
current_dir = "#{track_dir}/current"
|
||||
File.exist? current_dir
|
||||
end
|
||||
|
||||
def release_analyzer(track_slug)
|
||||
registry_endpoint = Pipeline.config["registry_endpoint"]
|
||||
|
||||
@@ -59,6 +65,7 @@ module Pipeline::Runtime
|
||||
end
|
||||
|
||||
def new_analysis(track_slug, exercise_slug, solution_slug)
|
||||
puts "AnalysisRun: #{track_slug} #{exercise_slug} #{solution_slug}"
|
||||
track_dir = "#{env_base}/#{track_slug}"
|
||||
AnalysisRun.new(track_dir, exercise_slug, solution_slug)
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user