kubernetes-client/csharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix flaky leader election tests (#676)

Browse Source 
* force max thread

* add log

* use countdown to sync

* add logs

* add thread
This commit is contained in:
Boshi Lian
2021-08-09 07:25:32 -07:00
committed by GitHub
parent 86e339a7f7
commit 72afeafe96
1 changed files with 33 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
tests/KubernetesClient.Tests/LeaderElection/LeaderElectionTests.cs
View File

@@ -7,13 +7,19 @@ using System.Threading.Tasks;
using k8s.LeaderElection;
using Moq;
using Xunit;
using Xunit.Abstractions;
namespace k8s.Tests.LeaderElection
{
public class LeaderElectionTests
{
public LeaderElectionTests()
private readonly ITestOutputHelper output;
public LeaderElectionTests(ITestOutputHelper output)
{
ThreadPool.SetMaxThreads(32, 32);
ThreadPool.SetMinThreads(32, 32);
this.output = output;
MockResourceLock.ResetGloablRecord();
}
@@ -81,6 +87,7 @@ namespace k8s.Tests.LeaderElection
{
var electionHistory = new List<string>();
var leadershipHistory = new List<string>();
var electionHistoryCountdown = new CountdownEvent(7);
var renewCountA = 3;
var mockLockA = new MockResourceLock("mockA") { UpdateWillFail = () => renewCountA <= 0 };
@@ -91,12 +98,14 @@ namespace k8s.Tests.LeaderElection
electionHistory.Add("A creates record");
leadershipHistory.Add("A gets leadership");
electionHistoryCountdown.Signal();
};
mockLockA.OnUpdate += (_) =>
{
renewCountA--;
electionHistory.Add("A updates record");
electionHistoryCountdown.Signal();
};
mockLockA.OnChange += (_) => { leadershipHistory.Add("A gets leadership"); };
@@ -116,6 +125,7 @@ namespace k8s.Tests.LeaderElection
renewCountB--;
electionHistory.Add("B creates record");
electionHistoryCountdown.Signal();
leadershipHistory.Add("B gets leadership");
};
@@ -123,6 +133,7 @@ namespace k8s.Tests.LeaderElection
{
renewCountB--;
electionHistory.Add("B updates record");
electionHistoryCountdown.Signal();
};
mockLockB.OnChange += (_) => { leadershipHistory.Add("B gets leadership"); };
@@ -179,7 +190,8 @@ namespace k8s.Tests.LeaderElection
leaderElector.RunAsync().Wait();
});
testLeaderElectionLatch.Wait(TimeSpan.FromSeconds(10));
testLeaderElectionLatch.Wait(TimeSpan.FromSeconds(15));
electionHistoryCountdown.Wait(TimeSpan.FromSeconds(15));
Assert.Equal(7, electionHistory.Count);
@@ -204,6 +216,7 @@ namespace k8s.Tests.LeaderElection
{
var electionHistory = new List<string>();
var leadershipHistory = new List<string>();
var electionHistoryCountdown = new CountdownEvent(9);
var renewCount = 3;
var mockLock = new MockResourceLock("mock") { UpdateWillFail = () => renewCount <= 0, };
@@ -213,17 +226,27 @@ namespace k8s.Tests.LeaderElection
renewCount--;
electionHistory.Add("create record");
leadershipHistory.Add("get leadership");
electionHistoryCountdown.Signal();
};
mockLock.OnUpdate += _ =>
{
renewCount--;
electionHistory.Add("update record");
electionHistoryCountdown.Signal();
};
mockLock.OnChange += _ => { electionHistory.Add("change record"); };
mockLock.OnChange += _ =>
{
electionHistory.Add("change record");
electionHistoryCountdown.Signal();
};
mockLock.OnTryUpdate += _ => { electionHistory.Add("try update record"); };
mockLock.OnTryUpdate += _ =>
{
electionHistory.Add("try update record");
electionHistoryCountdown.Signal();
};
var leaderElectionConfig = new LeaderElectionConfig(mockLock)
@@ -253,21 +276,15 @@ namespace k8s.Tests.LeaderElection
leaderElector.RunAsync().Wait();
});
countdown.Wait(TimeSpan.FromSeconds(10));
countdown.Wait(TimeSpan.FromSeconds(15));
electionHistoryCountdown.Wait(TimeSpan.FromSeconds(15));
// TODO flasky
// Assert.Equal(9, electionHistory.Count);
output.WriteLine(string.Join(",", electionHistory));
// Assert.True(electionHistory.SequenceEqual(new[]
// {
// "create record", "try update record", "update record", "try update record", "update record",
// "try update record", "try update record", "try update record", "try update record",
// }));
Assert.True(electionHistory.Take(7).SequenceEqual(new[]
Assert.True(electionHistory.Take(9).SequenceEqual(new[]
{
"create record", "try update record", "update record", "try update record", "update record",
"try update record", "try update record",
"try update record", "try update record", "try update record", "try update record",
}));
Assert.True(leadershipHistory.SequenceEqual(new[] { "get leadership", "start leading", "stop leading" }));