kubernetes-client/csharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix flaky leader election tests (#676)

Browse Source 
* force max thread

* add log

* use countdown to sync

* add logs

* add thread
This commit is contained in:
Boshi Lian
2021-08-09 07:25:32 -07:00
committed by GitHub
parent 86e339a7f7
commit 72afeafe96
1 changed files with 33 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
tests/KubernetesClient.Tests/LeaderElection/LeaderElectionTests.cs
View File

@@ -7,13 +7,19 @@ using System.Threading.Tasks;
using k8s.LeaderElection;
using Moq;
using Xunit;
using Xunit.Abstractions;
namespace k8s.Tests.LeaderElection
{
public class LeaderElectionTests
{
public LeaderElectionTests()
private readonly ITestOutputHelper output;
public LeaderElectionTests(ITestOutputHelper output)
{
ThreadPool.SetMaxThreads(32, 32);
ThreadPool.SetMinThreads(32, 32);
this.output = output;
MockResourceLock.ResetGloablRecord();
}
@@ -81,6 +87,7 @@ namespace k8s.Tests.LeaderElection
{
var electionHistory = new List<string>();
var leadershipHistory = new List<string>();
var electionHistoryCountdown = new CountdownEvent(7);
var renewCountA = 3;
var mockLockA = new MockResourceLock("mockA") { UpdateWillFail = () => renewCountA <= 0 };
@@ -91,12 +98,14 @@ namespace k8s.Tests.LeaderElection
electionHistory.Add("A creates record");
leadershipHistory.Add("A gets leadership");
electionHistoryCountdown.Signal();
};
mockLockA.OnUpdate += (_) =>
{
renewCountA--;
electionHistory.Add("A updates record");
electionHistoryCountdown.Signal();
};
mockLockA.OnChange += (_) => { leadershipHistory.Add("A gets leadership"); };
@@ -116,6 +125,7 @@ namespace k8s.Tests.LeaderElection
renewCountB--;
electionHistory.Add("B creates record");
electionHistoryCountdown.Signal();
leadershipHistory.Add("B gets leadership");
};
@@ -123,6 +133,7 @@ namespace k8s.Tests.LeaderElection
{
renewCountB--;
electionHistory.Add("B updates record");
electionHistoryCountdown.Signal();
};
mockLockB.OnChange += (_) => { leadershipHistory.Add("B gets leadership"); };
@@ -179,7 +190,8 @@ namespace k8s.Tests.LeaderElection
leaderElector.RunAsync().Wait();
});
testLeaderElectionLatch.Wait(TimeSpan.FromSeconds(10));
testLeaderElectionLatch.Wait(TimeSpan.FromSeconds(15));
electionHistoryCountdown.Wait(TimeSpan.FromSeconds(15));
Assert.Equal(7, electionHistory.Count);
@@ -204,6 +216,7 @@ namespace k8s.Tests.LeaderElection
{
var electionHistory = new List<string>();
var leadershipHistory = new List<string>();
var electionHistoryCountdown = new CountdownEvent(9);
var renewCount = 3;
var mockLock = new MockResourceLock("mock") { UpdateWillFail = () => renewCount <= 0, };
@@ -213,17 +226,27 @@ namespace k8s.Tests.LeaderElection
renewCount--;
electionHistory.Add("create record");
leadershipHistory.Add("get leadership");
electionHistoryCountdown.Signal();
};
mockLock.OnUpdate += _ =>
{
renewCount--;
electionHistory.Add("update record");
electionHistoryCountdown.Signal();
};
mockLock.OnChange += _ => { electionHistory.Add("change record"); };
mockLock.OnChange += _ =>
{
electionHistory.Add("change record");
electionHistoryCountdown.Signal();
};
mockLock.OnTryUpdate += _ => { electionHistory.Add("try update record"); };
mockLock.OnTryUpdate += _ =>
{
electionHistory.Add("try update record");
electionHistoryCountdown.Signal();
};
var leaderElectionConfig = new LeaderElectionConfig(mockLock)
@@ -253,21 +276,15 @@ namespace k8s.Tests.LeaderElection
leaderElector.RunAsync().Wait();
});
countdown.Wait(TimeSpan.FromSeconds(10));
countdown.Wait(TimeSpan.FromSeconds(15));
electionHistoryCountdown.Wait(TimeSpan.FromSeconds(15));
// TODO flasky
// Assert.Equal(9, electionHistory.Count);
output.WriteLine(string.Join(",", electionHistory));
// Assert.True(electionHistory.SequenceEqual(new[]
// {
// "create record", "try update record", "update record", "try update record", "update record",
// "try update record", "try update record", "try update record", "try update record",
// }));
Assert.True(electionHistory.Take(7).SequenceEqual(new[]
Assert.True(electionHistory.Take(9).SequenceEqual(new[]
{
"create record", "try update record", "update record", "try update record", "update record",
"try update record", "try update record",
"create record", "try update record", "update record", "try update record", "update record",
"try update record", "try update record", "try update record", "try update record",
}));
Assert.True(leadershipHistory.SequenceEqual(new[] { "get leadership", "start leading", "stop leading" }));