fix flaky leader election tests (#676)

* force max thread

* add log

* use countdown to sync

* add logs

* add thread

tests/KubernetesClient.Tests/LeaderElection/LeaderElectionTests.cs

@@ -7,13 +7,19 @@ using System.Threading.Tasks;
 using k8s.LeaderElection;
 using Moq;
 using Xunit;
+using Xunit.Abstractions;
 
 namespace k8s.Tests.LeaderElection
 {
     public class LeaderElectionTests
     {
-        public LeaderElectionTests()
+        private readonly ITestOutputHelper output;
+
+        public LeaderElectionTests(ITestOutputHelper output)
         {
+            ThreadPool.SetMaxThreads(32, 32);
+            ThreadPool.SetMinThreads(32, 32);
+            this.output = output;
             MockResourceLock.ResetGloablRecord();
         }
 
@@ -81,6 +87,7 @@ namespace k8s.Tests.LeaderElection
         {
             var electionHistory = new List<string>();
             var leadershipHistory = new List<string>();
+            var electionHistoryCountdown = new CountdownEvent(7);
 
             var renewCountA = 3;
             var mockLockA = new MockResourceLock("mockA") { UpdateWillFail = () => renewCountA <= 0 };
@@ -91,12 +98,14 @@ namespace k8s.Tests.LeaderElection
 
                 electionHistory.Add("A creates record");
                 leadershipHistory.Add("A gets leadership");
+                electionHistoryCountdown.Signal();
             };
 
             mockLockA.OnUpdate += (_) =>
             {
                 renewCountA--;
                 electionHistory.Add("A updates record");
+                electionHistoryCountdown.Signal();
             };
 
             mockLockA.OnChange += (_) => { leadershipHistory.Add("A gets leadership"); };
@@ -116,6 +125,7 @@ namespace k8s.Tests.LeaderElection
                 renewCountB--;
 
                 electionHistory.Add("B creates record");
+                electionHistoryCountdown.Signal();
                 leadershipHistory.Add("B gets leadership");
             };
 
@@ -123,6 +133,7 @@ namespace k8s.Tests.LeaderElection
             {
                 renewCountB--;
                 electionHistory.Add("B updates record");
+                electionHistoryCountdown.Signal();
             };
 
             mockLockB.OnChange += (_) => { leadershipHistory.Add("B gets leadership"); };
@@ -179,7 +190,8 @@ namespace k8s.Tests.LeaderElection
                 leaderElector.RunAsync().Wait();
             });
 
-            testLeaderElectionLatch.Wait(TimeSpan.FromSeconds(10));
+            testLeaderElectionLatch.Wait(TimeSpan.FromSeconds(15));
+            electionHistoryCountdown.Wait(TimeSpan.FromSeconds(15));
 
             Assert.Equal(7, electionHistory.Count);
 
@@ -204,6 +216,7 @@ namespace k8s.Tests.LeaderElection
         {
             var electionHistory = new List<string>();
             var leadershipHistory = new List<string>();
+            var electionHistoryCountdown = new CountdownEvent(9);
 
             var renewCount = 3;
             var mockLock = new MockResourceLock("mock") { UpdateWillFail = () => renewCount <= 0, };
@@ -213,17 +226,27 @@ namespace k8s.Tests.LeaderElection
                 renewCount--;
                 electionHistory.Add("create record");
                 leadershipHistory.Add("get leadership");
+                electionHistoryCountdown.Signal();
             };
 
             mockLock.OnUpdate += _ =>
             {
                 renewCount--;
                 electionHistory.Add("update record");
+                electionHistoryCountdown.Signal();
             };
 
-            mockLock.OnChange += _ => { electionHistory.Add("change record"); };
+            mockLock.OnChange += _ =>
+            {
+                electionHistory.Add("change record");
+                electionHistoryCountdown.Signal();
+            };
 
-            mockLock.OnTryUpdate += _ => { electionHistory.Add("try update record"); };
+            mockLock.OnTryUpdate += _ =>
+            {
+                electionHistory.Add("try update record");
+                electionHistoryCountdown.Signal();
+            };
 
 
             var leaderElectionConfig = new LeaderElectionConfig(mockLock)
@@ -253,21 +276,15 @@ namespace k8s.Tests.LeaderElection
                 leaderElector.RunAsync().Wait();
             });
 
-            countdown.Wait(TimeSpan.FromSeconds(10));
+            countdown.Wait(TimeSpan.FromSeconds(15));
+            electionHistoryCountdown.Wait(TimeSpan.FromSeconds(15));
 
-            // TODO flasky
+            output.WriteLine(string.Join(",", electionHistory));
-            // Assert.Equal(9, electionHistory.Count);
 
-            // Assert.True(electionHistory.SequenceEqual(new[]
+            Assert.True(electionHistory.Take(9).SequenceEqual(new[]
-            // {
-            //     "create record", "try update record", "update record", "try update record", "update record",
-            //     "try update record", "try update record", "try update record", "try update record",
-            // }));
-
-            Assert.True(electionHistory.Take(7).SequenceEqual(new[]
             {
-                "create record", "try update record", "update record", "try update record", "update record",
+                 "create record", "try update record", "update record", "try update record", "update record",
-                "try update record", "try update record",
+                 "try update record", "try update record", "try update record", "try update record",
             }));
 
             Assert.True(leadershipHistory.SequenceEqual(new[] { "get leadership", "start leading", "stop leading" }));