fix e2e

Signed-off-by: liubo02 <[email protected]>

pkg/controllers/tidb/tasks/status.go

@@ -63,12 +63,17 @@ func TaskStatus(state *ReconcileContext, c client.Client) task.Task {
 			}
 		}
 
-		if state.PodIsTerminating {
-			return task.Retry(defaultTaskWaitDuration).With("pod may be terminating, requeue to retry")
-		}
-
 		if !healthy {
-			return task.Wait().With("tidb may not be healthy, wait")
+			// TODO(liubo02): delete pod should retrigger the events, try to change to wait
+			if state.PodIsTerminating {
+				return task.Retry(defaultTaskWaitDuration).With("pod may be terminating, requeue to retry")
+			}
+
+			if pod != nil && statefulset.IsPodRunningAndReady(pod) {
+				return task.Retry(defaultTaskWaitDuration).With("tidb is not healthy, requeue to retry")
+			}
+
+			return task.Wait().With("pod of tidb is not ready, wait")
 		}
 
 		return task.Complete().With("status is synced")

pkg/controllers/tidb/tasks/status_test.go

@@ -208,6 +208,61 @@ func TestTaskStatus(t *testing.T) {
 				return obj
 			}),
 		},
+		{
+			desc: "pod is ready but tidb is not healthy",
+			state: &ReconcileContext{
+				State: &state{
+					tidb: fake.FakeObj(fakeTiDBName, func(obj *v1alpha1.TiDB) *v1alpha1.TiDB {
+						obj.Generation = 3
+						obj.Labels = map[string]string{
+							v1alpha1.LabelKeyInstanceRevisionHash: newRevision,
+						}
+						return obj
+					}),
+					pod: fake.FakeObj("aaa-tidb-xxx", func(obj *corev1.Pod) *corev1.Pod {
+						obj.SetDeletionTimestamp(&now)
+						obj.Labels = map[string]string{
+							v1alpha1.LabelKeyInstanceRevisionHash: oldRevision,
+						}
+						obj.Status.Phase = corev1.PodRunning
+						obj.Status.Conditions = append(obj.Status.Conditions, corev1.PodCondition{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						})
+						return obj
+					}),
+				},
+			},
+
+			expectedStatus: task.SRetry,
+			expectedObj: fake.FakeObj(fakeTiDBName, func(obj *v1alpha1.TiDB) *v1alpha1.TiDB {
+				obj.Generation = 3
+				obj.Labels = map[string]string{
+					v1alpha1.LabelKeyInstanceRevisionHash: newRevision,
+				}
+
+				obj.Status.ObservedGeneration = 3
+				obj.Status.UpdateRevision = newRevision
+				obj.Status.Conditions = []metav1.Condition{
+					{
+						Type:               v1alpha1.CondHealth,
+						Status:             metav1.ConditionFalse,
+						ObservedGeneration: 3,
+						Reason:             "Unhealthy",
+						Message:            "instance is not healthy",
+					},
+					{
+						Type:               v1alpha1.CondSuspended,
+						Status:             metav1.ConditionFalse,
+						ObservedGeneration: 3,
+						Reason:             v1alpha1.ReasonUnsuspended,
+						Message:            "instace is not suspended",
+					},
+				}
+
+				return obj
+			}),
+		},
 		{
 			desc: "failed to update status",
 			state: &ReconcileContext{