Skip to content

Commit

Permalink
fix: deployer pod state update (#431)
Browse files Browse the repository at this point in the history
Pod status won't be updated correctly due to timestampe check.
The pod creation time won't change once it's set although the pod
status can change. Therefore, checking creation time to check whether
or not to decide status update prevents the correct status update. The
modified code updates the status regardless of timestamp.
  • Loading branch information
myungjin authored Jun 5, 2023
1 parent c7acbbb commit f1e70cb
Showing 1 changed file with 23 additions and 16 deletions.
39 changes: 23 additions & 16 deletions cmd/deployer/app/deployer/k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ func (deployer *K8sDeployer) GetMonitoredPodStatuses() (map[string]TaskHealthDet
} else if len(out.Items) == 0 {
pod.Status = v1.PodStatus{Phase: v1.PodUnknown}
pod.UnknownCount++
zap.S().Debugf("Items are empty")
} else {
if len(out.Items) > 1 {
sort.Slice(out.Items, func(i, j int) bool {
Expand All @@ -191,22 +192,28 @@ func (deployer *K8sDeployer) GetMonitoredPodStatuses() (map[string]TaskHealthDet

lastPod := out.Items[0]

podCreationTime := lastPod.ObjectMeta.CreationTimestamp.Time

if podCreationTime.After(pod.CreationTime) {
pod.CreationTime = podCreationTime

if lastPod.Status.Phase == v1.PodPending || lastPod.Status.Phase == v1.PodRunning {
pod.Status = lastPod.Status
// reset unknown count
pod.UnknownCount = 0
} else if lastPod.Status.Phase == v1.PodSucceeded {
deployer.DeleteTaskFromMonitoring(pod.TaskID)
} else if lastPod.Status.Phase == v1.PodUnknown {
pod.UnknownCount++
} else {
pod.Status = lastPod.Status
}
pod.CreationTime = lastPod.ObjectMeta.CreationTimestamp.Time

switch lastPod.Status.Phase {
case v1.PodPending:
fallthrough

case v1.PodRunning:
pod.Status = lastPod.Status
// reset unknown count
pod.UnknownCount = 0

case v1.PodSucceeded:
deployer.DeleteTaskFromMonitoring(pod.TaskID)

case v1.PodUnknown:
pod.UnknownCount++

case v1.PodFailed:
fallthrough

default:
pod.Status = lastPod.Status
}
}

Expand Down

0 comments on commit f1e70cb

Please sign in to comment.