Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add alert escalation policy as part of the check/config #922

Merged
merged 1 commit into from
Jan 16, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion examples/advanced-project/checkly.config.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { defineConfig } from 'checkly'
import { RetryStrategyBuilder } from 'checkly/constructs'
import { AlertEscalationBuilder, RetryStrategyBuilder } from 'checkly/constructs'

/**
* See https://www.checklyhq.com/docs/cli/project-structure/
Expand Down Expand Up @@ -27,6 +27,8 @@ const config = defineConfig({
runtimeId: '2023.09',
/* Failed check runs will be retried before triggering alerts */
retryStrategy: RetryStrategyBuilder.fixedStrategy({ baseBackoffSeconds: 60, maxRetries: 4, sameRegion: true }),
/* All checks will have this alert escalation policy defined */
alertEscalationPolicy: AlertEscalationBuilder.runBasedEscalation(1),
/* A glob pattern that matches the Checks inside your repo, see https://www.checklyhq.com/docs/cli/using-check-test-match/ */
checkMatch: '**/__checks__/**/*.check.ts',
browserChecks: {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { CheckGroup, RetryStrategyBuilder } from 'checkly/constructs'
import { AlertEscalationBuilder, CheckGroup, RetryStrategyBuilder } from 'checkly/constructs'
import { smsChannel, emailChannel } from '../alert-channels'
const alertChannels = [smsChannel, emailChannel]
/*
Expand All @@ -24,6 +24,8 @@ export const websiteGroup = new CheckGroup('website-check-group-1', {
apiCheckDefaults: {},
concurrency: 100,
alertChannels,
/* All checks on this check group will have this alert escalation policy */
alertEscalationPolicy: AlertEscalationBuilder.runBasedEscalation(1),
/*
* Failed check runs in this group will be retried before triggering alerts.
* The wait time between retries will increase linearly: 30 seconds, 60 seconds, and then 90 seconds between the retries.
Expand Down
59 changes: 59 additions & 0 deletions packages/cli/src/constructs/alert-escalation-policy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// eslint-disable-next-line no-restricted-syntax
enum AlertEscalationType {
RUN = 'RUN_BASED',
TIME = 'TIME_BASED'
}

export type Reminders = {
amount?: number,
interval?: number
}

export interface AlertEscalation {
escalationType?: AlertEscalationType,
runBasedEscalation?: {
failedRunThreshold?: number
},
timeBasedEscalation?: {
minutesFailingThreshold?: number
},
reminders?: Reminders
}

export type AlertEscalationOptions = Pick<AlertEscalation, 'runBasedEscalation' | 'timeBasedEscalation' | 'reminders'>

export class AlertEscalationBuilder {
private static DEFAULT_RUN_BASED_ESCALATION = { failedRunThreshold: 1 }
private static DEFAULT_TIME_BASED_ESCALATION = { minutesFailingThreshold: 5 }
private static DEFAULT_REMINDERS = { amount: 0, interval: 5 }

static runBasedEscalation (failedRunThreshold: number, reminders?: Reminders) {
const options: AlertEscalationOptions = {
runBasedEscalation: {
failedRunThreshold,
},
reminders,
}
return this.alertEscalation(AlertEscalationType.RUN, options)
}

static timeBasedEscalation (minutesFailingThreshold: number, reminders?: Reminders) {
const options: AlertEscalationOptions = {
timeBasedEscalation: {
minutesFailingThreshold,
},
reminders,
}
return this.alertEscalation(AlertEscalationType.TIME, options)
}

private static alertEscalation (escalationType: AlertEscalationType,
options: AlertEscalationOptions): AlertEscalation {
return {
escalationType,
runBasedEscalation: options.runBasedEscalation ?? this.DEFAULT_RUN_BASED_ESCALATION,
timeBasedEscalation: options.timeBasedEscalation ?? this.DEFAULT_TIME_BASED_ESCALATION,
reminders: options.reminders ?? this.DEFAULT_REMINDERS,
}
}
}
9 changes: 8 additions & 1 deletion packages/cli/src/constructs/check-group.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { pathToPosix } from '../services/util'
import type { Region } from '..'
import type { Frequency } from './frequency'
import type { RetryStrategy } from './retry-strategy'
import { AlertEscalation } from './alert-escalation-policy'

const defaultApiCheckDefaults: ApiCheckDefaultConfig = {
headers: [],
Expand Down Expand Up @@ -90,6 +91,7 @@ export interface CheckGroupProps {
alertChannels?: Array<AlertChannel>
browserChecks?: BrowserCheckConfig,
multiStepChecks?: MultiStepCheckConfig,
alertEscalationPolicy?: AlertEscalation,
/**
* A valid piece of Node.js code to run in the setup phase of an API check in this group.
* @deprecated use the "ApiCheck.setupScript" property instead and use common JS/TS code
Expand Down Expand Up @@ -141,6 +143,8 @@ export class CheckGroup extends Construct {
multiStepChecks?: MultiStepCheckConfig
retryStrategy?: RetryStrategy
runParallel?: boolean
alertSettings?: AlertEscalation
useGlobalAlertSettings?: boolean

static readonly __checklyType = 'check-group'

Expand All @@ -166,7 +170,8 @@ export class CheckGroup extends Construct {
// `frequency` is not a CheckGroup resource property. Not present in synthesize()
this.frequency = props.frequency
this.apiCheckDefaults = { ...defaultApiCheckDefaults, ...props.apiCheckDefaults }

this.alertSettings = props.alertEscalationPolicy
this.useGlobalAlertSettings = !this.alertSettings
Comment on lines +173 to +174
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tnolet what do you think about this? Basically we set useGlobalAlertSettings to true if alertSettings doesn't exist and vice versa

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@umutuzgur @ferrandiaz hmmmm, feels a bit iffy / hidden magic. Not sure if we have a great alternative.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just my 2 cents, but I think it's nice. As a user, I think that it would be confusing if I passed alertSettings, but they weren't actually applied because I need to also explicitly set useGlobalAlertSettings. I'm not sure there's a case in the CLI were I would pass alertSettings but not want it to actually be used.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@clample yes, I agree. This seems to be the best UX. @umutuzgur @ferrandiaz let's go with this.

this.environmentVariables = props.environmentVariables ?? []
this.environmentVariables.forEach(ev => {
// only empty string is checked because the KeyValuePair.value doesn't allow undefined or null.
Expand Down Expand Up @@ -280,6 +285,8 @@ export class CheckGroup extends Construct {
environmentVariables: this.environmentVariables,
retryStrategy: this.retryStrategy,
runParallel: this.runParallel,
alertSettings: this.alertSettings,
useGlobalAlertSettings: this.useGlobalAlertSettings,
}
}
}
13 changes: 12 additions & 1 deletion packages/cli/src/constructs/check.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import type { CheckGroup } from './check-group'
import { PrivateLocation } from './private-location'
import { PrivateLocationCheckAssignment } from './private-location-check-assignment'
import { RetryStrategy } from './retry-strategy'
import { AlertEscalation } from './alert-escalation-policy'

export interface CheckProps {
/**
Expand Down Expand Up @@ -77,7 +78,11 @@ export interface CheckProps {
/**
* List of alert channels to notify when the check fails or recovers.
*/
alertChannels?: Array<AlertChannel>
alertChannels?: Array<AlertChannel>,
/**
* Determines the alert escalation policy for that particular check
*/
alertEscalationPolicy?: AlertEscalation
/**
* Determines if the check is available only when 'test' runs (not included when 'deploy' is executed).
*/
Expand Down Expand Up @@ -111,6 +116,8 @@ export abstract class Check extends Construct {
alertChannels?: Array<AlertChannel>
testOnly?: boolean
retryStrategy?: RetryStrategy
alertSettings?: AlertEscalation
useGlobalAlertSettings?: boolean
runParallel?: boolean
__checkFilePath?: string // internal variable to filter by check file name from the CLI

Expand Down Expand Up @@ -148,6 +155,8 @@ export abstract class Check extends Construct {

this.testOnly = props.testOnly ?? false
this.retryStrategy = props.retryStrategy
this.alertSettings = props.alertEscalationPolicy
this.useGlobalAlertSettings = !this.alertSettings
this.runParallel = props.runParallel ?? false
this.__checkFilePath = Session.checkFilePath
}
Expand Down Expand Up @@ -225,6 +234,8 @@ export abstract class Check extends Construct {
groupId: this.groupId,
environmentVariables: this.environmentVariables,
retryStrategy: this.retryStrategy,
alertSettings: this.alertSettings,
useGlobalAlertSettings: this.useGlobalAlertSettings,
runParallel: this.runParallel,
}
}
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/constructs/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,4 @@ export * from './dashboard'
export * from './phone-call-alert-channel'
export * from './retry-strategy'
export * from './multi-step-check'
export * from './alert-escalation-policy'
2 changes: 1 addition & 1 deletion packages/cli/src/services/checkly-config-loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import * as fs from 'fs'

export type CheckConfigDefaults = Pick<CheckProps, 'activated' | 'muted' | 'doubleCheck'
| 'shouldFail' | 'runtimeId' | 'locations' | 'tags' | 'frequency' | 'environmentVariables'
| 'alertChannels' | 'privateLocations' | 'retryStrategy'>
| 'alertChannels' | 'privateLocations' | 'retryStrategy' | 'alertEscalationPolicy'>

export type BrowserCheckDefaults = Pick<BrowserPlaywrightDefaults, 'activated' | 'muted' | 'doubleCheck'
| 'shouldFail' | 'runtimeId' | 'locations' | 'tags' | 'frequency' | 'environmentVariables'
Expand Down
Loading