forked from mozilla/bigquery-etl
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Create new derived query for Firefox-CI worker costs
Issue: mozilla#5759
- Loading branch information
Showing
5 changed files
with
82 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -522,16 +522,16 @@ bqetl_desktop_platform: | |
bqetl_internal_tooling: | ||
description: | | ||
This DAG schedules queries for populating queries related to Mozilla's | ||
internal developer tooling (e.g. mozregression). | ||
internal developer tooling (e.g. mozregression and Firefox-CI). | ||
default_args: | ||
depends_on_past: false | ||
email: | ||
- wlachance@mozilla.com | ||
- ahalberstadt@mozilla.com | ||
- [email protected] | ||
email_on_failure: true | ||
email_on_retry: true | ||
end_date: null | ||
owner: wlachance@mozilla.com | ||
owner: ahalberstadt@mozilla.com | ||
retries: 2 | ||
retry_delay: 30m | ||
start_date: "2020-06-01" | ||
|
15 changes: 15 additions & 0 deletions
15
sql/moz-fx-data-shared-prod/fxci_derived/dataset_metadata.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
friendly_name: Fxci Derived | ||
description: |- | ||
Derived tables related to the Firefox-CI Taskcluster instance. | ||
dataset_base_acl: derived | ||
user_facing: false | ||
labels: {} | ||
default_table_workgroup_access: | ||
- role: roles/bigquery.dataViewer | ||
members: | ||
- workgroup:mozilla-confidential | ||
workgroup_access: | ||
- role: roles/bigquery.dataViewer | ||
members: | ||
- workgroup:mozilla-confidential | ||
syndication: {} |
23 changes: 23 additions & 0 deletions
23
sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/metadata.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
friendly_name: Firefox-CI Worker Costs | ||
description: |- | ||
Derived Firefox-CI worker cost data from the GCP billing export. | ||
owners: | ||
- [email protected] | ||
labels: | ||
incremental: true | ||
owner1: ahalberstadt | ||
scheduling: | ||
dag_name: bqetl_internal_tooling | ||
bigquery: | ||
time_partitioning: | ||
type: day | ||
field: 'usage_start_date' | ||
require_partition_filter: true | ||
expiration_days: null | ||
range_partitioning: null | ||
clustering: | ||
fields: | ||
- project | ||
- zone | ||
- instance_id | ||
references: {} |
25 changes: 25 additions & 0 deletions
25
sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/query.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
-- Query for fxci_derived.worker_costs_v1 | ||
-- For more information on writing queries see: | ||
-- https://docs.telemetry.mozilla.org/cookbooks/bigquery/querying.html | ||
SELECT | ||
project.id AS project, | ||
REGEXP_EXTRACT(resource.name, "/instances/(.+)$") AS name, | ||
REGEXP_EXTRACT(resource.global_name, "/zones/([^/]+)") AS zone, | ||
REGEXP_EXTRACT(resource.global_name, "/instances/([^/]+)") AS instance_id, | ||
DATE(usage_start_time) as usage_start_date, | ||
SUM(cost) + SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) AS total_cost | ||
FROM | ||
`moz-fx-data-shared-prod.billing_syndicate.gcp_billing_export_resource_v1_01E7D5_97288E_E2EBA0` | ||
WHERE | ||
usage_start_date = @submission_date | ||
AND ( | ||
project.id = "fxci-production-level3-workers" | ||
OR project.id = "fxci-production-level1-workers" | ||
) | ||
AND resource.global_name LIKE "%/instances/%" | ||
GROUP BY | ||
name, | ||
project, | ||
zone, | ||
instance_id, | ||
usage_start_date |
16 changes: 16 additions & 0 deletions
16
sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/schema.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
fields: | ||
- name: project | ||
type: STRING | ||
mode: NULLABLE | ||
- name: name | ||
type: STRING | ||
mode: NULLABLE | ||
- name: zone | ||
type: STRING | ||
mode: NULLABLE | ||
- name: instance_id | ||
type: STRING | ||
mode: NULLABLE | ||
- name: total_cost | ||
type: FLOAT | ||
mode: NULLABLE |