Skip to content

Commit

Permalink
feat: Create new derived query for Firefox-CI worker costs
Browse files Browse the repository at this point in the history
  • Loading branch information
ahal committed Jun 11, 2024
1 parent 824f830 commit 7416d6e
Show file tree
Hide file tree
Showing 5 changed files with 82 additions and 3 deletions.
6 changes: 3 additions & 3 deletions dags.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -522,16 +522,16 @@ bqetl_desktop_platform:
bqetl_internal_tooling:
description: |
This DAG schedules queries for populating queries related to Mozilla's
internal developer tooling (e.g. mozregression).
internal developer tooling (e.g. mozregression and Firefox-CI).
default_args:
depends_on_past: false
email:
- wlachance@mozilla.com
- ahalberstadt@mozilla.com
- [email protected]
email_on_failure: true
email_on_retry: true
end_date: null
owner: wlachance@mozilla.com
owner: ahalberstadt@mozilla.com
retries: 2
retry_delay: 30m
start_date: "2020-06-01"
Expand Down
15 changes: 15 additions & 0 deletions sql/moz-fx-data-shared-prod/fxci_derived/dataset_metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
friendly_name: Fxci Derived
description: |-
Derived tables related to the Firefox-CI Taskcluster instance.
dataset_base_acl: derived
user_facing: false
labels: {}
default_table_workgroup_access:
- role: roles/bigquery.dataViewer
members:
- workgroup:mozilla-confidential
workgroup_access:
- role: roles/bigquery.dataViewer
members:
- workgroup:mozilla-confidential
syndication: {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
friendly_name: Firefox-CI Worker Costs
description: |-
Derived Firefox-CI worker cost data from the GCP billing export.
owners:
- [email protected]
labels:
incremental: true
owner1: ahalberstadt
scheduling:
dag_name: bqetl_internal_tooling
bigquery:
time_partitioning:
type: day
field: 'usage_start_date'
require_partition_filter: true
expiration_days: null
range_partitioning: null
clustering:
fields:
- project
- zone
- instance_id
references: {}
25 changes: 25 additions & 0 deletions sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/query.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
-- Query for fxci_derived.worker_costs_v1
-- For more information on writing queries see:
-- https://docs.telemetry.mozilla.org/cookbooks/bigquery/querying.html
SELECT
project.id AS project,
REGEXP_EXTRACT(resource.name, "/instances/(.+)$") AS name,
REGEXP_EXTRACT(resource.global_name, "/zones/([^/]+)") AS zone,
REGEXP_EXTRACT(resource.global_name, "/instances/([^/]+)") AS instance_id,
DATE(usage_start_time) as usage_start_date,
SUM(cost) + SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) AS total_cost
FROM
`moz-fx-data-shared-prod.billing_syndicate.gcp_billing_export_resource_v1_01E7D5_97288E_E2EBA0`
WHERE
usage_start_date = @submission_date
AND (
project.id = "fxci-production-level3-workers"
OR project.id = "fxci-production-level1-workers"
)
AND resource.global_name LIKE "%/instances/%"
GROUP BY
name,
project,
zone,
instance_id,
usage_start_date
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
fields:
- name: project
type: STRING
mode: NULLABLE
- name: name
type: STRING
mode: NULLABLE
- name: zone
type: STRING
mode: NULLABLE
- name: instance_id
type: STRING
mode: NULLABLE
- name: total_cost
type: FLOAT
mode: NULLABLE

0 comments on commit 7416d6e

Please sign in to comment.