From 8bcbadc041ac20b67090b27c447c64b76b3b9d2c Mon Sep 17 00:00:00 2001
From: Andrew Halberstadt <ahal@mozilla.com>
Date: Wed, 5 Jun 2024 09:35:03 -0400
Subject: [PATCH] feat: Create new derived query for Firefox-CI worker costs

Issue: #5759
---
 dags.yaml                                     |  6 ++---
 .../fxci_derived/dataset_metadata.yaml        | 15 +++++++++++
 .../worker_costs_v1/metadata.yaml             | 23 +++++++++++++++++
 .../fxci_derived/worker_costs_v1/query.sql    | 25 +++++++++++++++++++
 .../fxci_derived/worker_costs_v1/schema.yaml  | 19 ++++++++++++++
 5 files changed, 85 insertions(+), 3 deletions(-)
 create mode 100644 sql/moz-fx-data-shared-prod/fxci_derived/dataset_metadata.yaml
 create mode 100644 sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/metadata.yaml
 create mode 100644 sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/query.sql
 create mode 100644 sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/schema.yaml

diff --git a/dags.yaml b/dags.yaml
index 4d66cbdd12a..fbc88695a6a 100644
--- a/dags.yaml
+++ b/dags.yaml
@@ -522,16 +522,16 @@ bqetl_desktop_platform:
 bqetl_internal_tooling:
   description: |
     This DAG schedules queries for populating queries related to Mozilla's
-    internal developer tooling (e.g. mozregression).
+    internal developer tooling (e.g. mozregression and Firefox-CI).
   default_args:
     depends_on_past: false
     email:
-      - wlachance@mozilla.com
+      - ahalberstadt@mozilla.com
       - telemetry-alerts@mozilla.com
     email_on_failure: true
     email_on_retry: true
     end_date: null
-    owner: wlachance@mozilla.com
+    owner: ahalberstadt@mozilla.com
     retries: 2
     retry_delay: 30m
     start_date: "2020-06-01"
diff --git a/sql/moz-fx-data-shared-prod/fxci_derived/dataset_metadata.yaml b/sql/moz-fx-data-shared-prod/fxci_derived/dataset_metadata.yaml
new file mode 100644
index 00000000000..c0d381edef7
--- /dev/null
+++ b/sql/moz-fx-data-shared-prod/fxci_derived/dataset_metadata.yaml
@@ -0,0 +1,15 @@
+friendly_name: Fxci Derived
+description: |-
+  Derived tables related to the Firefox-CI Taskcluster instance.
+dataset_base_acl: derived
+user_facing: false
+labels: {}
+default_table_workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+syndication: {}
diff --git a/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/metadata.yaml
new file mode 100644
index 00000000000..8b32d9b3065
--- /dev/null
+++ b/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/metadata.yaml
@@ -0,0 +1,23 @@
+friendly_name: Firefox-CI Worker Costs
+description: |-
+  Derived Firefox-CI worker cost data from the GCP billing export.
+owners:
+- ahalberstadt@mozilla.com
+labels:
+  incremental: true
+  owner1: ahalberstadt
+scheduling:
+  dag_name: bqetl_internal_tooling
+bigquery:
+  time_partitioning:
+    type: day
+    field: 'usage_start_date'
+    require_partition_filter: true
+    expiration_days: null
+  range_partitioning: null
+  clustering:
+    fields:
+    - project
+    - zone
+    - instance_id
+references: {}
diff --git a/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/query.sql b/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/query.sql
new file mode 100644
index 00000000000..c7d21122a2e
--- /dev/null
+++ b/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/query.sql
@@ -0,0 +1,25 @@
+-- Query for fxci_derived.worker_costs_v1
+            -- For more information on writing queries see:
+            -- https://docs.telemetry.mozilla.org/cookbooks/bigquery/querying.html
+SELECT
+  project.id AS project,
+  REGEXP_EXTRACT(resource.name, "/instances/(.+)$") AS name,
+  REGEXP_EXTRACT(resource.global_name, "/zones/([^/]+)") AS zone,
+  REGEXP_EXTRACT(resource.global_name, "/instances/([^/]+)") AS instance_id,
+  DATE(usage_start_time) AS usage_start_date,
+  SUM(cost) + SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) AS total_cost
+FROM
+  `moz-fx-data-shared-prod.billing_syndicate.gcp_billing_export_resource_v1_01E7D5_97288E_E2EBA0`
+WHERE
+  DATE(usage_start_time) = @submission_date
+  AND (
+    project.id = "fxci-production-level3-workers"
+    OR project.id = "fxci-production-level1-workers"
+  )
+  AND resource.global_name LIKE "%/instances/%"
+GROUP BY
+  name,
+  project,
+  zone,
+  instance_id,
+  usage_start_date
diff --git a/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/schema.yaml b/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/schema.yaml
new file mode 100644
index 00000000000..ef16b0eb5a7
--- /dev/null
+++ b/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/schema.yaml
@@ -0,0 +1,19 @@
+fields:
+- name: project
+  type: STRING
+  mode: NULLABLE
+- name: name
+  type: STRING
+  mode: NULLABLE
+- name: zone
+  type: STRING
+  mode: NULLABLE
+- name: instance_id
+  type: STRING
+  mode: NULLABLE
+- name: total_cost
+  type: FLOAT
+  mode: NULLABLE
+- name: usage_start_date
+  type: DATE
+  mode: NULLABLE