Remove uri_count and active_hours from all the active_users_aggregate…

…s views. Stop updating active_users_aggregates_v1. Separate the scheduled update of Desktop and Mobile. (mozilla#5808)
ahal · Jun 19, 2024 · 51f9870 · 51f9870
1 parent 1cbbf4f
commit 51f9870
Show file tree

Hide file tree

Showing 7 changed files with 60 additions and 18 deletions.
diff --git a/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates/view.sql b/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates/view.sql
@@ -26,8 +26,6 @@ SELECT
   daily_users,
   weekly_users,
   monthly_users,
-  uri_count,
-  active_hours,
   app_name,
   app_version,
   app_version_major,
@@ -63,8 +61,6 @@ SELECT
   daily_users,
   weekly_users,
   monthly_users,
-  uri_count,
-  active_hours,
   app_name,
   app_version,
   app_version_major,

diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_v1/metadata.yaml
@@ -13,9 +13,6 @@ owners:
 labels:
   incremental: true
   deprecated: true
-scheduling:
-  dag_name: bqetl_analytics_aggregations
-  task_name: active_users_aggregates_v1
 bigquery:
   time_partitioning:
     type: day

diff --git a/sql_generators/active_users_aggregates_v3/__init__.py b/sql_generators/active_users_aggregates_v3/__init__.py
@@ -98,7 +98,8 @@ def generate(target_project, output_dir, use_cloud_function):
     mobile_view_template = env.get_template("mobile_view.sql")
     view_template = env.get_template("view.sql")
     # metadata template
-    metadata_template = "metadata.yaml"
+    mobile_metadata_template = "mobile_metadata.yaml"
+    desktop_metadata_template = "desktop_metadata.yaml"
     # schema template
     desktop_schema_template = "desktop_schema.yaml"
     mobile_schema_template = "mobile_schema.yaml"
@@ -162,17 +163,29 @@ def generate(target_project, output_dir, use_cloud_function):
             skip_existing=False,
         )
 
-        write_sql(
-            output_dir=output_dir,
-            full_table_id=f"{target_project}.{browser.name}_derived.{TABLE_NAME}",
-            basename="metadata.yaml",
-            sql=render(
-                metadata_template,
+        # create metadata files
+        if browser.name == "firefox_desktop":
+            metadata_file = render(
+                desktop_metadata_template,
                 template_folder=THIS_PATH / "templates",
                 app_value=browser.value,
                 app_name=browser.name,
                 format=False,
-            ),
+            )
+        else:
+            metadata_file = render(
+                mobile_metadata_template,
+                template_folder=THIS_PATH / "templates",
+                app_value=browser.value,
+                app_name=browser.name,
+                format=False,
+            )
+
+        write_sql(
+            output_dir=output_dir,
+            full_table_id=f"{target_project}.{browser.name}_derived.{TABLE_NAME}",
+            basename="metadata.yaml",
+            sql=metadata_file,
             skip_existing=False,
         )
 

diff --git a/sql_generators/active_users_aggregates_v3/templates/desktop_metadata.yaml b/sql_generators/active_users_aggregates_v3/templates/desktop_metadata.yaml
@@ -0,0 +1,38 @@
+friendly_name: {{ app_value }} Active Users Aggregates
+description: |-
+  This table contains daily/weekly/monthly active users,
+  new profiles, searches and ad_clicks for {{ app_value }},
+  aggregated by submission_date, attribution, channel,
+  country, city, device model, distribution_id, os details
+  and activity segment.
+
+  - dau is counting the users who reported a ping on the date and
+  are qualified as active users.
+  - daily_users counts all the users who reported a ping on the date.
+  Only dau is exposed in the view telemetry.active_users_aggregates.
+
+  The table is labeled as "change_controlled", which implies
+  that changes require the approval of at least one owner.
+
+  Proposal:
+  https://docs.google.com/document/d/1qvWO49Lr_Z_WErh3I3058A3B1YuiuURx19K3aTdmejM/edit?usp=sharing
+owners:
+  - [email protected]
+  - mozilla/kpi_table_reviewers
+labels:
+  incremental: true
+  change_controlled: true
+scheduling:
+  dag_name: bqetl_analytics_aggregations
+  task_name: {{ app_name }}_active_users_aggregates
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+      - country
+      - app_name
+      - attribution_medium
+      - channel
diff --git a/...ers_aggregates_v3/templates/metadata.yaml → ...regates_v3/templates/mobile_metadata.yaml b/...ers_aggregates_v3/templates/metadata.yaml → ...regates_v3/templates/mobile_metadata.yaml
diff --git a/sql_generators/active_users_aggregates_v3/templates/mobile_view.sql b/sql_generators/active_users_aggregates_v3/templates/mobile_view.sql
@@ -37,8 +37,6 @@ CREATE OR REPLACE VIEW
       dau,
       wau,
       mau,
-      uri_count,
-      active_hours,
       app_name,
       app_version,
       app_version_major,

diff --git a/sql_generators/active_users_aggregates_v3/templates/view.sql b/sql_generators/active_users_aggregates_v3/templates/view.sql
@@ -3,7 +3,7 @@ CREATE OR REPLACE VIEW
   `{{ project_id }}.{{ app_name }}.active_users_aggregates`
 AS
 SELECT
-  * EXCEPT (app_version, app_name),
+  * EXCEPT (app_version, app_name, uri_count, active_hours),
   app_name,
   app_version,
   `mozfun.norm.browser_version_info`(app_version).major_version AS app_version_major,