Refactor SQL (#83)

* Update ttfb.sql * Update ttfb.json * Update ttfb.json * Update ttfb.sql * Update ttfb.json * escape * Update ttfb.sql
rviscomi · Oct 25, 2023 · 8010fb1 · 8010fb1
1 parent f6e8261
commit 8010fb1
Show file tree

Hide file tree

Showing 2 changed files with 98 additions and 73 deletions.
diff --git a/ttfb.json b/ttfb.json
@@ -218,10 +218,10 @@
 }, {
   "platform": "Automattic",
   "client": "desktop",
-  "n": "50785",
-  "fast": "0.46056906566899675",
-  "avg": "0.4654917790686226",
-  "slow": "0.073939155262380624"
+  "n": "51013",
+  "fast": "0.46109815145159078",
+  "avg": "0.46474428086958225",
+  "slow": "0.074157567678826961"
 }, {
   "platform": "WP Engine",
   "client": "desktop",
@@ -302,10 +302,10 @@
 }, {
   "platform": "Automattic",
   "client": "mobile",
-  "n": "108505",
-  "fast": "0.19125385926915811",
-  "avg": "0.73906271600387075",
-  "slow": "0.0696834247269711"
+  "n": "108981",
+  "fast": "0.19355667501674603",
+  "avg": "0.736743102008607",
+  "slow": "0.069700222974646953"
 }, {
   "platform": "Hostinger",
   "client": "mobile",
@@ -334,4 +334,4 @@
   "fast": "0.1245619020555082",
   "avg": "0.4578005115089514",
   "slow": "0.41763758643554039"
-}]
+}]
diff --git a/ttfb.sql b/ttfb.sql
@@ -1,12 +1,61 @@
-# Update these two subqueries monthly.
+# Update this monthly.
+DECLARE QUERY_DATE DATE DEFAULT '2023-09-01';
+
+# Add/edit platforms in alphabetical order here.
+DECLARE PLATFORMS ARRAY<STRUCT<regex STRING, name STRING>> DEFAULT [
+  ('awex', '000webhost'),
+  ('x-ah-environment', 'Acquia'),
+  ('alproxy', 'AlwaysData'),
+  ('automattic.com/work-with-us', 'Automattic'),
+  ('wpvip.com/careers', 'Automattic'),
+  ('wordpress.com', 'Automattic'),
+  ('a9130478a60e5f9135f765b23f26593b', 'Automattic'),
+  ('flywheel', 'Flywheel'),
+  ('x-github-request', 'GitHub'),
+  ('dps/', 'GoDaddy Website Builder'),
+  ('hostinger', 'Hostinger'),
+  ('zyro.com', 'Hostinger Website Builder'),
+  ('hubspot', 'HubSpot'),
+  ('x-kinsta-cache', 'Kinsta'),
+  ('x-lw-cache', 'Liquid Web'),
+  ('netlify', 'Netlify'),
+  ('x-pantheon-styx-hostname', 'Pantheon'),
+  ('seravo', 'Seravo'),
+  ('x-shopify-stage', 'Shopify'),
+  ('b7440e60b07ee7b8044761568fab26e8', 'SiteGround'),
+  ('624d5be7be38418a3e2a818cc8b7029b', 'SiteGround'),
+  ('6b7412fb82ca5edfd0917e3957f05d89', 'SiteGround'),
+  ('squarespace', 'Squarespace'),
+  ('x-vercel-id', 'Vercel'),
+  ('weebly', 'Weebly'),
+  ('x-wix-request-id', 'Wix'),
+  ('wp-cloud', 'WP-Cloud'),
+  ('wpe-backend', 'WP Engine'),
+  ('wp engine atlas', 'WP Engine Atlas'),
+  ('wp engine', 'WP Engine'),
+  ('zoneos', 'Zone.eu')
+];
+
+
 WITH crux AS (
   SELECT
-    *
+    IF(device = 'desktop', 'desktop', 'mobile') AS client,
+    CONCAT(origin, '/') AS url,
+    CASE
+      WHEN SAFE_DIVIDE(fast_ttfb, (fast_ttfb + avg_ttfb + slow_ttfb)) >= 0.75 THEN 'Good'
+      WHEN SAFE_DIVIDE(slow_ttfb, (fast_ttfb + avg_ttfb + slow_ttfb)) >= 0.25 THEN 'Poor'
+      WHEN fast_ttfb IS NOT NULL THEN 'Needs Improvement'
+      ELSE NULL
+    END AS ttfb
   FROM
     `chrome-ux-report.materialized.device_summary`
   WHERE
-    date = '2023-09-01'
-), requests AS (
+    date = QUERY_DATE AND
+    device IN ('desktop', 'phone') AND
+    fast_ttfb IS NOT NULL
+),
+
+requests AS (
   SELECT
     client,
     root_page,
@@ -17,77 +66,53 @@ WITH crux AS (
   FROM
     `httparchive.all.requests`
   WHERE
-    date = '2023-09-01' AND
+    date = QUERY_DATE AND
     is_main_document
+),
+
+platform_regex AS (
+  SELECT
+    STRING_AGG(
+      # Escape special chars
+      REGEXP_REPLACE(regex, r'([\.\*\+\?\|\(\)\[\]\{\}\^\$])', r'\\\1'),
+      r'|'
+    ) AS pattern
+  FROM
+    UNNEST(PLATFORMS)
+),
+
+detected_platforms AS (
+  SELECT
+    client,
+    url,
+    name AS platform
+  FROM
+    UNNEST(PLATFORMS)
+  JOIN (
+    SELECT
+      client,
+      root_page AS url,
+      REGEXP_EXTRACT(LOWER(CONCAT(IFNULL(respOtherHeaders, ''), IFNULL(resp_x_powered_by, ''), IFNULL(resp_via, ''), IFNULL(resp_server, ''))), (SELECT pattern FROM platform_regex)) AS regex
+    FROM
+      requests)
+  USING
+    (regex)
 )
 
+
 SELECT DISTINCT
-  CASE
-   WHEN platform = 'zyro.com' THEN 'Hostinger Website Builder'
-   WHEN platform = 'hostinger' THEN 'Hostinger'
-   WHEN platform = 'zoneos' THEN 'Zone.eu'
-   WHEN platform = 'seravo' THEN 'Seravo'
-   WHEN platform = 'automattic.com/work-with-us' THEN 'Automattic'
-   WHEN platform = 'wpvip.com/careers' THEN 'Automattic'
-   WHEN platform = 'wordpress.com' THEN 'Automattic'
-   WHEN platform = 'a9130478a60e5f9135f765b23f26593b' then 'Automattic'
-   WHEN platform = 'x-ah-environment' THEN 'Acquia'
-   WHEN platform = 'x-pantheon-styx-hostname' THEN 'Pantheon'
-   WHEN platform = 'wp engine atlas' THEN 'WP Engine Atlas'
-   WHEN platform = 'wpe-backend' THEN 'WP Engine'
-   WHEN platform = 'wp engine' THEN 'WP Engine'
-   WHEN platform = 'x-kinsta-cache' THEN 'Kinsta'
-   WHEN platform = 'hubspot' THEN 'HubSpot'
-   WHEN platform = 'b7440e60b07ee7b8044761568fab26e8' THEN 'SiteGround'
-   WHEN platform = '624d5be7be38418a3e2a818cc8b7029b' THEN 'SiteGround'
-   WHEN platform = '6b7412fb82ca5edfd0917e3957f05d89' THEN 'SiteGround'
-   WHEN platform = 'x-github-request' THEN 'GitHub'
-   WHEN platform = 'alproxy' THEN 'AlwaysData'
-   WHEN platform = 'netlify' THEN 'Netlify'
-   WHEN platform = 'x-lw-cache' THEN 'Liquid Web'
-   WHEN platform = 'squarespace' THEN 'Squarespace'
-   WHEN platform = 'x-wix-request-id' THEN 'Wix'
-   WHEN platform = 'x-shopify-stage' THEN 'Shopify'
-   WHEN platform = 'x-vercel-id' THEN 'Vercel'
-   WHEN platform = 'flywheel' THEN 'Flywheel'
-   WHEN platform = 'weebly' THEN 'Weebly'
-   WHEN platform = 'dps/' THEN 'GoDaddy Website Builder'
-   WHEN platform = 'awex' THEN '000webhost'
-   WHEN platform = 'wp-cloud' THEN 'WP-Cloud'
-   ELSE NULL
-  END AS platform,
+  platform,
   client,
   COUNT(DISTINCT url) AS n,
   COUNT(DISTINCT IF(ttfb = 'Good', url, NULL)) / COUNT(DISTINCT url) AS fast,
   COUNT(DISTINCT IF(ttfb = 'Needs Improvement', url, NULL)) / COUNT(DISTINCT url) AS avg,
   COUNT(DISTINCT IF(ttfb = 'Poor', url, NULL)) / COUNT(DISTINCT url) AS slow
-FROM (
-  SELECT
-    IF(device = 'desktop', 'desktop', 'mobile') AS client,
-    CONCAT(origin, '/') AS url,
-    CASE
-      WHEN SAFE_DIVIDE(fast_ttfb, (fast_ttfb + avg_ttfb + slow_ttfb)) >= 0.75 THEN 'Good'
-      WHEN SAFE_DIVIDE(slow_ttfb, (fast_ttfb + avg_ttfb + slow_ttfb)) >= 0.25 THEN 'Poor'
-      WHEN fast_ttfb IS NOT NULL THEN 'Needs Improvement'
-      ELSE NULL
-    END AS ttfb
-  FROM
-    crux
-  WHERE
-    device IN ('desktop', 'phone') AND
-    fast_ttfb IS NOT NULL)
-JOIN (
-  SELECT
-    client,
-    root_page AS url,
-    REGEXP_EXTRACT(LOWER(CONCAT(IFNULL(respOtherHeaders, ''), IFNULL(resp_x_powered_by, ''), IFNULL(resp_via, ''), IFNULL(resp_server, ''))),
-      r'(zoneos|seravo|x-kinsta-cache|automattic.com/work-with-us|wpvip.com/careers|wordpress\.com|a9130478a60e5f9135f765b23f26593b|x-ah-environment|x-pantheon-styx-hostname|wp engine atlas|wpe-backend|wp engine|hubspot|b7440e60b07ee7b8044761568fab26e8|624d5be7be38418a3e2a818cc8b7029b|6b7412fb82ca5edfd0917e3957f05d89|x-github-request|alproxy|netlify|x-lw-cache|squarespace|x-wix-request-id|x-shopify-stage|x-vercel-id|flywheel|weebly|dps/|hostinger|awex|wp-cloud|zyro.com)') AS platform
-  FROM
-    requests)
+FROM
+  detected_platforms
+JOIN
+  crux
 USING
   (client, url)
-WHERE
-  platform IS NOT NULL
 GROUP BY
   platform,
   client