From 8010fb15785c56fedfc7ba04082db120a8565b50 Mon Sep 17 00:00:00 2001 From: Rick Viscomi Date: Wed, 25 Oct 2023 12:23:30 -0400 Subject: [PATCH] Refactor SQL (#83) * Update ttfb.sql * Update ttfb.json * Update ttfb.json * Update ttfb.sql * Update ttfb.json * escape * Update ttfb.sql --- ttfb.json | 18 +++---- ttfb.sql | 153 +++++++++++++++++++++++++++++++----------------------- 2 files changed, 98 insertions(+), 73 deletions(-) diff --git a/ttfb.json b/ttfb.json index 273aa02..313d8ce 100644 --- a/ttfb.json +++ b/ttfb.json @@ -218,10 +218,10 @@ }, { "platform": "Automattic", "client": "desktop", - "n": "50785", - "fast": "0.46056906566899675", - "avg": "0.4654917790686226", - "slow": "0.073939155262380624" + "n": "51013", + "fast": "0.46109815145159078", + "avg": "0.46474428086958225", + "slow": "0.074157567678826961" }, { "platform": "WP Engine", "client": "desktop", @@ -302,10 +302,10 @@ }, { "platform": "Automattic", "client": "mobile", - "n": "108505", - "fast": "0.19125385926915811", - "avg": "0.73906271600387075", - "slow": "0.0696834247269711" + "n": "108981", + "fast": "0.19355667501674603", + "avg": "0.736743102008607", + "slow": "0.069700222974646953" }, { "platform": "Hostinger", "client": "mobile", @@ -334,4 +334,4 @@ "fast": "0.1245619020555082", "avg": "0.4578005115089514", "slow": "0.41763758643554039" -}] \ No newline at end of file +}] diff --git a/ttfb.sql b/ttfb.sql index 13ecc15..6e2a6f3 100644 --- a/ttfb.sql +++ b/ttfb.sql @@ -1,12 +1,61 @@ -# Update these two subqueries monthly. +# Update this monthly. +DECLARE QUERY_DATE DATE DEFAULT '2023-09-01'; + +# Add/edit platforms in alphabetical order here. +DECLARE PLATFORMS ARRAY> DEFAULT [ + ('awex', '000webhost'), + ('x-ah-environment', 'Acquia'), + ('alproxy', 'AlwaysData'), + ('automattic.com/work-with-us', 'Automattic'), + ('wpvip.com/careers', 'Automattic'), + ('wordpress.com', 'Automattic'), + ('a9130478a60e5f9135f765b23f26593b', 'Automattic'), + ('flywheel', 'Flywheel'), + ('x-github-request', 'GitHub'), + ('dps/', 'GoDaddy Website Builder'), + ('hostinger', 'Hostinger'), + ('zyro.com', 'Hostinger Website Builder'), + ('hubspot', 'HubSpot'), + ('x-kinsta-cache', 'Kinsta'), + ('x-lw-cache', 'Liquid Web'), + ('netlify', 'Netlify'), + ('x-pantheon-styx-hostname', 'Pantheon'), + ('seravo', 'Seravo'), + ('x-shopify-stage', 'Shopify'), + ('b7440e60b07ee7b8044761568fab26e8', 'SiteGround'), + ('624d5be7be38418a3e2a818cc8b7029b', 'SiteGround'), + ('6b7412fb82ca5edfd0917e3957f05d89', 'SiteGround'), + ('squarespace', 'Squarespace'), + ('x-vercel-id', 'Vercel'), + ('weebly', 'Weebly'), + ('x-wix-request-id', 'Wix'), + ('wp-cloud', 'WP-Cloud'), + ('wpe-backend', 'WP Engine'), + ('wp engine atlas', 'WP Engine Atlas'), + ('wp engine', 'WP Engine'), + ('zoneos', 'Zone.eu') +]; + + WITH crux AS ( SELECT - * + IF(device = 'desktop', 'desktop', 'mobile') AS client, + CONCAT(origin, '/') AS url, + CASE + WHEN SAFE_DIVIDE(fast_ttfb, (fast_ttfb + avg_ttfb + slow_ttfb)) >= 0.75 THEN 'Good' + WHEN SAFE_DIVIDE(slow_ttfb, (fast_ttfb + avg_ttfb + slow_ttfb)) >= 0.25 THEN 'Poor' + WHEN fast_ttfb IS NOT NULL THEN 'Needs Improvement' + ELSE NULL + END AS ttfb FROM `chrome-ux-report.materialized.device_summary` WHERE - date = '2023-09-01' -), requests AS ( + date = QUERY_DATE AND + device IN ('desktop', 'phone') AND + fast_ttfb IS NOT NULL +), + +requests AS ( SELECT client, root_page, @@ -17,77 +66,53 @@ WITH crux AS ( FROM `httparchive.all.requests` WHERE - date = '2023-09-01' AND + date = QUERY_DATE AND is_main_document +), + +platform_regex AS ( + SELECT + STRING_AGG( + # Escape special chars + REGEXP_REPLACE(regex, r'([\.\*\+\?\|\(\)\[\]\{\}\^\$])', r'\\\1'), + r'|' + ) AS pattern + FROM + UNNEST(PLATFORMS) +), + +detected_platforms AS ( + SELECT + client, + url, + name AS platform + FROM + UNNEST(PLATFORMS) + JOIN ( + SELECT + client, + root_page AS url, + REGEXP_EXTRACT(LOWER(CONCAT(IFNULL(respOtherHeaders, ''), IFNULL(resp_x_powered_by, ''), IFNULL(resp_via, ''), IFNULL(resp_server, ''))), (SELECT pattern FROM platform_regex)) AS regex + FROM + requests) + USING + (regex) ) + SELECT DISTINCT - CASE - WHEN platform = 'zyro.com' THEN 'Hostinger Website Builder' - WHEN platform = 'hostinger' THEN 'Hostinger' - WHEN platform = 'zoneos' THEN 'Zone.eu' - WHEN platform = 'seravo' THEN 'Seravo' - WHEN platform = 'automattic.com/work-with-us' THEN 'Automattic' - WHEN platform = 'wpvip.com/careers' THEN 'Automattic' - WHEN platform = 'wordpress.com' THEN 'Automattic' - WHEN platform = 'a9130478a60e5f9135f765b23f26593b' then 'Automattic' - WHEN platform = 'x-ah-environment' THEN 'Acquia' - WHEN platform = 'x-pantheon-styx-hostname' THEN 'Pantheon' - WHEN platform = 'wp engine atlas' THEN 'WP Engine Atlas' - WHEN platform = 'wpe-backend' THEN 'WP Engine' - WHEN platform = 'wp engine' THEN 'WP Engine' - WHEN platform = 'x-kinsta-cache' THEN 'Kinsta' - WHEN platform = 'hubspot' THEN 'HubSpot' - WHEN platform = 'b7440e60b07ee7b8044761568fab26e8' THEN 'SiteGround' - WHEN platform = '624d5be7be38418a3e2a818cc8b7029b' THEN 'SiteGround' - WHEN platform = '6b7412fb82ca5edfd0917e3957f05d89' THEN 'SiteGround' - WHEN platform = 'x-github-request' THEN 'GitHub' - WHEN platform = 'alproxy' THEN 'AlwaysData' - WHEN platform = 'netlify' THEN 'Netlify' - WHEN platform = 'x-lw-cache' THEN 'Liquid Web' - WHEN platform = 'squarespace' THEN 'Squarespace' - WHEN platform = 'x-wix-request-id' THEN 'Wix' - WHEN platform = 'x-shopify-stage' THEN 'Shopify' - WHEN platform = 'x-vercel-id' THEN 'Vercel' - WHEN platform = 'flywheel' THEN 'Flywheel' - WHEN platform = 'weebly' THEN 'Weebly' - WHEN platform = 'dps/' THEN 'GoDaddy Website Builder' - WHEN platform = 'awex' THEN '000webhost' - WHEN platform = 'wp-cloud' THEN 'WP-Cloud' - ELSE NULL - END AS platform, + platform, client, COUNT(DISTINCT url) AS n, COUNT(DISTINCT IF(ttfb = 'Good', url, NULL)) / COUNT(DISTINCT url) AS fast, COUNT(DISTINCT IF(ttfb = 'Needs Improvement', url, NULL)) / COUNT(DISTINCT url) AS avg, COUNT(DISTINCT IF(ttfb = 'Poor', url, NULL)) / COUNT(DISTINCT url) AS slow -FROM ( - SELECT - IF(device = 'desktop', 'desktop', 'mobile') AS client, - CONCAT(origin, '/') AS url, - CASE - WHEN SAFE_DIVIDE(fast_ttfb, (fast_ttfb + avg_ttfb + slow_ttfb)) >= 0.75 THEN 'Good' - WHEN SAFE_DIVIDE(slow_ttfb, (fast_ttfb + avg_ttfb + slow_ttfb)) >= 0.25 THEN 'Poor' - WHEN fast_ttfb IS NOT NULL THEN 'Needs Improvement' - ELSE NULL - END AS ttfb - FROM - crux - WHERE - device IN ('desktop', 'phone') AND - fast_ttfb IS NOT NULL) -JOIN ( - SELECT - client, - root_page AS url, - REGEXP_EXTRACT(LOWER(CONCAT(IFNULL(respOtherHeaders, ''), IFNULL(resp_x_powered_by, ''), IFNULL(resp_via, ''), IFNULL(resp_server, ''))), - r'(zoneos|seravo|x-kinsta-cache|automattic.com/work-with-us|wpvip.com/careers|wordpress\.com|a9130478a60e5f9135f765b23f26593b|x-ah-environment|x-pantheon-styx-hostname|wp engine atlas|wpe-backend|wp engine|hubspot|b7440e60b07ee7b8044761568fab26e8|624d5be7be38418a3e2a818cc8b7029b|6b7412fb82ca5edfd0917e3957f05d89|x-github-request|alproxy|netlify|x-lw-cache|squarespace|x-wix-request-id|x-shopify-stage|x-vercel-id|flywheel|weebly|dps/|hostinger|awex|wp-cloud|zyro.com)') AS platform - FROM - requests) +FROM + detected_platforms +JOIN + crux USING (client, url) -WHERE - platform IS NOT NULL GROUP BY platform, client