From ce7963125d850cd19a6b4a1c0f3dadf77c02aa86 Mon Sep 17 00:00:00 2001 From: Severin Ferrand <19885877+Seburan@users.noreply.github.com> Date: Thu, 26 Oct 2023 16:51:05 +0900 Subject: [PATCH 1/6] [changelog] add change to changelog --- CHANGELOG.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 571b2dd5..cb2a9927 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,9 @@ #### Enhancements - DSP Site : serve attestation file version2 on privacy-sandbox-demos-dsp.dev -- Tools : Add [Aggregatable Report Converter](https://github.com/privacysandbox/privacy-sandbox-demos/tree/main/tools/aggregatable_report_converter) to the tooling codebase. This tool helps developers to create debug aggregatable reports that can be used for Local Testing and AWS Aggregation Service testing. -- GitHub documentation : Add a changelog +- Use Case : Single-touch conversion Attribution. Move attribution code from SSP to DSP +- Tools : Add [Aggregatable Report Converter](https://github.com/privacysandbox/privacy-sandbox-demos/tree/main/tools/aggregatable_report_converter) to the tooling codebase. This tool helps developers to create debug aggregatable reports that can be used for Local Testing and AWS Aggregation Service testing +- GitHub documentation : Add a changelog. #### Bug Fixes @@ -18,13 +19,13 @@ - Shop site : refactor service using expressJS (from nextJS) - Shop site : fix issue with Firebase filtering session cookies - DSP Site : attestation file is served from privacy-sandbox-demos-dsp.dev -- Use Case : remarketing. Move `renderURL` from the SSP codebase to the DSP codebase. -- Use Case : VAST Video Protected Audience. Release v1. +- Use Case : remarketing. Move `renderURL` from the SSP codebase to the DSP codebase +- Use Case : VAST Video Protected Audience. Release v1 #### Bug Fixes - Use Case : Single-touch conversion Attribution. Fix registering attribution sources ("Attribution-Reporting-Eligible" is a Dictionary Structured Header and thus need to be decoded accordingly) -- Home site : fix Docusaurus and nginx configuration that prevented direct links to use cases. +- Home site : fix Docusaurus and nginx configuration that prevented direct links to use cases - Home site : fix broken links in documentation - GitHub documentation : update deployment guides From d4d7b8fad8fe09dee2d951718b6a437f7cfb068e Mon Sep 17 00:00:00 2001 From: Severin Ferrand <19885877+Seburan@users.noreply.github.com> Date: Fri, 27 Oct 2023 23:59:25 +0900 Subject: [PATCH 2/6] [dsp] migrate attribution reporting code to DSP 1st part of migration : add code to DSP - add dependencies in package.json - update typescript configuration - refactor code from javascript to typescript - on shop site update reference to trigger registration using dsp --- services/dsp/package-lock.json | 28 ++- services/dsp/package.json | 6 +- services/dsp/src/arapi.ts | 197 +++++++++++++++++++++ services/dsp/src/index.ts | 218 ++++++++++++++++++++++-- services/dsp/src/views/reports.html.ejs | 17 ++ services/dsp/tsconfig.json | 3 +- services/shop/src/index.js | 4 +- services/shop/src/index.ts | 2 +- 8 files changed, 453 insertions(+), 22 deletions(-) create mode 100644 services/dsp/src/arapi.ts create mode 100644 services/dsp/src/views/reports.html.ejs diff --git a/services/dsp/package-lock.json b/services/dsp/package-lock.json index d0f03b6b..d44e26e9 100644 --- a/services/dsp/package-lock.json +++ b/services/dsp/package-lock.json @@ -9,8 +9,10 @@ "version": "0.0.0", "license": "apache-2.0", "dependencies": { + "cbor": "^9.0.0", "ejs": "^3.1.9", - "express": "^4.18.2" + "express": "^4.18.2", + "structured-field-values": "^2.0.1" }, "devDependencies": { "@types/express": "^4.17.17", @@ -277,6 +279,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/cbor": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cbor/-/cbor-9.0.1.tgz", + "integrity": "sha512-/TQOWyamDxvVIv+DY9cOLNuABkoyz8K/F3QE56539pGVYohx0+MEA1f4lChFTX79dBTBS7R1PF6ovH7G+VtBfQ==", + "dependencies": { + "nofilter": "^3.1.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -711,6 +724,14 @@ "node": ">= 0.6" } }, + "node_modules/nofilter": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", + "integrity": "sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==", + "engines": { + "node": ">=12.19" + } + }, "node_modules/object-inspect": { "version": "1.12.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", @@ -898,6 +919,11 @@ "node": ">= 0.8" } }, + "node_modules/structured-field-values": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/structured-field-values/-/structured-field-values-2.0.1.tgz", + "integrity": "sha512-1VNk582THEQbA6X7pZF+0mjTH8jD1lvckzZx5gzUW3F6m0Bqi5EJ/WnM4S0egNz+KdKII7A8ArRCqMafZWh7pQ==" + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", diff --git a/services/dsp/package.json b/services/dsp/package.json index b2a3e910..d880f01f 100644 --- a/services/dsp/package.json +++ b/services/dsp/package.json @@ -6,12 +6,14 @@ "type": "module", "scripts": { "start": "npm run build && node ./build/index.js", - "build": "tsc && cp -r src/views build/views", + "build": "tsc && cp -r src/views build/", "ncu": "npx npm-check-updates -u" }, "dependencies": { + "cbor": "^9.0.0", "ejs": "^3.1.9", - "express": "^4.18.2" + "express": "^4.18.2", + "structured-field-values": "^2.0.1" }, "devDependencies": { "@types/express": "^4.17.17", diff --git a/services/dsp/src/arapi.ts b/services/dsp/src/arapi.ts new file mode 100644 index 00000000..0ab3c849 --- /dev/null +++ b/services/dsp/src/arapi.ts @@ -0,0 +1,197 @@ +/* + Copyright 2022 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +// type: 2bit +const SOURCE_TYPE: { click: number; view: number } = { + click: 0b10, + view: 0b11 +} + +// advertiser: 16bit +const ADVERTISER: { [index: string]: number } = {} +ADVERTISER[process.env.SHOP_HOST as string] = 0b0 +ADVERTISER[process.env.TRAVEL_HOST as string] = 0b1 + +// publisher: 16bit +const PUBLISHER: { [index: string]: number } = {} +PUBLISHER[process.env.NEWS_HOST as string] = 0b0 + +// dimension: 8bit +const DIMENSION: { quantity: number; gross: number } = { + quantity: 0b0, + gross: 0b1 +} + +// type 8bit +const TRIGGER_TYPE = { + quantity: 0b1000_0000, + gross: 0b1100_0000 +} + +export { SOURCE_TYPE, ADVERTISER, PUBLISHER, DIMENSION, TRIGGER_TYPE } + +type AggregationKeyStructure = { + type: number + advertiser: number + publisher: number + id: number + dimension: number +} + +export function sourceKeyPiece(ako: AggregationKeyStructure) { + console.log(ako) + const source = encodeSource(ako) + const uint64: bigint = new DataView(source).getBigUint64(0, false) + return `0x${(uint64 << 64n).toString(16)}` +} + +type AggregatableTriggerData = { + type: number + id: number + size: number + category: number + option: number +} + +export function triggerKeyPiece(atd: AggregatableTriggerData) { + console.log(atd) + const trigger = encodeTrigger(atd) + const uint64 = new DataView(trigger).getBigUint64(0, false) + return `0x${"0".repeat(16)}${uint64.toString(16)}` +} + +// type: 2bit +// dimension: 6bit +// id: 24bit +// advertiser: 16bit +// publisher: 16bit +// ----------------- +// 64bit +function encodeSource(ako: AggregationKeyStructure) { + const buffer = new ArrayBuffer(8) + const view = new DataView(buffer) + const first32 = (((ako.type << 6) + ako.dimension) << 24) + ako.id + view.setUint32(0, first32) + view.setUint16(4, ako.advertiser) + view.setUint16(6, ako.publisher) + return buffer +} + +function decodeSource(buffer: ArrayBuffer): AggregationKeyStructure { + const view = new DataView(buffer) + const first32 = view.getUint32(0) + const id = first32 & (2 ** 24 - 1) + const first8 = first32 >>> 24 + const dimension = first8 & 0b111111 + const type = first8 >>> 6 + const advertiser = view.getUint16(4) + const publisher = view.getUint16(6) + return { type, dimension, id, advertiser, publisher } +} + +// type: 8bit +// id: 24bit +// size: 8bit +// category: 8bit +// option: 16bit +// ----------------- +// 64bit +function encodeTrigger(atd: AggregatableTriggerData) { + const buffer = new ArrayBuffer(8) + const view = new DataView(buffer) + view.setUint32(0, (atd.type << 24) + atd.id) + view.setUint8(4, atd.size) + view.setUint8(5, atd.category) + view.setUint16(6, atd.option) + return buffer +} + +function decodeTrigger(buffer: ArrayBuffer): AggregatableTriggerData { + const view = new DataView(buffer) + const first32 = view.getUint32(0) + const type = first32 >>> 24 + const id = first32 & 0xffffff + const size = view.getUint8(4) + const category = view.getUint8(5) + const option = view.getUint16(6) + return { type, id, size, category, option } +} + +export function decodeBucket(buffer: ArrayBuffer) { + const u8a = new Uint8Array(buffer) + const sourceBuf = u8a.slice(0, u8a.length / 2) + const source: AggregationKeyStructure = decodeSource(sourceBuf.buffer) + const triggerBuf = u8a.slice(u8a.length / 2, u8a.length) + const trigger: AggregatableTriggerData = decodeTrigger(triggerBuf.buffer) + + const readableAggregationKey: { [index: string]: string } = {} + readableAggregationKey.type = key_from_value(SOURCE_TYPE, source.type) + readableAggregationKey.dimension = key_from_value(DIMENSION, source.dimension) + readableAggregationKey.id = source.id.toString(16) + readableAggregationKey.advertiser = key_from_value(ADVERTISER, source.advertiser) + readableAggregationKey.publisher = key_from_value(PUBLISHER, source.publisher) + + const readableAggregatableTriggerData: { [index: string]: string } = {} + readableAggregatableTriggerData.type = key_from_value(TRIGGER_TYPE, trigger.type) + readableAggregatableTriggerData.id = trigger.id.toString(16) + + return { + readableAggregationKey, + readableAggregatableTriggerData + } +} + +export function sourceEventId() { + // 64bit dummy value + return ((1n << 64n) - 1n).toString() +} + +export function debugKey(): string { + // 64bit dummy value + return ((1n << 64n) - 2n).toString() +} + +function key_from_value(object: any, value: any) { + const key: string = Object.keys(object).find((key) => object[key] === value) as string + + return key +} + +function test() { + const advertiser = ADVERTISER["shop"] + const publisher = PUBLISHER["news"] + const id = 0xff + const dimension = DIMENSION["gross"] + const size = (26.5 - 20) * 10 + const category = 1 + const source_type = SOURCE_TYPE.click + const trigger_type = TRIGGER_TYPE.gross + const option = 2 + + const source_key = sourceKeyPiece({ type: source_type, dimension, id, advertiser, publisher }) + console.log({ source_key }) + + const trigger_key = triggerKeyPiece({ type: trigger_type, id, size, category, option }) + console.log({ trigger_key }) + + const source = encodeSource({ type: source_type, dimension, id, advertiser, publisher }) + console.log(decodeSource(source)) + + const trigger = encodeTrigger({ type: trigger_type, id, size, category, option }) + console.log(decodeTrigger(trigger)) +} + +test() diff --git a/services/dsp/src/index.ts b/services/dsp/src/index.ts index 23161c42..0ae8b78e 100644 --- a/services/dsp/src/index.ts +++ b/services/dsp/src/index.ts @@ -16,9 +16,31 @@ // DSP import express, { Application, Request, Response } from "express" +import cbor from "cbor" +import { decodeDict } from "structured-field-values" +import { + debugKey, + sourceEventId, + sourceKeyPiece, + triggerKeyPiece, + ADVERTISER, + PUBLISHER, + DIMENSION, + decodeBucket, + SOURCE_TYPE, + TRIGGER_TYPE +} from "./arapi.js" const { EXTERNAL_PORT, PORT, DSP_HOST, DSP_TOKEN, DSP_DETAIL, SSP_HOST, SHOP_HOST } = process.env +// in-memory storage for debug reports +const Reports: any[] = [] + +// clear in-memory storage every 10 min +setInterval(() => { + Reports.length = 0 +}, 1000 * 60 * 10) + const app: Application = express() app.use((req, res, next) => { @@ -26,7 +48,7 @@ app.use((req, res, next) => { next() }) -app.use(express.urlencoded({extended: true})); +app.use(express.urlencoded({ extended: true })) app.use(express.json()) // To parse the incoming requests with JSON payloads app.use( @@ -64,7 +86,7 @@ app.get("/ads", async (req, res) => { const creative = new URL(`https://${advertiser}:${EXTERNAL_PORT}/ads/${id}`) - const registerSource = new URL(`https://${SSP_HOST}:${EXTERNAL_PORT}/register-source`) + const registerSource = new URL(`https://${DSP_HOST}:${EXTERNAL_PORT}/register-source`) registerSource.searchParams.append("advertiser", advertiser as string) registerSource.searchParams.append("id", id as string) @@ -142,33 +164,199 @@ app.get("/bidding_signal.json", async (req: Request, res: Response) => { // app.get("/daily_update_url", async (req: Request, res: Response) => { // }) -app.post("/.well-known/private-aggregation/report-shared-storage", (req, res) => { +// ************************************************************************ +// [START] Section for Attribution Reporting API Code *** +// ************************************************************************ +app.get("/register-source", async (req: Request, res: Response) => { + const advertiser: string = req.query.advertiser as string + const id: string = req.query.id as string + + console.log("Registering source attribution for", { advertiser, id }) + if (req.headers["attribution-reporting-eligible"]) { + //const are = req.headers["attribution-reporting-eligible"].split(",").map((e) => e.trim()) + const are = decodeDict(req.headers["attribution-reporting-eligible"] as string) + + // register navigation source + if ("navigation-source" in are) { + const destination = `https://${advertiser}` + const source_event_id = sourceEventId() + const debug_key = debugKey() + const AttributionReportingRegisterSource = { + destination, + source_event_id, + debug_key, + aggregation_keys: { + quantity: sourceKeyPiece({ + type: SOURCE_TYPE["click"], // click attribution + advertiser: ADVERTISER[advertiser], + publisher: PUBLISHER["news"], + id: Number(`0x${id}`), + dimension: DIMENSION["quantity"] + }), + gross: sourceKeyPiece({ + type: SOURCE_TYPE["click"], // click attribution + advertiser: ADVERTISER[advertiser], + publisher: PUBLISHER["news"], + id: Number(`0x${id}`), + dimension: DIMENSION["gross"] + }) + } + } - console.log( `Received Aggregatable Report on live endpoint`); + console.log("Registering navigation source :", { AttributionReportingRegisterSource }) + res.setHeader("Attribution-Reporting-Register-Source", JSON.stringify(AttributionReportingRegisterSource)) + res.status(200).send("attribution nevigation (click) source registered") + } + + // register event source + else if ("event-source" in are) { + const destination = `https://${advertiser}` + const source_event_id = sourceEventId() + const debug_key = debugKey() + const AttributionReportingRegisterSource = { + destination, + source_event_id, + debug_key, + aggregation_keys: { + quantity: sourceKeyPiece({ + type: SOURCE_TYPE["view"], // view attribution + advertiser: ADVERTISER[advertiser], + publisher: PUBLISHER["news"], + id: Number(`0x${id}`), + dimension: DIMENSION["quantity"] + }), + gross: sourceKeyPiece({ + type: SOURCE_TYPE["view"], // view attribution + advertiser: ADVERTISER[advertiser], + publisher: PUBLISHER["news"], + id: Number(`0x${id}`), + dimension: DIMENSION["gross"] + }) + } + } - let aggregationReport = req.body; - console.log(req.body); + console.log("Registering event source :", { AttributionReportingRegisterSource }) + res.setHeader("Attribution-Reporting-Register-Source", JSON.stringify(AttributionReportingRegisterSource)) + res.status(200).send("attribution event (view) source registered") + } else { + res.status(400).send("'Attribution-Reporting-Eligible' header is malformed") // just send back response header. no content. + } + } else { + res.status(400).send("'Attribution-Reporting-Eligible' header is missing") // just send back response header. no content. + } +}) - res.sendStatus(200); +app.get("/register-trigger", async (req: Request, res: Response) => { + const id: string = req.query.id as string + const quantity: string = req.query.quantity as string + const size: string = req.query.size as string + const category: string = req.query.category as string + const gross: string = req.query.gross as string + const AttributionReportingRegisterTrigger = { + aggregatable_trigger_data: [ + { + key_piece: triggerKeyPiece({ + type: TRIGGER_TYPE["quantity"], + id: parseInt(id, 16), + size: Number(size), + category: Number(category), + option: 0 + }), + source_keys: ["quantity"] + }, + { + key_piece: triggerKeyPiece({ + type: TRIGGER_TYPE["gross"], + id: parseInt(id, 16), + size: Number(size), + category: Number(category), + option: 0 + }), + source_keys: ["gross"] + } + ], + aggregatable_values: { + // TODO: scaling + quantity: Number(quantity), + gross: Number(gross) + }, + debug_key: debugKey() + } + res.setHeader("Attribution-Reporting-Register-Trigger", JSON.stringify(AttributionReportingRegisterTrigger)) + res.sendStatus(200) }) -app.get("/private-aggregation", (req, res) => { - res.render('private-aggregation'); +app.post("/.well-known/attribution-reporting/debug/report-aggregate-attribution", async (req: Request, res: Response) => { + console.log(`Attribution Reporting - Received Aggregatable Report on debug endpoint`) + const debug_report = req.body + debug_report.shared_info = JSON.parse(debug_report.shared_info) + + console.log(JSON.stringify(debug_report, null, "\t")) + + debug_report.aggregation_service_payloads = debug_report.aggregation_service_payloads.map((e: any) => { + const plain = Buffer.from(e.debug_cleartext_payload, "base64") + const debug_cleartext_payload = cbor.decodeAllSync(plain) + e.debug_cleartext_payload = debug_cleartext_payload.map(({ data, operation }) => { + return { + operation, + data: data.map(({ value, bucket }: any) => { + return { + value: value.readUInt32BE(0), + bucket: decodeBucket(bucket) + } + }) + } + }) + return e + }) + + console.log(JSON.stringify(debug_report, null, "\t")) + + // save to global storage + Reports.push(debug_report) + + res.sendStatus(200) }) -app.post("/.well-known/private-aggregation/debug/report-shared-storage", (req, res) => { +app.post("/.well-known/attribution-reporting/report-aggregate-attribution", async (req: Request, res: Response) => { + console.log(`Attribution Reporting - Received Aggregatable Report on live endpoint`) + const report = req.body + report.shared_info = JSON.parse(report.shared_info) + console.log(JSON.stringify(report, null, "\t")) + res.sendStatus(200) +}) - let timeStr = new Date().toISOString(); - console.log( `Received Aggregatable Report on debug endpoint`); +app.get("/reports", async (req, res) => { + res.render("reports.html.ejs", { title: "Report", Reports }) +}) - let aggregationReport = req.body; +// ************************************************************************ +// [END] Section for Attribution Reporting API Code *** +// ************************************************************************ - console.log(aggregationReport); +app.post("/.well-known/private-aggregation/report-shared-storage", (req, res) => { + console.log(`Private Aggregation for Shared Storage - Received Aggregatable Report on live endpoint`) + + let aggregationReport = req.body + console.log(req.body) + + res.sendStatus(200) +}) + +app.get("/private-aggregation", (req, res) => { + res.render("private-aggregation") +}) + +app.post("/.well-known/private-aggregation/debug/report-shared-storage", (req, res) => { + let timeStr = new Date().toISOString() + console.log(`Private Aggregation for Shared Storage - Received Aggregatable Report on debug endpoint`) + let aggregationReport = req.body - res.sendStatus(200); + console.log(aggregationReport) + res.sendStatus(200) }) app.get("/", async (req: Request, res: Response) => { diff --git a/services/dsp/src/views/reports.html.ejs b/services/dsp/src/views/reports.html.ejs new file mode 100644 index 00000000..f20f01a9 --- /dev/null +++ b/services/dsp/src/views/reports.html.ejs @@ -0,0 +1,17 @@ + + + + + + + <%= title %> + + + + + +

<%= title %>

+ 
<%= JSON.stringify(Reports, " ", " ") %>
+ + + \ No newline at end of file diff --git a/services/dsp/tsconfig.json b/services/dsp/tsconfig.json index ff6e0782..9c07f0c9 100644 --- a/services/dsp/tsconfig.json +++ b/services/dsp/tsconfig.json @@ -1,9 +1,10 @@ { "compilerOptions": { "target": "es2022", - "module": "ES2022", + "module": "NodeNext", "rootDir": "./src", "outDir": "./build", + "moduleResolution": "NodeNext", "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "strict": true, diff --git a/services/shop/src/index.js b/services/shop/src/index.js index 2ee8ab12..384077a0 100644 --- a/services/shop/src/index.js +++ b/services/shop/src/index.js @@ -16,7 +16,7 @@ import express from "express" import session from "express-session" import MemoryStoreFactory from "memorystore" -import { DSP_HOST, EXTERNAL_PORT, PORT, SHOP_DETAIL, SHOP_HOST, SSP_HOST } from "./env.js" +import { DSP_HOST, EXTERNAL_PORT, PORT, SHOP_DETAIL, SHOP_HOST } from "./env.js" import { addOrder, displayCategory, fromSize, getItem, getItems, removeOrder, updateOrder } from "./lib/items.js" const app = express() app.set("trust proxy", 1) // required for Set-Cookie with Secure @@ -58,7 +58,7 @@ app.locals = { displayCategory, register_trigger: (order) => { const { item, size, quantity } = order - const register_trigger = new URL(`https://${SSP_HOST}:${EXTERNAL_PORT}`) + const register_trigger = new URL(`https://${DSP_HOST}:${EXTERNAL_PORT}`) register_trigger.pathname = "/register-trigger" register_trigger.searchParams.append("id", item.id) register_trigger.searchParams.append("category", `${item.category}`) diff --git a/services/shop/src/index.ts b/services/shop/src/index.ts index 2c808aee..49da95f0 100644 --- a/services/shop/src/index.ts +++ b/services/shop/src/index.ts @@ -72,7 +72,7 @@ app.locals = { displayCategory, register_trigger: (order: Order) => { const { item, size, quantity } = order - const register_trigger = new URL(`https://${SSP_HOST}:${EXTERNAL_PORT}`) + const register_trigger = new URL(`https://${DSP_HOST}:${EXTERNAL_PORT}`) register_trigger.pathname = "/register-trigger" register_trigger.searchParams.append("id", item.id) register_trigger.searchParams.append("category", `${item.category}`) From 294be3bf5d44acc0597b6b673a00f69f192f3e9c Mon Sep 17 00:00:00 2001 From: Severin Ferrand <19885877+Seburan@users.noreply.github.com> Date: Tue, 31 Oct 2023 14:54:02 +0900 Subject: [PATCH 3/6] [dsp] migrate attribution reporting code to DSP Enable debug reports with ar_debug cookies Update debug report output --- services/dsp/src/arapi.ts | 27 +++++++++++++++------------ services/dsp/src/index.ts | 11 +++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/services/dsp/src/arapi.ts b/services/dsp/src/arapi.ts index 0ab3c849..35529419 100644 --- a/services/dsp/src/arapi.ts +++ b/services/dsp/src/arapi.ts @@ -137,20 +137,23 @@ export function decodeBucket(buffer: ArrayBuffer) { const triggerBuf = u8a.slice(u8a.length / 2, u8a.length) const trigger: AggregatableTriggerData = decodeTrigger(triggerBuf.buffer) - const readableAggregationKey: { [index: string]: string } = {} - readableAggregationKey.type = key_from_value(SOURCE_TYPE, source.type) - readableAggregationKey.dimension = key_from_value(DIMENSION, source.dimension) - readableAggregationKey.id = source.id.toString(16) - readableAggregationKey.advertiser = key_from_value(ADVERTISER, source.advertiser) - readableAggregationKey.publisher = key_from_value(PUBLISHER, source.publisher) - - const readableAggregatableTriggerData: { [index: string]: string } = {} - readableAggregatableTriggerData.type = key_from_value(TRIGGER_TYPE, trigger.type) - readableAggregatableTriggerData.id = trigger.id.toString(16) + const aggregation_keys: { [index: string]: string } = {} + aggregation_keys.type = key_from_value(SOURCE_TYPE, source.type) + aggregation_keys.dimension = key_from_value(DIMENSION, source.dimension) + aggregation_keys.id = source.id.toString(16) + aggregation_keys.advertiser = key_from_value(ADVERTISER, source.advertiser) + aggregation_keys.publisher = key_from_value(PUBLISHER, source.publisher) + + const aggregatable_trigger_data: { [index: string]: string } = {} + aggregatable_trigger_data.type = key_from_value(TRIGGER_TYPE, trigger.type) + aggregatable_trigger_data.id = trigger.id.toString(16) + aggregatable_trigger_data.size = trigger.size.toString() + aggregatable_trigger_data.category = trigger.category.toString() + aggregatable_trigger_data.option = trigger.option.toString() return { - readableAggregationKey, - readableAggregatableTriggerData + aggregation_keys, + aggregatable_trigger_data } } diff --git a/services/dsp/src/index.ts b/services/dsp/src/index.ts index 0ae8b78e..ca5822ad 100644 --- a/services/dsp/src/index.ts +++ b/services/dsp/src/index.ts @@ -49,8 +49,19 @@ app.use((req, res, next) => { }) app.use(express.urlencoded({ extended: true })) + app.use(express.json()) // To parse the incoming requests with JSON payloads +app.use((req, res, next) => { + // enable transitional debugging reports (https://github.com/WICG/attribution-reporting-api/blob/main/EVENT.md#optional-transitional-debugging-reports) + res.cookie("ar_debug", "1", { + sameSite: "none", + secure: true, + httpOnly: true + }) + next() +}) + app.use( express.static("src/public", { setHeaders: (res: Response, path, stat) => { From 5f905f53080abed7397278198de7b76642313d0e Mon Sep 17 00:00:00 2001 From: Severin Ferrand <19885877+Seburan@users.noreply.github.com> Date: Tue, 31 Oct 2023 15:51:44 +0900 Subject: [PATCH 4/6] [dsp] migrate attribution reporting code to DSP update use case documentation ToDo : update permalink once commit is merged --- CHANGELOG.md | 10 ++- .../single-touch-conversion-attribution.md | 73 +++++++++++-------- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb2a9927..3baf8bf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,27 +3,28 @@ ## v1.x.x (2023/xx/xx) #### Enhancements + - DSP Site : serve attestation file version2 on privacy-sandbox-demos-dsp.dev -- Use Case : Single-touch conversion Attribution. Move attribution code from SSP to DSP +- Use Case : Single-touch conversion Attribution. Move attribution code from SSP to DSP and update documentation - Tools : Add [Aggregatable Report Converter](https://github.com/privacysandbox/privacy-sandbox-demos/tree/main/tools/aggregatable_report_converter) to the tooling codebase. This tool helps developers to create debug aggregatable reports that can be used for Local Testing and AWS Aggregation Service testing - GitHub documentation : Add a changelog. #### Bug Fixes - --- ## v1.1 (2023/10/13) #### Enhancements -- Shop site : refactor service using expressJS (from nextJS) + +- Shop site : refactor service using expressJS (from nextJS) - Shop site : fix issue with Firebase filtering session cookies - DSP Site : attestation file is served from privacy-sandbox-demos-dsp.dev - Use Case : remarketing. Move `renderURL` from the SSP codebase to the DSP codebase - Use Case : VAST Video Protected Audience. Release v1 - #### Bug Fixes + - Use Case : Single-touch conversion Attribution. Fix registering attribution sources ("Attribution-Reporting-Eligible" is a Dictionary Structured Header and thus need to be decoded accordingly) - Home site : fix Docusaurus and nginx configuration that prevented direct links to use cases - Home site : fix broken links in documentation @@ -32,6 +33,7 @@ --- ## v1.0.0 (2023/06/30) + - Launch Privacy Sandbox Demos [Github Repository](https://github.com/privacysandbox/privacy-sandbox-demos) - Launch hosted demos [https://privacy-sandbox-demos.dev/ ](https://privacy-sandbox-demos.dev/) - Publication of a [blog post ](https://developer.chrome.com/blog/privacy-sandbox-demos/)on [developers.chrome.com](http://developers.chrome.com/) diff --git a/services/home/docs/demos/single-touch-conversion-attribution.md b/services/home/docs/demos/single-touch-conversion-attribution.md index 44e98a83..06fecbfd 100644 --- a/services/home/docs/demos/single-touch-conversion-attribution.md +++ b/services/home/docs/demos/single-touch-conversion-attribution.md @@ -71,6 +71,7 @@ Below is a general introduction of Single-Touch conversion Attribution using Pri --> ```mermaid + sequenceDiagram Title: Single-touch conversion Attribution - User Journey 1 @@ -80,25 +81,33 @@ participant SSP participant Advertiser participant DSP -Browser-)Publisher:visits a publisher site and sees an ad -Browser->>SSP:Load ad creative -SSP-->>Browser:Attribution-Reporting-Register-Source:{...} json config +Browser-)SSP:User visits a publisher site +SSP-->>Browser: SSP runs auction and load winning bid renderURL into an iframe* + +Note right of Browser:* we skip the auction flow
to focus on measurement with ARA + +Browser->>DSP:Browser loads ad creative +DSP-->>Browser:Attribution-Reporting-Register-Source:{...} json config Browser-->>Browser:Register Attribution Source -Browser-)Advertiser:visits the advertiser site and check out -Browser->>SSP: Load attribution pixel -SSP-->>Browser:Attribution-Reporting-Register-Trigger:{...} json config +Browser-)Advertiser:User visits the advertiser site (from click to the ad or from different journey) + +Browser-)Advertiser:User navigates to check out page +Browser->>DSP: Browser loads attribution pixel +DSP-->>Browser:Attribution-Reporting-Register-Trigger:{...} json config Browser-->>Browser:Register Attribution Trigger Browser-->>Browser:Attribution logic & create report -Note right of Browser: debug reports
are sent immediately -Browser-)SSP:sends aggregatable report (Debug Report) -Note over SSP:Scenario 1 stops here
where we visualize
debug reports +Browser-)DSP:Browser sends aggregatable report (Debug Repor are sent immediately) +Browser-)DSP:Browser sends aggregatable report (Live Report are deferred) + +Note over DSP:Scenario 1 stops here
where we visualize
debug reports + ``` @@ -133,7 +142,7 @@ Note over SSP:Scenario 1 stops here
where we visualize
debug reports 6. Navigate to chrome://attribution-internals/ and click the `Active Sources` tab -- At the bottom of the page, you will see 2 **sources** with the status `Attributable`, the `source origin` is the **news** site, the `destination` is the **shop** site and the `reporting origin` is the **SSP** service. One of the `source type` is **event** (for view-through) and the other one is **navigation** (for click-through) This reference will be used later to attribute (match) the conversion (here the. purchase of an item on the **shop** site) to a previous event (here. The user saw/clicked an ad on the **news** site) +- At the bottom of the page, you will see 2 **sources** with the status `Attributable`, the `source origin` is the **news** site, the `destination` is the **shop** site and the `reporting origin` is the **DSP** service. One of the `source type` is **event** (for view-through) and the other one is **navigation** (for click-through) This reference will be used later to attribute (match) the conversion (here the. purchase of an item on the **shop** site) to a previous event (here. The user saw/clicked an ad on the **news** site) 7. On the product page, click “Add to cart” 8. On the cart page, click “Checkout” @@ -143,11 +152,11 @@ Note over SSP:Scenario 1 stops here
where we visualize
debug reports 9. Navigate to chrome://attribution-internals/ and click the `Trigger Registration` tab -- At the bottom of the page, you will see 1 **trigger** . the `destination` is the **shop** site and the `reporting origin` is the **SSP** service. The `Registration JSON` contains information about the conversion event. In this scenario the advertiser chose to report the gross price and the quantity of the product item purchased. the `Aggregatable Status` indicates **Success: Report stored**, it means Attribution Reporting API has now stored this report in the browser. It will then be scheduled for sending to the `reporting origin` at a later time. +- At the bottom of the page, you will see 1 **trigger** . the `destination` is the **shop** site and the `reporting origin` is the **DSP** service. The `Registration JSON` contains information about the conversion event. In this scenario the advertiser chose to report the gross price and the quantity of the product item purchased. the `Aggregatable Status` indicates **Success: Report stored**, it means Attribution Reporting API has now stored this report in the browser. It will then be scheduled for sending to the `reporting origin` at a later time. -10. Navigate to [SSP service report visualization page](https://privacy-sandbox-demos-ssp.dev/reports) +10. Navigate to [DSP service report visualization page](https://privacy-sandbox-demos-dsp.dev/reports) -- on this page you can see the aggregatable report sent by the browser to the SSP. In a production environment, the aggregatable report is encrypted by the browser and sent to the SSP. There, they will be batched and sent to the Aggregation Service where they will be aggregated and noised to preserve privacy. However for development and testing purposes, you can also send an unencrypted version called **debug report**. This is what you are seeing now. +- on this page you can see the aggregatable report sent by the browser to the DSP. In a production environment, the aggregatable report is encrypted by the browser and sent to the DSP. There, they will be batched and sent to the Aggregation Service where they will be aggregated and noised to preserve privacy. However for development and testing purposes, you can also send an unencrypted version called **debug report**. This is what you are seeing now. - The report shows aggregation data on 2 dimensions : gross with a value of 180 and quantity with a value of 1. ### Implementation details @@ -158,23 +167,25 @@ First on the Attribution Source registration side. Look at the [code](https://github.com/privacysandbox/privacy-sandbox-demos/blob/cd28aba4e85b641d50d6ee999019d25607c439fc/services/ssp/src/views/ads.html.ejs#L29) displaying the ad creative ```html - - - - + + + + + + ``` The `img` tag also specifies the `attributionsrc` attribute. It means that showing this ad will register an attribution source of type `event` in the browser. @@ -200,7 +211,7 @@ The checkout page contains a 1 pixel image loaded from the code alt="" width="1" height="1" - src="https://privacy-sandbox-demos-ssp.dev/register-trigger?id=1f45e&category=1&quantity=2&size=50&gross=180" + src="https://privacy-sandbox-demos-dsp.dev/register-trigger?id=1f45e&category=1&quantity=2&size=50&gross=180" /> ``` From d363150c2409446bfbe57f639b48cf2627204783 Mon Sep 17 00:00:00 2001 From: Severin Ferrand <19885877+Seburan@users.noreply.github.com> Date: Tue, 31 Oct 2023 16:11:49 +0900 Subject: [PATCH 5/6] [remarketing] update documentation update permalink and align content with code migration to DSP --- .../docs/demos/retargeting-remarketing.md | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/services/home/docs/demos/retargeting-remarketing.md b/services/home/docs/demos/retargeting-remarketing.md index a4540905..aceaf86b 100644 --- a/services/home/docs/demos/retargeting-remarketing.md +++ b/services/home/docs/demos/retargeting-remarketing.md @@ -102,8 +102,8 @@ note right of Browser:for each candidate ad in the auction Browser-)Browser:scoreAd(...) note right of Browser:Winning ad is displayed in a fenced-frame -Browser->>SSP:Request ad creative -SSP-->>Browser:Return ad creative +Browser->>DSP:Request ad creative +DSP-->>Browser:Return ad creative Note right of Browser:Scenario 1 stops here @@ -242,19 +242,31 @@ The result of the auction is displayed within a Fenced Frame by specifying the a note that Fenced Frame attribute `mode` must be set to “[opaque-ads](https://github.com/WICG/fenced-frame/blob/master/explainer/use_cases.md#opaque-ads)” to make the url opaque to the embedding context. Fenced Frame size (width and height) only allow pre-defined values, please refer to the allow-list from the documentation. -The request to the `src` url[ returns the ad creative](https://github.com/privacysandbox/privacy-sandbox-demos/blob/8a33afb7433ed70e639047316c5bff30d61be58b/services/ssp/src/index.js#L87) to be displayed +The request to the `renderURL`from the winning Interest Group [returns the ad creative](https://github.com/privacysandbox/privacy-sandbox-demos/blob/9f8578e6d99da4dd52007843a283c97885c07146/services/dsp/src/index.ts#L57) to be displayed ```html - + - + - - + ``` -This code contains the `img` tag with `src` attribute specifying the product the user might be interested in. The `advertiser` and `id` are resolved by the SSP which returns the product image and product url to the shopping site. +This code contains the `img` tag with `src` attribute specifying the product the user might be interested in, and a link “href” pointing to the advertiser page. ### Related API documentation From 24cc361f5427c7e301380cefa1cc5a0d35e9a9b3 Mon Sep 17 00:00:00 2001 From: Severin Ferrand <19885877+Seburan@users.noreply.github.com> Date: Wed, 1 Nov 2023 00:06:36 +0900 Subject: [PATCH 6/6] [ssp] remove measurement code from ssp comment out code then clean once it's tested --- services/ssp/src/arapi.js | 328 +++++++++++++++++----------------- services/ssp/src/index.js | 364 +++++++++++++++++++------------------- 2 files changed, 346 insertions(+), 346 deletions(-) diff --git a/services/ssp/src/arapi.js b/services/ssp/src/arapi.js index df596219..0f55f349 100644 --- a/services/ssp/src/arapi.js +++ b/services/ssp/src/arapi.js @@ -14,167 +14,167 @@ limitations under the License. */ -// type: 2bit -export const SOURCE_TYPE = { - click: 0b10, - view: 0b11 -} - -// advertiser: 16bit -export const ADVERTISER = { - [process.env.SHOP_HOST]: 0b0, - [process.env.TRAVEL_HOST]: 0b1 -} - -// publisher: 16bit -export const PUBLISHER = { - [process.env.NEWS_HOST]: 0b0 -} - -// dimension: 8bit -export const DIMENSION = { - quantity: 0b0, - gross: 0b1 -} - -// type 8bit -export const TRIGGER_TYPE = { - quantity: 0b1000_0000, - gross: 0b1100_0000 -} - -export function sourceKeyPiece({ type, dimension, id, advertiser, publisher }) { - console.log({ type, dimension, id, advertiser, publisher }) - const source = encodeSource({ type, dimension, id, advertiser, publisher }) - const uint64 = new DataView(source).getBigUint64() - return `0x${(uint64 << 64n).toString(16)}` -} - -export function triggerKeyPiece({ type, id, size, category, option }) { - console.log({ type, id, size, category, option }) - const trigger = encodeTrigger({ type, id, size, category, option }) - const uint64 = new DataView(trigger).getBigUint64() - return `0x${"0".repeat(16)}${uint64.toString(16)}` -} - -// type: 2bit -// dimension: 6bit -// id: 24bit -// advertiser: 16bit -// publisher: 16bit -// ----------------- -// 64bit -function encodeSource({ type, dimension, id, advertiser, publisher }) { - const buffer = new ArrayBuffer(8) - const view = new DataView(buffer) - const first32 = (((type << 6) + dimension) << 24) + id - view.setUint32(0, first32) - view.setUint16(4, advertiser) - view.setUint16(6, publisher) - return buffer -} - -function decodeSource(buffer) { - const view = new DataView(buffer) - const first32 = view.getUint32(0) - const id = first32 & (2 ** 24 - 1) - const first8 = first32 >>> 24 - const dimension = first8 & 0b111111 - const type = first8 >>> 6 - const advertiser = view.getUint16(4) - const publisher = view.getUint16(6) - return { type, dimension, id, advertiser, publisher } -} - -// type: 8bit -// id: 24bit -// size: 8bit -// category: 8bit -// option: 16bit -// ----------------- -// 64bit -function encodeTrigger({ type, id, size, category, option }) { - const buffer = new ArrayBuffer(8) - const view = new DataView(buffer) - view.setUint32(0, (type << 24) + id) - view.setUint8(4, size) - view.setUint8(5, category) - view.setUint16(6, option) - return buffer -} - -function decodeTrigger(buffer) { - const view = new DataView(buffer) - const first32 = view.getUint32(0) - const type = first32 >>> 24 - const id = first32 & 0xffffff - const size = view.getUint8(4) - const category = view.getUint8(5) - const option = view.getUint16(6) - return { type, id, size, category, option } -} - -export function decodeBucket(buffer) { - const u8a = new Uint8Array(buffer) - const sourceBuf = u8a.slice(0, u8a.length / 2) - const source = decodeSource(sourceBuf.buffer) - const triggerBuf = u8a.slice(u8a.length / 2, u8a.length) - const trigger = decodeTrigger(triggerBuf.buffer) - - source.type = key_from_value(SOURCE_TYPE, source.type) - source.dimension = key_from_value(DIMENSION, source.dimension) - source.id = source.id.toString(16) - source.advertiser = key_from_value(ADVERTISER, source.advertiser) - source.publisher = key_from_value(PUBLISHER, source.publisher) - - trigger.type = key_from_value(TRIGGER_TYPE, trigger.type) - trigger.id = trigger.id.toString(16) - - return { - source, - trigger - } -} - -export function sourceEventId() { - // 64bit dummy value - return ((1n << 64n) - 1n).toString() -} - -export function debugKey() { - // 64bit dummy value - return ((1n << 64n) - 2n).toString() -} - -function key_from_value(obj, value) { - return Object.entries(obj) - .filter(([k, v]) => v === value) - .at(0) - .at(0) -} - -function test() { - const advertiser = ADVERTISER["shop"] - const publisher = PUBLISHER["news"] - const id = 0xff - const dimension = DIMENSION["gross"] - const size = (26.5 - 20) * 10 - const category = 1 - const source_type = SOURCE_TYPE.click - const trigger_type = TRIGGER_TYPE.gross - const option = 2 - - const source_key = sourceKeyPiece({ type: source_type, dimension, id, advertiser, publisher }) - console.log({ source_key }) - - const trigger_key = triggerKeyPiece({ type: trigger_type, id, size, category, option }) - console.log({ trigger_key }) - - const source = encodeSource({ type: source_type, dimension, id, advertiser, publisher }) - console.log(decodeSource(source)) - - const trigger = encodeTrigger({ type: trigger_type, id, size, category, option }) - console.log(decodeTrigger(trigger)) -} - -test() +// // type: 2bit +// export const SOURCE_TYPE = { +// click: 0b10, +// view: 0b11 +// } + +// // advertiser: 16bit +// export const ADVERTISER = { +// [process.env.SHOP_HOST]: 0b0, +// [process.env.TRAVEL_HOST]: 0b1 +// } + +// // publisher: 16bit +// export const PUBLISHER = { +// [process.env.NEWS_HOST]: 0b0 +// } + +// // dimension: 8bit +// export const DIMENSION = { +// quantity: 0b0, +// gross: 0b1 +// } + +// // type 8bit +// export const TRIGGER_TYPE = { +// quantity: 0b1000_0000, +// gross: 0b1100_0000 +// } + +// export function sourceKeyPiece({ type, dimension, id, advertiser, publisher }) { +// console.log({ type, dimension, id, advertiser, publisher }) +// const source = encodeSource({ type, dimension, id, advertiser, publisher }) +// const uint64 = new DataView(source).getBigUint64() +// return `0x${(uint64 << 64n).toString(16)}` +// } + +// export function triggerKeyPiece({ type, id, size, category, option }) { +// console.log({ type, id, size, category, option }) +// const trigger = encodeTrigger({ type, id, size, category, option }) +// const uint64 = new DataView(trigger).getBigUint64() +// return `0x${"0".repeat(16)}${uint64.toString(16)}` +// } + +// // type: 2bit +// // dimension: 6bit +// // id: 24bit +// // advertiser: 16bit +// // publisher: 16bit +// // ----------------- +// // 64bit +// function encodeSource({ type, dimension, id, advertiser, publisher }) { +// const buffer = new ArrayBuffer(8) +// const view = new DataView(buffer) +// const first32 = (((type << 6) + dimension) << 24) + id +// view.setUint32(0, first32) +// view.setUint16(4, advertiser) +// view.setUint16(6, publisher) +// return buffer +// } + +// function decodeSource(buffer) { +// const view = new DataView(buffer) +// const first32 = view.getUint32(0) +// const id = first32 & (2 ** 24 - 1) +// const first8 = first32 >>> 24 +// const dimension = first8 & 0b111111 +// const type = first8 >>> 6 +// const advertiser = view.getUint16(4) +// const publisher = view.getUint16(6) +// return { type, dimension, id, advertiser, publisher } +// } + +// // type: 8bit +// // id: 24bit +// // size: 8bit +// // category: 8bit +// // option: 16bit +// // ----------------- +// // 64bit +// function encodeTrigger({ type, id, size, category, option }) { +// const buffer = new ArrayBuffer(8) +// const view = new DataView(buffer) +// view.setUint32(0, (type << 24) + id) +// view.setUint8(4, size) +// view.setUint8(5, category) +// view.setUint16(6, option) +// return buffer +// } + +// function decodeTrigger(buffer) { +// const view = new DataView(buffer) +// const first32 = view.getUint32(0) +// const type = first32 >>> 24 +// const id = first32 & 0xffffff +// const size = view.getUint8(4) +// const category = view.getUint8(5) +// const option = view.getUint16(6) +// return { type, id, size, category, option } +// } + +// export function decodeBucket(buffer) { +// const u8a = new Uint8Array(buffer) +// const sourceBuf = u8a.slice(0, u8a.length / 2) +// const source = decodeSource(sourceBuf.buffer) +// const triggerBuf = u8a.slice(u8a.length / 2, u8a.length) +// const trigger = decodeTrigger(triggerBuf.buffer) + +// source.type = key_from_value(SOURCE_TYPE, source.type) +// source.dimension = key_from_value(DIMENSION, source.dimension) +// source.id = source.id.toString(16) +// source.advertiser = key_from_value(ADVERTISER, source.advertiser) +// source.publisher = key_from_value(PUBLISHER, source.publisher) + +// trigger.type = key_from_value(TRIGGER_TYPE, trigger.type) +// trigger.id = trigger.id.toString(16) + +// return { +// source, +// trigger +// } +// } + +// export function sourceEventId() { +// // 64bit dummy value +// return ((1n << 64n) - 1n).toString() +// } + +// export function debugKey() { +// // 64bit dummy value +// return ((1n << 64n) - 2n).toString() +// } + +// function key_from_value(obj, value) { +// return Object.entries(obj) +// .filter(([k, v]) => v === value) +// .at(0) +// .at(0) +// } + +// function test() { +// const advertiser = ADVERTISER["shop"] +// const publisher = PUBLISHER["news"] +// const id = 0xff +// const dimension = DIMENSION["gross"] +// const size = (26.5 - 20) * 10 +// const category = 1 +// const source_type = SOURCE_TYPE.click +// const trigger_type = TRIGGER_TYPE.gross +// const option = 2 + +// const source_key = sourceKeyPiece({ type: source_type, dimension, id, advertiser, publisher }) +// console.log({ source_key }) + +// const trigger_key = triggerKeyPiece({ type: trigger_type, id, size, category, option }) +// console.log({ trigger_key }) + +// const source = encodeSource({ type: source_type, dimension, id, advertiser, publisher }) +// console.log(decodeSource(source)) + +// const trigger = encodeTrigger({ type: trigger_type, id, size, category, option }) +// console.log(decodeTrigger(trigger)) +// } + +// test() diff --git a/services/ssp/src/index.js b/services/ssp/src/index.js index 0477f2b0..741620be 100644 --- a/services/ssp/src/index.js +++ b/services/ssp/src/index.js @@ -17,29 +17,29 @@ // SSP import express from "express" import url from "url" -import cbor from "cbor" -import { decodeDict } from "structured-field-values" -import { - debugKey, - sourceEventId, - sourceKeyPiece, - triggerKeyPiece, - ADVERTISER, - PUBLISHER, - DIMENSION, - decodeBucket, - SOURCE_TYPE, - TRIGGER_TYPE -} from "./arapi.js" +// import cbor from "cbor" +// import { decodeDict } from "structured-field-values" +// import { +// debugKey, +// sourceEventId, +// sourceKeyPiece, +// triggerKeyPiece, +// ADVERTISER, +// PUBLISHER, +// DIMENSION, +// decodeBucket, +// SOURCE_TYPE, +// TRIGGER_TYPE +// } from "./arapi.js" const { EXTERNAL_PORT, PORT, SSP_HOST, SSP_DETAIL, SSP_TOKEN, DSP_HOST, SHOP_HOST } = process.env -// global memory storage -const Reports = [] -// clear storage -setInterval(() => { - Reports.length = 0 -}, 1000 * 60 * 10) +// // in-memory storage for debug reports +// const Reports = []; +// // clear in-memory storage every 10 min +// setInterval(() => { +// Reports.length = 0; +// }, 1000 * 60 * 10); const app = express() @@ -50,15 +50,15 @@ app.use((req, res, next) => { app.use(express.json()) -app.use((req, res, next) => { - // enable debug mode - res.cookie("ar_debug", "1", { - sameSite: "none", - secure: true, - httpOnly: true - }) - next() -}) +// app.use((req, res, next) => { +// // enable transitional debugging reports (https://github.com/WICG/attribution-reporting-api/blob/main/EVENT.md#optional-transitional-debugging-reports) +// res.cookie("ar_debug", "1", { +// sameSite: "none", +// secure: true, +// httpOnly: true +// }); +// next(); +// }); app.use((req, res, next) => { // opt-in fencedframe @@ -89,119 +89,119 @@ app.get("/", async (req, res) => { res.render("index.html.ejs", { title, DSP_HOST, SSP_HOST, EXTERNAL_PORT, SHOP_HOST }) }) -app.get("/register-source", async (req, res) => { - const { advertiser, id } = req.query - console.log("Registering source attribution for", { advertiser, id }) - if (req.headers["attribution-reporting-eligible"]) { - //const are = req.headers["attribution-reporting-eligible"].split(",").map((e) => e.trim()) - const are = decodeDict(req.headers["attribution-reporting-eligible"]) - - // register navigation source - if ("navigation-source" in are) { - const destination = `https://${advertiser}` - const source_event_id = sourceEventId() - const debug_key = debugKey() - const AttributionReportingRegisterSource = { - destination, - source_event_id, - debug_key, - aggregation_keys: { - quantity: sourceKeyPiece({ - type: SOURCE_TYPE["click"], // click attribution - advertiser: ADVERTISER[advertiser], - publisher: PUBLISHER["news"], - id: Number(`0x${id}`), - dimension: DIMENSION["quantity"] - }), - gross: sourceKeyPiece({ - type: SOURCE_TYPE["click"], // click attribution - advertiser: ADVERTISER[advertiser], - publisher: PUBLISHER["news"], - id: Number(`0x${id}`), - dimension: DIMENSION["gross"] - }) - } - } - - console.log("Registering navigation source :", { AttributionReportingRegisterSource }) - res.setHeader("Attribution-Reporting-Register-Source", JSON.stringify(AttributionReportingRegisterSource)) - res.status(200).send("attribution nevigation (click) source registered") - } - - // register event source - else if ("event-source" in are) { - const destination = `https://${advertiser}` - const source_event_id = sourceEventId() - const debug_key = debugKey() - const AttributionReportingRegisterSource = { - destination, - source_event_id, - debug_key, - aggregation_keys: { - quantity: sourceKeyPiece({ - type: SOURCE_TYPE["view"], // view attribution - advertiser: ADVERTISER[advertiser], - publisher: PUBLISHER["news"], - id: Number(`0x${id}`), - dimension: DIMENSION["quantity"] - }), - gross: sourceKeyPiece({ - type: SOURCE_TYPE["view"], // view attribution - advertiser: ADVERTISER[advertiser], - publisher: PUBLISHER["news"], - id: Number(`0x${id}`), - dimension: DIMENSION["gross"] - }) - } - } - - console.log("Registering event source :", { AttributionReportingRegisterSource }) - res.setHeader("Attribution-Reporting-Register-Source", JSON.stringify(AttributionReportingRegisterSource)) - res.status(200).send("attribution event (view) source registered") - } else { - res.status(400).send("'Attribution-Reporting-Eligible' header is malformed") // just send back response header. no content. - } - } else { - res.status(400).send("'Attribution-Reporting-Eligible' header is missing") // just send back response header. no content. - } -}) - -app.get("/register-trigger", async (req, res) => { - const { id, quantity, size, category, gross } = req.query - - const AttributionReportingRegisterTrigger = { - aggregatable_trigger_data: [ - { - key_piece: triggerKeyPiece({ - type: TRIGGER_TYPE["quantity"], - id: parseInt(id, 16), - size: Number(size), - category: Number(category), - option: 0 - }), - source_keys: ["quantity"] - }, - { - key_piece: triggerKeyPiece({ - type: TRIGGER_TYPE["gross"], - id: parseInt(id, 16), - size: Number(size), - category: Number(category), - option: 0 - }), - source_keys: ["gross"] - } - ], - aggregatable_values: { - // TODO: scaling - quantity: Number(quantity), - gross: Number(gross) - }, - debug_key: debugKey() - } - res.setHeader("Attribution-Reporting-Register-Trigger", JSON.stringify(AttributionReportingRegisterTrigger)) - res.sendStatus(200) -}) +// app.get("/register-source", async (req, res) => { +// const { advertiser, id } = req.query +// console.log("Registering source attribution for", { advertiser, id }) +// if (req.headers["attribution-reporting-eligible"]) { +// //const are = req.headers["attribution-reporting-eligible"].split(",").map((e) => e.trim()) +// const are = decodeDict(req.headers["attribution-reporting-eligible"]) + +// // register navigation source +// if ("navigation-source" in are) { +// const destination = `https://${advertiser}` +// const source_event_id = sourceEventId() +// const debug_key = debugKey() +// const AttributionReportingRegisterSource = { +// destination, +// source_event_id, +// debug_key, +// aggregation_keys: { +// quantity: sourceKeyPiece({ +// type: SOURCE_TYPE["click"], // click attribution +// advertiser: ADVERTISER[advertiser], +// publisher: PUBLISHER["news"], +// id: Number(`0x${id}`), +// dimension: DIMENSION["quantity"] +// }), +// gross: sourceKeyPiece({ +// type: SOURCE_TYPE["click"], // click attribution +// advertiser: ADVERTISER[advertiser], +// publisher: PUBLISHER["news"], +// id: Number(`0x${id}`), +// dimension: DIMENSION["gross"] +// }) +// } +// } + +// console.log("Registering navigation source :", { AttributionReportingRegisterSource }) +// res.setHeader("Attribution-Reporting-Register-Source", JSON.stringify(AttributionReportingRegisterSource)) +// res.status(200).send("attribution nevigation (click) source registered") +// } + +// // register event source +// else if ("event-source" in are) { +// const destination = `https://${advertiser}` +// const source_event_id = sourceEventId() +// const debug_key = debugKey() +// const AttributionReportingRegisterSource = { +// destination, +// source_event_id, +// debug_key, +// aggregation_keys: { +// quantity: sourceKeyPiece({ +// type: SOURCE_TYPE["view"], // view attribution +// advertiser: ADVERTISER[advertiser], +// publisher: PUBLISHER["news"], +// id: Number(`0x${id}`), +// dimension: DIMENSION["quantity"] +// }), +// gross: sourceKeyPiece({ +// type: SOURCE_TYPE["view"], // view attribution +// advertiser: ADVERTISER[advertiser], +// publisher: PUBLISHER["news"], +// id: Number(`0x${id}`), +// dimension: DIMENSION["gross"] +// }) +// } +// } + +// console.log("Registering event source :", { AttributionReportingRegisterSource }) +// res.setHeader("Attribution-Reporting-Register-Source", JSON.stringify(AttributionReportingRegisterSource)) +// res.status(200).send("attribution event (view) source registered") +// } else { +// res.status(400).send("'Attribution-Reporting-Eligible' header is malformed") // just send back response header. no content. +// } +// } else { +// res.status(400).send("'Attribution-Reporting-Eligible' header is missing") // just send back response header. no content. +// } +// }) + +// app.get("/register-trigger", async (req, res) => { +// const { id, quantity, size, category, gross } = req.query + +// const AttributionReportingRegisterTrigger = { +// aggregatable_trigger_data: [ +// { +// key_piece: triggerKeyPiece({ +// type: TRIGGER_TYPE["quantity"], +// id: parseInt(id, 16), +// size: Number(size), +// category: Number(category), +// option: 0 +// }), +// source_keys: ["quantity"] +// }, +// { +// key_piece: triggerKeyPiece({ +// type: TRIGGER_TYPE["gross"], +// id: parseInt(id, 16), +// size: Number(size), +// category: Number(category), +// option: 0 +// }), +// source_keys: ["gross"] +// } +// ], +// aggregatable_values: { +// // TODO: scaling +// quantity: Number(quantity), +// gross: Number(gross) +// }, +// debug_key: debugKey() +// } +// res.setHeader("Attribution-Reporting-Register-Trigger", JSON.stringify(AttributionReportingRegisterTrigger)) +// res.sendStatus(200) +// }) app.get("/ad-tag.html", async (req, res) => { res.render("ad-tag.html.ejs") @@ -211,9 +211,9 @@ app.get("/video-ad-tag.html", async (req, res) => { res.render("video-ad-tag.html.ejs") }) -app.get("/reports", async (req, res) => { - res.render("reports.html.ejs", { title: "Report", Reports }) -}) +// app.get("/reports", async (req, res) => { +// res.render("reports.html.ejs", { title: "Report", Reports }) +// }) app.get("/auction-config.json", async (req, res) => { const DSP = new URL(`https://${DSP_HOST}:${EXTERNAL_PORT}`) @@ -254,43 +254,43 @@ app.get("/auction-config.json", async (req, res) => { res.json(auctionConfig) }) -app.post("/.well-known/attribution-reporting/debug/report-aggregate-attribution", async (req, res) => { - const debug_report = req.body - debug_report.shared_info = JSON.parse(debug_report.shared_info) - - console.log(JSON.stringify(debug_report, " ", " ")) - - debug_report.aggregation_service_payloads = debug_report.aggregation_service_payloads.map((e) => { - const plain = Buffer.from(e.debug_cleartext_payload, "base64") - const debug_cleartext_payload = cbor.decodeAllSync(plain) - e.debug_cleartext_payload = debug_cleartext_payload.map(({ data, operation }) => { - return { - operation, - data: data.map(({ value, bucket }) => { - return { - value: value.readUInt32BE(0), - bucket: decodeBucket(bucket) - } - }) - } - }) - return e - }) - - console.log(JSON.stringify(debug_report, " ", " ")) - - // save to global storage - Reports.push(debug_report) - - res.sendStatus(200) -}) - -app.post("/.well-known/attribution-reporting/report-aggregate-attribution", async (req, res) => { - const report = req.body - report.shared_info = JSON.parse(report.shared_info) - console.log(JSON.stringify(report, " ", " ")) - res.sendStatus(200) -}) +// app.post("/.well-known/attribution-reporting/debug/report-aggregate-attribution", async (req, res) => { +// const debug_report = req.body +// debug_report.shared_info = JSON.parse(debug_report.shared_info) + +// console.log(JSON.stringify(debug_report, " ", " ")) + +// debug_report.aggregation_service_payloads = debug_report.aggregation_service_payloads.map((e) => { +// const plain = Buffer.from(e.debug_cleartext_payload, "base64") +// const debug_cleartext_payload = cbor.decodeAllSync(plain) +// e.debug_cleartext_payload = debug_cleartext_payload.map(({ data, operation }) => { +// return { +// operation, +// data: data.map(({ value, bucket }) => { +// return { +// value: value.readUInt32BE(0), +// bucket: decodeBucket(bucket) +// } +// }) +// } +// }) +// return e +// }) + +// console.log(JSON.stringify(debug_report, " ", " ")) + +// // save to global storage +// Reports.push(debug_report) + +// res.sendStatus(200) +// }) + +// app.post("/.well-known/attribution-reporting/report-aggregate-attribution", async (req, res) => { +// const report = req.body +// report.shared_info = JSON.parse(report.shared_info) +// console.log(JSON.stringify(report, " ", " ")) +// res.sendStatus(200) +// }) app.listen(PORT, function () { console.log(`Listening on port ${PORT}`)