Skip to content

Commit

Permalink
App Hosting Emulator: Init (#7937)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mathu97
mathu97 authored Nov 20, 2024
1 parent 877a889 commit d19be77
Show file tree
Hide file tree
Showing 13 changed files with 271 additions and 203 deletions.
144 changes: 138 additions & 6 deletions src/apphosting/config.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { resolve, join, dirname } from "path";
import { resolve, join, dirname, basename } from "path";
import { writeFileSync } from "fs";
import * as yaml from "yaml";

Expand All @@ -7,6 +7,11 @@ import { NodeType } from "yaml/dist/nodes/Node";
import * as prompt from "../prompt";
import * as dialogs from "./secrets/dialogs";
import { AppHostingYamlConfig } from "./yaml";
import { FirebaseError } from "../error";
import { promptForAppHostingYaml } from "./utils";
import { fetchSecrets } from "./secrets";
import { logger } from "../logger";
import { getOrPromptProject } from "../management/projects";

export const APPHOSTING_BASE_YAML_FILE = "apphosting.yaml";
export const APPHOSTING_LOCAL_YAML_FILE = "apphosting.local.yaml";
Expand Down Expand Up @@ -37,6 +42,9 @@ export interface Config {
env?: Env[];
}

const SECRET_CONFIG = "Secret";
const EXPORTABLE_CONFIG = [SECRET_CONFIG];

/**
* Returns the absolute path for an app hosting backend root.
*
Expand Down Expand Up @@ -65,8 +73,8 @@ export function discoverBackendRoot(cwd: string): string | null {

/**
* Returns paths of apphosting config files in the given path
* */
export function listAppHostingFilesInPath(path: string) {
*/
export function listAppHostingFilesInPath(path: string): string[] {
return fs
.listFiles(path)
.filter((file) => APPHOSTING_YAML_FILE_REGEX.test(file))
Expand Down Expand Up @@ -172,15 +180,79 @@ export async function maybeAddSecretToYaml(secretName: string): Promise<void> {
dynamicDispatch.store(path, projectYaml);
}

/**
* Reads userGivenConfigFile and exports the secrets defined in that file by
* hitting Google Secret Manager. The secrets are written in plain text to an
* apphosting.local.yaml file as environment variables.
*
* If userGivenConfigFile is not given, user is prompted to select one of the
* discovered app hosting yaml files.
*/
export async function exportConfig(
cwd: string,
backendRoot: string,
projectId?: string,
userGivenConfigFile?: string,
): Promise<void> {
const choices = await prompt.prompt({}, [
{
type: "checkbox",
name: "configurations",
message: "What configs would you like to export?",
choices: EXPORTABLE_CONFIG,
},
]);

/**
* TODO: Update when supporting additional configurations. Currently only
* Secrets are exportable.
*/
if (!choices.configurations.includes(SECRET_CONFIG)) {
logger.info("No configs selected to export");
return;
}

if (!projectId) {
const project = await getOrPromptProject({});
projectId = project.projectId;
}

let localAppHostingConfig: AppHostingYamlConfig = AppHostingYamlConfig.empty();

const localAppHostingConfigPath = resolve(backendRoot, APPHOSTING_LOCAL_YAML_FILE);
if (fs.fileExistsSync(localAppHostingConfigPath)) {
localAppHostingConfig = await AppHostingYamlConfig.loadFromFile(localAppHostingConfigPath);
}

const configToExport = await loadConfigToExportSecrets(cwd, userGivenConfigFile);
const secretsToExport = configToExport.secrets;
if (!secretsToExport) {
logger.info("No secrets found to export in the chosen App Hosting config files");
return;
}

const secretMaterial = await fetchSecrets(projectId, secretsToExport);
for (const [key, value] of secretMaterial) {
localAppHostingConfig.addEnvironmentVariable({
variable: key,
value: value,
availability: ["RUNTIME"],
});
}

// update apphosting.local.yaml
await localAppHostingConfig.upsertFile(localAppHostingConfigPath);
logger.info(`Wrote secrets as environment variables to ${APPHOSTING_LOCAL_YAML_FILE}.`);
}

/**
* Given apphosting yaml config paths this function returns the
* appropriate combined configuration.
*
* Environment specific config (i.e apphosting.<environment>.yaml) will
* take precedence over the base config (apphosting.yaml).
*
* @param envYamlPath: Example: "/home/my-project/apphosting.staging.yaml"
* @param baseYamlPath: Example: "/home/my-project/apphosting.yaml"
* @param envYamlPath Example: "/home/my-project/apphosting.staging.yaml"
* @param baseYamlPath Example: "/home/my-project/apphosting.yaml"
*/
export async function loadConfigForEnvironment(
envYamlPath: string,
Expand All @@ -199,3 +271,63 @@ export async function loadConfigForEnvironment(

return envYamlConfig;
}

/**
* Returns the appropriate App Hosting YAML configuration for exporting secrets.
* @return The final merged config
*/
export async function loadConfigToExportSecrets(
cwd: string,
userGivenConfigFile?: string,
): Promise<AppHostingYamlConfig> {
if (userGivenConfigFile && !APPHOSTING_YAML_FILE_REGEX.test(userGivenConfigFile)) {
throw new FirebaseError(
"Invalid apphosting yaml config file provided. File must be in format: 'apphosting.yaml' or 'apphosting.<environment>.yaml'",
);
}

const allConfigs = getValidConfigs(cwd);
let userGivenConfigFilePath: string;
if (userGivenConfigFile) {
if (!allConfigs.has(userGivenConfigFile)) {
throw new FirebaseError(
`The provided app hosting config file "${userGivenConfigFile}" does not exist`,
);
}

userGivenConfigFilePath = allConfigs.get(userGivenConfigFile)!;
} else {
userGivenConfigFilePath = await promptForAppHostingYaml(
allConfigs,
"Which environment would you like to export secrets from Secret Manager for?",
);
}

if (userGivenConfigFile === APPHOSTING_BASE_YAML_FILE) {
return AppHostingYamlConfig.loadFromFile(allConfigs.get(APPHOSTING_BASE_YAML_FILE)!);
}

const baseFilePath = allConfigs.get(APPHOSTING_BASE_YAML_FILE)!;
return await loadConfigForEnvironment(userGivenConfigFilePath, baseFilePath);
}

/**
* Gets all apphosting yaml configs excluding apphosting.local.yaml and returns
* a map in the format {"apphosting.staging.yaml" => "/cwd/apphosting.staging.yaml"}.
*/
function getValidConfigs(cwd: string): Map<string, string> {
const appHostingConfigPaths = listAppHostingFilesInPath(cwd).filter(
(path) => !path.endsWith(APPHOSTING_LOCAL_YAML_FILE),
);
if (appHostingConfigPaths.length === 0) {
throw new FirebaseError("No apphosting.*.yaml configs found");
}

const fileNameToPathMap: Map<string, string> = new Map();
for (const path of appHostingConfigPaths) {
const fileName = basename(path);
fileNameToPathMap.set(fileName, path);
}

return fileNameToPathMap;
}
45 changes: 0 additions & 45 deletions src/apphosting/secrets/index.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@ import * as gce from "../../gcp/computeEngine";
import * as gcsmImport from "../../gcp/secretManager";
import * as utilsImport from "../../utils";
import * as promptImport from "../../prompt";
import * as configImport from "../config";

import { Secret } from "../yaml";
import { FirebaseError } from "../../error";

describe("secrets", () => {
let gcsm: sinon.SinonStubbedInstance<typeof gcsmImport>;
Expand Down Expand Up @@ -297,47 +295,4 @@ describe("secrets", () => {
);
});
});

describe("loadConfigToExport", () => {
let apphostingConfigs: sinon.SinonStubbedInstance<typeof configImport>;

const baseYamlPath = "/parent/cwd/apphosting.yaml";

beforeEach(() => {
apphostingConfigs = sinon.stub(configImport);
apphostingConfigs.listAppHostingFilesInPath.returns([
baseYamlPath,
"/parent/apphosting.staging.yaml",
]);
});

afterEach(() => {
sinon.verifyAndRestore();
});

it("returns throws an error if an invalid apphosting yaml if provided", async () => {
await expect(secrets.loadConfigToExport("/parent/cwd", "blah.txt")).to.be.rejectedWith(
FirebaseError,
"Invalid apphosting yaml config file provided. File must be in format: 'apphosting.yaml' or 'apphosting.<environment>.yaml'",
);
});

it("does not prompt user if an userGivenConfigFile is provided", async () => {
await secrets.loadConfigToExport("/parent/cwd", "apphosting.staging.yaml");
expect(prompt.promptOnce).to.not.be.called;
});

it("prompts user if userGivenConfigFile not provided", async () => {
await secrets.loadConfigToExport("/parent/cwd");
expect(prompt.promptOnce).to.be.called;
});

it("should throw an error if userGivenConfigFile could not be found", async () => {
await expect(
secrets.loadConfigToExport("/parent/cwd", "apphosting.preview.yaml"),
).to.be.rejectedWith(
'The provided app hosting config file "apphosting.preview.yaml" does not exist',
);
});
});
});
71 changes: 1 addition & 70 deletions src/apphosting/secrets/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,7 @@ import { FIREBASE_MANAGED } from "../../gcp/secretManager";
import { isFunctionsManaged } from "../../gcp/secretManager";
import * as utils from "../../utils";
import * as prompt from "../../prompt";
import { basename } from "path";
import {
APPHOSTING_BASE_YAML_FILE,
APPHOSTING_LOCAL_YAML_FILE,
APPHOSTING_YAML_FILE_REGEX,
listAppHostingFilesInPath,
loadConfigForEnvironment,
} from "../config";
import { promptForAppHostingYaml } from "../utils";
import { AppHostingYamlConfig, Secret } from "../yaml";
import { Secret } from "../yaml";

/** Interface for holding the service account pair for a given Backend. */
export interface ServiceAccounts {
Expand Down Expand Up @@ -203,66 +194,6 @@ export async function fetchSecrets(
return secretsKeyValuePairs;
}

/**
* Returns the appropriate App Hosting YAML configuration for exporting secrets.
*
* @returns The final merged config
*/
export async function loadConfigToExport(
cwd: string,
userGivenConfigFile?: string,
): Promise<AppHostingYamlConfig> {
if (userGivenConfigFile && !APPHOSTING_YAML_FILE_REGEX.test(userGivenConfigFile)) {
throw new FirebaseError(
"Invalid apphosting yaml config file provided. File must be in format: 'apphosting.yaml' or 'apphosting.<environment>.yaml'",
);
}

const allConfigs = getValidConfigs(cwd);
let userGivenConfigFilePath: string;
if (userGivenConfigFile) {
if (!allConfigs.has(userGivenConfigFile)) {
throw new FirebaseError(
`The provided app hosting config file "${userGivenConfigFile}" does not exist`,
);
}

userGivenConfigFilePath = allConfigs.get(userGivenConfigFile)!;
} else {
userGivenConfigFilePath = await promptForAppHostingYaml(
allConfigs,
"Which environment would you like to export secrets from Secret Manager for?",
);
}

if (userGivenConfigFile === APPHOSTING_BASE_YAML_FILE) {
return AppHostingYamlConfig.loadFromFile(allConfigs.get(APPHOSTING_BASE_YAML_FILE)!);
}

const baseFilePath = allConfigs.get(APPHOSTING_BASE_YAML_FILE)!;
return await loadConfigForEnvironment(userGivenConfigFilePath, baseFilePath);
}

/**
* Gets all apphosting yaml configs excluding apphosting.local.yaml and returns
* a map in the format {"apphosting.staging.yaml" => "/cwd/apphosting.staging.yaml"}.
*/
function getValidConfigs(cwd: string): Map<string, string> {
const appHostingConfigPaths = listAppHostingFilesInPath(cwd).filter(
(path) => !path.endsWith(APPHOSTING_LOCAL_YAML_FILE),
);
if (appHostingConfigPaths.length === 0) {
throw new FirebaseError("No apphosting.*.yaml configs found");
}

const fileNameToPathMap: Map<string, string> = new Map();
for (const path of appHostingConfigPaths) {
const fileName = basename(path);
fileNameToPathMap.set(fileName, path);
}

return fileNameToPathMap;
}
/**
* secret expected to be in format "myApiKeySecret@5",
* "projects/test-project/secrets/secretID", or
Expand Down
34 changes: 2 additions & 32 deletions src/commands/apphosting-config-export.ts
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,10 @@
import { Command } from "../command";
import { logger } from "../logger";
import { Options } from "../options";
import { needProjectId } from "../projectUtils";
import { requireAuth } from "../requireAuth";
import * as secretManager from "../gcp/secretManager";
import { requirePermissions } from "../requirePermissions";
import { APPHOSTING_LOCAL_YAML_FILE, discoverBackendRoot } from "../apphosting/config";
import { fetchSecrets, loadConfigToExport } from "../apphosting/secrets";
import { resolve } from "path";
import * as fs from "../fsutils";
import { AppHostingYamlConfig } from "../apphosting/yaml";
import { discoverBackendRoot, exportConfig } from "../apphosting/config";
import { FirebaseError } from "../error";

export const command = new Command("apphosting:config:export")
Expand All @@ -28,37 +23,12 @@ export const command = new Command("apphosting:config:export")
const environmentConfigFile = options.secrets as string | undefined;
const cwd = process.cwd();

// Load apphosting.local.yaml file if it exists. Secrets should be added to the env list in this object and written back to the apphosting.local.yaml
let localAppHostingConfig: AppHostingYamlConfig = AppHostingYamlConfig.empty();
const backendRoot = discoverBackendRoot(cwd);
if (!backendRoot) {
throw new FirebaseError(
"Missing apphosting.yaml: This command requires an apphosting.yaml configuration file. Please run 'firebase init apphosting' and try again.",
);
}

const localAppHostingConfigPath = resolve(backendRoot, APPHOSTING_LOCAL_YAML_FILE);
if (fs.fileExistsSync(localAppHostingConfigPath)) {
localAppHostingConfig = await AppHostingYamlConfig.loadFromFile(localAppHostingConfigPath);
}

const configToExport = await loadConfigToExport(cwd, environmentConfigFile);
const secretsToExport = configToExport.secrets;
if (!secretsToExport) {
logger.warn("No secrets found to export in the chosen App Hosting config files");
return;
}

const secretMaterial = await fetchSecrets(projectId, secretsToExport);
for (const [key, value] of secretMaterial) {
localAppHostingConfig.addEnvironmentVariable({
variable: key,
value: value,
availability: ["RUNTIME"],
});
}

// update apphosting.local.yaml
localAppHostingConfig.upsertFile(localAppHostingConfigPath);
logger.info(`Wrote secrets as environment variables to ${APPHOSTING_LOCAL_YAML_FILE}.`);
await exportConfig(cwd, backendRoot, projectId, environmentConfigFile);
});
2 changes: 1 addition & 1 deletion src/emulator/apphosting/config.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import * as path from "path";
import * as utils from "./utils";
import * as utils from "./developmentServer";

import * as sinon from "sinon";
import { expect } from "chai";
Expand Down
Loading

0 comments on commit d19be77

Please sign in to comment.