diff --git a/.gitignore b/.gitignore index 8bb22129f381..26fc540918a1 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,5 @@ doc/source/tutorial/services.rst # Pyenv .python-version +.env diff --git a/awscli/customizations/cloudformation/artifact_exporter.py b/awscli/customizations/cloudformation/artifact_exporter.py index 7fba8dddbba3..cad3cacb3817 100644 --- a/awscli/customizations/cloudformation/artifact_exporter.py +++ b/awscli/customizations/cloudformation/artifact_exporter.py @@ -25,11 +25,15 @@ from awscli.customizations.cloudformation import exceptions from awscli.customizations.cloudformation.yamlhelper import yaml_dump, \ yaml_parse +from awscli.customizations.cloudformation import modules +from awscli.customizations.cloudformation import module_constants import jmespath LOG = logging.getLogger(__name__) +MODULES = "Modules" +RESOURCES = "Resources" def is_path_value_valid(path): return isinstance(path, str) @@ -139,6 +143,9 @@ def upload_local_artifacts(resource_id, resource_dict, property_name, local_path = make_abs_path(parent_dir, local_path) + if uploader is None: + raise exceptions.PackageBucketRequiredError() + # Or, pointing to a folder. Zip the folder and upload if is_local_folder(local_path): return zip_and_upload(local_path, uploader) @@ -154,6 +161,8 @@ def upload_local_artifacts(resource_id, resource_dict, property_name, def zip_and_upload(local_path, uploader): + if uploader is None: + raise exceptions.PackageBucketRequiredError() with zip_folder(local_path) as zipfile: return uploader.upload_with_dedup(zipfile) @@ -472,6 +481,9 @@ def do_export(self, resource_id, resource_dict, parent_dir): exported_template_str = yaml_dump(exported_template_dict) + if self.uploader is None: + raise exceptions.PackageBucketRequiredError() + with mktempfile() as temporary_file: temporary_file.write(exported_template_str) temporary_file.flush() @@ -558,6 +570,9 @@ def include_transform_export_handler(template_dict, uploader, parent_dir): return template_dict # We are confident at this point that `include_location` is a string containing the local path + if uploader is None: + raise exceptions.PackageBucketRequiredError() + abs_include_location = os.path.join(parent_dir, include_location) if is_local_file(abs_include_location): template_dict["Parameters"]["Location"] = uploader.upload_with_dedup(abs_include_location) @@ -591,8 +606,8 @@ def __init__(self, template_path, parent_dir, uploader, raise ValueError("parent_dir parameter must be " "an absolute path to a folder {0}" .format(parent_dir)) - abs_template_path = make_abs_path(parent_dir, template_path) + self.module_parent_path = abs_template_path template_dir = os.path.dirname(abs_template_path) with open(abs_template_path, "r") as handle: @@ -603,6 +618,8 @@ def __init__(self, template_path, parent_dir, uploader, self.resources_to_export = resources_to_export self.metadata_to_export = metadata_to_export self.uploader = uploader + self.no_metrics = False # TODO - New parameter for package command + self.no_source_map = False # TODO def export_global_artifacts(self, template_dict): """ @@ -646,19 +663,43 @@ def export_metadata(self, template_dict): def export(self): """ Exports the local artifacts referenced by the given template to an - s3 bucket. + s3 bucket, and/or packages a template with module. - :return: The template with references to artifacts that have been - exported to s3. + :return: The modified template (which is altered in place) """ - self.template_dict = self.export_metadata(self.template_dict) - if "Resources" not in self.template_dict: - return self.template_dict + # Process constants + try: + constants = module_constants.process_constants(self.template_dict) + if constants is not None: + module_constants.replace_constants(constants, self.template_dict) + except Exception as e: + msg=f"Failed to process Constants section: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + + # Process modules + try: + self.template_dict = modules.process_module_section( + self.template_dict, + self.template_dir, + self.module_parent_path, + None, + self.no_metrics, + self.no_source_map) + except Exception as e: + msg=f"Failed to process Modules section: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + + self.template_dict = self.export_metadata(self.template_dict) self.template_dict = self.export_global_artifacts(self.template_dict) - self.export_resources(self.template_dict["Resources"]) + if RESOURCES not in self.template_dict: + return self.template_dict + + self.export_resources(self.template_dict[RESOURCES]) return self.template_dict diff --git a/awscli/customizations/cloudformation/exceptions.py b/awscli/customizations/cloudformation/exceptions.py index b2625cdd27f9..68f478fb9bf5 100644 --- a/awscli/customizations/cloudformation/exceptions.py +++ b/awscli/customizations/cloudformation/exceptions.py @@ -53,7 +53,15 @@ class DeployBucketRequiredError(CloudFormationCommandError): "via an S3 Bucket. Please add the --s3-bucket parameter to your " "command. The local template will be copied to that S3 bucket and " "then deployed.") - + +class PackageBucketRequiredError(CloudFormationCommandError): + fmt = "Add the --s3-bucket parameter to your command to upload artifacts to S3" class InvalidForEachIntrinsicFunctionError(CloudFormationCommandError): fmt = 'The value of {resource_id} has an invalid "Fn::ForEach::" format: Must be a list of three entries' + +class InvalidModulePathError(CloudFormationCommandError): + fmt = 'The value of {source} is not a valid path to a local file' + +class InvalidModuleError(CloudFormationCommandError): + fmt = 'Invalid module: {msg}' diff --git a/awscli/customizations/cloudformation/module_conditions.py b/awscli/customizations/cloudformation/module_conditions.py new file mode 100644 index 000000000000..925a07cc133c --- /dev/null +++ b/awscli/customizations/cloudformation/module_conditions.py @@ -0,0 +1,124 @@ +# Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +# pylint: disable=fixme + +""" +Parse the Conditions section in a module + +This section is not emitted into the output. +We have to be able to fully resolve it locally. +""" + +from collections import OrderedDict +from awscli.customizations.cloudformation import exceptions + +AND = "Fn::And" +EQUALS = "Fn::Equals" +IF = "Fn::If" +NOT = "Fn::Not" +OR = "Fn::Or" +REF = "Ref" +CONDITION = "Condition" + + +def parse_conditions(d, find_ref): + """Parse conditions and return a map of name:boolean""" + retval = {} + + for k, v in d.items(): + retval[k] = istrue(v, find_ref, retval) + + return retval + + +def resolve_if(v, find_ref, prior): + "Resolve Fn::If" + msg = f"If expression should be a list with 3 elements: {v}" + if not isinstance(v, list): + raise exceptions.InvalidModuleError(msg=msg) + if len(v) != 3: + raise exceptions.InvalidModuleError(msg=msg) + if istrue(v[0], find_ref, prior): + return v[1] + return v[2] + + +# pylint: disable=too-many-branches,too-many-statements +def istrue(v, find_ref, prior): + "Recursive function to evaluate a Condition" + if not isdict(v): + return False + retval = False + if EQUALS in v: + eq = v[EQUALS] + if len(eq) == 2: + val0 = eq[0] + val1 = eq[1] + if isdict(val0) and IF in val0: + val0 = resolve_if(val0[IF], find_ref, prior) + if isdict(val1) and IF in val1: + val1 = resolve_if(val1[IF], find_ref, prior) + if isdict(val0) and REF in val0: + val0 = find_ref(val0[REF]) + if isdict(val1) and REF in val1: + val1 = find_ref(val1[REF]) + retval = val0 == val1 + else: + msg = f"Equals expression should be a list with 2 elements: {eq}" + raise exceptions.InvalidModuleError(msg=msg) + if NOT in v: + if not isinstance(v[NOT], list): + msg = f"Not expression should be a list with 1 element: {v[NOT]}" + raise exceptions.InvalidModuleError(msg=msg) + retval = not istrue(v[NOT][0], find_ref, prior) + if AND in v: + vand = v[AND] + msg = f"And expression should be a list with 2 elements: {vand}" + if not isinstance(vand, list): + raise exceptions.InvalidModuleError(msg=msg) + if len(vand) != 2: + raise exceptions.InvalidModuleError(msg=msg) + retval = istrue(vand[0], find_ref, prior) and istrue( + vand[1], find_ref, prior + ) + if OR in v: + vor = v[OR] + msg = f"Or expression should be a list with 2 elements: {vor}" + if not isinstance(vor, list): + raise exceptions.InvalidModuleError(msg=msg) + if len(vor) != 2: + raise exceptions.InvalidModuleError(msg=msg) + retval = istrue(vor[0], find_ref, prior) or istrue( + vor[1], find_ref, prior + ) + if IF in v: + # Shouldn't ever see an IF here + msg = f"Unexpected If: {v[IF]}" + raise exceptions.InvalidModuleError(msg=msg) + if CONDITION in v: + condition_name = v[CONDITION] + if condition_name in prior: + retval = prior[condition_name] + else: + msg = f"Condition {condition_name} was not evaluated yet" + raise exceptions.InvalidModuleError(msg=msg) + # TODO: Should we re-order the conditions? + # We are depending on the author putting them in order + + return retval + + +def isdict(v): + "Returns True if the type is a dict or OrderedDict" + return isinstance(v, (dict, OrderedDict)) diff --git a/awscli/customizations/cloudformation/module_constants.py b/awscli/customizations/cloudformation/module_constants.py new file mode 100644 index 000000000000..a3c32d7b3581 --- /dev/null +++ b/awscli/customizations/cloudformation/module_constants.py @@ -0,0 +1,125 @@ +# Copyright 2012-2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +""" +Module constants. + +Add a Constants section to the module or the parent template for +string constants, to help reduce copy-paste within the template. + +Refer to constants in Sub strings later in the template using ${Constant::name} + +Constants can refer to other constants that were defined previously. + +Adding constants to a template has side effects, since we have to +re-write all Subs in the template! Ideally nothing will change but +it's possible. + +Example: + + Constants: + foo: bar + baz: abc-${AWS::AccountId}-${Constant::foo} + + Resources: + Bucket: + Type: AWS::S3::Bucket + Metadata: + Test: !Sub ${Constant:baz} + Properties: + BucketName: !Sub ${Constant::foo} + +""" + +from collections import OrderedDict +from awscli.customizations.cloudformation.parse_sub import WordType +from awscli.customizations.cloudformation.parse_sub import parse_sub +from awscli.customizations.cloudformation.parse_sub import is_sub_needed +from awscli.customizations.cloudformation.module_visitor import Visitor +from awscli.customizations.cloudformation import exceptions + +CONSTANTS = "Constants" +SUB = "Fn::Sub" + + +def process_constants(d): + """ + Look for a Constants item in d and if it's found, return it + as a dict. Looks for references to previously defined constants + and substitutes them. + Deletes the Constants item from d. + Returns a dict of the constants. + """ + if CONSTANTS not in d: + return None + + constants = {} + for k, v in d[CONSTANTS].items(): + s = replace_constants(constants, v) + constants[k] = s + + del d[CONSTANTS] + + return constants + + +def replace_constants(constants, s): + """ + Replace all constants in a string or in an entire dictionary. + If s is a string, returns the modified string. + If s is a dictionary, modifies the dictionary in place. + """ + if isinstance(s, str): + retval = "" + words = parse_sub(s) + for w in words: + if w.t == WordType.STR: + retval += w.w + if w.t == WordType.REF: + retval += f"${{{w.w}}}" + if w.t == WordType.AWS: + retval += f"${{AWS::{w.w}}}" + if w.t == WordType.GETATT: + retval += f"${{{w.w}}}" + if w.t == WordType.CONSTANT: + if w.w in constants: + retval += constants[w.w] + else: + msg = f"Unknown constant: {w.w}" + raise exceptions.InvalidModuleError(msg=msg) + return retval + + if isdict(s): + + # Recursively dive into d and replace all string constants + # that are found in Subs. Error if the constant does not exist. + + def vf(v): + if isdict(v.d) and SUB in v.d and v.p is not None: + s = v.d[SUB] + if isinstance(s, str): + newval = replace_constants(constants, s) + if is_sub_needed(newval): + v.p[v.k] = {SUB: newval} + else: + v.p[v.k] = newval + + v = Visitor(s) + v.visit(vf) + + return None + + +def isdict(d): + "Returns true if d is a dict" + return isinstance(d, (dict, OrderedDict)) diff --git a/awscli/customizations/cloudformation/module_read.py b/awscli/customizations/cloudformation/module_read.py new file mode 100644 index 000000000000..9433a6546bd2 --- /dev/null +++ b/awscli/customizations/cloudformation/module_read.py @@ -0,0 +1,71 @@ +# Copyright 2012-2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +""" +Read CloudFormation Module source files. +""" + +import os +import urllib + +from awscli.customizations.cloudformation import exceptions +from awscli.customizations.cloudformation import yamlhelper + + +def is_url(p): + "Returns true if the path looks like a URL instead of a local file" + return p.startswith("https") + + +def read_source(source): + """ + Read the source file and return the content as a string, + plus a dictionary with line numbers. + """ + + if not isinstance(source, str): + raise exceptions.InvalidModulePathError(source=source) + + if is_url(source): + try: + with urllib.request.urlopen(source) as response: + return response.read() + except Exception as e: + print(e) + raise exceptions.InvalidModulePathError(source=source) + + if not os.path.isfile(source): + raise exceptions.InvalidModulePathError(source=source) + + content = "" + with open(source, "r", encoding="utf-8") as s: + content = s.read() + + node = yamlhelper.yaml_compose(content) + lines = {} + read_line_numbers(node, lines) + + return content, lines + + +def read_line_numbers(node, lines): + """ + Read resource line numbers from a yaml node, + using the logical id as the key. + """ + for n in node.value: + if n[0].value == "Resources": + resource_map = n[1].value + for r in resource_map: + logical_id = r[0].value + lines[logical_id] = r[1].start_mark.line diff --git a/awscli/customizations/cloudformation/module_visitor.py b/awscli/customizations/cloudformation/module_visitor.py new file mode 100644 index 000000000000..43223c492982 --- /dev/null +++ b/awscli/customizations/cloudformation/module_visitor.py @@ -0,0 +1,67 @@ +# Copyright 2012-2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +""" +Visitor pattern for recursively modifying all elements in a dictionary. +""" + +from collections import OrderedDict + + +class Visitor: + """ + This class implements a visitor pattern, to run a function on + all elements of a template or subset. + + Example of a visitor that replaces all strings: + + v = Visitor(template_dict) + def vf(v): + if isinstance(v.d, string) and v.p is not None: + v.p[v.k] = "replacement" + v.vist(vf) + """ + + def __init__(self, d, p=None, k=""): + """ + Initialize the visitor with a dictionary. This can be the entire + template or a subset. + + :param d A dict or OrderedDict + :param p The parent dictionary (default is None for the template) + :param k the key for d (p[k] = d) (default is "" for the template) + """ + self.d = d + self.p = p + self.k = k + + def __str__(self): + return f"d: {self.d}, p: {self.p}, k: {self.k}" + + def visit(self, visit_func): + """ + Run the specified function on all nodes. + + :param visit_func A function that accepts a Visitor + """ + + def walk(visitor): + visit_func(visitor) + if isinstance(visitor.d, (dict, OrderedDict)): + for k, v in visitor.d.copy().items(): + walk(Visitor(v, visitor.d, k)) + if isinstance(visitor.d, list): + for i, v in enumerate(visitor.d): + walk(Visitor(v, visitor.d, i)) + + walk(self) diff --git a/awscli/customizations/cloudformation/modules.py b/awscli/customizations/cloudformation/modules.py new file mode 100644 index 000000000000..446049cf6251 --- /dev/null +++ b/awscli/customizations/cloudformation/modules.py @@ -0,0 +1,1011 @@ +# Copyright 2012-2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +""" +This file implements local module support for the package command + +See tests/unit/customizations/cloudformation/modules for examples of what the +Modules section of a template looks like. + +Modules are imported with a Source attribute pointing to a local file, a +Properties attribute that corresponds to Inputs in the modules, and an +Overrides attribute that can override module output. + +The `Modules` section. + +```yaml +Modules: + Content: + Source: ./module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +``` + +A module is itself basically a CloudFormation template, with an Inputs +section and Resources that are injected into the parent template. The +Properties defined in the Modules section correspond to the Inputs in the +module. These modules operate in a similar way to registry modules. + +The name of the module in the Modules section is used as a prefix to logical +ids that are defined in the module. + +In addition to the parent setting Properties, all attributes of the module can +be overridden with Overrides, which require the consumer to know how the module +is structured. This "escape hatch" is considered a first class citizen in the +design, to avoid excessive Parameter definitions to cover every possible use +case. One caveat is that using Overrides is less stable, since the module +author might change logical ids. Using module References can mitigate this. + +Module Inputs (set by Properties in the parent) are referenced with Refs, +Subs, and GetAtts in the module. These are handled in a way that fixes +references to match module prefixes, fully resolving values that are actually +strings and leaving others to be resolved at deploy time. + +Modules can contain other modules, with no enforced limit to the levels of +nesting. + +Modules can define References, which are key-value pairs that can be referenced +by the parent using Sub and GetAtt. + +When using modules, you can use a comma-delimited list to create a number of +similar resources with the Map attribute. This is simpler than using +`Fn::ForEach` but has the limitation of requiring the list to be resolved at +build time. See +tests/unit/customizations/cloudformation/modules/vpc-module.yaml. + +An example of a Map is defining subnets in a VPC. + +```yaml +Parameters: + CidrBlock: + Type: String + PrivateCidrBlocks: + Type: CommaDelimitedList + PublicCidrBlocks: + Type: CommaDelimitedList +Modules: + PublicSubnet: + Map: !Ref PublicCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + VpcId: !Ref VPC + PrivateSubnet: + Map: !Ref PrivateCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + VpcId: !Ref VPC +Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: !Ref CidrBlock + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default +``` + +""" + +# pylint: disable=fixme,too-many-instance-attributes,too-many-lines + +import copy +import logging +import os +from collections import OrderedDict + +from awscli.customizations.cloudformation import exceptions +from awscli.customizations.cloudformation import yamlhelper +from awscli.customizations.cloudformation.module_constants import ( + process_constants, + replace_constants, +) + +from awscli.customizations.cloudformation.parse_sub import WordType +from awscli.customizations.cloudformation.parse_sub import ( + parse_sub, + is_sub_needed, +) +from awscli.customizations.cloudformation.module_conditions import ( + parse_conditions, +) +from awscli.customizations.cloudformation.module_visitor import Visitor +from awscli.customizations.cloudformation.module_read import ( + read_source, + is_url, +) + +LOG = logging.getLogger(__name__) + +RESOURCES = "Resources" +METADATA = "Metadata" +OVERRIDES = "Overrides" +DEPENDSON = "DependsOn" +PROPERTIES = "Properties" +CREATIONPOLICY = "CreationPolicy" +UPDATEPOLICY = "UpdatePolicy" +DELETIONPOLICY = "DeletionPolicy" +UPDATEREPLACEPOLICY = "UpdateReplacePolicy" +DEFAULT = "Default" +NAME = "Name" +SOURCE = "Source" +REF = "Ref" +SUB = "Fn::Sub" +GETATT = "Fn::GetAtt" +PARAMETERS = "Parameters" +MODULES = "Modules" +TYPE = "Type" +LOCAL_MODULE = "LocalModule" +OUTPUTS = "Outputs" +MAP = "Map" +MAP_PLACEHOLDER = "$MapValue" +INDEX_PLACEHOLDER = "$MapIndex" +CONDITIONS = "Conditions" +CONDITION = "Condition" +IF = "Fn::If" +INPUTS = "Inputs" +REFERENCES = "References" +AWSTOOLSMETRICS = "AWSToolsMetrics" +CLOUDFORMATION_PACKAGE = "CloudForCloudFormationPackage" +SOURCE_MAP = "SourceMap" +NO_SOURCE_MAP = "NoSourceMap" + + +# pylint:disable=too-many-arguments,too-many-positional-arguments +def make_module(template, name, config, base_path, parent_path, no_source_map): + "Create an instance of a module based on a template and the module config" + module_config = {} + module_config[NAME] = name + if SOURCE not in config: + msg = f"{name} missing {SOURCE}" + raise exceptions.InvalidModulePathError(msg=msg) + + source_path = config[SOURCE] + + if not is_url(source_path): + relative_path = source_path + module_config[SOURCE] = os.path.join(base_path, relative_path) + module_config[SOURCE] = os.path.normpath(module_config[SOURCE]) + else: + module_config[SOURCE] = source_path + + if module_config[SOURCE] == parent_path: + msg = f"Module refers to itself: {parent_path}" + raise exceptions.InvalidModuleError(msg=msg) + if PROPERTIES in config: + module_config[PROPERTIES] = config[PROPERTIES] + if OVERRIDES in config: + module_config[OVERRIDES] = config[OVERRIDES] + module_config[NO_SOURCE_MAP] = no_source_map + return Module(template, module_config) + + +def map_placeholders(i, token, val): + "Replace $MapValue and $MapIndex" + if SUB in val: + sub = val[SUB] + r = sub.replace(MAP_PLACEHOLDER, token) + r = r.replace(INDEX_PLACEHOLDER, f"{i}") + words = parse_sub(r) + need_sub = False + for word in words: + if word.t != WordType.STR: + need_sub = True + break + if need_sub: + return {SUB: r} + return r + r = val.replace(MAP_PLACEHOLDER, token) + r = r.replace(INDEX_PLACEHOLDER, f"{i}") + return r + + +def add_metrics_metadata(template): + "Add metadata to the template so we know modules were used" + if METADATA not in template: + template[METADATA] = {} + metadata = template[METADATA] + if AWSTOOLSMETRICS not in metadata: + metadata[AWSTOOLSMETRICS] = {} + metrics = metadata[AWSTOOLSMETRICS] + if CLOUDFORMATION_PACKAGE not in metrics: + metrics[CLOUDFORMATION_PACKAGE] = {} + metrics[CLOUDFORMATION_PACKAGE]["Modules"] = "true" + + +# pylint:disable=too-many-arguments,too-many-positional-arguments +def process_module_section( + template, base_path, parent_path, parent_module, no_metrics, no_source_map +): + "Recursively process the Modules section of a template" + + if MODULES not in template: + return template + + if not isdict(template[MODULES]): + msg = "Modules section is invalid" + raise exceptions.InvalidModuleError(msg=msg) + + if parent_module is None: + + # This is the actual parent template, not a module + + if not no_metrics: + add_metrics_metadata(template) + + # Make a fake Module instance to handle find_ref for Maps + # The only valid way to do this at the template level + # is to specify a default for a Parameter, since we need to + # resolve the actual value client-side + parent_module = Module(template, {NAME: "", SOURCE: ""}) + + # The actual parent template will have Parameters + if PARAMETERS in template: + parent_module.inputs = template[PARAMETERS] + + # A module will have Inputs instead of Parameters + if INPUTS in template: + parent_module.inputs = template[INPUTS] + + # First, pre-process local modules that are looping over a list + process_module_maps(template, parent_module) + + # Process each Module node separately after processing Maps + modules = template[MODULES] + for k, v in modules.items(): + module = make_module( + template, k, v, base_path, parent_path, no_source_map + ) + template = module.process() + + # Remove the Modules section from the template + del template[MODULES] + + # Lift the created resources up to the parent + for k, v in template[RESOURCES].items(): + parent_module.template[RESOURCES][parent_module.name + k] = v + + return template + + +def process_module_maps(template, parent_module): + "Loop over Maps in modules" + modules = template[MODULES] + for k, v in modules.copy().items(): + if MAP in v: + # Expect Map to be a CSV or ref to a CSV + m = v[MAP] + if isdict(m) and REF in m: + m = parent_module.find_ref(m[REF]) + if m is None: + msg = f"{k} has an invalid Map Ref" + raise exceptions.InvalidModuleError(msg=msg) + tokens = m.split(",") + for i, token in enumerate(tokens): + # Make a new module + module_id = f"{k}{i}" + copied_module = copy.deepcopy(v) + del copied_module[MAP] + # Replace $Map and $Index placeholders + if PROPERTIES in copied_module: + for prop, val in copied_module[PROPERTIES].copy().items(): + copied_module[PROPERTIES][prop] = map_placeholders( + i, token, val + ) + modules[module_id] = copied_module + + del modules[k] + + +def isdict(v): + "Returns True if the type is a dict or OrderedDict" + return isinstance(v, (dict, OrderedDict)) + + +def merge_props(original, overrides): + """ + This function merges dicts, replacing values in the original with + overrides. This function is recursive and can act on lists and scalars. + See the unit tests for example merges. + See tests/unit/customizations/cloudformation/modules/policy-*.yaml + + :return A new value with the overridden properties + """ + original_type = type(original) + override_type = type(overrides) + if not isdict(overrides) and override_type is not list: + return overrides + + if original_type is not override_type: + return overrides + + if isdict(original): + retval = original.copy() + for k in original: + if k in overrides: + retval[k] = merge_props(retval[k], overrides[k]) + for k in overrides: + if k not in original: + retval[k] = overrides[k] + return retval + + # original and overrides are lists + new_list = [] + for item in original: + new_list.append(item) + for item in overrides: + new_list.append(item) + return new_list + + +class Module: + """ + Process client-side modules. + + """ + + def __init__(self, template, module_config): + """ + Initialize the module with values from the parent template + + :param template The parent template dictionary + :param module_config The configuration from the parent Modules section + """ + + # The parent template dictionary + self.template = template + if RESOURCES not in self.template: + # The parent might only have Modules + self.template[RESOURCES] = {} + + # The name of the module, which is used as a logical id prefix + self.name = module_config[NAME] + + # The location of the source for the module, a URI string + self.source = module_config[SOURCE] + + # The Properties from the parent template + self.props = {} + if PROPERTIES in module_config: + self.props = module_config[PROPERTIES] + + # The Overrides from the parent template + self.overrides = {} + if OVERRIDES in module_config: + self.overrides = module_config[OVERRIDES] + + # Resources defined in the module + self.resources = {} + + # Input parameters defined in the module + self.inputs = {} + + # Outputs defined in the module + self.references = {} + + # Conditions defined in the module + self.conditions = {} + + # Line numbers for resources + self.lines = {} + self.no_source_map = False + if module_config.get(NO_SOURCE_MAP, False): + self.no_source_map = True + + def __str__(self): + "Print out a string with module details for logs" + return ( + f"module name: {self.name}, " + + f"source: {self.source}, props: {self.props}" + ) + + # pylint: disable=too-many-branches + def process(self): + """ + Read the module source process it. + + :return: The modified parent template dictionary + """ + + content, lines = read_source(self.source) + self.lines = lines + + module_dict = yamlhelper.yaml_parse(content) + + # Process constants + constants = process_constants(module_dict) + if constants is not None: + replace_constants(constants, module_dict) + + if RESOURCES not in module_dict: + # The module may only have sub modules in the Modules section + self.resources = {} + else: + self.resources = module_dict[RESOURCES] + + if INPUTS in module_dict: + self.inputs = module_dict[INPUTS] + + if REFERENCES in module_dict: + self.references = module_dict[REFERENCES] + + # Read the Conditions section and store it as a dict of string:boolean + if CONDITIONS in module_dict: + cs = module_dict[CONDITIONS] + + def find_ref(v): + return self.find_ref(v) + + try: + self.conditions = parse_conditions(cs, find_ref) + except Exception as e: + msg = f"Failed to process conditions in {self.source}: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + + # Check each resource to see if a Condition omits it + for logical_id, resource in self.resources.copy().items(): + if CONDITION in resource: + if resource[CONDITION] in self.conditions: + if self.conditions[resource[CONDITION]] is False: + del self.resources[logical_id] + else: + del self.resources[logical_id][CONDITION] + + # Do the same for modules in the Modules section + if MODULES in module_dict: + for k, v in module_dict[MODULES].copy().items(): + if CONDITION in v: + if v[CONDITION] in self.conditions: + if self.conditions[v[CONDITION]] is False: + del module_dict[MODULES][k] + + # Recurse on nested modules + bp = os.path.dirname(self.source) + try: + process_module_section( + module_dict, bp, self.source, self, True, self.no_source_map + ) + except Exception as e: + msg = f"Failed to process {self.source} Modules section: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + + self.validate_overrides() + + for logical_id, resource in self.resources.items(): + self.process_resource(logical_id, resource) + + self.process_references() + + return self.template + + def process_references(self): + """ + Fix parent template GetAtt and Sub that point to module references. + + In the parent you can !GetAtt ModuleName.OutputName + This will be converted so that it's correct in the packaged template. + + The parent can also refer to module Properties as a convenience, + so !GetAtt ModuleName.PropertyName will simply copy the + configured property value. + + Recurse over all sections in the parent template looking for + GetAtts and Subs that reference a module References value. + """ + sections = [RESOURCES, REFERENCES, MODULES, OUTPUTS] + for section in sections: + if section not in self.template: + continue + for k, v in self.template[section].items(): + self.resolve_references(k, v, self.template, section) + + def resolve_references(self, k, v, d, n): + """ + Recursively resolve GetAtts and Subs that reference module references. + + :param name The name of the reference + :param k The name of the node + :param v The value of the node + :param d The dict that holds the parent of k + :param n The name of the node that holds k + + If a reference is found, this function sets the value of d[n] + """ + if k == SUB: + self.resolve_reference_sub(v, d, n) + elif k == GETATT: + self.resolve_reference_getatt(v, d, n) + else: + if isdict(v): + for k2, v2 in v.copy().items(): + self.resolve_references(k2, v2, d[n], k) + elif isinstance(v, list): + idx = -1 + for v2 in v: + idx = idx + 1 + if isdict(v2): + for k3, v3 in v2.copy().items(): + self.resolve_references(k3, v3, v, idx) + + def resolve_reference_sub_getatt(self, w): + """ + Resolve a reference to a module in a Sub GetAtt word. + + For example, the parent has + + Modules: + A: + Properties: + Name: foo + Outputs: + B: + Value: !Sub ${A.MyName} + C: + Value: !Sub ${A.Name} + + The module has: + + Inputs: + Name: + Type: String + Resources: + X: + Properties: + Y: !Ref Name + References: + MyName: !GetAtt Y.Name + + The resulting output: + B: !Sub ${AX.Name} + C: foo + + """ + + resolved = "${" + w + "}" + tokens = w.split(".", 1) + if len(tokens) < 2: + msg = f"GetAtt {w} has unexpected number of tokens" + raise exceptions.InvalidModuleError(msg=msg) + + # !Sub ${Content.BucketArn} -> !Sub ${ContentBucket.Arn} + + r = None + if tokens[0] == self.name: + if tokens[1] in self.references: + # We're referring to the module's References + r = self.references[tokens[1]] + elif tokens[1] in self.props: + # We're referring to parent Module Properties + r = self.props[tokens[1]] + if r is not None: + if GETATT in r: + getatt = r[GETATT] + resolved = "${" + self.name + ".".join(getatt) + "}" + elif SUB in r: + resolved = "${" + self.name + r[SUB] + "}" + elif REF in r: + resolved = "${" + self.name + r[REF] + "}" + else: + # Handle scalar properties + resolved = r + + return resolved + + def resolve_reference_sub(self, v, d, n): + "Resolve a Sub that refers to a module reference or property" + words = parse_sub(v, True) + sub = "" + for word in words: + if word.t == WordType.STR: + sub += word.w + elif word.t == WordType.AWS: + sub += "${AWS::" + word.w + "}" + elif word.t == WordType.REF: + # A ref to a module reference has to be a getatt + resolved = "${" + word.w + "}" + sub += resolved + elif word.t == WordType.GETATT: + sub += self.resolve_reference_sub_getatt(word.w) + + if is_sub_needed(sub): + d[n] = {SUB: sub} + else: + d[n] = sub + + # pylint:disable=too-many-branches + def resolve_reference_getatt(self, v, d, n): + """ + Resolve a GetAtt that refers to a module Reference. + + :param v The value + :param d The dictionary + :param n The name of the node + + This function sets d[n] + """ + if not isinstance(v, list) or len(v) < 2: + msg = f"GetAtt {v} invalid" + raise exceptions.InvalidModuleError(msg=msg) + + r = None + if v[0] == self.name: + if v[1] in self.references: + r = self.references[v[1]] + elif v[1] in self.props: + r = self.props[v[1]] + if r is not None: + if REF in r: + ref = r[REF] + d[n] = {REF: self.name + ref} + elif GETATT in r: + getatt = r[GETATT] + if len(getatt) < 2: + msg = f"GetAtt {getatt} in Output {v[1]} is invalid" + raise exceptions.InvalidModuleError(msg=msg) + d[n] = {GETATT: [self.name + getatt[0], getatt[1]]} + elif SUB in r: + # Parse the Sub in the module reference + words = parse_sub(r[SUB], True) + sub = "" + for word in words: + if word.t == WordType.STR: + sub += word.w + elif word.t == WordType.AWS: + sub += "${AWS::" + word.w + "}" + elif word.t == WordType.REF: + # This is a ref to a param or resource + # TODO: If it's a ref to a param...? is this allowed? + # If it's a resource, concatenante the name + resolved = "${" + word.w + "}" + if word.w in self.resources: + resolved = "${" + self.name + word.w + "}" + sub += resolved + elif word.t == WordType.GETATT: + resolved = "${" + word.w + "}" + tokens = word.w.split(".", 1) + if len(tokens) < 2: + msg = f"GetAtt {word.w} unexpected length" + raise exceptions.InvalidModuleError(msg=msg) + if tokens[0] in self.resources: + resolved = "${" + self.name + word.w + "}" + sub += resolved + d[n] = {SUB: sub} + else: + # Handle scalars in Properties + d[n] = r + + def validate_overrides(self): + "Make sure resources referenced by overrides actually exist" + for logical_id in self.overrides: + if logical_id not in self.resources: + msg = f"Override {logical_id} not found in {self.source}" + raise exceptions.InvalidModuleError(msg=msg) + + def process_resource(self, logical_id, resource): + "Process a single resource" + + # For each property (and property-like attribute), + # replace the value if it appears in parent overrides. + attrs = [ + PROPERTIES, + CREATIONPOLICY, + METADATA, + UPDATEPOLICY, + DELETIONPOLICY, + CONDITION, + UPDATEREPLACEPOLICY, + DEPENDSON, + ] + for a in attrs: + self.process_overrides(logical_id, resource, a) + + # Resolve refs, subs, and getatts + # (Process module Inputs and parent Properties) + container = {} + # We need the container for the first iteration of the recursion + container[RESOURCES] = self.resources + self.resolve(logical_id, resource, container, RESOURCES) + self.template[RESOURCES][self.name + logical_id] = resource + self.process_resource_conditions() + + # Add the source map to Metadata + if not self.no_source_map: + if METADATA not in resource: + resource[METADATA] = {} + metadata = resource[METADATA] + metadata[SOURCE_MAP] = self.get_source_map(logical_id) + + def get_source_map(self, logical_id): + "Get the string to put in the resource metadata source map" + n = self.lines.get(logical_id, 0) + return f"{self.source}:{logical_id}:{n}" + + def process_resource_conditions(self): + "Visit all resources to look for Fn::If conditions" + + # Example + # + # Resources + # Foo: + # Properties: + # Something: + # Fn::If: + # - ConditionName + # - AnObject + # - !Ref AWS::NoValue + # + # In this case, delete the 'Something' node entirely + # Otherwise replace the Fn::If with the correct value + def vf(v): + if isdict(v.d) and IF in v.d and v.p is not None: + conditional = v.d[IF] + if len(conditional) != 3: + msg = f"Invalid conditional in {self.name}: {conditional}" + raise exceptions.InvalidModuleError(msg=msg) + condition_name = conditional[0] + trueval = conditional[1] + falseval = conditional[2] + if condition_name not in self.conditions: + return # Assume this is a parent template condition? + if self.conditions[condition_name]: + v.p[v.k] = trueval + else: + v.p[v.k] = falseval + newval = v.p[v.k] + if isdict(newval) and REF in newval: + if newval[REF] == "AWS::NoValue": + del v.p[v.k] + + v = Visitor(self.resources) + v.visit(vf) + v = Visitor(self.references) + v.visit(vf) + + def process_overrides(self, logical_id, resource, attr_name): + """ + Replace overridden values in a property-like attribute of a resource. + + (Properties, Metadata, CreationPolicy, and UpdatePolicy) + + Overrides are a way to customize modules without needing a Parameter. + + Example template.yaml: + + Modules: + Foo: + Source: ./module.yaml + Overrides: + Bar: + Properties: + Name: bbb + + Example module.yaml: + + Resources: + Bar: + Type: A::B::C + Properties: + Name: aaa + + Output yaml: + + Resources: + Bar: + Type: A::B::C + Properties: + Name: bbb + """ + + if logical_id not in self.overrides: + return + + resource_overrides = self.overrides[logical_id] + if resource_overrides is None: + return + if attr_name not in resource_overrides: + return + + # Might be overriding something that's not in the module at all, + # like a Bucket with no Properties + if attr_name not in resource: + if attr_name in resource_overrides: + resource[attr_name] = resource_overrides[attr_name] + else: + return + + original = resource[attr_name] + overrides = resource_overrides[attr_name] + resource[attr_name] = merge_props(original, overrides) + + def resolve(self, k, v, d, n): + """ + Resolve Refs, Subs, and GetAtts recursively. + + :param k The name of the node + :param v The value of the node + :param d The dict that is the parent of the dict that holds k, v + :param n The name of the dict that holds k, v + + Example + + Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name + + In the above example, + k = !Ref, v = Name, d = Properties{}, n = BucketName + + So we can set d[n] = resolved_value (which replaces {k,v}) + + In the prior iteration, + k = BucketName, v = {!Ref, Name}, d = Bucket{}, n = Properties + """ + + if k == REF: + self.resolve_ref(v, d, n) + elif k == SUB: + self.resolve_sub(v, d, n) + elif k == GETATT: + self.resolve_getatt(v, d, n) + else: + if isdict(v): + vc = v.copy() + for k2, v2 in vc.items(): + self.resolve(k2, v2, d[n], k) + elif isinstance(v, list): + idx = -1 + for v2 in v: + idx = idx + 1 + if isdict(v2): + v2c = v2.copy() + for k3, v3 in v2c.items(): + self.resolve(k3, v3, v, idx) + + def resolve_ref(self, v, d, n): + """ + Look for the Ref in the parent template Properties if it matches + a module Parameter name. If it's not there, use the default if + there is one. If not, raise an error. + + If there is no matching Parameter, look for a resource with that + name in this module and fix the logical id so it has the prefix. + + Otherwise just leave it be and assume the module author is + expecting the parent template to have that Reference. + """ + if not isinstance(v, str): + msg = f"Ref should be a string: {v}" + raise exceptions.InvalidModuleError(msg=msg) + + found = self.find_ref(v) + if found is not None: + d[n] = found + + def resolve_sub_ref(self, w): + "Resolve a ref inside of a Sub string" + resolved = "${" + w + "}" + found = self.find_ref(w) + if found is not None: + if isinstance(found, str): + resolved = found + else: + if REF in found: + resolved = "${" + found[REF] + "}" + elif SUB in found: + resolved = found[SUB] + elif GETATT in found: + tokens = found[GETATT] + if len(tokens) < 2: + msg = ( + "Invalid Sub referencing a GetAtt. " + + f"{w}: {found}" + ) + raise exceptions.InvalidModuleError(msg=msg) + resolved = "${" + ".".join(tokens) + "}" + return resolved + + def resolve_sub_getatt(self, w): + "Resolve a GetAtt ('A.B') inside a Sub string" + resolved = "${" + w + "}" + tokens = w.split(".", 1) + if len(tokens) < 2: + msg = f"GetAtt {w} has unexpected number of tokens" + raise exceptions.InvalidModuleError(msg=msg) + if tokens[0] in self.resources: + tokens[0] = self.name + tokens[0] + resolved = "${" + tokens[0] + "." + tokens[1] + "}" + return resolved + + # pylint: disable=too-many-branches,unused-argument + def resolve_sub(self, v, d, n): + """ + Parse the Sub string and break it into tokens. + + If we can fully resolve it, we can replace it with a string. + + Use the same logic as with resolve_ref. + """ + words = parse_sub(v, True) + sub = "" + for word in words: + if word.t == WordType.STR: + sub += word.w + elif word.t == WordType.AWS: + sub += "${AWS::" + word.w + "}" + elif word.t == WordType.REF: + sub += self.resolve_sub_ref(word.w) + elif word.t == WordType.GETATT: + sub += self.resolve_sub_getatt(word.w) + + need_sub = is_sub_needed(sub) + if need_sub: + d[n] = {SUB: sub} + else: + d[n] = sub + + def resolve_getatt(self, v, d, n): + """ + Resolve a GetAtt. All we do here is add the prefix. + + !GetAtt Foo.Bar becomes !GetAtt ModuleNameFoo.Bar + """ + if not isinstance(v, list): + msg = f"GetAtt {v} is not a list" + raise exceptions.InvalidModuleError(msg=msg) + logical_id = self.name + v[0] + d[n] = {GETATT: [logical_id, v[1]]} + + def find_ref(self, name): + """ + Find a Ref. + + A Ref might be to a module Parameter with a matching parent + template Property, or a Parameter Default. It could also + be a reference to another resource in this module. + + :param name The name to search for + :return The referenced element or None + """ + if name in self.props: + if name not in self.inputs: + # The parent tried to set a property that doesn't exist + # in the Inputs section of this module + msg = f"{name} not found in module Inputs: {self.source}" + raise exceptions.InvalidModuleError(msg=msg) + return self.props[name] + + if name in self.inputs: + param = self.inputs[name] + if DEFAULT in param: + # Use the default value of the Parameter + return param[DEFAULT] + msg = f"{name} does not have a Default and is not a Property" + raise exceptions.InvalidModuleError(msg=msg) + + for logical_id in self.resources: + if name == logical_id: + # Simply rename local references to include the module name + return {REF: self.name + logical_id} + + return None diff --git a/awscli/customizations/cloudformation/package.py b/awscli/customizations/cloudformation/package.py index 9bc7464d442e..2a4f861f074e 100644 --- a/awscli/customizations/cloudformation/package.py +++ b/awscli/customizations/cloudformation/package.py @@ -57,7 +57,6 @@ class PackageCommand(BasicCommand): { 'name': 's3-bucket', - 'required': True, 'help_text': ( 'The name of the S3 bucket where this command uploads' ' the artifacts that are referenced in your template.' @@ -124,11 +123,7 @@ class PackageCommand(BasicCommand): ] def _run_main(self, parsed_args, parsed_globals): - s3_client = self._session.create_client( - "s3", - config=Config(signature_version='s3v4'), - region_name=parsed_globals.region, - verify=parsed_globals.verify_ssl) + template_path = parsed_args.template_file if not os.path.isfile(template_path): @@ -137,13 +132,25 @@ def _run_main(self, parsed_args, parsed_globals): bucket = parsed_args.s3_bucket - self.s3_uploader = S3Uploader(s3_client, - bucket, - parsed_args.s3_prefix, - parsed_args.kms_key_id, - parsed_args.force_upload) - # attach the given metadata to the artifacts to be uploaded - self.s3_uploader.artifact_metadata = parsed_args.metadata + # Only create the s3 uploaded if we need it, + # since this command now also supports local modules. + # Local modules should be able to run without credentials. + if bucket: + s3_client = self._session.create_client( + "s3", + config=Config(signature_version='s3v4'), + region_name=parsed_globals.region, + verify=parsed_globals.verify_ssl) + + self.s3_uploader = S3Uploader(s3_client, + bucket, + parsed_args.s3_prefix, + parsed_args.kms_key_id, + parsed_args.force_upload) + # attach the given metadata to the artifacts to be uploaded + self.s3_uploader.artifact_metadata = parsed_args.metadata + else: + self.s3_uploader = None output_file = parsed_args.output_template_file use_json = parsed_args.use_json diff --git a/awscli/customizations/cloudformation/parse_sub.py b/awscli/customizations/cloudformation/parse_sub.py new file mode 100644 index 000000000000..7dcae4364341 --- /dev/null +++ b/awscli/customizations/cloudformation/parse_sub.py @@ -0,0 +1,144 @@ +""" +This module parses CloudFormation Sub strings. + +For example: + + !Sub abc-${AWS::Region}-def-${Foo} + +The string is broken down into "words" that are one of four types: + + String: A literal string component + Ref: A reference to another resource or paramter like ${Foo} + AWS: An AWS pseudo-parameter like ${AWS::Region} + GetAtt: A reference to an attribute like ${Foo.Bar} +""" +#pylint: disable=too-few-public-methods + +from enum import Enum + +DATA = ' ' # Any other character +DOLLAR = '$' +OPEN = '{' +CLOSE = '}' +BANG = '!' +SPACE = ' ' + +class WordType(Enum): + "Word type enumeration" + STR = 0 # A literal string fragment + REF = 1 # ${ParamOrResourceName} + AWS = 2 # ${AWS::X} + GETATT = 3 # ${X.Y} + CONSTANT = 4 # ${Constant::name} + +class State(Enum): + "State machine enumeration" + READSTR = 0 + READVAR = 1 + MAYBE = 2 + READLIT = 3 + +class SubWord: + "A single word with a type and the word itself" + def __init__(self, word_type, word): + self.t = word_type + self.w = word # Does not include the ${} if it's not a STR + + def __str__(self): + return f"{self.t} {self.w}" + +def is_sub_needed(s): + "Returns true if the string has any Sub variables" + words = parse_sub(s) + for w in words: + if w.t != WordType.STR: + return True + return False + +#pylint: disable=too-many-branches,too-many-statements +def parse_sub(sub_str, leave_bang=False): + """ + Parse a Sub string + + :param leave_bang If this is True, leave the ! in literals + :return list of words + """ + words = [] + state = State.READSTR + buf = '' + last = '' + for i, char in enumerate(sub_str): + if char == DOLLAR: + if state != State.READVAR: + state = State.MAYBE + else: + # This is a literal $ inside a variable: "${AB$C}" + buf += char + elif char == OPEN: + if state == State.MAYBE: + # Peek to see if we're about to start a LITERAL ! + if len(sub_str) > i+1 and sub_str[i+1] == BANG: + # Treat this as part of the string, not a var + buf += "${" + state = State.READLIT + else: + state = State.READVAR + # We're about to start reading a variable. + # Append the last word in the buffer if it's not empty + if buf: + words.append(SubWord(WordType.STR, buf)) + buf = '' + else: + buf += char + elif char == CLOSE: + if state == State.READVAR: + # Figure out what type it is + if buf.startswith("AWS::"): + word_type = WordType.AWS + elif buf.startswith("Constant::") or buf.startswith("Constants::"): + word_type = WordType.CONSTANT + elif '.' in buf: + word_type = WordType.GETATT + else: + word_type = WordType.REF + buf = buf.replace("AWS::", "", 1) + buf = buf.replace("Constant::", "", 1) + # Very common typo to put Constants instead of Constant + buf = buf.replace("Constants::", "", 1) + words.append(SubWord(word_type, buf)) + buf = '' + state = State.READSTR + else: + buf += char + elif char == BANG: + # ${!LITERAL} becomes ${LITERAL} + if state == State.READLIT: + # Don't write the ! to the string + state = State.READSTR + if leave_bang: + # Unless we actually want it + buf += char + else: + # This is a ! somewhere not related to a LITERAL + buf += char + elif char == SPACE: + # Ignore spaces around Refs. ${ ABC } == ${ABC} + if state != State.READVAR: + buf += char + else: + if state == State.MAYBE: + buf += last # Put the $ back on the buffer + state = State.READSTR + buf += char + + last = char + + if buf: + words.append(SubWord(WordType.STR, buf)) + + # Handle malformed strings, like "ABC${XYZ" + if state != State.READSTR: + # Ended the string in the middle of a variable? + raise ValueError("invalid string, unclosed variable") + + return words diff --git a/awscli/customizations/cloudformation/yamlhelper.py b/awscli/customizations/cloudformation/yamlhelper.py index 61603603e669..4bdcc38215ab 100644 --- a/awscli/customizations/cloudformation/yamlhelper.py +++ b/awscli/customizations/cloudformation/yamlhelper.py @@ -79,10 +79,15 @@ def _dict_constructor(loader, node): class SafeLoaderWrapper(yaml.SafeLoader): - """Isolated safe loader to allow for customizations without global changes. + """ + Isolated safe loader to allow for customizations without global changes. """ - pass +# def construct_mapping(self, node, deep=False): +# "Adds support for line numbers" +# mapping = super().construct_mapping(node, deep=deep) +# mapping["__line__"] = node.start_mark.line + 1 +# return mapping def yaml_parse(yamlstr): """Parse a yaml string""" @@ -96,8 +101,17 @@ def yaml_parse(yamlstr): loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _dict_constructor) loader.add_multi_constructor("!", intrinsics_multi_constructor) + return yaml.load(yamlstr, loader) +def yaml_compose(yamlstr): + """Returns a tree after parsing the string""" + + loader = SafeLoaderWrapper + loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + _dict_constructor) + loader.add_multi_constructor("!", intrinsics_multi_constructor) + return yaml.compose(yamlstr, loader) class FlattenAliasDumper(yaml.SafeDumper): def ignore_aliases(self, data): diff --git a/awscli/examples/cloudformation/_package_description.rst b/awscli/examples/cloudformation/_package_description.rst index f47ec2212916..f75a788f2a53 100644 --- a/awscli/examples/cloudformation/_package_description.rst +++ b/awscli/examples/cloudformation/_package_description.rst @@ -1,10 +1,13 @@ -Packages the local artifacts (local paths) that your AWS CloudFormation template -references. The command uploads local artifacts, such as source code for an AWS -Lambda function or a Swagger file for an AWS API Gateway REST API, to an S3 -bucket. The command returns a copy of your template, replacing references to -local artifacts with the S3 location where the command uploaded the artifacts. +Packages the local artifacts (local paths) that your AWS CloudFormation +template references. The command uploads local artifacts, such as source code +for an AWS Lambda function or a Swagger file for an AWS API Gateway REST API, +to an S3 bucket. The command can also process local modules, which are +parameterized CloudFormation snippets that are merged into the parent template. +The command returns a copy of your template, replacing references to local +artifacts with the S3 location where the command uploaded the artifacts, or +replacing local and remote module references with the resources in the module. -Use this command to quickly upload local artifacts that might be required by +Use this command to quickly process local artifacts that might be required by your template. After you package your template's artifacts, run the ``deploy`` command to deploy the returned template. @@ -32,25 +35,42 @@ This command can upload local artifacts referenced in the following places: - ``S3`` property for the ``AWS::CodeCommit::Repository`` resource -To specify a local artifact in your template, specify a path to a local file or folder, -as either an absolute or relative path. The relative path is a location -that is relative to your template's location. +To specify a local artifact in your template, specify a path to a local file or +folder, as either an absolute or relative path. The relative path is a location +that is relative to your template's location. If the artifact is a module, the +path can be a local file or a remote URL starting with 'https'. For example, if your AWS Lambda function source code is in the -``/home/user/code/lambdafunction/`` folder, specify -``CodeUri: /home/user/code/lambdafunction`` for the -``AWS::Serverless::Function`` resource. The command returns a template and replaces -the local path with the S3 location: ``CodeUri: s3://mybucket/lambdafunction.zip``. +``/home/user/code/lambdafunction/`` folder, specify ``CodeUri: +/home/user/code/lambdafunction`` for the ``AWS::Serverless::Function`` +resource. The command returns a template and replaces the local path with the +S3 location: ``CodeUri: s3://mybucket/lambdafunction.zip``. If you specify a file, the command directly uploads it to the S3 bucket. If you specify a folder, the command zips the folder and then uploads the .zip file. -For most resources, if you don't specify a path, the command zips and uploads the -current working directory. The exception is ``AWS::ApiGateway::RestApi``; -if you don't specify a ``BodyS3Location``, this command will not upload an artifact to S3. +For most resources, if you don't specify a path, the command zips and uploads +the current working directory. The exception is ``AWS::ApiGateway::RestApi``; +if you don't specify a ``BodyS3Location``, this command will not upload an +artifact to S3. Before the command uploads artifacts, it checks if the artifacts are already present in the S3 bucket to prevent unnecessary uploads. The command uses MD5 checksums to compare files. If the values match, the command doesn't upload the -artifacts. Use the ``--force-upload flag`` to skip this check and always upload the -artifacts. +artifacts. Use the ``--force-upload flag`` to skip this check and always upload +the artifacts. + +Modules can be referenced either in the top level ``Modules`` section of the +template, or from a Resource with ``Type: LocalModule``. Module references have +a ``Source`` attribute pointing to the module, either a local file or an +``https`` URL, a ``Properties`` attribute that corresponds to the module's +parameters, and an ``Overrides`` attribute that can override module output. + +This command also allows you to add a ``Constants`` section to the template +or to a local module. This section is a simple set of key-value pairs that +can be used to reduce copy-paste within the template. Constants values are +strings that can be references within ``Fn::Sub`` functions using the format +``${Constant::NAME}``. + + + diff --git a/awscli/examples/cloudformation/package.rst b/awscli/examples/cloudformation/package.rst index 159b9b4f844e..2de7d63b1576 100644 --- a/awscli/examples/cloudformation/package.rst +++ b/awscli/examples/cloudformation/package.rst @@ -1,6 +1,69 @@ -Following command exports a template named ``template.json`` by uploading local +Following command exports a template named ``template.yaml`` by uploading local artifacts to S3 bucket ``bucket-name`` and writes the exported template to -``packaged-template.json``:: +``packaged-template.yaml``:: - aws cloudformation package --template-file /path_to_template/template.json --s3-bucket bucket-name --output-template-file packaged-template.json --use-json + aws cloudformation package --template-file /path_to_template/template.yaml --s3-bucket bucket-name --output-template-file packaged-template.yaml + + +The following is an example of a template with a ``Modules`` section:: + + Modules: + Content: + Source: ./module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Metadata: + OverrideMe: def + +A module configured as a Resource:: + + Resources: + Content: + Type: LocalModule + Source: ./module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Metadata: + OverrideMe: def + +An example module:: + + Parameters: + Name: + Type: String + Resources: + Bucket: + Type: AWS::S3::Bucket + Metadata: + OverrideMe: abc + Properties: + BucketName: !Ref Name + Outputs: + BucketArn: !GetAtt Bucket.Arn + +Packaging the template with this module would result in the following output:: + + Resources: + ContentBucket: + Type: AWS::S3::Bucket + Metadata: + OverrideMe: def + Properties: + BucketName: foo + +The following is an example of adding constants to a template:: + + Constants: + foo: bar + baz: ${Constant:foo}-xyz-${AWS::AccountId} + + Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub ${Constant::baz} diff --git a/tests/unit/customizations/cloudformation/modules/basic-expect.yaml b/tests/unit/customizations/cloudformation/modules/basic-expect.yaml new file mode 100644 index 000000000000..862fea4ed601 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/basic-expect.yaml @@ -0,0 +1,19 @@ +Resources: + OtherResource: + Type: AWS::S3::Bucket + Metadata: + TestOutputSub: + Fn::Sub: ${ContentBucket} + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + OverrideMe: def + SubName: foo + ContentBucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: + Fn::GetAtt: + - ContentBucket + - Something diff --git a/tests/unit/customizations/cloudformation/modules/basic-module.yaml b/tests/unit/customizations/cloudformation/modules/basic-module.yaml new file mode 100644 index 000000000000..78a7dcd5c517 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/basic-module.yaml @@ -0,0 +1,16 @@ +Inputs: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name + OverrideMe: abc + SubName: !Sub ${Name} + Bucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: !GetAtt Bucket.Something +References: + Name: !Ref Bucket diff --git a/tests/unit/customizations/cloudformation/modules/basic-template.yaml b/tests/unit/customizations/cloudformation/modules/basic-template.yaml new file mode 100644 index 000000000000..fa0dfb3305c7 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/basic-template.yaml @@ -0,0 +1,14 @@ +Modules: + Content: + Source: ./basic-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +Resources: + OtherResource: + Type: AWS::S3::Bucket + Metadata: + TestOutputSub: !Sub ${Content.Name} diff --git a/tests/unit/customizations/cloudformation/modules/cond-intrinsics-expect.yaml b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-expect.yaml new file mode 100644 index 000000000000..649ec70a9de9 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-expect.yaml @@ -0,0 +1,7 @@ +Resources: + ABucket0: + Type: AWS::S3::Bucket + ABucket1: + Type: AWS::S3::Bucket + ABucket2: + Type: AWS::S3::Bucket diff --git a/tests/unit/customizations/cloudformation/modules/cond-intrinsics-module.yaml b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-module.yaml new file mode 100644 index 000000000000..498029573d59 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-module.yaml @@ -0,0 +1,40 @@ +Inputs: + Foo: + Type: String + +Conditions: + Show0: + Fn::Equals: + - !Ref Foo + - bar + Show1: + Fn::And: + - Fn::Equals: + - !Ref Foo + - bar + - Condition: Show0 + Show2: + Fn::Not: + - Fn::Or: + - Fn::Equals: + - !Ref Foo + - baz + - Fn::Equals: + - Fn::If: + - Fn::Equals: + - a + - a + - x + - y + - y + +Resources: + Bucket0: + Type: AWS::S3::Bucket + Condition: Show0 + Bucket1: + Type: AWS::S3::Bucket + Condition: Show1 + Bucket2: + Type: AWS::S3::Bucket + Condition: Show2 diff --git a/tests/unit/customizations/cloudformation/modules/cond-intrinsics-template.yaml b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-template.yaml new file mode 100644 index 000000000000..c3daddf7c8f2 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/cond-intrinsics-template.yaml @@ -0,0 +1,9 @@ +Modules: + A: + Source: ./cond-intrinsics-module.yaml + Properties: + Foo: bar + B: + Source: ./cond-intrinsics-module.yaml + Properties: + Foo: baz diff --git a/tests/unit/customizations/cloudformation/modules/conditional-expect.yaml b/tests/unit/customizations/cloudformation/modules/conditional-expect.yaml new file mode 100644 index 000000000000..ad1f15200d9c --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/conditional-expect.yaml @@ -0,0 +1,57 @@ +Resources: + AHideMeFoo: + Type: A::B::C + ALambdaPolicy: + Type: AWS::IAM::RolePolicy + Metadata: + Comment: This resource is created only if the LambdaRoleArn is set + Properties: + PolicyDocument: + Statement: + - Action: + - dynamodb:BatchGetItem + - dynamodb:GetItem + - dynamodb:Query + - dynamodb:Scan + - dynamodb:BatchWriteItem + - dynamodb:PutItem + - dynamodb:UpdateItem + Effect: Allow + Resource: + - Fn::GetAtt: + - ATable + - Arn + PolicyName: table-a-policy + RoleName: my-role + ATable: + Type: AWS::DynamoDB::Table + Properties: + BillingMode: PAY_PER_REQUEST + TableName: table-a + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + ATestResourceConditional: + Type: A::B::C + Properties: + Show: Foo + ShowA: A + BTable: + Type: AWS::DynamoDB::Table + Properties: + BillingMode: PAY_PER_REQUEST + TableName: table-b + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + BTestResourceConditional: + Type: A::B::C + Properties: + Hide: Foo + ShowA: B diff --git a/tests/unit/customizations/cloudformation/modules/conditional-module.yaml b/tests/unit/customizations/cloudformation/modules/conditional-module.yaml new file mode 100644 index 000000000000..021b10afe3d8 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/conditional-module.yaml @@ -0,0 +1,76 @@ +Inputs: + + TableName: + Type: String + + LambdaRoleName: + Type: String + Description: If set, allow the lambda function to access this table + Default: "" + +Conditions: + IfLambdaRoleIsSet: + Fn::Not: + - Fn::Equals: + - !Ref LambdaRoleName + - "" + +Modules: + HideMe: + Condition: IfLambdaRoleIsSet + Source: conditional-module2.yaml + +Resources: + Table: + Type: AWS::DynamoDB::Table + Properties: + BillingMode: PAY_PER_REQUEST + TableName: !Sub ${TableName} + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + + TestResourceConditional: + Type: A::B::C + Properties: + Show: + Fn::If: + - IfLambdaRoleIsSet + - Foo + - !Ref AWS::NoValue + Hide: + Fn::If: + - IfLambdaRoleIsSet + - !Ref AWS::NoValue + - Foo + ShowA: + Fn::If: + - IfLambdaRoleIsSet + - A + - B + + LambdaPolicy: + Type: AWS::IAM::RolePolicy + Condition: IfLambdaRoleIsSet + Metadata: + Comment: This resource is created only if the LambdaRoleArn is set + Properties: + PolicyDocument: + Statement: + - Action: + - dynamodb:BatchGetItem + - dynamodb:GetItem + - dynamodb:Query + - dynamodb:Scan + - dynamodb:BatchWriteItem + - dynamodb:PutItem + - dynamodb:UpdateItem + Effect: Allow + Resource: + - !GetAtt Table.Arn + PolicyName: !Sub ${TableName}-policy + RoleName: !Ref LambdaRoleName + diff --git a/tests/unit/customizations/cloudformation/modules/conditional-module2.yaml b/tests/unit/customizations/cloudformation/modules/conditional-module2.yaml new file mode 100644 index 000000000000..3ec62e417fd0 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/conditional-module2.yaml @@ -0,0 +1,3 @@ +Resources: + Foo: + Type: A::B::C diff --git a/tests/unit/customizations/cloudformation/modules/conditional-template.yaml b/tests/unit/customizations/cloudformation/modules/conditional-template.yaml new file mode 100644 index 000000000000..3599ca93bc97 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/conditional-template.yaml @@ -0,0 +1,11 @@ +Modules: + A: + Source: ./conditional-module.yaml + Properties: + TableName: table-a + LambdaRoleName: my-role + B: + Source: ./conditional-module.yaml + Properties: + TableName: table-b + diff --git a/tests/unit/customizations/cloudformation/modules/constant-expect.yaml b/tests/unit/customizations/cloudformation/modules/constant-expect.yaml new file mode 100644 index 000000000000..d9074de4d355 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/constant-expect.yaml @@ -0,0 +1,10 @@ +Metadata: + test: zzz +Resources: + ContentBucket: + Type: AWS::S3::Bucket + Metadata: + Test: + Fn::Sub: bar-abc-${AWS::AccountId}-xyz + Properties: + BucketName: bar diff --git a/tests/unit/customizations/cloudformation/modules/constant-module.yaml b/tests/unit/customizations/cloudformation/modules/constant-module.yaml new file mode 100644 index 000000000000..768e11418294 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/constant-module.yaml @@ -0,0 +1,12 @@ +Constants: + foo: bar + sub: "${Constant::foo}-abc-${AWS::AccountId}" + +Resources: + Bucket: + Type: AWS::S3::Bucket + Metadata: + Test: !Sub ${Constant::sub}-xyz + Properties: + BucketName: !Sub ${Constant::foo} + diff --git a/tests/unit/customizations/cloudformation/modules/constant-template.yaml b/tests/unit/customizations/cloudformation/modules/constant-template.yaml new file mode 100644 index 000000000000..f0aacecfc156 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/constant-template.yaml @@ -0,0 +1,8 @@ +Constants: + yyy: zzz +Metadata: + test: !Sub ${Constant::yyy} +Modules: + Content: + Source: ./constant-module.yaml + diff --git a/tests/unit/customizations/cloudformation/modules/example-expect.yaml b/tests/unit/customizations/cloudformation/modules/example-expect.yaml new file mode 100644 index 000000000000..0314736240bd --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/example-expect.yaml @@ -0,0 +1,7 @@ +Resources: + ContentBucket: + Type: AWS::S3::Bucket + Metadata: + OverrideMe: def + Properties: + BucketName: foo diff --git a/tests/unit/customizations/cloudformation/modules/example-module.yaml b/tests/unit/customizations/cloudformation/modules/example-module.yaml new file mode 100644 index 000000000000..fa3f21e3897f --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/example-module.yaml @@ -0,0 +1,11 @@ +Inputs: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Metadata: + OverrideMe: abc + Properties: + BucketName: !Ref Name + diff --git a/tests/unit/customizations/cloudformation/modules/example-template.yaml b/tests/unit/customizations/cloudformation/modules/example-template.yaml new file mode 100644 index 000000000000..1949bd339a0e --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/example-template.yaml @@ -0,0 +1,9 @@ +Modules: + Content: + Source: ./example-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Metadata: + OverrideMe: def diff --git a/tests/unit/customizations/cloudformation/modules/getatt-expect.yaml b/tests/unit/customizations/cloudformation/modules/getatt-expect.yaml new file mode 100644 index 000000000000..ae9d1c69e557 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/getatt-expect.yaml @@ -0,0 +1,15 @@ +Outputs: + BucketName: + Value: + Fn::GetAtt: + - ContentBucket + - BucketName + PropGetAtt: + Value: foo + PropSub: + Value: foo +Resources: + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo diff --git a/tests/unit/customizations/cloudformation/modules/getatt-module.yaml b/tests/unit/customizations/cloudformation/modules/getatt-module.yaml new file mode 100644 index 000000000000..f242ef434720 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/getatt-module.yaml @@ -0,0 +1,10 @@ +Inputs: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name +References: + BucketName: !GetAtt "Bucket.BucketName" diff --git a/tests/unit/customizations/cloudformation/modules/getatt-template.yaml b/tests/unit/customizations/cloudformation/modules/getatt-template.yaml new file mode 100644 index 000000000000..11818a481523 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/getatt-template.yaml @@ -0,0 +1,13 @@ +Modules: + Content: + Type: LocalModule + Source: ./getatt-module.yaml + Properties: + Name: foo +Outputs: + BucketName: + Value: !GetAtt Content.BucketName + PropGetAtt: + Value: !GetAtt Content.Name + PropSub: + Value: !Sub ${Content.Name} diff --git a/tests/unit/customizations/cloudformation/modules/map-expect.yaml b/tests/unit/customizations/cloudformation/modules/map-expect.yaml new file mode 100644 index 000000000000..b20a99bc160f --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/map-expect.yaml @@ -0,0 +1,17 @@ +Inputs: + List: + Type: CommaDelimitedList + Default: A,B,C +Resources: + Content0Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: my-bucket-A + Content1Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: my-bucket-B + Content2Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: my-bucket-C diff --git a/tests/unit/customizations/cloudformation/modules/map-module.yaml b/tests/unit/customizations/cloudformation/modules/map-module.yaml new file mode 100644 index 000000000000..79968ef0ae5f --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/map-module.yaml @@ -0,0 +1,9 @@ +Inputs: + Name: + Type: String + +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name diff --git a/tests/unit/customizations/cloudformation/modules/map-template.yaml b/tests/unit/customizations/cloudformation/modules/map-template.yaml new file mode 100644 index 000000000000..3c42f626577c --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/map-template.yaml @@ -0,0 +1,11 @@ +Inputs: + List: + Type: CommaDelimitedList + Default: A,B,C + +Modules: + Content: + Source: ./map-module.yaml + Map: !Ref List + Properties: + Name: !Sub my-bucket-$MapValue diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml new file mode 100644 index 000000000000..211896562671 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml @@ -0,0 +1,14 @@ +Inputs: + ParentVal: + Type: String + AppName: + Type: String +Resources: + MySubBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: mod-in-mod-bucket + XName: + Fn::Sub: ${ParentVal}-abc + YName: + Fn::Sub: ${AppName}-xyz diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-module.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-module.yaml new file mode 100644 index 000000000000..7ddee957ed62 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-module.yaml @@ -0,0 +1,10 @@ +Inputs: + AppName: + Type: String +Modules: + Sub: + Source: "./modinmod-submodule.yaml" + Properties: + X: !Sub ${ParentVal}-abc + Y: !Sub ${AppName}-xyz + diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml new file mode 100644 index 000000000000..eee066288f54 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml @@ -0,0 +1,12 @@ +Inputs: + X: + Type: String + Y: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: mod-in-mod-bucket + XName: !Ref X + YName: !Ref Y diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-template.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-template.yaml new file mode 100644 index 000000000000..41f7d99594ec --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-template.yaml @@ -0,0 +1,10 @@ +Inputs: + ParentVal: + Type: String + AppName: + Type: String +Modules: + My: + Source: ./modinmod-module.yaml + Properties: + AppName: !Ref AppName diff --git a/tests/unit/customizations/cloudformation/modules/output-expect.yaml b/tests/unit/customizations/cloudformation/modules/output-expect.yaml new file mode 100644 index 000000000000..a66ea404de8d --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-expect.yaml @@ -0,0 +1,35 @@ +Resources: + A: + Type: D::E::F + Properties: + Object: + Arn: + Fn::Sub: ${ContentBucket.Arn} + ArnGetAtt: + Fn::GetAtt: + - ContentBucket + - Arn + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + AnotherModuleFoo: + Type: A::B::C + Properties: + AName: + Fn::Sub: another-${ContentBucket.Arn} +Outputs: + ExampleOutput: + Value: + Fn::GetAtt: + - ContentBucket + - Arn + ExampleSub: + Value: + Fn::Sub: ${ContentBucket.Arn} + ExampleGetSub: + Value: + Fn::Sub: ${ContentBucket.Arn} + ExampleRef: + Value: + Ref: ContentBucket diff --git a/tests/unit/customizations/cloudformation/modules/output-module.yaml b/tests/unit/customizations/cloudformation/modules/output-module.yaml new file mode 100644 index 000000000000..2efb24e47d33 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-module.yaml @@ -0,0 +1,13 @@ +Inputs: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name +References: + BucketArn: !GetAtt Bucket.Arn + BucketArnSub: !Sub ${Bucket.Arn} + BucketRef: !Ref Bucket + diff --git a/tests/unit/customizations/cloudformation/modules/output-module2.yaml b/tests/unit/customizations/cloudformation/modules/output-module2.yaml new file mode 100644 index 000000000000..c6b7e58c5163 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-module2.yaml @@ -0,0 +1,9 @@ +Inputs: + Name: + Type: String +Resources: + Foo: + Type: A::B::C + Properties: + AName: !Sub another-${Name} + diff --git a/tests/unit/customizations/cloudformation/modules/output-template.yaml b/tests/unit/customizations/cloudformation/modules/output-template.yaml new file mode 100644 index 000000000000..3f756701407e --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-template.yaml @@ -0,0 +1,26 @@ +Modules: + Content: + Source: ./output-module.yaml + Properties: + Name: foo + AnotherModule: + Source: ./output-module2.yaml + Properties: + Name: !GetAtt ContentBucket.Arn + +Resources: + A: + Type: D::E::F + Properties: + Object: + Arn: !Sub ${Content.BucketArn} + ArnGetAtt: !GetAtt Content.BucketArn +Outputs: + ExampleOutput: + Value: !GetAtt Content.BucketArn + ExampleSub: + Value: !Sub ${Content.BucketArn} + ExampleGetSub: + Value: !GetAtt Content.BucketArnSub + ExampleRef: + Value: !GetAtt Content.BucketRef diff --git a/tests/unit/customizations/cloudformation/modules/policy-expect.yaml b/tests/unit/customizations/cloudformation/modules/policy-expect.yaml new file mode 100644 index 000000000000..8d524c1fb4ce --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/policy-expect.yaml @@ -0,0 +1,16 @@ +Resources: + AccessPolicy: + Type: AWS::IAM::RolePolicy + Properties: + PolicyDocument: + Statement: + - Effect: ALLOW + Action: s3:List* + Resource: + - arn:aws:s3:::foo + - arn:aws:s3:::foo/* + - Effect: DENY + Action: s3:Put* + Resource: arn:aws:s3:::bar + PolicyName: my-policy + RoleName: foo diff --git a/tests/unit/customizations/cloudformation/modules/policy-module.yaml b/tests/unit/customizations/cloudformation/modules/policy-module.yaml new file mode 100644 index 000000000000..1f39d8853751 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/policy-module.yaml @@ -0,0 +1,19 @@ +Inputs: + Role: + Type: String + BucketName: + Default: foo +Resources: + Policy: + Type: AWS::IAM::RolePolicy + Properties: + PolicyDocument: + Statement: + - Effect: ALLOW + Action: s3:List* + Resource: + - !Sub arn:aws:s3:::${BucketName} + - !Sub arn:aws:s3:::${BucketName}/* + PolicyName: my-policy + RoleName: !Ref Role + diff --git a/tests/unit/customizations/cloudformation/modules/policy-template.yaml b/tests/unit/customizations/cloudformation/modules/policy-template.yaml new file mode 100644 index 000000000000..9f587f7bda9d --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/policy-template.yaml @@ -0,0 +1,13 @@ +Modules: + Access: + Source: ./policy-module.yaml + Properties: + Role: foo + Overrides: + Policy: + Properties: + PolicyDocument: + Statement: + - Effect: DENY + Action: s3:Put* + Resource: arn:aws:s3:::bar diff --git a/tests/unit/customizations/cloudformation/modules/proparray-expect.yaml b/tests/unit/customizations/cloudformation/modules/proparray-expect.yaml new file mode 100644 index 000000000000..bb13421109cf --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/proparray-expect.yaml @@ -0,0 +1,7 @@ +Resources: + FooBaz: + Type: A::B::C + Properties: + AnArray: + - a + - b diff --git a/tests/unit/customizations/cloudformation/modules/proparray-module.yaml b/tests/unit/customizations/cloudformation/modules/proparray-module.yaml new file mode 100644 index 000000000000..d2450977ad59 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/proparray-module.yaml @@ -0,0 +1,9 @@ +Inputs: + Bar: + Type: Array + +Resources: + Baz: + Type: A::B::C + Properties: + AnArray: !Ref Bar diff --git a/tests/unit/customizations/cloudformation/modules/proparray-template.yaml b/tests/unit/customizations/cloudformation/modules/proparray-template.yaml new file mode 100644 index 000000000000..02c9213c4a42 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/proparray-template.yaml @@ -0,0 +1,7 @@ +Modules: + Foo: + Source: proparray-module.yaml + Properties: + Bar: + - a + - b diff --git a/tests/unit/customizations/cloudformation/modules/sub-expect.yaml b/tests/unit/customizations/cloudformation/modules/sub-expect.yaml new file mode 100644 index 000000000000..79cb95080ccc --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/sub-expect.yaml @@ -0,0 +1,21 @@ +Resources: + MyBucket1: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + X: + Fn::Sub: ${Foo} + Y: + - Fn::Sub: noparent0-${Foo} + - Fn::Sub: noparent1-${Foo} + Z: + - Fn::Sub: ${Foo} + - Ref: MyBucket2 + - ZZ: + ZZZ: + ZZZZ: + Fn::Sub: ${Foo} + MyBucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: bar diff --git a/tests/unit/customizations/cloudformation/modules/sub-module.yaml b/tests/unit/customizations/cloudformation/modules/sub-module.yaml new file mode 100644 index 000000000000..e066bb8381ff --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/sub-module.yaml @@ -0,0 +1,23 @@ +Inputs: + Name: + Type: String + SubName: + Type: String +Resources: + Bucket1: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub ${Name} + X: !Sub "${SubName}" + Y: + - !Sub noparent0-${SubName} + - !Sub noparent1-${SubName} + Z: + - !Ref SubName + - !Ref Bucket2 + - ZZ: + ZZZ: + ZZZZ: !Sub "${SubName}" + Bucket2: + Type: AWS::S3::Bucket + diff --git a/tests/unit/customizations/cloudformation/modules/sub-template.yaml b/tests/unit/customizations/cloudformation/modules/sub-template.yaml new file mode 100644 index 000000000000..41de33dfd4f8 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/sub-template.yaml @@ -0,0 +1,10 @@ +Modules: + My: + Source: ./sub-module.yaml + Properties: + Name: foo + SubName: !Sub ${Foo} + Overrides: + Bucket2: + Properties: + BucketName: bar diff --git a/tests/unit/customizations/cloudformation/modules/subnet-module.yaml b/tests/unit/customizations/cloudformation/modules/subnet-module.yaml new file mode 100644 index 000000000000..bd7cafa25f1e --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/subnet-module.yaml @@ -0,0 +1,59 @@ +Inputs: + + AZSelection: + Type: Number + + SubnetCidrBlock: + Type: String + +Resources: + + Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: !Select [!Ref AZSelection, Fn::GetAZs: !Ref "AWS::Region"] + CidrBlock: !Ref SubnetCidrBlock + MapIpOnLaunch: true + VpcId: !Ref VPC + + RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + + RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: !Ref SubnetRouteTable + SubnetId: !Ref Subnet + + DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + RouteTableId: !Ref SubnetRouteTable + DependsOn: VPCGW + + EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + + NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: !GetAtt SubnetEIP.AllocationId + SubnetId: !Ref Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + diff --git a/tests/unit/customizations/cloudformation/modules/type-expect.yaml b/tests/unit/customizations/cloudformation/modules/type-expect.yaml new file mode 100644 index 000000000000..1010e68cf134 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/type-expect.yaml @@ -0,0 +1,16 @@ +Resources: + OtherResource: + Type: AWS::S3::Bucket + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + OverrideMe: def + SubName: foo + ContentBucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: + Fn::GetAtt: + - ContentBucket + - Something diff --git a/tests/unit/customizations/cloudformation/modules/type-module.yaml b/tests/unit/customizations/cloudformation/modules/type-module.yaml new file mode 100644 index 000000000000..ddf7d65217ec --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/type-module.yaml @@ -0,0 +1,14 @@ +Inputs: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name + OverrideMe: abc + SubName: !Sub ${Name} + Bucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: !GetAtt Bucket.Something diff --git a/tests/unit/customizations/cloudformation/modules/type-template.yaml b/tests/unit/customizations/cloudformation/modules/type-template.yaml new file mode 100644 index 000000000000..2af5a5069bf9 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/type-template.yaml @@ -0,0 +1,12 @@ +Modules: + Content: + Source: ./type-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +Resources: + OtherResource: + Type: AWS::S3::Bucket diff --git a/tests/unit/customizations/cloudformation/modules/url-template.yaml b/tests/unit/customizations/cloudformation/modules/url-template.yaml new file mode 100644 index 000000000000..febd0aee8ca0 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/url-template.yaml @@ -0,0 +1,14 @@ +Constants: + ModuleSource: https://raw.githubusercontent.com/aws/aws-cli/2f0143bab567386b930322b2b0e845740f7adfd0/tests/unit/customizations/cloudformation/modules +Modules: + Content: + Source: !Sub ${Constant::ModuleSource}/basic-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +Resources: + OtherResource: + Type: AWS::S3::Bucket diff --git a/tests/unit/customizations/cloudformation/modules/vpc-expect.yaml b/tests/unit/customizations/cloudformation/modules/vpc-expect.yaml new file mode 100644 index 000000000000..2fe748a175de --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/vpc-expect.yaml @@ -0,0 +1,236 @@ +Resources: + NetworkPrivateSubnet0DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPrivateSubnet0EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPrivateSubnet0NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPrivateSubnet0SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPrivateSubnet0Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPrivateSubnet0RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet0RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPrivateSubnet0Subnet + NetworkPrivateSubnet0Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '0' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.0.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet1DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPrivateSubnet1EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPrivateSubnet1NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPrivateSubnet1SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPrivateSubnet1Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPrivateSubnet1RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet1RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPrivateSubnet1Subnet + NetworkPrivateSubnet1Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '1' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.64.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet0DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPublicSubnet0EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPublicSubnet0NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPublicSubnet0SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPublicSubnet0Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPublicSubnet0RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet0RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPublicSubnet0Subnet + NetworkPublicSubnet0Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '0' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.128.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet1DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPublicSubnet1EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPublicSubnet1NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPublicSubnet1SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPublicSubnet1Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPublicSubnet1RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet1RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPublicSubnet1Subnet + NetworkPublicSubnet1Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '1' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.192.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkVPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/16 + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default diff --git a/tests/unit/customizations/cloudformation/modules/vpc-module.yaml b/tests/unit/customizations/cloudformation/modules/vpc-module.yaml new file mode 100644 index 000000000000..0e2be0039295 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/vpc-module.yaml @@ -0,0 +1,37 @@ +Inputs: + + CidrBlock: + Type: String + + PrivateCidrBlocks: + Type: CommaDelimitedList + + PublicCidrBlocks: + Type: CommaDelimitedList + +Modules: + PublicSubnet: + Map: !Ref PublicCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + + PrivateSubnet: + Map: !Ref PrivateCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + +Resources: + + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: !Ref CidrBlock + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default + + diff --git a/tests/unit/customizations/cloudformation/modules/vpc-template.yaml b/tests/unit/customizations/cloudformation/modules/vpc-template.yaml new file mode 100644 index 000000000000..e8ab1ce9a296 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/vpc-template.yaml @@ -0,0 +1,7 @@ +Modules: + Network: + Source: ./vpc-module.yaml + Properties: + CidrBlock: 10.0.0.0/16 + PrivateCidrBlocks: 10.0.0.0/18,10.0.64.0/18 + PublicCidrBlocks: 10.0.128.0/18,10.0.192.0/18 diff --git a/tests/unit/customizations/cloudformation/test_modules.py b/tests/unit/customizations/cloudformation/test_modules.py new file mode 100644 index 000000000000..95e504ee0735 --- /dev/null +++ b/tests/unit/customizations/cloudformation/test_modules.py @@ -0,0 +1,163 @@ +"Tests for module support in the package command" + +# pylint: disable=fixme + +from awscli.testutils import unittest +from awscli.customizations.cloudformation import yamlhelper +from awscli.customizations.cloudformation import modules +from awscli.customizations.cloudformation.parse_sub import SubWord, WordType +from awscli.customizations.cloudformation.parse_sub import parse_sub +from awscli.customizations.cloudformation.module_visitor import Visitor +from awscli.customizations.cloudformation.module_constants import ( + process_constants, + replace_constants, +) +from awscli.customizations.cloudformation.module_read import read_source + +MODULES = "Modules" +RESOURCES = "Resources" +TYPE = "Type" +LOCAL_MODULE = "LocalModule" + + +class TestPackageModules(unittest.TestCase): + "Module tests" + + def setUp(self): + "Initialize the tests" + + def test_parse_sub(self): + "Test the parse_sub function" + cases = { + "ABC": [SubWord(WordType.STR, "ABC")], + "ABC-${XYZ}-123": [ + SubWord(WordType.STR, "ABC-"), + SubWord(WordType.REF, "XYZ"), + SubWord(WordType.STR, "-123"), + ], + "ABC-${!Literal}-1": [SubWord(WordType.STR, "ABC-${Literal}-1")], + "${ABC}": [SubWord(WordType.REF, "ABC")], + "${ABC.XYZ}": [SubWord(WordType.GETATT, "ABC.XYZ")], + "ABC${AWS::AccountId}XYZ": [ + SubWord(WordType.STR, "ABC"), + SubWord(WordType.AWS, "AccountId"), + SubWord(WordType.STR, "XYZ"), + ], + "BAZ${ABC$XYZ}FOO$BAR": [ + SubWord(WordType.STR, "BAZ"), + SubWord(WordType.REF, "ABC$XYZ"), + SubWord(WordType.STR, "FOO$BAR"), + ], + "${ ABC }": [SubWord(WordType.REF, "ABC")], + "${ ABC }": [SubWord(WordType.REF, "ABC")], + " ABC ": [SubWord(WordType.STR, " ABC ")], + } + + for sub, expect in cases.items(): + words = parse_sub(sub, False) + self.assertEqual( + len(expect), + len(words), + f'"{sub}": words len is {len(words)}, expected {len(expect)}', + ) + for i, w in enumerate(expect): + self.assertEqual( + words[i].t, w.t, f'"{sub}": got {words[i]}, expected {w}' + ) + self.assertEqual( + words[i].w, w.w, f'"{sub}": got {words[i]}, expected {w}' + ) + + # Invalid strings should fail + sub = "${AAA" + with self.assertRaises(Exception, msg=f'"{sub}": should have failed'): + parse_sub(sub, False) + + def test_merge_props(self): + "Test the merge_props function" + + original = {"b": "c", "d": {"e": "f", "i": [1, 2, 3]}} + overrides = {"b": "cc", "d": {"e": "ff", "g": "h", "i": [4, 5]}} + expect = {"b": "cc", "d": {"e": "ff", "g": "h", "i": [1, 2, 3, 4, 5]}} + merged = modules.merge_props(original, overrides) + self.assertEqual(merged, expect) + + def test_main(self): + "Run tests on sample templates that include local modules" + + # pylint: disable=invalid-name + self.maxDiff = None + + # The tests are in the modules directory. + # Each test has 3 files: + # test-template.yaml, test-module.yaml, and test-expect.yaml + tests = [ + "basic", + "type", + "sub", + "modinmod", + "output", + "policy", + "vpc", + "map", + "conditional", + "cond-intrinsics", + "example", + "getatt", + "constant", + "proparray", + ] + for test in tests: + base = "unit/customizations/cloudformation/modules" + t, _ = modules.read_source(f"{base}/{test}-template.yaml") + td = yamlhelper.yaml_parse(t) + e, _ = modules.read_source(f"{base}/{test}-expect.yaml") + + constants = process_constants(td) + if constants is not None: + replace_constants(constants, td) + + # Modules section + td = modules.process_module_section(td, base, t, None, True, True) + + processed = yamlhelper.yaml_dump(td) + self.assertEqual(e, processed, f"{test} failed") + + def test_visitor(self): + "Test module_visitor" + + template_dict = {} + resources = {} + template_dict["Resources"] = resources + resources["Bucket"] = { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketName": "foo", + "TestList": [ + "Item0", + {"BucketName": "foo"}, + {"Item2": {"BucketName": "foo"}}, + ], + }, + } + v = Visitor(template_dict, None, "") + + def vf(v): + if isinstance(v.d, str) and v.p is not None: + if v.k == "BucketName": + v.p[v.k] = "bar" + + v.visit(vf) + self.assertEqual( + resources["Bucket"]["Properties"]["BucketName"], "bar" + ) + test_list = resources["Bucket"]["Properties"]["TestList"] + self.assertEqual(test_list[1]["BucketName"], "bar") + self.assertEqual(test_list[2]["Item2"]["BucketName"], "bar") + + def test_read_source(self): + "Test reading a source file" + + base = "unit/customizations/cloudformation/modules" + _, lines = read_source(f"{base}/example-module.yaml") + self.assertEqual(lines["Bucket"], 5)