Recursive modules

awscli/customizations/cloudformation/artifact_exporter.py

@@ -34,8 +34,6 @@
 
 MODULES = "Modules"
 RESOURCES = "Resources"
-TYPE = "Type"
-LOCAL_MODULE = "LocalModule"
 
 def is_path_value_valid(path):
     return isinstance(path, str)
@@ -659,18 +657,9 @@ def export(self):
 
         # Process modules
         try:
-            if MODULES in self.template_dict:
-                # Process each Module node separately
-                for module_name, module_config in self.template_dict[MODULES].items():
-                    module_config[modules.NAME] = module_name
-                    # Fix the source path
-                    relative_path = module_config[modules.SOURCE]
-                    module_config[modules.SOURCE] = f"{self.template_dir}/{relative_path}"
-                    module = modules.Module(self.template_dict, module_config)
-                    self.template_dict = module.process()
-
-                # Remove the Modules section from the template
-                del self.template_dict[MODULES]
+            self.template_dict = modules.process_module_section(
+                    self.template_dict, 
+                    self.template_dir)
         except Exception as e:
             traceback.print_exc()
             msg=f"Failed to process Modules section: {e}"
@@ -683,22 +672,9 @@ def export(self):
 
         # Process modules that are specified as Resources, not in Modules
         try:
-            for k, v in self.template_dict[RESOURCES].copy().items():
-                if TYPE in v and v[TYPE] == LOCAL_MODULE:
-                    module_config = {}
-                    module_config[modules.NAME] = k
-                    if modules.SOURCE not in v:
-                        msg = f"{k} missing {modules.SOURCE}"
-                        raise exceptions.InvalidModulePathError(msg=msg)
-                    relative_path = v[modules.SOURCE]
-                    module_config[modules.SOURCE] = f"{self.template_dir}/{relative_path}"
-                    if modules.PROPERTIES in v:
-                        module_config[modules.PROPERTIES] = v[modules.PROPERTIES]
-                    if modules.OVERRIDES in v:
-                        module_config[modules.OVERRIDES] = v[modules.OVERRIDES]
-                    module = modules.Module(self.template_dict, module_config)
-                    self.template_dict = module.process()
-                    del self.template_dict[RESOURCES][k]
+            self.template_dict = modules.process_resources_section(
+                    self.template_dict, 
+                    self.template_dir)
         except Exception as e:
             traceback.print_exc()
             msg=f"Failed to process modules in Resources: {e}"

awscli/customizations/cloudformation/modules.py

@@ -16,6 +16,7 @@
 # pylint: disable=fixme
 
 import os
+import traceback
 from collections import OrderedDict
 
 from awscli.customizations.cloudformation import exceptions
@@ -41,6 +42,51 @@
 SUB = "Fn::Sub"
 GETATT = "Fn::GetAtt"
 PARAMETERS = "Parameters"
+MODULES = "Modules"
+TYPE = "Type"
+LOCAL_MODULE = "LocalModule"
+
+
+def process_module_section(template, base_path):
+    "Recursively process the Modules section of a template"
+    if MODULES in template:
+
+        # Process each Module node separately
+        for module_name, module_config in template[MODULES].items():
+            module_config[NAME] = module_name
+            # Fix the source path
+            relative_path = module_config[SOURCE]
+            module_config[SOURCE] = os.path.join(base_path, relative_path)
+            module_config[SOURCE] = os.path.normpath(module_config[SOURCE])
+            module = Module(template, module_config)
+            template = module.process()
+
+        # Remove the Modules section from the template
+        del template[MODULES]
+
+    return template
+
+
+def process_resources_section(template, base_path):
+    "Recursively process the Resources section of the template"
+    for k, v in template[RESOURCES].copy().items():
+        if TYPE in v and v[TYPE] == LOCAL_MODULE:
+            module_config = {}
+            module_config[NAME] = k
+            if SOURCE not in v:
+                msg = f"{k} missing {SOURCE}"
+                raise exceptions.InvalidModulePathError(msg=msg)
+            relative_path = v[SOURCE]
+            module_config[SOURCE] = os.path.join(base_path, relative_path)
+            module_config[SOURCE] = os.path.normpath(module_config[SOURCE])
+            if PROPERTIES in v:
+                module_config[PROPERTIES] = v[PROPERTIES]
+            if OVERRIDES in v:
+                module_config[OVERRIDES] = v[OVERRIDES]
+            module = Module(template, module_config)
+            template = module.process()
+            del template[RESOURCES][k]
+    return template
 
 
 def isdict(v):
@@ -202,14 +248,26 @@ def process(self):
 
         module_dict = yamlhelper.yaml_parse(content)
         if RESOURCES not in module_dict:
-            msg = "Modules must have a Resources section"
-            raise exceptions.InvalidModuleError(msg=msg)
-        self.resources = module_dict[RESOURCES]
+            # The module may only have sub modules in the Modules section
+            self.resources = {}
+        else:
+            self.resources = module_dict[RESOURCES]
 
         if PARAMETERS in module_dict:
             self.params = module_dict[PARAMETERS]
 
-        # TODO: Recurse on nested modules
+        # Recurse on nested modules
+        base_path = os.path.dirname(self.source)
+        section = ""
+        try:
+            section = MODULES
+            module_dict = process_module_section(module_dict, base_path)
+            section = RESOURCES
+            module_dict = process_resources_section(module_dict, base_path)
+        except Exception as e:
+            traceback.print_exc()
+            msg = f"Failed to process {section} section: {e}"
+            raise exceptions.InvalidModuleError(msg=msg)
 
         self.validate_overrides()
 
@@ -427,13 +485,14 @@ def resolve_sub(self, v, d, n):
                 sub += "${AWS::" + word.w + "}"
                 need_sub = True
             elif word.t == WordType.REF:
-                resolved = f"${word.w}"
+                resolved = "${" + word.w + "}"
+                need_sub = True
                 found = self.find_ref(word.w)
                 if found is not None:
                     if isinstance(found, str):
+                        need_sub = False
                         resolved = found
                     else:
-                        need_sub = True
                         if REF in found:
                             resolved = "${" + found[REF] + "}"
                         elif SUB in found:

tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml

@@ -0,0 +1,14 @@
+Parameters:
+  ParentVal:
+    Type: String
+  AppName:
+    Type: String
+Resources:
+  MySubBucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: mod-in-mod-bucket
+      XName:
+        Fn::Sub: ${ParentVal}-abc
+      YName:
+        Fn::Sub: ${AppName}-xyz

tests/unit/customizations/cloudformation/modules/modinmod-module.yaml

@@ -0,0 +1,11 @@
+Parameters:
+  AppName:
+    Type: String
+Resources:
+  Sub:
+    Type: LocalModule 
+    Source: "./modinmod-submodule.yaml"
+    Properties:
+      X: !Sub ${ParentVal}-abc
+      Y: !Sub ${AppName}-xyz
+

tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml

@@ -0,0 +1,12 @@
+Parameters:
+  X:
+    Type: String
+  Y:
+    Type: String
+Resources:
+  Bucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: mod-in-mod-bucket
+      XName: !Ref X
+      YName: !Ref Y

tests/unit/customizations/cloudformation/modules/modinmod-template.yaml

@@ -0,0 +1,11 @@
+Parameters:
+  ParentVal:
+    Type: String
+  AppName:
+    Type: String
+Resources:
+  My:
+    Type: LocalModule 
+    Source: ./modinmod-module.yaml
+    Properties:
+      AppName: !Ref AppName

tests/unit/customizations/cloudformation/test_modules.py

@@ -82,7 +82,7 @@ def test_main(self):
         # The tests are in the modules directory.
         # Each test has 3 files:
         # test-template.yaml, test-module.yaml, and test-expect.yaml
-        tests = ["basic", "type", "sub"]
+        tests = ["basic", "type", "sub", "modinmod"]
         for test in tests:
             base = "unit/customizations/cloudformation/modules"
             t = modules.read_source(f"{base}/{test}-template.yaml")