From 98b60321ea5d65acb2ae28b492ca71a17171227d Mon Sep 17 00:00:00 2001 From: Levko Burburas <62853952+levkohimins@users.noreply.github.com> Date: Wed, 24 Jul 2024 19:29:47 +0200 Subject: [PATCH] Fixing generation of `provider_installation` block used for provider caching (#3290) --- cli/provider_cache.go | 9 +++-- terraform/cliconfig/config.go | 3 +- terraform/cliconfig/provider_installation.go | 34 +++++++++++++++++++ .../{mirror => filesystem-mirror}/app/main.tf | 4 --- .../app/terragrunt.hcl | 0 .../network-mirror/app/main.tf | 12 +++++++ .../network-mirror/app/terragrunt.hcl | 1 + test/integration_serial_test.go | 23 ++++++------- test/integration_test.go | 3 +- 9 files changed, 66 insertions(+), 23 deletions(-) rename test/fixture-provider-cache/{mirror => filesystem-mirror}/app/main.tf (86%) rename test/fixture-provider-cache/{mirror => filesystem-mirror}/app/terragrunt.hcl (100%) create mode 100644 test/fixture-provider-cache/network-mirror/app/main.tf create mode 100644 test/fixture-provider-cache/network-mirror/app/terragrunt.hcl diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 7b2d06f95e..58b876cfd4 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -105,19 +105,17 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e for _, registryName := range opts.ProviderCacheRegistryNames { excludeAddrs = append(excludeAddrs, fmt.Sprintf("%s/*/*", registryName)) } + for _, method := range cliCfg.ProviderInstallation.Methods { switch method := method.(type) { case *cliconfig.ProviderInstallationFilesystemMirror: providerHandlers = append(providerHandlers, handlers.NewProviderFilesystemMirrorHandler(providerService, cacheProviderHTTPStatusCode, method)) - method.AppendExclude(excludeAddrs) case *cliconfig.ProviderInstallationNetworkMirror: providerHandlers = append(providerHandlers, handlers.NewProviderNetworkMirrorHandler(providerService, cacheProviderHTTPStatusCode, method)) - method.AppendExclude(excludeAddrs) case *cliconfig.ProviderInstallationDirect: providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, cacheProviderHTTPStatusCode, method)) - method.RemoveExclude(excludeAddrs) - continue } + method.AppendExclude(excludeAddrs) } providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, cacheProviderHTTPStatusCode, new(cliconfig.ProviderInstallationDirect))) @@ -181,7 +179,6 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti if err != nil { return nil, err } - if err := getproviders.UpdateLockfile(ctx, opts.WorkingDir, caches); err != nil { return nil, err } @@ -250,6 +247,8 @@ func (cache *ProviderCache) createLocalCLIConfig(opts *options.TerragruntOptions cfg.AddProviderInstallationMethods( cliconfig.NewProviderInstallationFilesystemMirror(opts.ProviderCacheDir, providerInstallationIncludes, nil), ) + } else { + cfg.ProviderInstallation = nil } cfg.AddProviderInstallationMethods( diff --git a/terraform/cliconfig/config.go b/terraform/cliconfig/config.go index 2855c99103..7c7064bd7b 100644 --- a/terraform/cliconfig/config.go +++ b/terraform/cliconfig/config.go @@ -80,7 +80,7 @@ func (cfg *Config) AddHost(name string, services map[string]string) { }) } -// AddProviderInstallationMethods adds installation methods, https://developer.hashicorp.com/terraform/cli/config/config-file#provider-installation +// AddProviderInstallationMethods merges new installation methods with the current ones, https://developer.hashicorp.com/terraform/cli/config/config-file#provider-installation // // provider_installation { // filesystem_mirror { @@ -91,6 +91,7 @@ func (cfg *Config) AddHost(name string, services map[string]string) { // exclude = ["example.com/*/*"] // } // } + func (cfg *Config) AddProviderInstallationMethods(newMethods ...ProviderInstallationMethod) { if cfg.ProviderInstallation == nil { cfg.ProviderInstallation = &ProviderInstallation{} diff --git a/terraform/cliconfig/provider_installation.go b/terraform/cliconfig/provider_installation.go index 2703cb3792..eb40405695 100644 --- a/terraform/cliconfig/provider_installation.go +++ b/terraform/cliconfig/provider_installation.go @@ -52,6 +52,7 @@ type ProviderInstallationMethod interface { fmt.Stringer AppendInclude(addrs []string) AppendExclude(addrs []string) + RemoveInclude(addrs []string) RemoveExclude(addrs []string) Merge(with ProviderInstallationMethod) bool } @@ -123,6 +124,17 @@ func (method *ProviderInstallationDirect) RemoveExclude(addrs []string) { } } +func (method *ProviderInstallationDirect) RemoveInclude(addrs []string) { + if len(addrs) == 0 || method.Include == nil { + return + } + *method.Include = util.RemoveSublistFromList(*method.Include, addrs) + + if len(*method.Include) == 0 { + method.Include = nil + } +} + func (method *ProviderInstallationDirect) String() string { b, _ := json.Marshal(method) //nolint:errcheck return string(b) @@ -197,6 +209,17 @@ func (method *ProviderInstallationFilesystemMirror) RemoveExclude(addrs []string } } +func (method *ProviderInstallationFilesystemMirror) RemoveInclude(addrs []string) { + if len(addrs) == 0 || method.Include == nil { + return + } + *method.Include = util.RemoveSublistFromList(*method.Include, addrs) + + if len(*method.Include) == 0 { + method.Include = nil + } +} + func (method *ProviderInstallationFilesystemMirror) String() string { b, _ := json.Marshal(method) //nolint:errcheck return string(b) @@ -271,6 +294,17 @@ func (method *ProviderInstallationNetworkMirror) RemoveExclude(addrs []string) { } } +func (method *ProviderInstallationNetworkMirror) RemoveInclude(addrs []string) { + if len(addrs) == 0 || method.Include == nil { + return + } + *method.Include = util.RemoveSublistFromList(*method.Include, addrs) + + if len(*method.Include) == 0 { + method.Include = nil + } +} + func (method *ProviderInstallationNetworkMirror) String() string { b, _ := json.Marshal(method) //nolint:errcheck return string(b) diff --git a/test/fixture-provider-cache/mirror/app/main.tf b/test/fixture-provider-cache/filesystem-mirror/app/main.tf similarity index 86% rename from test/fixture-provider-cache/mirror/app/main.tf rename to test/fixture-provider-cache/filesystem-mirror/app/main.tf index 3db6ee5fcd..1c4f049b95 100644 --- a/test/fixture-provider-cache/mirror/app/main.tf +++ b/test/fixture-provider-cache/filesystem-mirror/app/main.tf @@ -13,7 +13,3 @@ terraform { } } } - -provider "null" { - # Configuration options -} diff --git a/test/fixture-provider-cache/mirror/app/terragrunt.hcl b/test/fixture-provider-cache/filesystem-mirror/app/terragrunt.hcl similarity index 100% rename from test/fixture-provider-cache/mirror/app/terragrunt.hcl rename to test/fixture-provider-cache/filesystem-mirror/app/terragrunt.hcl diff --git a/test/fixture-provider-cache/network-mirror/app/main.tf b/test/fixture-provider-cache/network-mirror/app/main.tf new file mode 100644 index 0000000000..46d2418f07 --- /dev/null +++ b/test/fixture-provider-cache/network-mirror/app/main.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + aws = { + source = "example.com/hashicorp/aws" + version = "5.59.0" + } + azurerm = { + source = "example.com/hashicorp/azurerm" + version = "3.113.0" + } + } +} diff --git a/test/fixture-provider-cache/network-mirror/app/terragrunt.hcl b/test/fixture-provider-cache/network-mirror/app/terragrunt.hcl new file mode 100644 index 0000000000..bb7b160deb --- /dev/null +++ b/test/fixture-provider-cache/network-mirror/app/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index 22b6db1244..d788e0db3f 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -34,9 +34,9 @@ import ( func TestTerragruntProviderCacheWithFilesystemMirror(t *testing.T) { // In this test we use os.Setenv to set the Terraform env var TF_CLI_CONFIG_FILE. - cleanupTerraformFolder(t, TEST_FIXTURE_PROVIDER_CACHE_MIRROR) - tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PROVIDER_CACHE_MIRROR) - rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PROVIDER_CACHE_MIRROR) + cleanupTerraformFolder(t, TEST_FIXTURE_PROVIDER_CACHE_FILESYSTEM_MIRROR) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PROVIDER_CACHE_FILESYSTEM_MIRROR) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PROVIDER_CACHE_FILESYSTEM_MIRROR) appPath := filepath.Join(rootPath, "app") providersMirrorPath := filepath.Join(rootPath, "providers-mirror") @@ -86,9 +86,9 @@ func TestTerragruntProviderCacheWithFilesystemMirror(t *testing.T) { } createCLIConfig(t, cliConfigFilename, cliConfigSettings) - runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appPath)) + runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.opentofu.org --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appPath)) - expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/*/*"] exclude = ["example.com/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.terraform.io/*/*"] } "direct" { } }` + expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/*/*"] exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "direct" { } }` expectedProviderInstallation = fmt.Sprintf(strings.Join(strings.Fields(expectedProviderInstallation), " "), providersMirrorPath, providerCacheDir) terraformrcBytes, err := os.ReadFile(filepath.Join(appPath, ".terraformrc")) @@ -101,9 +101,9 @@ func TestTerragruntProviderCacheWithFilesystemMirror(t *testing.T) { func TestTerragruntProviderCacheWithNetworkMirror(t *testing.T) { // In this test we use os.Setenv to set the Terraform env var TF_CLI_CONFIG_FILE. - cleanupTerraformFolder(t, TEST_FIXTURE_PROVIDER_CACHE_MIRROR) - tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PROVIDER_CACHE_MIRROR) - rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PROVIDER_CACHE_MIRROR) + cleanupTerraformFolder(t, TEST_FIXTURE_PROVIDER_CACHE_NETWORK_MIRROR) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PROVIDER_CACHE_NETWORK_MIRROR) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PROVIDER_CACHE_NETWORK_MIRROR) appPath := filepath.Join(rootPath, "app") providersNetkworMirrorPath := filepath.Join(rootPath, "providers-network-mirror") @@ -170,16 +170,15 @@ func TestTerragruntProviderCacheWithNetworkMirror(t *testing.T) { NetworkMirrorMethods: []CLIConfigProviderInstallationNetworkMirror{ { URL: networkMirrorURL.String(), - Include: []string{"example.com/hashicorp/aws"}, + Exclude: []string{"example.com/hashicorp/azurerm"}, }, }, } createCLIConfig(t, cliConfigFilename, cliConfigSettings) - runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appPath)) - - expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/hashicorp/azurerm"] exclude = ["example.com/*/*", "registry.terraform.io/*/*"] } "network_mirror" { url = "%s" include = ["example.com/hashicorp/aws"] exclude = ["example.com/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.terraform.io/*/*"] } "direct" { } }` + runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.opentofu.org --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appPath)) + expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/hashicorp/azurerm"] exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "network_mirror" { url = "%s" exclude = ["example.com/hashicorp/azurerm", "example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "direct" { exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } }` expectedProviderInstallation = fmt.Sprintf(strings.Join(strings.Fields(expectedProviderInstallation), " "), providersFilesystemMirrorPath, networkMirrorURL.String(), providerCacheDir) terraformrcBytes, err := os.ReadFile(filepath.Join(appPath, ".terraformrc")) diff --git a/test/integration_test.go b/test/integration_test.go index 0b2dd48aed..95b999d53c 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -65,7 +65,8 @@ const ( TEST_FIXTURE_INIT_ONCE = "fixture-init-once" TEST_FIXTURE_PROVIDER_CACHE_MULTIPLE_PLATFORMS = "fixture-provider-cache/multiple-platforms" TEST_FIXTURE_PROVIDER_CACHE_DIRECT = "fixture-provider-cache/direct" - TEST_FIXTURE_PROVIDER_CACHE_MIRROR = "fixture-provider-cache/mirror" + TEST_FIXTURE_PROVIDER_CACHE_NETWORK_MIRROR = "fixture-provider-cache/network-mirror" + TEST_FIXTURE_PROVIDER_CACHE_FILESYSTEM_MIRROR = "fixture-provider-cache/filesystem-mirror" TEST_FIXTURE_DESTROY_ORDER = "fixture-destroy-order" TEST_FIXTURE_CODEGEN_PATH = "fixture-codegen" TEST_FIXTURE_GCS_PATH = "fixture-gcs/"