From 439a222ff27997e0c87e22ccbf0d657da877d9a5 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Wed, 12 Apr 2023 14:00:40 -0700 Subject: [PATCH 01/11] Watch for env file changes --- cli/azd/pkg/environment/environment.go | 147 +++++++++++++------- cli/azd/pkg/environment/environment_test.go | 30 ++-- cli/azd/pkg/ext/hooks_runner.go | 5 - go.mod | 1 + go.sum | 3 + 5 files changed, 119 insertions(+), 67 deletions(-) diff --git a/cli/azd/pkg/environment/environment.go b/cli/azd/pkg/environment/environment.go index 95d64b36916..46eead6fc33 100644 --- a/cli/azd/pkg/environment/environment.go +++ b/cli/azd/pkg/environment/environment.go @@ -6,6 +6,7 @@ package environment import ( "errors" "fmt" + "log" "os" "path/filepath" "regexp" @@ -16,6 +17,7 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/config" "github.com/azure/azure-dev/cli/azd/pkg/environment/azdcontext" "github.com/azure/azure-dev/cli/azd/pkg/osutil" + "github.com/fsnotify/fsnotify" "github.com/joho/godotenv" ) @@ -55,6 +57,8 @@ type Environment struct { // will not be persisted when `Save` is called. This allows the zero value to be used // for testing. Root string + + watcher *fsnotify.Watcher } type EnvironmentResolver func() (*Environment, error) @@ -79,7 +83,11 @@ func FromRoot(root string) (*Environment, error) { Root: root, } - if err := env.Reload(); err != nil { + if err := env.watchForChanges(); err != nil { + return EmptyWithRoot(root), fmt.Errorf("failed watching environment for changes, %w", err) + } + + if err := env.reload(); err != nil { return EmptyWithRoot(root), err } @@ -132,40 +140,6 @@ func (e *Environment) Getenv(key string) string { return os.Getenv(key) } -// Reloads environment variables and configuration -func (e *Environment) Reload() error { - // Reload env values - envPath := filepath.Join(e.Root, azdcontext.DotEnvFileName) - if envMap, err := godotenv.Read(envPath); errors.Is(err, os.ErrNotExist) { - e.Values = make(map[string]string) - } else if err != nil { - return fmt.Errorf("loading .env: %w", err) - } else { - e.Values = envMap - } - - // Reload env config - cfgPath := filepath.Join(e.Root, azdcontext.ConfigFileName) - cfgMgr := config.NewManager() - if cfg, err := cfgMgr.Load(cfgPath); errors.Is(err, os.ErrNotExist) { - e.Config = config.NewConfig(nil) - } else if err != nil { - return fmt.Errorf("loading config: %w", err) - } else { - e.Config = cfg - } - - if e.GetEnvName() != "" { - telemetry.SetUsageAttributes(fields.StringHashed(fields.EnvNameKey, e.GetEnvName())) - } - - if e.GetSubscriptionId() != "" { - telemetry.SetGlobalAttributes(fields.SubscriptionIdKey.String(e.GetSubscriptionId())) - } - - return nil -} - // If `Root` is set, Save writes the current contents of the environment to // the given directory, creating it and any intermediate directories as needed. func (e *Environment) Save() error { @@ -179,17 +153,6 @@ func (e *Environment) Save() error { return fmt.Errorf("saving config: %w", err) } - // Cache current values & reload to get any new env vars - currentValues := e.Values - if err := e.Reload(); err != nil { - return fmt.Errorf("failed reloading env vars, %w", err) - } - - // Overlay current values before saving - for key, value := range currentValues { - e.Values[key] = value - } - err := os.MkdirAll(e.Root, osutil.PermissionDirectory) if err != nil { return fmt.Errorf("failed to create a directory: %w", err) @@ -201,6 +164,14 @@ func (e *Environment) Save() error { } telemetry.SetUsageAttributes(fields.StringHashed(fields.EnvNameKey, e.GetEnvName())) + + // If this was an empty (aka new) environment initialize the watcher + if e.watcher == nil { + if err := e.watchForChanges(); err != nil { + return err + } + } + return nil } @@ -232,10 +203,6 @@ func (e *Environment) SetLocation(location string) { e.Values[LocationEnvVarName] = location } -func normalize(key string) string { - return strings.ReplaceAll(strings.ToUpper(key), "-", "_") -} - // Returns the value of a service-namespaced property in the environment. func (e *Environment) GetServiceProperty(serviceName string, propertyName string) string { return e.Values[fmt.Sprintf("SERVICE_%s_%s", normalize(serviceName), propertyName)] @@ -256,3 +223,83 @@ func (e *Environment) Environ() []string { return envVars } + +func (e *Environment) watchForChanges() error { + if e.watcher != nil { + return nil + } + + envFilePath := filepath.Join(e.Root, azdcontext.DotEnvFileName) + + watcher, err := fsnotify.NewWatcher() + if err != nil { + return fmt.Errorf("failed creating file watcher, %w", err) + } + + e.watcher = watcher + + if err := e.watcher.Add(envFilePath); err != nil { + return fmt.Errorf("failed watching environment file %s, %e", envFilePath, err) + } + + // Watch for changes to the environment file + go func() { + for { + select { + case event, ok := <-e.watcher.Events: + if !ok { + return + } + if event.Op&fsnotify.Write == fsnotify.Write { + log.Println("Environment file changed, reloading") + + if err := e.reload(); err != nil { + panic("error reloading environment") + } + } + case err := <-e.watcher.Errors: + fmt.Println("Error watching environment file:", err) + } + } + }() + + return nil +} + +// Reloads environment variables and configuration +func (e *Environment) reload() error { + // Reload env values + envPath := filepath.Join(e.Root, azdcontext.DotEnvFileName) + if envMap, err := godotenv.Read(envPath); errors.Is(err, os.ErrNotExist) { + e.Values = make(map[string]string) + } else if err != nil { + return fmt.Errorf("loading .env: %w", err) + } else { + e.Values = envMap + } + + // Reload env config + cfgPath := filepath.Join(e.Root, azdcontext.ConfigFileName) + cfgMgr := config.NewManager() + if cfg, err := cfgMgr.Load(cfgPath); errors.Is(err, os.ErrNotExist) { + e.Config = config.NewConfig(nil) + } else if err != nil { + return fmt.Errorf("loading config: %w", err) + } else { + e.Config = cfg + } + + if e.GetEnvName() != "" { + telemetry.SetUsageAttributes(fields.StringHashed(fields.EnvNameKey, e.GetEnvName())) + } + + if e.GetSubscriptionId() != "" { + telemetry.SetGlobalAttributes(fields.SubscriptionIdKey.String(e.GetSubscriptionId())) + } + + return nil +} + +func normalize(key string) string { + return strings.ReplaceAll(strings.ToUpper(key), "-", "_") +} diff --git a/cli/azd/pkg/environment/environment_test.go b/cli/azd/pkg/environment/environment_test.go index 5e1fe3ae5cb..8dc8293152f 100644 --- a/cli/azd/pkg/environment/environment_test.go +++ b/cli/azd/pkg/environment/environment_test.go @@ -114,26 +114,32 @@ func Test_SaveAndReload(t *testing.T) { tempDir := t.TempDir() ostest.Chdir(t, tempDir) - env, err := FromRoot(tempDir) + env := EmptyWithRoot(tempDir) require.NotNil(t, env) - require.NoError(t, err) env.SetLocation("eastus2") env.SetSubscriptionId("SUBSCRIPTION_ID") - err = env.Save() + err := env.Save() require.NoError(t, err) - // Simulate another process updating the .env file - envPath := filepath.Join(tempDir, azdcontext.DotEnvFileName) - envMap, err := godotenv.Read(envPath) - require.NotNil(t, envMap) - require.NoError(t, err) + asyncDone := make(chan bool) - // This entry does not exist in the current env state but is added as part of the reload process - envMap["SERVICE_API_ENDPOINT_URL"] = "http://api.example.com" - err = godotenv.Write(envMap, envPath) - require.NoError(t, err) + go func() { + // Simulate another process updating the .env file + envPath := filepath.Join(tempDir, azdcontext.DotEnvFileName) + envMap, err := godotenv.Read(envPath) + require.NotNil(t, envMap) + require.NoError(t, err) + + // This entry does not exist in the current env state but is added as part of the reload process + envMap["SERVICE_API_ENDPOINT_URL"] = "http://api.example.com" + err = godotenv.Write(envMap, envPath) + require.NoError(t, err) + asyncDone <- true + }() + + <-asyncDone // Set a new property in the env env.SetServiceProperty("web", "ENDPOINT_URL", "http://web.example.com") diff --git a/cli/azd/pkg/ext/hooks_runner.go b/cli/azd/pkg/ext/hooks_runner.go index 1d0702d065f..2dfb0136751 100644 --- a/cli/azd/pkg/ext/hooks_runner.go +++ b/cli/azd/pkg/ext/hooks_runner.go @@ -83,11 +83,6 @@ func (h *HooksRunner) RunHooks(ctx context.Context, hookType HookType, commands return fmt.Errorf("failed running scripts for hooks '%s', %w", strings.Join(commands, ","), err) } - // Reload env vars before execution to enable support for hooks to generate new env vars between commands - if err := h.env.Reload(); err != nil { - return fmt.Errorf("failed reloading env values, %w", err) - } - for _, hookConfig := range hooks { err := h.execHook(ctx, hookConfig) if err != nil { diff --git a/go.mod b/go.mod index d239847c633..7e2539d9529 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/bradleyjkemp/cupaloy/v2 v2.8.0 github.com/drone/envsubst v1.0.3 github.com/fatih/color v1.13.0 + github.com/fsnotify/fsnotify v1.6.0 github.com/gofrs/flock v0.8.1 github.com/golobby/container/v3 v3.3.1 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 2dcf74bbfe1..653714275c3 100644 --- a/go.sum +++ b/go.sum @@ -162,6 +162,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -683,6 +685,7 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From 60f017d19ccd3919dbd589fcd4442e20451763f7 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Wed, 12 Apr 2023 16:50:02 -0700 Subject: [PATCH 02/11] Updates unit tests for environment --- cli/azd/pkg/environment/environment.go | 51 +++++++++++++++++---- cli/azd/pkg/environment/environment_test.go | 31 +++++++------ 2 files changed, 58 insertions(+), 24 deletions(-) diff --git a/cli/azd/pkg/environment/environment.go b/cli/azd/pkg/environment/environment.go index 46eead6fc33..09e0117c5c8 100644 --- a/cli/azd/pkg/environment/environment.go +++ b/cli/azd/pkg/environment/environment.go @@ -7,10 +7,13 @@ import ( "errors" "fmt" "log" + "math" "os" "path/filepath" "regexp" "strings" + "sync" + "time" "github.com/azure/azure-dev/cli/azd/internal/telemetry" "github.com/azure/azure-dev/cli/azd/internal/telemetry/fields" @@ -58,7 +61,11 @@ type Environment struct { // for testing. Root string + // File watcher is configured to detect changes to the underlying .env file and automatically reload the instance watcher *fsnotify.Watcher + + // Only used for unit testing to have a consistent way to be notified of reload activities + reloadCallback func() } type EnvironmentResolver func() (*Environment, error) @@ -239,26 +246,52 @@ func (e *Environment) watchForChanges() error { e.watcher = watcher if err := e.watcher.Add(envFilePath); err != nil { - return fmt.Errorf("failed watching environment file %s, %e", envFilePath, err) + return fmt.Errorf("failed watching environment file '%s', %w", envFilePath, err) } // Watch for changes to the environment file go func() { + var mutex sync.Mutex + var timer *time.Timer + waitFor := 100 * time.Millisecond + for { select { case event, ok := <-e.watcher.Events: - if !ok { - return + if !ok || !event.Has(fsnotify.Write) { + continue } - if event.Op&fsnotify.Write == fsnotify.Write { - log.Println("Environment file changed, reloading") - if err := e.reload(); err != nil { - panic("error reloading environment") - } + // Dedup multiple write events that happen within a short window + if timer == nil { + newTimer := time.AfterFunc(math.MaxInt64, func() { + if err := e.reload(); err != nil { + log.Printf("error reloading environment, %s\n", err.Error()) + return + } + + // This is primarily used to support unit test scenarios so we can avoid adding + // arbitrary sleeps timers into the test code. + if e.reloadCallback != nil { + e.reloadCallback() + } + + log.Println("environment reloaded") + + mutex.Lock() + timer = nil + mutex.Unlock() + }) + newTimer.Stop() + + mutex.Lock() + timer = newTimer + mutex.Unlock() } + + timer.Reset(waitFor) case err := <-e.watcher.Errors: - fmt.Println("Error watching environment file:", err) + log.Printf("error watching environment file: %s\n", err.Error()) } } }() diff --git a/cli/azd/pkg/environment/environment_test.go b/cli/azd/pkg/environment/environment_test.go index 8dc8293152f..c6633f37ce9 100644 --- a/cli/azd/pkg/environment/environment_test.go +++ b/cli/azd/pkg/environment/environment_test.go @@ -111,35 +111,36 @@ func TestFromRoot(t *testing.T) { } func Test_SaveAndReload(t *testing.T) { + reloadComplete := make(chan bool, 1) + tempDir := t.TempDir() ostest.Chdir(t, tempDir) env := EmptyWithRoot(tempDir) require.NotNil(t, env) + env.reloadCallback = func() { + reloadComplete <- true + } + env.SetLocation("eastus2") env.SetSubscriptionId("SUBSCRIPTION_ID") err := env.Save() require.NoError(t, err) - asyncDone := make(chan bool) - - go func() { - // Simulate another process updating the .env file - envPath := filepath.Join(tempDir, azdcontext.DotEnvFileName) - envMap, err := godotenv.Read(envPath) - require.NotNil(t, envMap) - require.NoError(t, err) + // Simulate another process writing to .env file + envPath := filepath.Join(tempDir, azdcontext.DotEnvFileName) + envMap, err := godotenv.Read(envPath) + require.NotNil(t, envMap) + require.NoError(t, err) - // This entry does not exist in the current env state but is added as part of the reload process - envMap["SERVICE_API_ENDPOINT_URL"] = "http://api.example.com" - err = godotenv.Write(envMap, envPath) - require.NoError(t, err) - asyncDone <- true - }() + // This entry does not exist in the current env state but is added as part of the reload process + envMap["SERVICE_API_ENDPOINT_URL"] = "http://api.example.com" + err = godotenv.Write(envMap, envPath) + require.NoError(t, err) - <-asyncDone + <-reloadComplete // Set a new property in the env env.SetServiceProperty("web", "ENDPOINT_URL", "http://web.example.com") From 9367beee0fe2d247789c97df5f1b0447a514fb09 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Wed, 12 Apr 2023 17:00:05 -0700 Subject: [PATCH 03/11] Don't setup file watcher if .env doesn't exist --- cli/azd/pkg/environment/environment.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/azd/pkg/environment/environment.go b/cli/azd/pkg/environment/environment.go index 09e0117c5c8..c7024c13493 100644 --- a/cli/azd/pkg/environment/environment.go +++ b/cli/azd/pkg/environment/environment.go @@ -236,7 +236,11 @@ func (e *Environment) watchForChanges() error { return nil } + // Don't setup the watcher if the .env file doesn't exist yet envFilePath := filepath.Join(e.Root, azdcontext.DotEnvFileName) + if _, err := os.Stat(envFilePath); errors.Is(err, os.ErrNotExist) { + return nil + } watcher, err := fsnotify.NewWatcher() if err != nil { From 8e34e18dc3195521a301915690727f5d7f7d8a7e Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Thu, 13 Apr 2023 09:29:39 -0700 Subject: [PATCH 04/11] Sync env instances --- cli/azd/cmd/container.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cli/azd/cmd/container.go b/cli/azd/cmd/container.go index 2641e20898d..135109cca12 100644 --- a/cli/azd/cmd/container.go +++ b/cli/azd/cmd/container.go @@ -167,6 +167,7 @@ func registerCommonDependencies(container *ioc.NestedContainer) { container.RegisterSingleton( func(ctx context.Context, azdContext *azdcontext.AzdContext, + lazyEnv *lazy.Lazy[*environment.Environment], envFlags envFlag, console input.Console, ) (*environment.Environment, error) { @@ -182,6 +183,10 @@ func registerCommonDependencies(container *ioc.NestedContainer) { return nil, fmt.Errorf("loading environment: %w", err) } + // Reset lazy env value after loading or creating environment + // This allows any previous lazy instances (such as hooks) to now point to the same instance + lazyEnv.SetValue(env) + return env, nil }, ) From 2cd538f730fbb26185a50b86c4ca3d4ffa391943 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Thu, 13 Apr 2023 09:56:56 -0700 Subject: [PATCH 05/11] Fixes spelling lint issues --- cli/azd/.vscode/cspell-azd-dictionary.txt | 2 ++ cli/azd/pkg/environment/environment.go | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cli/azd/.vscode/cspell-azd-dictionary.txt b/cli/azd/.vscode/cspell-azd-dictionary.txt index fa434884027..c2f7400bfd6 100644 --- a/cli/azd/.vscode/cspell-azd-dictionary.txt +++ b/cli/azd/.vscode/cspell-azd-dictionary.txt @@ -38,6 +38,7 @@ contoso csharpapp csharpapptest cupaloy +dedup deletedservices devel docf @@ -51,6 +52,7 @@ errcheck errorinfo errorlint executil +fsnotify funcapp functestapp functionapp diff --git a/cli/azd/pkg/environment/environment.go b/cli/azd/pkg/environment/environment.go index c7024c13493..156bf493e11 100644 --- a/cli/azd/pkg/environment/environment.go +++ b/cli/azd/pkg/environment/environment.go @@ -247,12 +247,12 @@ func (e *Environment) watchForChanges() error { return fmt.Errorf("failed creating file watcher, %w", err) } - e.watcher = watcher - - if err := e.watcher.Add(envFilePath); err != nil { + if err := watcher.Add(envFilePath); err != nil { return fmt.Errorf("failed watching environment file '%s', %w", envFilePath, err) } + e.watcher = watcher + // Watch for changes to the environment file go func() { var mutex sync.Mutex @@ -267,6 +267,7 @@ func (e *Environment) watchForChanges() error { } // Dedup multiple write events that happen within a short window + // A single file write can spawn multiple OS level write events if timer == nil { newTimer := time.AfterFunc(math.MaxInt64, func() { if err := e.reload(); err != nil { From 15bcecddbfc1a1033d265f8c52ecfc26d18ae205 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Fri, 14 Apr 2023 09:51:06 -0700 Subject: [PATCH 06/11] Fixes env slice validation to ignore ordering --- cli/azd/pkg/ext/hooks_runner_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/pkg/ext/hooks_runner_test.go b/cli/azd/pkg/ext/hooks_runner_test.go index 1d3afe42043..c815985bb93 100644 --- a/cli/azd/pkg/ext/hooks_runner_test.go +++ b/cli/azd/pkg/ext/hooks_runner_test.go @@ -60,7 +60,7 @@ func Test_Hooks_Execute(t *testing.T) { ranPreHook = true require.Equal(t, "scripts/precommand.sh", args.Args[0]) require.Equal(t, cwd, args.Cwd) - require.Equal(t, env.Environ(), args.Env) + require.ElementsMatch(t, env.Environ(), args.Env) require.Equal(t, false, args.Interactive) return exec.NewRunResult(0, "", ""), nil @@ -86,7 +86,7 @@ func Test_Hooks_Execute(t *testing.T) { ranPostHook = true require.Equal(t, "scripts/postcommand.sh", args.Args[0]) require.Equal(t, cwd, args.Cwd) - require.Equal(t, env.Environ(), args.Env) + require.ElementsMatch(t, env.Environ(), args.Env) require.Equal(t, false, args.Interactive) return exec.NewRunResult(0, "", ""), nil @@ -112,7 +112,7 @@ func Test_Hooks_Execute(t *testing.T) { ranPostHook = true require.Equal(t, "scripts/preinteractive.sh", args.Args[0]) require.Equal(t, cwd, args.Cwd) - require.Equal(t, env.Environ(), args.Env) + require.ElementsMatch(t, env.Environ(), args.Env) require.Equal(t, true, args.Interactive) return exec.NewRunResult(0, "", ""), nil From e575332b831993d33fdd4444cc526262d3b3daf1 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Mon, 17 Apr 2023 11:29:23 -0700 Subject: [PATCH 07/11] Addresses PR feedback --- cli/azd/pkg/azdo/service_connection.go | 2 +- cli/azd/pkg/commands/pipeline/azdo_provider.go | 2 +- cli/azd/pkg/environment/environment.go | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cli/azd/pkg/azdo/service_connection.go b/cli/azd/pkg/azdo/service_connection.go index 6101b12af09..d7dd5266609 100644 --- a/cli/azd/pkg/azdo/service_connection.go +++ b/cli/azd/pkg/azdo/service_connection.go @@ -81,7 +81,7 @@ func CreateServiceConnection( ctx context.Context, connection *azuredevops.Connection, projectId string, - azdEnvironment environment.Environment, + azdEnvironment *environment.Environment, credentials AzureServicePrincipalCredentials, console input.Console) error { diff --git a/cli/azd/pkg/commands/pipeline/azdo_provider.go b/cli/azd/pkg/commands/pipeline/azdo_provider.go index 2dd95fa6e7b..49e2f2bea8a 100644 --- a/cli/azd/pkg/commands/pipeline/azdo_provider.go +++ b/cli/azd/pkg/commands/pipeline/azdo_provider.go @@ -632,7 +632,7 @@ func (p *AzdoCiProvider) configureConnection( if err != nil { return err } - err = azdo.CreateServiceConnection(ctx, connection, details.projectId, *p.Env, *p.credentials, p.console) + err = azdo.CreateServiceConnection(ctx, connection, details.projectId, p.Env, *p.credentials, p.console) if err != nil { return err } diff --git a/cli/azd/pkg/environment/environment.go b/cli/azd/pkg/environment/environment.go index 156bf493e11..33e56763095 100644 --- a/cli/azd/pkg/environment/environment.go +++ b/cli/azd/pkg/environment/environment.go @@ -62,7 +62,8 @@ type Environment struct { Root string // File watcher is configured to detect changes to the underlying .env file and automatically reload the instance - watcher *fsnotify.Watcher + watcher *fsnotify.Watcher + watchMutex sync.Mutex // Only used for unit testing to have a consistent way to be notified of reload activities reloadCallback func() @@ -236,6 +237,9 @@ func (e *Environment) watchForChanges() error { return nil } + e.watchMutex.Lock() + defer e.watchMutex.Unlock() + // Don't setup the watcher if the .env file doesn't exist yet envFilePath := filepath.Join(e.Root, azdcontext.DotEnvFileName) if _, err := os.Stat(envFilePath); errors.Is(err, os.ErrNotExist) { @@ -262,7 +266,13 @@ func (e *Environment) watchForChanges() error { for { select { case event, ok := <-e.watcher.Events: - if !ok || !event.Has(fsnotify.Write) { + if !ok { + log.Printf("environment watcher channel was closed") + return + } + + // Ignore any non-write operations + if !event.Has(fsnotify.Write) { continue } @@ -296,7 +306,7 @@ func (e *Environment) watchForChanges() error { timer.Reset(waitFor) case err := <-e.watcher.Errors: - log.Printf("error watching environment file: %s\n", err.Error()) + log.Printf("non-terminating error while watching environment file: %s\n", err.Error()) } } }() From b47d42099819934042478156503652774a5251cf Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Mon, 17 Apr 2023 17:44:00 -0700 Subject: [PATCH 08/11] Adds additional checks for race conditions during environment reload --- cli/azd/pkg/environment/environment.go | 121 +++++++++++++------------ 1 file changed, 65 insertions(+), 56 deletions(-) diff --git a/cli/azd/pkg/environment/environment.go b/cli/azd/pkg/environment/environment.go index 33e56763095..9d5a2944a8e 100644 --- a/cli/azd/pkg/environment/environment.go +++ b/cli/azd/pkg/environment/environment.go @@ -91,7 +91,7 @@ func FromRoot(root string) (*Environment, error) { Root: root, } - if err := env.watchForChanges(); err != nil { + if err := env.initWatcher(); err != nil { return EmptyWithRoot(root), fmt.Errorf("failed watching environment for changes, %w", err) } @@ -175,7 +175,7 @@ func (e *Environment) Save() error { // If this was an empty (aka new) environment initialize the watcher if e.watcher == nil { - if err := e.watchForChanges(); err != nil { + if err := e.initWatcher(); err != nil { return err } } @@ -232,7 +232,7 @@ func (e *Environment) Environ() []string { return envVars } -func (e *Environment) watchForChanges() error { +func (e *Environment) initWatcher() error { if e.watcher != nil { return nil } @@ -257,59 +257,7 @@ func (e *Environment) watchForChanges() error { e.watcher = watcher - // Watch for changes to the environment file - go func() { - var mutex sync.Mutex - var timer *time.Timer - waitFor := 100 * time.Millisecond - - for { - select { - case event, ok := <-e.watcher.Events: - if !ok { - log.Printf("environment watcher channel was closed") - return - } - - // Ignore any non-write operations - if !event.Has(fsnotify.Write) { - continue - } - - // Dedup multiple write events that happen within a short window - // A single file write can spawn multiple OS level write events - if timer == nil { - newTimer := time.AfterFunc(math.MaxInt64, func() { - if err := e.reload(); err != nil { - log.Printf("error reloading environment, %s\n", err.Error()) - return - } - - // This is primarily used to support unit test scenarios so we can avoid adding - // arbitrary sleeps timers into the test code. - if e.reloadCallback != nil { - e.reloadCallback() - } - - log.Println("environment reloaded") - - mutex.Lock() - timer = nil - mutex.Unlock() - }) - newTimer.Stop() - - mutex.Lock() - timer = newTimer - mutex.Unlock() - } - - timer.Reset(waitFor) - case err := <-e.watcher.Errors: - log.Printf("non-terminating error while watching environment file: %s\n", err.Error()) - } - } - }() + go watchForChanges(e) return nil } @@ -351,3 +299,64 @@ func (e *Environment) reload() error { func normalize(key string) string { return strings.ReplaceAll(strings.ToUpper(key), "-", "_") } + +// Watch for changes to the environment file +func watchForChanges(env *Environment) { + var timerReadMutex sync.Mutex + var timerSetMutex sync.Mutex + var timer *time.Timer + waitFor := 100 * time.Millisecond + + for { + select { + case event, ok := <-env.watcher.Events: + if !ok { + log.Printf("environment watcher channel was closed") + return + } + + // Ignore any non-write operations + if !event.Has(fsnotify.Write) { + continue + } + + // Dedup multiple write events that happen within a short window + // A single file write can spawn multiple OS level write events + timerReadMutex.Lock() + if timer == nil { + newTimer := time.AfterFunc(math.MaxInt64, func() { + if err := env.reload(); err != nil { + log.Printf("error reloading environment, %s\n", err.Error()) + return + } + + // This is primarily used to support unit test scenarios so we can avoid adding + // arbitrary sleeps timers into the test code. + if env.reloadCallback != nil { + env.reloadCallback() + } + + log.Println("environment reloaded") + + timerSetMutex.Lock() + timer = nil + timerSetMutex.Unlock() + }) + newTimer.Stop() + + timerSetMutex.Lock() + timer = newTimer + timerSetMutex.Unlock() + } + + // Double check that the timer has not been set to nil between the time we checked above and + // when the timer `AfterFunc` callback was invoked. + if timer != nil { + timer.Reset(waitFor) + } + timerReadMutex.Unlock() + case err := <-env.watcher.Errors: + log.Printf("non-terminating error while watching environment file: %s\n", err.Error()) + } + } +} From b3ff91adf3dea2ffcae6819d465c5b6e9718648d Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Tue, 18 Apr 2023 08:48:40 -0700 Subject: [PATCH 09/11] WIP --- cli/azd/pkg/azdo/service_connection.go | 2 +- .../pkg/commands/pipeline/azdo_provider.go | 2 +- cli/azd/pkg/environment/environment.go | 204 +++++------------- cli/azd/pkg/environment/environment_test.go | 12 +- cli/azd/pkg/ext/hooks_runner.go | 8 + 5 files changed, 67 insertions(+), 161 deletions(-) diff --git a/cli/azd/pkg/azdo/service_connection.go b/cli/azd/pkg/azdo/service_connection.go index d7dd5266609..6101b12af09 100644 --- a/cli/azd/pkg/azdo/service_connection.go +++ b/cli/azd/pkg/azdo/service_connection.go @@ -81,7 +81,7 @@ func CreateServiceConnection( ctx context.Context, connection *azuredevops.Connection, projectId string, - azdEnvironment *environment.Environment, + azdEnvironment environment.Environment, credentials AzureServicePrincipalCredentials, console input.Console) error { diff --git a/cli/azd/pkg/commands/pipeline/azdo_provider.go b/cli/azd/pkg/commands/pipeline/azdo_provider.go index 49e2f2bea8a..2dd95fa6e7b 100644 --- a/cli/azd/pkg/commands/pipeline/azdo_provider.go +++ b/cli/azd/pkg/commands/pipeline/azdo_provider.go @@ -632,7 +632,7 @@ func (p *AzdoCiProvider) configureConnection( if err != nil { return err } - err = azdo.CreateServiceConnection(ctx, connection, details.projectId, p.Env, *p.credentials, p.console) + err = azdo.CreateServiceConnection(ctx, connection, details.projectId, *p.Env, *p.credentials, p.console) if err != nil { return err } diff --git a/cli/azd/pkg/environment/environment.go b/cli/azd/pkg/environment/environment.go index 9d5a2944a8e..95d64b36916 100644 --- a/cli/azd/pkg/environment/environment.go +++ b/cli/azd/pkg/environment/environment.go @@ -6,21 +6,16 @@ package environment import ( "errors" "fmt" - "log" - "math" "os" "path/filepath" "regexp" "strings" - "sync" - "time" "github.com/azure/azure-dev/cli/azd/internal/telemetry" "github.com/azure/azure-dev/cli/azd/internal/telemetry/fields" "github.com/azure/azure-dev/cli/azd/pkg/config" "github.com/azure/azure-dev/cli/azd/pkg/environment/azdcontext" "github.com/azure/azure-dev/cli/azd/pkg/osutil" - "github.com/fsnotify/fsnotify" "github.com/joho/godotenv" ) @@ -60,13 +55,6 @@ type Environment struct { // will not be persisted when `Save` is called. This allows the zero value to be used // for testing. Root string - - // File watcher is configured to detect changes to the underlying .env file and automatically reload the instance - watcher *fsnotify.Watcher - watchMutex sync.Mutex - - // Only used for unit testing to have a consistent way to be notified of reload activities - reloadCallback func() } type EnvironmentResolver func() (*Environment, error) @@ -91,11 +79,7 @@ func FromRoot(root string) (*Environment, error) { Root: root, } - if err := env.initWatcher(); err != nil { - return EmptyWithRoot(root), fmt.Errorf("failed watching environment for changes, %w", err) - } - - if err := env.reload(); err != nil { + if err := env.Reload(); err != nil { return EmptyWithRoot(root), err } @@ -148,6 +132,40 @@ func (e *Environment) Getenv(key string) string { return os.Getenv(key) } +// Reloads environment variables and configuration +func (e *Environment) Reload() error { + // Reload env values + envPath := filepath.Join(e.Root, azdcontext.DotEnvFileName) + if envMap, err := godotenv.Read(envPath); errors.Is(err, os.ErrNotExist) { + e.Values = make(map[string]string) + } else if err != nil { + return fmt.Errorf("loading .env: %w", err) + } else { + e.Values = envMap + } + + // Reload env config + cfgPath := filepath.Join(e.Root, azdcontext.ConfigFileName) + cfgMgr := config.NewManager() + if cfg, err := cfgMgr.Load(cfgPath); errors.Is(err, os.ErrNotExist) { + e.Config = config.NewConfig(nil) + } else if err != nil { + return fmt.Errorf("loading config: %w", err) + } else { + e.Config = cfg + } + + if e.GetEnvName() != "" { + telemetry.SetUsageAttributes(fields.StringHashed(fields.EnvNameKey, e.GetEnvName())) + } + + if e.GetSubscriptionId() != "" { + telemetry.SetGlobalAttributes(fields.SubscriptionIdKey.String(e.GetSubscriptionId())) + } + + return nil +} + // If `Root` is set, Save writes the current contents of the environment to // the given directory, creating it and any intermediate directories as needed. func (e *Environment) Save() error { @@ -161,6 +179,17 @@ func (e *Environment) Save() error { return fmt.Errorf("saving config: %w", err) } + // Cache current values & reload to get any new env vars + currentValues := e.Values + if err := e.Reload(); err != nil { + return fmt.Errorf("failed reloading env vars, %w", err) + } + + // Overlay current values before saving + for key, value := range currentValues { + e.Values[key] = value + } + err := os.MkdirAll(e.Root, osutil.PermissionDirectory) if err != nil { return fmt.Errorf("failed to create a directory: %w", err) @@ -172,14 +201,6 @@ func (e *Environment) Save() error { } telemetry.SetUsageAttributes(fields.StringHashed(fields.EnvNameKey, e.GetEnvName())) - - // If this was an empty (aka new) environment initialize the watcher - if e.watcher == nil { - if err := e.initWatcher(); err != nil { - return err - } - } - return nil } @@ -211,6 +232,10 @@ func (e *Environment) SetLocation(location string) { e.Values[LocationEnvVarName] = location } +func normalize(key string) string { + return strings.ReplaceAll(strings.ToUpper(key), "-", "_") +} + // Returns the value of a service-namespaced property in the environment. func (e *Environment) GetServiceProperty(serviceName string, propertyName string) string { return e.Values[fmt.Sprintf("SERVICE_%s_%s", normalize(serviceName), propertyName)] @@ -231,132 +256,3 @@ func (e *Environment) Environ() []string { return envVars } - -func (e *Environment) initWatcher() error { - if e.watcher != nil { - return nil - } - - e.watchMutex.Lock() - defer e.watchMutex.Unlock() - - // Don't setup the watcher if the .env file doesn't exist yet - envFilePath := filepath.Join(e.Root, azdcontext.DotEnvFileName) - if _, err := os.Stat(envFilePath); errors.Is(err, os.ErrNotExist) { - return nil - } - - watcher, err := fsnotify.NewWatcher() - if err != nil { - return fmt.Errorf("failed creating file watcher, %w", err) - } - - if err := watcher.Add(envFilePath); err != nil { - return fmt.Errorf("failed watching environment file '%s', %w", envFilePath, err) - } - - e.watcher = watcher - - go watchForChanges(e) - - return nil -} - -// Reloads environment variables and configuration -func (e *Environment) reload() error { - // Reload env values - envPath := filepath.Join(e.Root, azdcontext.DotEnvFileName) - if envMap, err := godotenv.Read(envPath); errors.Is(err, os.ErrNotExist) { - e.Values = make(map[string]string) - } else if err != nil { - return fmt.Errorf("loading .env: %w", err) - } else { - e.Values = envMap - } - - // Reload env config - cfgPath := filepath.Join(e.Root, azdcontext.ConfigFileName) - cfgMgr := config.NewManager() - if cfg, err := cfgMgr.Load(cfgPath); errors.Is(err, os.ErrNotExist) { - e.Config = config.NewConfig(nil) - } else if err != nil { - return fmt.Errorf("loading config: %w", err) - } else { - e.Config = cfg - } - - if e.GetEnvName() != "" { - telemetry.SetUsageAttributes(fields.StringHashed(fields.EnvNameKey, e.GetEnvName())) - } - - if e.GetSubscriptionId() != "" { - telemetry.SetGlobalAttributes(fields.SubscriptionIdKey.String(e.GetSubscriptionId())) - } - - return nil -} - -func normalize(key string) string { - return strings.ReplaceAll(strings.ToUpper(key), "-", "_") -} - -// Watch for changes to the environment file -func watchForChanges(env *Environment) { - var timerReadMutex sync.Mutex - var timerSetMutex sync.Mutex - var timer *time.Timer - waitFor := 100 * time.Millisecond - - for { - select { - case event, ok := <-env.watcher.Events: - if !ok { - log.Printf("environment watcher channel was closed") - return - } - - // Ignore any non-write operations - if !event.Has(fsnotify.Write) { - continue - } - - // Dedup multiple write events that happen within a short window - // A single file write can spawn multiple OS level write events - timerReadMutex.Lock() - if timer == nil { - newTimer := time.AfterFunc(math.MaxInt64, func() { - if err := env.reload(); err != nil { - log.Printf("error reloading environment, %s\n", err.Error()) - return - } - - // This is primarily used to support unit test scenarios so we can avoid adding - // arbitrary sleeps timers into the test code. - if env.reloadCallback != nil { - env.reloadCallback() - } - - log.Println("environment reloaded") - - timerSetMutex.Lock() - timer = nil - timerSetMutex.Unlock() - }) - newTimer.Stop() - - timerSetMutex.Lock() - timer = newTimer - timerSetMutex.Unlock() - } - - // Double check that the timer has not been set to nil between the time we checked above and - // when the timer `AfterFunc` callback was invoked. - if timer != nil { - timer.Reset(waitFor) - } - timerReadMutex.Unlock() - case err := <-env.watcher.Errors: - log.Printf("non-terminating error while watching environment file: %s\n", err.Error()) - } - } -} diff --git a/cli/azd/pkg/environment/environment_test.go b/cli/azd/pkg/environment/environment_test.go index c6633f37ce9..50d1d41bf02 100644 --- a/cli/azd/pkg/environment/environment_test.go +++ b/cli/azd/pkg/environment/environment_test.go @@ -111,7 +111,7 @@ func TestFromRoot(t *testing.T) { } func Test_SaveAndReload(t *testing.T) { - reloadComplete := make(chan bool, 1) + //reloadComplete := make(chan bool, 1) tempDir := t.TempDir() ostest.Chdir(t, tempDir) @@ -119,9 +119,9 @@ func Test_SaveAndReload(t *testing.T) { env := EmptyWithRoot(tempDir) require.NotNil(t, env) - env.reloadCallback = func() { - reloadComplete <- true - } + // env.reloadCallback = func() { + // reloadComplete <- true + // } env.SetLocation("eastus2") env.SetSubscriptionId("SUBSCRIPTION_ID") @@ -140,7 +140,9 @@ func Test_SaveAndReload(t *testing.T) { err = godotenv.Write(envMap, envPath) require.NoError(t, err) - <-reloadComplete + err = env.Reload() + require.NoError(t, err) + //<-reloadComplete // Set a new property in the env env.SetServiceProperty("web", "ENDPOINT_URL", "http://web.example.com") diff --git a/cli/azd/pkg/ext/hooks_runner.go b/cli/azd/pkg/ext/hooks_runner.go index 2dfb0136751..fdbc1c26478 100644 --- a/cli/azd/pkg/ext/hooks_runner.go +++ b/cli/azd/pkg/ext/hooks_runner.go @@ -84,10 +84,18 @@ func (h *HooksRunner) RunHooks(ctx context.Context, hookType HookType, commands } for _, hookConfig := range hooks { + if err := h.env.Reload(); err != nil { + return fmt.Errorf("reloading environment before running hook: %w", err) + } + err := h.execHook(ctx, hookConfig) if err != nil { return err } + + if err := h.env.Reload(); err != nil { + return fmt.Errorf("reloading environment after running hook: %w", err) + } } return nil From 4e3d6670b817468b26f5c306cd937828d165ff22 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Tue, 18 Apr 2023 09:30:34 -0700 Subject: [PATCH 10/11] Updates unit tests --- cli/azd/pkg/environment/environment_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cli/azd/pkg/environment/environment_test.go b/cli/azd/pkg/environment/environment_test.go index 50d1d41bf02..bfb15bd445c 100644 --- a/cli/azd/pkg/environment/environment_test.go +++ b/cli/azd/pkg/environment/environment_test.go @@ -111,18 +111,12 @@ func TestFromRoot(t *testing.T) { } func Test_SaveAndReload(t *testing.T) { - //reloadComplete := make(chan bool, 1) - tempDir := t.TempDir() ostest.Chdir(t, tempDir) env := EmptyWithRoot(tempDir) require.NotNil(t, env) - // env.reloadCallback = func() { - // reloadComplete <- true - // } - env.SetLocation("eastus2") env.SetSubscriptionId("SUBSCRIPTION_ID") @@ -142,7 +136,6 @@ func Test_SaveAndReload(t *testing.T) { err = env.Reload() require.NoError(t, err) - //<-reloadComplete // Set a new property in the env env.SetServiceProperty("web", "ENDPOINT_URL", "http://web.example.com") From 2ad68356d3e5e888a9e374b5a6d281bf800c9ba6 Mon Sep 17 00:00:00 2001 From: Wallace Breza Date: Tue, 18 Apr 2023 09:32:12 -0700 Subject: [PATCH 11/11] Revert changes --- cli/azd/.vscode/cspell-azd-dictionary.txt | 2 -- go.mod | 1 - go.sum | 3 --- 3 files changed, 6 deletions(-) diff --git a/cli/azd/.vscode/cspell-azd-dictionary.txt b/cli/azd/.vscode/cspell-azd-dictionary.txt index c2f7400bfd6..fa434884027 100644 --- a/cli/azd/.vscode/cspell-azd-dictionary.txt +++ b/cli/azd/.vscode/cspell-azd-dictionary.txt @@ -38,7 +38,6 @@ contoso csharpapp csharpapptest cupaloy -dedup deletedservices devel docf @@ -52,7 +51,6 @@ errcheck errorinfo errorlint executil -fsnotify funcapp functestapp functionapp diff --git a/go.mod b/go.mod index 7e2539d9529..d239847c633 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/bradleyjkemp/cupaloy/v2 v2.8.0 github.com/drone/envsubst v1.0.3 github.com/fatih/color v1.13.0 - github.com/fsnotify/fsnotify v1.6.0 github.com/gofrs/flock v0.8.1 github.com/golobby/container/v3 v3.3.1 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 653714275c3..2dcf74bbfe1 100644 --- a/go.sum +++ b/go.sum @@ -162,8 +162,6 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -685,7 +683,6 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=