Ensure docker-compose runs down process if any error setting up service or agents (#3207)

Ensures that the tear down process (specially docker-compose down
command) is executed if there is any failure in the SetUp process of
Service and Agent deployers based on docker-compose. Running this
tear down process ensures that there are no stale docker resources, 
e.g. docker networks, if containers failed to start.

Author: mrodm

internal/agentdeployer/agent.go

@@ -177,6 +177,26 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI
 		ExtraArgs: []string{"--build", "-d"},
 	}
 
+	defer func() {
+		if err == nil {
+			return
+		}
+		// If running with --setup or --tear-down flags or a regular test system execution,
+		// force to tear down the service in case of setup error.
+		if d.runTestsOnly {
+			// In case of running only tests (--no-provision flag), container logs are still useful for debugging.
+			processAgentContainerLogs(context.WithoutCancel(ctx), p, compose.CommandOptions{
+				Env: opts.Env,
+			}, agentInfo.Name)
+			logger.Debug("Skipping tearing down service due to runTestsOnly flag")
+			return
+		}
+		logger.Debug("Tearing down service due to setup error")
+		// Update svcInfo with the latest info before tearing down
+		agent.agentInfo = agentInfo
+		agent.TearDown(context.WithoutCancel(ctx))
+	}()
+
 	if d.runTestsOnly || d.runTearDown {
 		logger.Debug("Skipping bringing up docker-compose project and connect container to network (non setup steps)")
 	} else {
@@ -194,9 +214,6 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI
 	// requires to be connected the service to the stack network
 	err = p.WaitForHealthy(ctx, opts)
 	if err != nil {
-		processAgentContainerLogs(ctx, p, compose.CommandOptions{
-			Env: opts.Env,
-		}, agentName)
 		return nil, fmt.Errorf("service is unhealthy: %w", err)
 	}
 

internal/cobraext/flags.go

@@ -275,7 +275,7 @@ const (
 	TearDownFlagDescription = "trigger just the tear-down phase of testing"
 
 	NoProvisionFlagName        = "no-provision"
-	NoProvisionFlagDescription = "trigger just system tests wihout setup nor teardown"
+	NoProvisionFlagDescription = "trigger just system tests without setup nor teardown"
 
 	UpdateScriptTestArchiveFlagName        = "update"
 	UpdateScriptTestArchiveFlagDescription = "update archive file if a cmp fails"

internal/servicedeployer/compose.go

@@ -118,6 +118,26 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic
 		ExtraArgs: []string{"--build", "-d"},
 	}
 
+	defer func() {
+		if err == nil {
+			return
+		}
+		// If running with --setup or --tear-down flags or a regular test system execution,
+		// force to tear down the service in case of setup error.
+		if d.runTestsOnly {
+			// In case of running only tests (--no-provision flag), container logs are still useful for debugging.
+			processServiceContainerLogs(context.WithoutCancel(ctx), p, compose.CommandOptions{
+				Env: opts.Env,
+			}, svcInfo.Name)
+			logger.Debug("Skipping tearing down service due to runTestsOnly flag")
+			return
+		}
+		logger.Debug("Tearing down service due to setup error")
+		// Update svcInfo with the latest info before tearing down
+		service.svcInfo = svcInfo
+		service.TearDown(context.WithoutCancel(ctx))
+	}()
+
 	serviceName := svcInfo.Name
 	if d.runTearDown || d.runTestsOnly {
 		logger.Debug("Skipping bringing up docker-compose custom agent project")
@@ -130,9 +150,6 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic
 
 	err = p.WaitForHealthy(ctx, opts)
 	if err != nil {
-		processServiceContainerLogs(context.WithoutCancel(ctx), p, compose.CommandOptions{
-			Env: opts.Env,
-		}, svcInfo.Name)
 		return nil, fmt.Errorf("service is unhealthy: %w", err)
 	}
 

internal/servicedeployer/custom_agent.go

@@ -156,6 +156,26 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D
 		ExtraArgs: []string{"--build", "-d"},
 	}
 
+	defer func() {
+		if err == nil {
+			return
+		}
+		// If running with --setup or --tear-down flags or a regular test system execution,
+		// force to tear down the service in case of setup error.
+		if d.runTestsOnly {
+			// In case of running only tests (--no-provision flag), container logs are still useful for debugging.
+			processServiceContainerLogs(context.WithoutCancel(ctx), p, compose.CommandOptions{
+				Env: opts.Env,
+			}, svcInfo.Name)
+			logger.Debug("Skipping tearing down service due to runTestsOnly flag")
+			return
+		}
+		logger.Debug("Tearing down service due to setup error")
+		// Update svcInfo with the latest info before tearing down
+		service.svcInfo = svcInfo
+		service.TearDown(context.WithoutCancel(ctx))
+	}()
+
 	if d.runTestsOnly || d.runTearDown {
 		logger.Debug("Skipping bringing up docker-compose project and connect container to network (non setup steps)")
 	} else {
@@ -177,9 +197,6 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D
 	// requires to be connected the service to the stack network
 	err = p.WaitForHealthy(ctx, opts)
 	if err != nil {
-		processServiceContainerLogs(ctx, p, compose.CommandOptions{
-			Env: opts.Env,
-		}, svcInfo.Name)
 		return nil, fmt.Errorf("service is unhealthy: %w", err)
 	}
 

internal/servicedeployer/terraform.go

@@ -147,6 +147,16 @@ func (tsd TerraformServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceIn
 		return nil, fmt.Errorf("can't build Terraform aliases: %w", err)
 	}
 
+	defer func() {
+		if err == nil {
+			return
+		}
+		logger.Debug("Tearing down service due to setup error")
+		// Update svcInfo with the latest info before tearing down
+		service.svcInfo = svcInfo
+		service.TearDown(context.WithoutCancel(ctx))
+	}()
+
 	// Boot up service
 	opts = compose.CommandOptions{
 		Env:       service.env,
@@ -159,9 +169,6 @@ func (tsd TerraformServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceIn
 
 	err = p.WaitForHealthy(ctx, opts)
 	if err != nil {
-		processServiceContainerLogs(ctx, p, compose.CommandOptions{
-			Env: opts.Env,
-		}, svcInfo.Name)
 		//lint:ignore ST1005 error starting with product name can be capitalized
 		return nil, fmt.Errorf("Terraform deployer is unhealthy: %w", err)
 	}