From 3c4e1083e9806477fbc237c5d98e2754e0de63f3 Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Wed, 15 Jan 2025 17:04:24 +0200 Subject: [PATCH 1/2] fix: auto_merge not working properly iwith orchestrator backend When running with Digger backend mode, PR is merged when `auto_merge` is enabled even though not all impacted projects have been applied. This is caused because Digger batch does not have context whether all the needed jobs have been triggered when using `digger apply` with `-p` flag. Unlike in backendless mode, where all the jobs are run sequentially and the parent one does have context if all the impacted projects have been applied, in backend mode jobs are triggered via the Batch one, which does have this context. This PR addresses the issue by adding one more column to store this context, which will then be used in the subsequent run. --- backend/controllers/github.go | 31 +++++++++++---------- backend/controllers/github_test.go | 11 ++++---- backend/controllers/projects.go | 15 +++++----- backend/controllers/projects_test.go | 27 +++++++++++++----- backend/migrations/20250115155205.sql | 2 ++ backend/migrations/atlas.sum | 3 +- backend/models/scheduler.go | 36 ++++++++++++------------ backend/models/storage.go | 40 ++++++++++++++------------- backend/models/storage_test.go | 11 ++++---- backend/tasks/runs_test.go | 11 ++++---- backend/utils/graphs.go | 9 +++--- ee/backend/controllers/gitlab.go | 23 +++++++-------- ee/backend/hooks/github.go | 13 +++++---- 13 files changed, 130 insertions(+), 102 deletions(-) create mode 100644 backend/migrations/20250115155205.sql diff --git a/backend/controllers/github.go b/backend/controllers/github.go index a91297bb9..769ab9515 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -6,6 +6,18 @@ import ( "encoding/json" "errors" "fmt" + "log" + "math/rand" + "net/http" + "net/url" + "os" + "path/filepath" + "reflect" + "runtime/debug" + "slices" + "strconv" + "strings" + "github.com/davecgh/go-spew/spew" "github.com/diggerhq/digger/backend/ci_backends" config2 "github.com/diggerhq/digger/backend/config" @@ -29,17 +41,6 @@ import ( "github.com/samber/lo" "golang.org/x/oauth2" "gorm.io/gorm" - "log" - "math/rand" - "net/http" - "net/url" - "os" - "path/filepath" - "reflect" - "runtime/debug" - "slices" - "strconv" - "strings" ) type IssueCommentHook func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error @@ -397,7 +398,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR return fmt.Errorf("error processing event") } - jobsForImpactedProjects, _, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config, false) + jobsForImpactedProjects, coverAllImpactedProjects, err := dg_github.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config, false) if err != nil { log.Printf("Error converting event to jobsForImpactedProjects: %v", err) commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error converting event to jobsForImpactedProjects: %v", err)) @@ -522,7 +523,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR aiSummaryCommentId = aiSummaryComment.Id } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) @@ -827,7 +828,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu } log.Printf("GitHub IssueComment event processed successfully\n") - jobs, _, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, requestedProject, config.Workflows, prBranchName, defaultBranch) + jobs, coverAllImpactedProjects, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, requestedProject, config.Workflows, prBranchName, defaultBranch) if err != nil { log.Printf("Error converting event to jobs: %v", err) commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error converting event to jobs: %v", err)) @@ -930,7 +931,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu aiSummaryCommentId = aiSummaryComment.Id } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs, coverAllImpactedProjects) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/backend/controllers/github_test.go b/backend/controllers/github_test.go index 39b57cfe7..67304a6a6 100644 --- a/backend/controllers/github_test.go +++ b/backend/controllers/github_test.go @@ -2,12 +2,13 @@ package controllers import ( "encoding/json" - orchestrator "github.com/diggerhq/digger/libs/scheduler" "log" "os" "strings" "testing" + orchestrator "github.com/diggerhq/digger/libs/scheduler" + "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" configuration "github.com/diggerhq/digger/libs/digger_config" @@ -731,7 +732,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) { graph, err := configuration.CreateProjectDependencyGraph(projects) assert.NoError(t, err) - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false, true) assert.NoError(t, err) assert.Equal(t, 1, len(result)) parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID) @@ -760,7 +761,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true) assert.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -793,7 +794,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) { projectMap["dev"] = project1 projectMap["prod"] = project2 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true) assert.NoError(t, err) assert.Equal(t, 2, len(result)) parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID) @@ -838,7 +839,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) { projectMap["555"] = project5 projectMap["666"] = project6 - _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false) + _, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false, true) assert.NoError(t, err) assert.Equal(t, 6, len(result)) parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["111"].DiggerJobID) diff --git a/backend/controllers/projects.go b/backend/controllers/projects.go index 667cc52a8..a856555ab 100644 --- a/backend/controllers/projects.go +++ b/backend/controllers/projects.go @@ -4,6 +4,13 @@ import ( "encoding/json" "errors" "fmt" + "log" + "net/http" + "os" + "strconv" + "strings" + "time" + "github.com/diggerhq/digger/backend/middleware" "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/services" @@ -15,12 +22,6 @@ import ( orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" "github.com/gin-gonic/gin" "gorm.io/gorm" - "log" - "net/http" - "os" - "strconv" - "strings" - "time" ) func ListProjects(c *gin.Context) { @@ -730,7 +731,7 @@ func AutomergePRforBatchIfEnabled(gh utils.GithubClientProvider, batch *models.D } else { automerge = false } - if batch.Status == orchestrator_scheduler.BatchJobSucceeded && batch.BatchType == orchestrator_scheduler.DiggerCommandApply && automerge == true { + if batch.Status == orchestrator_scheduler.BatchJobSucceeded && batch.BatchType == orchestrator_scheduler.DiggerCommandApply && batch.CoverAllImpactedProjects == true && automerge == true { prService, err := GetPrServiceFromBatch(batch, gh) if err != nil { log.Printf("Error getting github service: %v", err) diff --git a/backend/controllers/projects_test.go b/backend/controllers/projects_test.go index a4ace792b..213e43cee 100644 --- a/backend/controllers/projects_test.go +++ b/backend/controllers/projects_test.go @@ -1,6 +1,9 @@ package controllers import ( + "net/http" + "testing" + "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" @@ -8,8 +11,6 @@ import ( "github.com/google/uuid" "github.com/migueleliasweb/go-github-mock/src/mock" "github.com/stretchr/testify/assert" - "net/http" - "testing" ) func TestAutomergeWhenBatchIsSuccessfulStatus(t *testing.T) { @@ -68,11 +69,12 @@ func TestAutomergeWhenBatchIsSuccessfulStatus(t *testing.T) { " - name: dev\n" + " dir: dev\n" + "auto_merge: false", - GithubInstallationId: int64(41584295), - RepoFullName: "diggerhq/github-job-scheduler", - RepoOwner: "diggerhq", - RepoName: "github-job-scheduler", - BatchType: orchestrator_scheduler.DiggerCommandApply, + GithubInstallationId: int64(41584295), + RepoFullName: "diggerhq/github-job-scheduler", + RepoOwner: "diggerhq", + RepoName: "github-job-scheduler", + BatchType: orchestrator_scheduler.DiggerCommandApply, + CoverAllImpactedProjects: true, } err := AutomergePRforBatchIfEnabled(gh, &batch) assert.NoError(t, err) @@ -98,4 +100,15 @@ func TestAutomergeWhenBatchIsSuccessfulStatus(t *testing.T) { assert.NoError(t, err) assert.True(t, isMergeCalled) + batch.DiggerConfig = "" + + "projects:\n" + + " - name: dev\n" + + " dir: dev\n" + + "auto_merge: true" + batch.BatchType = orchestrator_scheduler.DiggerCommandApply + batch.CoverAllImpactedProjects = false + err = AutomergePRforBatchIfEnabled(gh, &batch) + assert.NoError(t, err) + assert.False(t, isMergeCalled) + } diff --git a/backend/migrations/20250115155205.sql b/backend/migrations/20250115155205.sql new file mode 100644 index 000000000..825fd34fd --- /dev/null +++ b/backend/migrations/20250115155205.sql @@ -0,0 +1,2 @@ +-- Modify "digger_batches" table +ALTER TABLE "public"."digger_batches" ADD COLUMN "cover_all_impacted_projects" boolean NULL; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index 4b13a3e4e..f71bd3474 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:kHWmqYJMe9ZbdcztvM02SVEs5lyw/RFbKzZmqgbmSIk= +h1:KSPayCJw5aDjgANi5Nb0yhR8pvu1f0fOrSNRQ1U/VJU= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -35,3 +35,4 @@ h1:kHWmqYJMe9ZbdcztvM02SVEs5lyw/RFbKzZmqgbmSIk= 20241107172343.sql h1:E1j+7R5TZlyCKEpyYmH1mJ2zh+y5hVbtQ/PuEMJR7us= 20241114202249.sql h1:P2DhJK8MLe8gSAAz+Y5KNmsvKVw8KfLQPCncynYXEfM= 20241229112312.sql h1:Fr06uwt7LcQoLh6bjGzKB+uy9i8+uk8m6jfi+OBBbP4= +20250115155205.sql h1:zXoB4jz7dc6OpzNcEdA7Ln1j/FMTyXtwyES1Y2DjvRM= diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index c371b106f..34733784a 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -3,11 +3,12 @@ package models import ( "encoding/json" "fmt" + "log" + "time" + orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler" "github.com/google/uuid" "gorm.io/gorm" - "log" - "time" ) type DiggerJobParentLink struct { @@ -22,21 +23,22 @@ const DiggerVCSGithub DiggerVCSType = "github" const DiggerVCSGitlab DiggerVCSType = "gitlab" type DiggerBatch struct { - ID uuid.UUID `gorm:"primary_key"` - VCS DiggerVCSType - PrNumber int - CommentId *int64 - AiSummaryCommentId string - Status orchestrator_scheduler.DiggerBatchStatus - BranchName string - DiggerConfig string - GithubInstallationId int64 - GitlabProjectId int - RepoFullName string - RepoOwner string - RepoName string - BatchType orchestrator_scheduler.DiggerCommand - ReportTerraformOutputs bool + ID uuid.UUID `gorm:"primary_key"` + VCS DiggerVCSType + PrNumber int + CommentId *int64 + AiSummaryCommentId string + Status orchestrator_scheduler.DiggerBatchStatus + BranchName string + DiggerConfig string + GithubInstallationId int64 + GitlabProjectId int + RepoFullName string + RepoOwner string + RepoName string + BatchType orchestrator_scheduler.DiggerCommand + ReportTerraformOutputs bool + CoverAllImpactedProjects bool // used for module source grouping comments SourceDetails []byte } diff --git a/backend/models/storage.go b/backend/models/storage.go index 146ada683..7a492af7e 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -4,6 +4,10 @@ import ( "encoding/json" "errors" "fmt" + "log" + "net/http" + "time" + "github.com/dchest/uniuri" configuration "github.com/diggerhq/digger/libs/digger_config" scheduler "github.com/diggerhq/digger/libs/scheduler" @@ -11,9 +15,6 @@ import ( "github.com/google/uuid" "github.com/samber/lo" "gorm.io/gorm" - "log" - "net/http" - "time" ) func (db *Database) GetProjectsFromContext(c *gin.Context, orgIdKey string) ([]Project, bool) { @@ -617,24 +618,25 @@ func (db *Database) GetDiggerBatch(batchId *uuid.UUID) (*DiggerBatch, error) { return batch, nil } -func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool) (*DiggerBatch, error) { +func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutputs bool, coverAllImpactedProjects bool) (*DiggerBatch, error) { uid := uuid.New() batch := &DiggerBatch{ - ID: uid, - VCS: vcsType, - GithubInstallationId: githubInstallationId, - RepoOwner: repoOwner, - RepoName: repoName, - RepoFullName: repoFullname, - PrNumber: PRNumber, - CommentId: commentId, - Status: scheduler.BatchJobCreated, - BranchName: branchName, - DiggerConfig: diggerConfig, - BatchType: batchType, - GitlabProjectId: gitlabProjectId, - AiSummaryCommentId: aiSummaryCommentId, - ReportTerraformOutputs: reportTerraformOutputs, + ID: uid, + VCS: vcsType, + GithubInstallationId: githubInstallationId, + RepoOwner: repoOwner, + RepoName: repoName, + RepoFullName: repoFullname, + PrNumber: PRNumber, + CommentId: commentId, + Status: scheduler.BatchJobCreated, + BranchName: branchName, + DiggerConfig: diggerConfig, + BatchType: batchType, + GitlabProjectId: gitlabProjectId, + AiSummaryCommentId: aiSummaryCommentId, + ReportTerraformOutputs: reportTerraformOutputs, + CoverAllImpactedProjects: coverAllImpactedProjects, } result := db.GormDB.Save(batch) if result.Error != nil { diff --git a/backend/models/storage_test.go b/backend/models/storage_test.go index d9d06ba72..359bc8bdd 100644 --- a/backend/models/storage_test.go +++ b/backend/models/storage_test.go @@ -1,15 +1,16 @@ package models import ( + "log" + "os" + "strings" + "testing" + "github.com/diggerhq/digger/libs/scheduler" "github.com/stretchr/testify/assert" "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" - "log" - "os" - "strings" - "testing" ) func setupSuite(tb testing.TB) (func(tb testing.TB), *Database, *Organisation) { @@ -151,7 +152,7 @@ func TestGetDiggerJobsForBatchPreloadsSummary(t *testing.T) { resourcesUpdated := uint(2) resourcesDeleted := uint(3) - batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false) + batch, err := DB.CreateDiggerBatch(DiggerVCSGithub, 123, repoOwner, repoName, repoFullName, prNumber, diggerconfig, branchName, batchType, &commentId, 0, "", false, true) assert.NoError(t, err) job, err := DB.CreateDiggerJob(batch.ID, []byte(jobSpec), "workflow_file.yml") diff --git a/backend/tasks/runs_test.go b/backend/tasks/runs_test.go index 9e42f6cee..205dc366b 100644 --- a/backend/tasks/runs_test.go +++ b/backend/tasks/runs_test.go @@ -2,6 +2,11 @@ package main import ( "fmt" + "log" + "os" + "strings" + "testing" + "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/utils" github2 "github.com/diggerhq/digger/libs/ci/github" @@ -10,10 +15,6 @@ import ( "github.com/stretchr/testify/assert" "gorm.io/driver/sqlite" "gorm.io/gorm" - "log" - "os" - "strings" - "testing" ) func init() { @@ -143,7 +144,7 @@ func TestThatRunQueueItemMovesFromQueuedToPlanningAfterPickup(t *testing.T) { for i, testParam := range testParameters { ciService := github2.MockCiService{} - batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false) + batch, _ := models.DB.CreateDiggerBatch(models.DiggerVCSGithub, 123, "", "", "", 22, "", "", "", nil, 0, "", false, true) project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), nil, nil, false, false) planStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) applyStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) diff --git a/backend/utils/graphs.go b/backend/utils/graphs.go index 227c31907..5117ec53e 100644 --- a/backend/utils/graphs.go +++ b/backend/utils/graphs.go @@ -4,17 +4,18 @@ import ( "encoding/json" "errors" "fmt" + "log" + "os" + "github.com/diggerhq/digger/backend/models" configuration "github.com/diggerhq/digger/libs/digger_config" "github.com/diggerhq/digger/libs/scheduler" "github.com/dominikbraun/graph" "github.com/google/uuid" - "log" - "os" ) // ConvertJobsToDiggerJobs jobs is map with project name as a key and a Job as a value -func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool) (*uuid.UUID, map[string]*models.DiggerJob, error) { +func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int, aiSummaryCommentId string, reportTerraformOutput bool, coverAllImpactedProjects bool) (*uuid.UUID, map[string]*models.DiggerJob, error) { result := make(map[string]*models.DiggerJob) organisation, err := models.DB.GetOrganisationById(organisationId) if err != nil { @@ -43,7 +44,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig log.Printf("marshalledJobsMap: %v\n", marshalledJobsMap) - batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput) + batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId, aiSummaryCommentId, reportTerraformOutput, coverAllImpactedProjects) if err != nil { return nil, nil, fmt.Errorf("failed to create batch: %v", err) } diff --git a/ee/backend/controllers/gitlab.go b/ee/backend/controllers/gitlab.go index e7c71aa24..7f4337e18 100644 --- a/ee/backend/controllers/gitlab.go +++ b/ee/backend/controllers/gitlab.go @@ -3,6 +3,14 @@ package controllers import ( "encoding/json" "fmt" + "io" + "log" + "net/http" + "os" + "reflect" + "strconv" + "strings" + "github.com/diggerhq/digger/backend/ci_backends" "github.com/diggerhq/digger/backend/controllers" "github.com/diggerhq/digger/backend/locking" @@ -18,13 +26,6 @@ import ( "github.com/diggerhq/digger/libs/scheduler" "github.com/gin-gonic/gin" "github.com/xanzy/go-gitlab" - "io" - "log" - "net/http" - "os" - "reflect" - "strconv" - "strings" ) type DiggerEEController struct { @@ -237,7 +238,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab return fmt.Errorf("error processing event") } - jobsForImpactedProjects, _, err := gitlab2.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config) + jobsForImpactedProjects, coverAllImpactedProjects, err := gitlab2.ConvertGithubPullRequestEventToJobs(payload, impactedProjects, nil, *config) if err != nil { log.Printf("Error converting event to jobsForImpactedProjects: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: Error converting event to jobsForImpactedProjects: %v", err)) @@ -333,7 +334,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab log.Printf("strconv.ParseInt error: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId, "", false) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId, "", false, coverAllImpactedProjects) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) @@ -480,7 +481,7 @@ func handleIssueCommentEvent(gitlabProvider utils.GitlabProvider, payload *gitla return nil } - jobs, _, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, requestedProject, config.Workflows, prBranchName, defaultBranch) + jobs, coverAllImpactedProjects, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, requestedProject, config.Workflows, prBranchName, defaultBranch) if err != nil { log.Printf("Error converting event to jobs: %v", err) utils.InitCommentReporter(glService, issueNumber, fmt.Sprintf(":x: Error converting event to jobs: %v", err)) @@ -524,7 +525,7 @@ func handleIssueCommentEvent(gitlabProvider utils.GitlabProvider, payload *gitla log.Printf("ParseInt err: %v", err) return fmt.Errorf("parseint error: %v", err) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId, "", false) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId, "", false, coverAllImpactedProjects) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(glService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) diff --git a/ee/backend/hooks/github.go b/ee/backend/hooks/github.go index d6fb07dc7..512e93b75 100644 --- a/ee/backend/hooks/github.go +++ b/ee/backend/hooks/github.go @@ -2,6 +2,11 @@ package hooks import ( "fmt" + "log" + "regexp" + "strconv" + "strings" + "github.com/diggerhq/digger/backend/ci_backends" ce_controllers "github.com/diggerhq/digger/backend/controllers" "github.com/diggerhq/digger/backend/locking" @@ -15,10 +20,6 @@ import ( "github.com/diggerhq/digger/libs/scheduler" "github.com/google/go-github/v61/github" "github.com/samber/lo" - "log" - "regexp" - "strconv" - "strings" ) var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error { @@ -120,7 +121,7 @@ var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.Git } impactedProjects := config.GetProjects(projectName) - jobs, _, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, nil, config.Workflows, defaultBranch, defaultBranch) + jobs, coverAllImpactedProjects, err := generic.ConvertIssueCommentEventToJobs(repoFullName, actor, issueNumber, commentBody, impactedProjects, nil, config.Workflows, defaultBranch, defaultBranch) if err != nil { log.Printf("Error converting event to jobs: %v", err) utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: Error converting event to jobs: %v", err)) @@ -151,7 +152,7 @@ var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.Git utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: could not handle commentId: %v", err)) } - batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0, "", false) + batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0, "", false, coverAllImpactedProjects) if err != nil { log.Printf("ConvertJobsToDiggerJobs error: %v", err) utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err)) From da512e3dc066b3b5bf2904f4d82277577dc53cf6 Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Wed, 15 Jan 2025 18:17:03 +0200 Subject: [PATCH 2/2] fix: backend test --- backend/controllers/projects_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/controllers/projects_test.go b/backend/controllers/projects_test.go index 213e43cee..96d1b2cb9 100644 --- a/backend/controllers/projects_test.go +++ b/backend/controllers/projects_test.go @@ -96,9 +96,10 @@ func TestAutomergeWhenBatchIsSuccessfulStatus(t *testing.T) { " dir: dev\n" + "auto_merge: true" batch.BatchType = orchestrator_scheduler.DiggerCommandApply + batch.CoverAllImpactedProjects = false err = AutomergePRforBatchIfEnabled(gh, &batch) assert.NoError(t, err) - assert.True(t, isMergeCalled) + assert.False(t, isMergeCalled) batch.DiggerConfig = "" + "projects:\n" + @@ -106,9 +107,8 @@ func TestAutomergeWhenBatchIsSuccessfulStatus(t *testing.T) { " dir: dev\n" + "auto_merge: true" batch.BatchType = orchestrator_scheduler.DiggerCommandApply - batch.CoverAllImpactedProjects = false + batch.CoverAllImpactedProjects = true err = AutomergePRforBatchIfEnabled(gh, &batch) assert.NoError(t, err) - assert.False(t, isMergeCalled) - + assert.True(t, isMergeCalled) }