Merge branch 'kubernetes-sigs:main' into kep/kueue-workflow

.github/ISSUE_TEMPLATE/NEW_RELEASE.md

@@ -2,7 +2,7 @@
 name: New Release
 about: Propose a new release
 title: Release v0.x.0
-assignees: ahg-g, alculquicondor, tenzen-y
+assignees: alculquicondor, mimowo, tenzen-y
 
 ---
 
@@ -24,38 +24,39 @@ Please do not remove items from the checklist
   - [ ] Update `RELEASE_BRANCH` and `RELEASE_VERSION` in `Makefile` and run `make prepare-release-branch`
   - [ ] Update the `CHANGELOG`
   - [ ] Submit a pull request with the changes: <!-- example #211 #214 -->
-- [ ] An OWNER [prepares a draft release](https://github.com/kubernetes-sigs/kueue/releases)
-  - [ ] Write the change log into the draft release.
-  - [ ] Run
-      `make artifacts IMAGE_REGISTRY=registry.k8s.io/kueue GIT_TAG=$VERSION`
-      to generate the artifacts and upload the files in the `artifacts` folder
-      to the draft release.
 - [ ] An OWNER creates a signed tag running
      `git tag -s $VERSION`
       and inserts the changelog into the tag description.
       To perform this step, you need [a PGP key registered on github](https://docs.github.com/en/authentication/managing-commit-signature-verification/checking-for-existing-gpg-keys).
 - [ ] An OWNER pushes the tag with
-      `git push $VERSION`
+      `git push upstream $VERSION`
   - Triggers prow to build and publish a staging container image
       `us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:$VERSION`
+- [ ] An OWNER [prepares a draft release](https://github.com/kubernetes-sigs/kueue/releases)
+  - [ ] Create the draft release poiting out to the created tag.
+  - [ ] Write the change log into the draft release.
+  - [ ] Run
+      `make artifacts IMAGE_REGISTRY=registry.k8s.io/kueue GIT_TAG=$VERSION`
+      to generate the artifacts in the `artifacts` folder.
+  - [ ] Upload the files in the `artifacts` folder to the draft release - either
+      via UI or `gh release upload <tag> artifacts/*`.
 - [ ] Submit a PR against [k8s.io](https://github.com/kubernetes/k8s.io),
       updating `registry.k8s.io/images/k8s-staging-kueue/images.yaml` to
-      [promote the container images](https://github.com/kubernetes/k8s.io/tree/main/k8s.gcr.io#image-promoter)
+      [promote the container images](https://github.com/kubernetes/k8s.io/tree/main/registry.k8s.io#image-promoter)
       to production: <!-- example kubernetes/k8s.io#3612-->
 - [ ] Wait for the PR to be merged and verify that the image `registry.k8s.io/kueue/kueue:$VERSION` is available.
 - [ ] Publish the draft release prepared at the [GitHub releases page](https://github.com/kubernetes-sigs/kueue/releases).
       Link: <!-- example https://github.com/kubernetes-sigs/kueue/releases/tag/v0.1.0 -->
 - [ ] Run the [openvex action](https://github.com/kubernetes-sigs/kueue/actions/workflows/openvex.yaml) to generate openvex data. The action will add the file to the release artifacts.
+- [ ] Update the `main` branch :
+  - [ ] Update `RELEASE_VERSION` in `Makefile` and run `make prepare-release-branch`
+  - [ ] Release notes in the `CHANGELOG`
+  - [ ] `SECURITY-INSIGHTS.yaml` values by running `make update-security-insights GIT_TAG=$VERSION`
+  - [ ] Submit a pull request with the changes: <!-- example #3007 -->
+  - [ ] Cherry-pick the pull request onto the `website` branch
 - [ ] Run the [SBOM action](https://github.com/kubernetes-sigs/kueue/actions/workflows/sbom.yaml) to generate the SBOM and add it to the release.
 - [ ] For major or minor releases, merge the `main` branch into the `website` branch to publish the updated documentation.
 - [ ] Send an announcement email to `[email protected]` and `[email protected]` with the subject `[ANNOUNCE] kueue $VERSION is released`.   <!--Link: example https://groups.google.com/a/kubernetes.io/g/wg-batch/c/-gZOrSnwDV4 -->
-- [ ] Update the below files with respective values in `main` branch :
-  - Latest version in `README.md`
-  - Latest version in `cmd/experimental/kjobctl/docs/installation.md`
-  - Release notes in the `CHANGELOG`
-  - `version` in `site/hugo.toml`
-  - `appVersion` in `charts/kueue/Chart.yaml`
-  - `SECURITY-INSIGHTS.yaml` values by running `make update-security-insights GIT_TAG=$VERSION`
 - [ ] For a major or minor release, prepare the repo for the next version:
   - [ ] Create an unannotated _devel_ tag in the
         `main` branch, on the first commit that gets merged after the release

.golangci.yaml

@@ -9,11 +9,17 @@ linters-settings:
     disabled-checks:
       - appendAssign
       - exitAfterDefer
-  goimports:
-    local-prefixes: sigs.k8s.io/kueue
   govet:
     enable:
       - nilness
+  gci:
+    sections:
+      - standard # Standard section: captures all standard packages.
+      - default # Default section: contains all imports that could not be matched to another section type.
+      - prefix(sigs.k8s.io/kueue) # Custom section: groups all imports with the specified Prefix.
+      - blank # Blank section: contains all blank imports. This section is not present unless explicitly enabled.
+      - dot # Dot section: contains all dot imports.
+    skip-generated: true # Skip generated files.
   perfsprint:
     int-conversion: false
     errorf: true
@@ -34,9 +40,9 @@ linters:
     - dupword
     - durationcheck
     - fatcontext
+    - gci
     - ginkgolinter
     - gocritic
-    - goimports
     - govet
     - loggercheck
     - misspell
@@ -58,8 +64,14 @@ issues:
       # TODO(#768): Drop when incrementing the API version.
       text: "SA1019: constants.QueueAnnotation is deprecated"
     - linters:
-      - unused
-      path: "^pkg/hierarchy.cohort.go"
+        - staticcheck
+      text: "SA1019: j.Status.State is deprecated: the State field is replaced by the Conditions field."
+    - linters:
+        - staticcheck
+      text: "SA1019: j.Status.RayClusterStatus.State is deprecated: the State field is replaced by the Conditions field."
+    - linters:
+        - fatcontext
+      path: "^test/*"
   # Show all issues from a linter
   max-issues-per-linter: 0
   # Show all issues with the same text

CHANGELOG/CHANGELOG-0.8.md

@@ -1,3 +1,25 @@
+## v0.8.1
+
+Changes since `v0.8.0`:
+
+### Feature
+
+- Add gauge metric admission_cycle_preemption_skips that reports the number of Workloads in a ClusterQueue
+  that got preemptions candidates, but had to be skipped in the last cycle. (#2942, @alculquicondor)
+- Publish images via artifact registry (#2832, @alculquicondor)
+
+### Bug or Regression
+
+- CLI: Support `-` and `.` in the resource flavor name on `create cq` (#2706, @trasc)
+- Detect and enable support for job CRDs installed after Kueue starts. (#2991, @ChristianZaccaria)
+- Fix over-admission after deleting resources from borrowing ClusterQueue. (#2879, @mbobrovskyi)
+- Fix support for kuberay 1.2.x (#2983, @mbobrovskyi)
+- Helm: Fix a bug for "unclosed action error". (#2688, @mbobrovskyi)
+- Prevent infinite preemption loop when PrioritySortingWithinCohort=false
+  is used together with borrowWithinCohort. (#2831, @mimowo)
+- Support for helm charts in the us-central1-docker.pkg.dev/k8s-staging-images/charts repository (#2834, @IrvingMg)
+- Update Flavor selection logic to prefer Flavors which allow reclamation of lent nominal quota, over Flavors which require preempting workloads within the ClusterQueue. This matches the behavior in the single Flavor case. (#2829, @gabesaba)
+
 ## v0.8.0
 
 Changes since `v0.7.0`:

Dockerfile

@@ -1,5 +1,6 @@
-ARG BUILDER_IMAGE
-ARG BASE_IMAGE
+ARG BUILDER_IMAGE=golang:1.23
+ARG BASE_IMAGE=gcr.io/distroless/static:nonroot
+
 # Build the manager binary
 FROM --platform=${BUILDPLATFORM} ${BUILDER_IMAGE} AS builder
 

Makefile

@@ -76,7 +76,7 @@ LD_FLAGS += -X '$(version_pkg).GitCommit=$(shell git rev-parse HEAD)'
 
 # Update these variables when preparing a new release or a release branch.
 # Then run `make prepare-release-branch`
-RELEASE_VERSION=v0.8.0
+RELEASE_VERSION=v0.8.1
 RELEASE_BRANCH=main
 
 .PHONY: all
@@ -122,7 +122,7 @@ update-helm: manifests yq
 .PHONY: generate
 generate: gomod-download controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations and client-go libraries.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./apis/..."
-	./hack/update-codegen.sh $(GO_CMD)
+	TOOLS_DIR=${TOOLS_DIR} ./hack/update-codegen.sh $(GO_CMD)
 
 .PHONY: fmt
 fmt: ## Run go fmt against code.
@@ -265,20 +265,21 @@ artifacts: kustomize yq helm ## Generate release artifacts.
 	$(KUSTOMIZE) build config/dev -o artifacts/manifests-dev.yaml
 	$(KUSTOMIZE) build config/alpha-enabled -o artifacts/manifests-alpha-enabled.yaml
 	$(KUSTOMIZE) build config/prometheus -o artifacts/prometheus.yaml
-	$(KUSTOMIZE) build config/visibility -o artifacts/visibility-api.yaml
+	$(KUSTOMIZE) build config/visibility-apf/default -o artifacts/visibility-apf.yaml
+	$(KUSTOMIZE) build config/visibility-apf/1_28 -o artifacts/visibility-apf-1-28.yaml
 	@$(call clean-manifests)
 	# Update the image tag and policy
 	$(YQ)  e  '.controllerManager.manager.image.repository = "$(IMAGE_REPO)" | .controllerManager.manager.image.tag = "$(GIT_TAG)" | .controllerManager.manager.image.pullPolicy = "IfNotPresent"' -i charts/kueue/values.yaml
 	# create the package. TODO: consider signing it
 	$(HELM) package --version $(GIT_TAG) --app-version $(GIT_TAG) charts/kueue -d artifacts/
 	mv artifacts/kueue-$(GIT_TAG).tgz artifacts/kueue-chart-$(GIT_TAG).tgz
 	# Revert the image changes
-	$(YQ)  e  '.controllerManager.manager.image.repository = "$(IMAGE_REGISTRY)/$(IMAGE_NAME)" | .controllerManager.manager.image.tag = "main" | .controllerManager.manager.image.pullPolicy = "Always"' -i charts/kueue/values.yaml
+	$(YQ)  e  '.controllerManager.manager.image.repository = "$(IMAGE_REGISTRY)/$(IMAGE_NAME)" | del(.controllerManager.manager.image.tag) | .controllerManager.manager.image.pullPolicy = "Always"' -i charts/kueue/values.yaml
 	CGO_ENABLED=$(CGO_ENABLED) GO_CMD="$(GO_CMD)" LD_FLAGS="$(LD_FLAGS)" BUILD_DIR="artifacts" BUILD_NAME=kubectl-kueue PLATFORMS="$(CLI_PLATFORMS)" ./hack/multiplatform-build.sh ./cmd/kueuectl/main.go
 
 .PHONY: prepare-release-branch
 prepare-release-branch: yq kustomize ## Prepare the release branch with the release version.
-	$(SED) -r 's/v[0-9]+\.[0-9]+\.[0-9]+/$(RELEASE_VERSION)/g' -i README.md -i site/hugo.toml -i cmd/experimental/kjobctl/docs/installation.md
+	$(SED) -r 's/v[0-9]+\.[0-9]+\.[0-9]+/$(RELEASE_VERSION)/g' -i README.md -i site/hugo.toml
 	$(YQ) e '.appVersion = "$(RELEASE_VERSION)"' -i charts/kueue/Chart.yaml
 	@$(call clean-manifests)
 
@@ -340,5 +341,5 @@ generate-apiref: genref
 generate-kueuectl-docs: kueuectl-docs
 	rm -Rf $(PROJECT_DIR)/site/content/en/docs/reference/kubectl-kueue/commands/kueuectl*
 	$(PROJECT_DIR)/bin/kueuectl-docs \
-		$(PROJECT_DIR)/hack/internal/tools/kueuectl-docs/templates \
+		$(PROJECT_DIR)/cmd/kueuectl-docs/templates \
 		$(PROJECT_DIR)/site/content/en/docs/reference/kubectl-kueue/commands

Makefile-deps.mk

@@ -111,8 +111,19 @@ mpi-operator-crd: ## Copy the CRDs from the mpi-operator to the dep-crds directo
 KF_TRAINING_ROOT = $(shell $(GO_CMD) list -m -mod=readonly -f "{{.Dir}}" github.com/kubeflow/training-operator)
 .PHONY: kf-training-operator-crd
 kf-training-operator-crd: ## Copy the CRDs from the training-operator to the dep-crds directory.
-	mkdir -p $(EXTERNAL_CRDS_DIR)/training-operator/
-	cp -f $(KF_TRAINING_ROOT)/manifests/base/crds/* $(EXTERNAL_CRDS_DIR)/training-operator/
+	## Removing kubeflow.org_mpijobs.yaml is required as the version of MPIJob is conflicting between training-operator and mpi-operator - in integration tests.
+	mkdir -p $(EXTERNAL_CRDS_DIR)/training-operator-crds/
+	find $(KF_TRAINING_ROOT)/manifests/base/crds/* -type f -not -name "kubeflow.org_mpijobs.yaml" -exec cp -pf {} $(EXTERNAL_CRDS_DIR)/training-operator-crds/ \;
+
+.PHONY: kf-training-operator-manifests
+kf-training-operator-manifests: ## Copy whole manifests folder from the training-operator to the dep-crds directory.
+	## Full version of the manifest is required for e2e multikueue tests.
+	if [ -d "$(EXTERNAL_CRDS_DIR)/training-operator" ]; then \
+		chmod -R u+w "$(EXTERNAL_CRDS_DIR)/training-operator" && \
+		rm -rf "$(EXTERNAL_CRDS_DIR)/training-operator"; \
+	fi
+	mkdir -p "$(EXTERNAL_CRDS_DIR)/training-operator"
+	cp -rf "$(KF_TRAINING_ROOT)/manifests" "$(EXTERNAL_CRDS_DIR)/training-operator"
 
 RAY_ROOT = $(shell $(GO_CMD) list -m -mod=readonly -f "{{.Dir}}" github.com/ray-project/kuberay/ray-operator)
 .PHONY: ray-operator-crd
@@ -133,11 +144,9 @@ cluster-autoscaler-crd: ## Copy the CRDs from the cluster-autoscaler to the dep-
 	cp -f $(CLUSTER_AUTOSCALER_ROOT)/config/crd/* $(EXTERNAL_CRDS_DIR)/cluster-autoscaler/
 
 .PHONY: dep-crds
-dep-crds: mpi-operator-crd kf-training-operator-crd ray-operator-crd jobset-operator-crd cluster-autoscaler-crd ## Copy the CRDs from the external operators to the dep-crds directory.
+dep-crds: mpi-operator-crd kf-training-operator-crd ray-operator-crd jobset-operator-crd cluster-autoscaler-crd kf-training-operator-manifests ## Copy the CRDs from the external operators to the dep-crds directory.
 	@echo "Copying CRDs from external operators to dep-crds directory"
 
 .PHONY: kueuectl-docs
 kueuectl-docs:
-	cd $(PROJECT_DIR)/hack/internal/tools && \
-	go mod download && \
-	$(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o $(PROJECT_DIR)/bin/kueuectl-docs ./kueuectl-docs/main.go
+	$(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o $(PROJECT_DIR)/bin/kueuectl-docs ./cmd/kueuectl-docs/main.go