feat: Add kubernetes.azure.com/scalesetpriority to WellKnownLabels

pkg/apis/v1beta1/labels.go

@@ -65,6 +65,7 @@ var (
 
 		AKSLabelCluster,
 		AKSLabelMode,
+		AKSLabelScaleSetPriority,
 	)
 
 	RestrictedLabels = sets.New(
@@ -109,7 +110,8 @@ var (
 
 	AKSLabelCluster                 = AKSLabelDomain + "/cluster"
 	AKSLabelKubeletIdentityClientID = AKSLabelDomain + "/kubelet-identity-client-id"
-	AKSLabelMode                    = AKSLabelDomain + "/mode" // "system" or "user"
+	AKSLabelMode                    = AKSLabelDomain + "/mode"             // "system" or "user"
+	AKSLabelScaleSetPriority        = AKSLabelDomain + "/scalesetpriority" // "spot" or "regular". Note that "regular" is never written by AKS as a label but we write it to make scheduling easier
 
 	AnnotationAKSNodeClassHash        = apis.Group + "/aksnodeclass-hash"
 	AnnotationAKSNodeClassHashVersion = apis.Group + "/aksnodeclass-hash-version"
@@ -127,6 +129,11 @@ const (
 	AzureLinuxImageFamily = "AzureLinux"
 )
 
+const (
+	ScaleSetPriorityRegular = "regular"
+	ScaleSetPrioritySpot    = "spot"
+)
+
 var UbuntuFamilies = sets.New(
 	UbuntuImageFamily,
 	Ubuntu2204ImageFamily,

pkg/cloudprovider/cloudprovider.go

@@ -507,6 +507,7 @@ func (c *CloudProvider) vmInstanceToNodeClaim(ctx context.Context, vm *armcomput
 	}
 
 	labels[karpv1.CapacityTypeLabelKey] = instance.GetCapacityTypeFromVM(vm)
+	labels[v1beta1.AKSLabelScaleSetPriority] = instance.GetScaleSetPriorityLabelFromVM(vm)
 
 	if tag, ok := vm.Tags[launchtemplate.NodePoolTagKey]; ok {
 		labels[karpv1.NodePoolLabelKey] = *tag

pkg/providers/instance/offerings/offerings_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/Azure/karpenter-provider-azure/pkg/apis/v1beta1"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
@@ -89,6 +90,7 @@ func TestPickSkuSizePriorityAndZone(t *testing.T) {
 							Price: 0.05,
 							Requirements: scheduling.NewRequirements(
 								scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeSpot),
+								scheduling.NewRequirement(v1beta1.AKSLabelScaleSetPriority, corev1.NodeSelectorOpIn, v1beta1.ScaleSetPrioritySpot),
 								scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, "westus-1"),
 							),
 							Available: true,
@@ -97,6 +99,7 @@ func TestPickSkuSizePriorityAndZone(t *testing.T) {
 							Price: 0.1,
 							Requirements: scheduling.NewRequirements(
 								scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeOnDemand),
+								scheduling.NewRequirement(v1beta1.AKSLabelScaleSetPriority, corev1.NodeSelectorOpIn, v1beta1.ScaleSetPriorityRegular),
 								scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, "westus-1"),
 							),
 							Available: true,
@@ -128,6 +131,57 @@ func TestPickSkuSizePriorityAndZone(t *testing.T) {
 			expectedPriority:     karpv1.CapacityTypeSpot,
 			expectedZone:         "westus-1",
 		},
+		{
+			name: "Select spot instance when requested (via legacy kubernetes.azure.com/scalesetpriority label)",
+			instanceTypes: []*cloudprovider.InstanceType{
+				{
+					Name: "Standard_D2s_v3",
+					Offerings: []*cloudprovider.Offering{
+						{
+							Price: 0.05,
+							Requirements: scheduling.NewRequirements(
+								scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeSpot),
+								scheduling.NewRequirement(v1beta1.AKSLabelScaleSetPriority, corev1.NodeSelectorOpIn, v1beta1.ScaleSetPrioritySpot),
+								scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, "westus-1"),
+							),
+							Available: true,
+						},
+						{
+							Price: 0.1,
+							Requirements: scheduling.NewRequirements(
+								scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeOnDemand),
+								scheduling.NewRequirement(v1beta1.AKSLabelScaleSetPriority, corev1.NodeSelectorOpIn, v1beta1.ScaleSetPriorityRegular),
+								scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, "westus-1"),
+							),
+							Available: true,
+						},
+					},
+				},
+			},
+			nodeClaim: &karpv1.NodeClaim{
+				Spec: karpv1.NodeClaimSpec{
+					Requirements: []karpv1.NodeSelectorRequirementWithMinValues{
+						{
+							NodeSelectorRequirement: corev1.NodeSelectorRequirement{
+								Key:      v1beta1.AKSLabelScaleSetPriority,
+								Operator: corev1.NodeSelectorOpIn,
+								Values:   []string{v1beta1.ScaleSetPrioritySpot},
+							},
+						},
+						{
+							NodeSelectorRequirement: corev1.NodeSelectorRequirement{
+								Key:      corev1.LabelTopologyZone,
+								Operator: corev1.NodeSelectorOpIn,
+								Values:   []string{"westus-1"},
+							},
+						},
+					},
+				},
+			},
+			expectedInstanceType: "Standard_D2s_v3",
+			expectedPriority:     karpv1.CapacityTypeSpot,
+			expectedZone:         "westus-1",
+		},
 		{
 			name: "Multiple zones - should pick one of the available zones",
 			instanceTypes: []*cloudprovider.InstanceType{

pkg/providers/instance/vminstance.go

@@ -58,6 +58,15 @@ var (
 		armcompute.VirtualMachinePriorityTypesSpot:    karpv1.CapacityTypeSpot,
 		armcompute.VirtualMachinePriorityTypesRegular: karpv1.CapacityTypeOnDemand,
 	}
+	// Note that there is no ScaleSetPriorityToKarpCapacityType because the karpenter.sh/capacity-type
+	// label is the "official" label that we actually key priority off of. Selection still works though
+	// because when we list instance types on-demand offerings always have v1beta1.ScaleSetPriorityRegular
+	// and spot instances always have v1beta1.ScaleSetPrioritySpot, so the correct karpenter.sh/capacity-type
+	// label is still selected even if the user is using kubernetes.azure.com/scalesetpriority only on the NodePool.
+	VMPriorityToScaleSetPriority = map[armcompute.VirtualMachinePriorityTypes]string{
+		armcompute.VirtualMachinePriorityTypesSpot:    v1beta1.ScaleSetPrioritySpot,
+		armcompute.VirtualMachinePriorityTypesRegular: v1beta1.ScaleSetPriorityRegular,
+	}
 )
 
 const (
@@ -989,3 +998,10 @@ func GetCapacityTypeFromVM(vm *armcompute.VirtualMachine) string {
 	}
 	return ""
 }
+
+func GetScaleSetPriorityLabelFromVM(vm *armcompute.VirtualMachine) string {
+	if vm != nil && vm.Properties != nil && vm.Properties.Priority != nil {
+		return VMPriorityToScaleSetPriority[*vm.Properties.Priority]
+	}
+	return ""
+}

pkg/providers/instancetype/instancetype.go

@@ -175,6 +175,7 @@ func computeRequirements(
 		scheduling.NewRequirement(v1beta1.LabelSKUGPUName, corev1.NodeSelectorOpDoesNotExist),
 		scheduling.NewRequirement(v1beta1.AKSLabelCluster, corev1.NodeSelectorOpIn, labels.NormalizeClusterResourceGroupNameForLabel(opts.NodeResourceGroup)),
 		scheduling.NewRequirement(v1beta1.AKSLabelMode, corev1.NodeSelectorOpIn, v1beta1.ModeSystem, v1beta1.ModeUser),
+		scheduling.NewRequirement(v1beta1.AKSLabelScaleSetPriority, corev1.NodeSelectorOpIn, v1beta1.ScaleSetPriorityRegular, v1beta1.ScaleSetPrioritySpot),
 
 		// composites
 		scheduling.NewRequirement(v1beta1.LabelSKUName, corev1.NodeSelectorOpDoesNotExist),

pkg/providers/instancetype/instancetypes.go

@@ -228,6 +228,7 @@ func (p *DefaultProvider) createOfferings(sku *skewer.SKU, zones sets.Set[string
 		onDemandOffering := &cloudprovider.Offering{
 			Requirements: scheduling.NewRequirements(
 				scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeOnDemand),
+				scheduling.NewRequirement(v1beta1.AKSLabelScaleSetPriority, corev1.NodeSelectorOpIn, v1beta1.ScaleSetPriorityRegular),
 				scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, zone),
 			),
 			Price:     onDemandPrice,
@@ -237,6 +238,7 @@ func (p *DefaultProvider) createOfferings(sku *skewer.SKU, zones sets.Set[string
 		spotOffering := &cloudprovider.Offering{
 			Requirements: scheduling.NewRequirements(
 				scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeSpot),
+				scheduling.NewRequirement(v1beta1.AKSLabelScaleSetPriority, corev1.NodeSelectorOpIn, v1beta1.ScaleSetPrioritySpot),
 				scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, zone),
 			),
 			Price:     spotPrice,

pkg/providers/instancetype/suite_test.go

@@ -1074,6 +1074,7 @@ var _ = Describe("InstanceType Provider", func() {
 				Expect(*vm.Properties.StorageProfile.OSDisk.DiskSizeGB).To(Equal(int32(128)))
 				Expect(vm.Properties.StorageProfile.OSDisk.DiffDiskSettings).To(BeNil())
 			})
+
 			It("should select NvmeDisk for v6 skus with maxNvmeDiskSize > 0", func() {
 				nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{
 					NodeSelectorRequirement: v1.NodeSelectorRequirement{
@@ -2333,6 +2334,8 @@ var _ = Describe("InstanceType Provider", func() {
 				{Name: v1beta1.AKSLabelMemory, Label: v1beta1.AKSLabelMemory, ValueFunc: func() string { return "8192" }, ExpectedInKubeletLabels: true, ExpectedOnNode: true},
 				{Name: v1beta1.AKSLabelMode + "=user", Label: v1beta1.AKSLabelMode, ValueFunc: func() string { return "user" }, ExpectedInKubeletLabels: true, ExpectedOnNode: true},
 				{Name: v1beta1.AKSLabelMode + "=system", Label: v1beta1.AKSLabelMode, ValueFunc: func() string { return "system" }, ExpectedInKubeletLabels: true, ExpectedOnNode: true},
+				{Name: v1beta1.AKSLabelScaleSetPriority + "=regular", Label: v1beta1.AKSLabelScaleSetPriority, ValueFunc: func() string { return "regular" }, ExpectedInKubeletLabels: true, ExpectedOnNode: true},
+				{Name: v1beta1.AKSLabelScaleSetPriority + "=spot", Label: v1beta1.AKSLabelScaleSetPriority, ValueFunc: func() string { return "spot" }, ExpectedInKubeletLabels: true, ExpectedOnNode: true},
 				// Deprecated Labels -- note that these are not expected in kubelet labels or on the node.
 				// They are written by CloudProvider so don't need to be sent to kubelet, and they aren't required on the node object because Karpenter does a mapping from
 				// the new labels to the old labels for compatibility.

pkg/providers/labels/labels.go

@@ -102,6 +102,7 @@ func Get(
 	// Prevent race conditions with startup taints by telling Karpenter not to sync taints from the NodeClaim to the Node
 	// See https://github.com/kubernetes-sigs/karpenter/issues/1772
 	labels[karpv1.NodeDoNotSyncTaintsLabelKey] = "true"
+	labels[v1beta1.AKSLabelScaleSetPriority] = v1beta1.ScaleSetPriorityRegular
 
 	if opts.IsAzureCNIOverlay() {
 		// TODO: make conditional on pod subnet

pkg/providers/labels/labels_test.go

@@ -58,18 +58,20 @@ func TestGetAllSingleValuedRequirementLabels(t *testing.T) {
 			requirements: scheduling.NewRequirements(
 				scheduling.NewRequirement(corev1.LabelInstanceTypeStable, corev1.NodeSelectorOpIn, "Standard_D2s_v3"),
 				scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeOnDemand, karpv1.CapacityTypeSpot),
+				scheduling.NewRequirement(v1beta1.AKSLabelScaleSetPriority, corev1.NodeSelectorOpIn, v1beta1.ScaleSetPriorityRegular, v1beta1.ScaleSetPrioritySpot),
 				scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, "westus-1"),
 			),
 			expectedLabels: map[string]string{
 				corev1.LabelInstanceTypeStable: "Standard_D2s_v3",
 				corev1.LabelTopologyZone:       "westus-1",
-				// karpv1.CapacityTypeLabelKey should be excluded because it has multiple values
+				// karpv1.CapacityTypeLabelKey and v1beta1.AKSLabelScaleSetPriority should be excluded because they have multiple values
 			},
 		},
 		{
 			name: "No single-valued requirements",
 			requirements: scheduling.NewRequirements(
 				scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeOnDemand, karpv1.CapacityTypeSpot),
+				scheduling.NewRequirement(v1beta1.AKSLabelScaleSetPriority, corev1.NodeSelectorOpIn, v1beta1.ScaleSetPriorityRegular, v1beta1.ScaleSetPrioritySpot),
 				scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, "westus-1", "westus-2"),
 			),
 			expectedLabels: map[string]string{},

test/suites/scheduling/suite_test.go

@@ -133,6 +133,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() {
 				v1beta1.LabelSKUStorageEphemeralOSMaxSize: "53",
 				v1beta1.AKSLabelCluster:                   env.NodeResourceGroup,
 				v1beta1.AKSLabelMode:                      "system",
+				v1beta1.AKSLabelScaleSetPriority:          "regular",
 			}
 			selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels
 			requirements := lo.MapToSlice(nodeSelector, func(key string, value string) corev1.NodeSelectorRequirement {

test/suites/spot/suite_test.go

@@ -101,6 +101,7 @@ var _ = Describe("Spot", func() {
 		// Verify nodes are created with the spot capacity type label
 		nodes := env.ExpectCreatedNodeCount("==", 1)
 		Expect(nodes[0].Labels).To(HaveKeyWithValue(karpv1.CapacityTypeLabelKey, karpv1.CapacityTypeSpot))
+		Expect(nodes[0].Labels).To(HaveKeyWithValue(v1beta1.AKSLabelScaleSetPriority, v1beta1.ScaleSetPrioritySpot))
 
 		// Simulate spot eviction
 		env.SimulateVMEviction(nodes[0].Name)
@@ -116,3 +117,55 @@ var _ = Describe("Spot", func() {
 		env.ExpectDeleted(dep)
 	})
 })
+
+var _ = Describe("Spot (nonstandard node pool)", func() {
+	It("should provision spot nodes via kubernetes.azure.com/scalesetpriority label", func() {
+		// Create a node pool with spot requirement
+		nodePool = test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{
+			NodeSelectorRequirement: corev1.NodeSelectorRequirement{
+				Key:      v1beta1.AKSLabelScaleSetPriority,
+				Operator: corev1.NodeSelectorOpIn,
+				Values:   []string{v1beta1.ScaleSetPrioritySpot},
+			}})
+		nodePool.Spec.Template.Spec.Requirements = lo.Reject(nodePool.Spec.Template.Spec.Requirements, func(r karpv1.NodeSelectorRequirementWithMinValues, _ int) bool {
+			return r.Key == karpv1.CapacityTypeLabelKey
+		})
+		env.ExpectCreated(nodePool, nodeClass)
+
+		podLabels := map[string]string{"app": "spot-test"}
+		dep := test.Deployment(test.DeploymentOptions{
+			Replicas: 1,
+			PodOptions: test.PodOptions{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: podLabels,
+				},
+				NodeSelector: map[string]string{
+					v1beta1.AKSLabelScaleSetPriority: v1beta1.ScaleSetPrioritySpot,
+				},
+				Tolerations: []corev1.Toleration{
+					{
+						Key:      "kubernetes.azure.com/scalesetpriority",
+						Operator: corev1.TolerationOpEqual,
+						Value:    "spot",
+						Effect:   corev1.TaintEffectNoSchedule,
+					},
+				},
+				TerminationGracePeriodSeconds: lo.ToPtr(int64(0)),
+			},
+		})
+
+		// Create resources
+		env.ExpectCreated(dep)
+
+		// Verify pods are scheduled and running
+		env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(dep.Spec.Selector.MatchLabels), 1)
+
+		// Verify nodes are created with the spot capacity type label
+		nodes := env.ExpectCreatedNodeCount("==", 1)
+		Expect(nodes[0].Labels).To(HaveKeyWithValue(karpv1.CapacityTypeLabelKey, karpv1.CapacityTypeSpot))
+		Expect(nodes[0].Labels).To(HaveKeyWithValue(v1beta1.AKSLabelScaleSetPriority, v1beta1.ScaleSetPrioritySpot))
+
+		// Cleanup resources
+		env.ExpectDeleted(dep)
+	})
+})