diff --git a/anydatacenter/30-policy-demo/.terraform.lock.hcl b/anydatacenter/30-policy-demo/.terraform.lock.hcl new file mode 100644 index 0000000..7b6c569 --- /dev/null +++ b/anydatacenter/30-policy-demo/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.34.0" + constraints = "~> 2.34" + hashes = [ + "h1:QOiO85qZnkUm7kAtuPkfblchuKPWUqRdNVWE5agpr8k=", + "zh:076b451dc8629c49f4260de6d43595e98ac5f1bdbebb01d112659ef94d99451f", + "zh:0c29855dbd3c6ba82fce680fa5ac969d4e09e20fecb4ed40166b778bd19895a4", + "zh:583b4dfcea4d8392dd7904c00b2ff41bbae78d238e8b72e5ad580370a24a4ecb", + "zh:5e20844d8d1af052381d00de4febd4055ad0f3c3c02795c361265b9ef72a1075", + "zh:766b7ab7c4727c62b5887c3922e0467c4cc355ba0dc3aabe465ebb86bc1caabb", + "zh:776a5000b441d7c8262d17d4a4aa4aa9760ae64de4cb7172961d9e007e0be1e5", + "zh:7838f509235116e55adeeecbe6def3da1b66dd3c4ce0de02fc7dc66a60e1d630", + "zh:931e5581ec66c145c1d29198bd23fddc8d0c5cbf4cda22e02dba65644c7842f2", + "zh:95e728efa2a31a63b879fd093507466e509e3bfc9325eb35ea3dc28fed15c6f7", + "zh:972b9e3ca2b6a1057dcf5003fc78cabb0dd8847580bddeb52d885ebd64df38ea", + "zh:ef6114217965d55f5bddbd7a316b8f85f15b8a77c075fcbed95813039d522e0a", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/anydatacenter/30-policy-demo/kyvernoPolicies.tf b/anydatacenter/30-policy-demo/kyvernoPolicies.tf new file mode 100644 index 0000000..92a5314 --- /dev/null +++ b/anydatacenter/30-policy-demo/kyvernoPolicies.tf @@ -0,0 +1,11 @@ +resource "kubernetes_manifest" "kyverno_policy_topology_spread" { + # To ensure the correct namespace is used, we could do a patch on the manifest + # For the sake of the demo and to ensure the kyverno tests are running correctly + # the namespace is hardcoded in the policy + manifest = yamldecode(file("${path.module}/kyvernoPolicies/rossumTopologySpread.yaml")) + depends_on = [ + kubernetes_namespace.rossum, + kubernetes_deployment.pre_policy_sleeper, + kubernetes_deployment.pre_policy_sleeper_without_topology_spread, + ] +} diff --git a/anydatacenter/30-policy-demo/kyvernoPolicies/rossumTopologySpread.yaml b/anydatacenter/30-policy-demo/kyvernoPolicies/rossumTopologySpread.yaml new file mode 100644 index 0000000..713713d --- /dev/null +++ b/anydatacenter/30-policy-demo/kyvernoPolicies/rossumTopologySpread.yaml @@ -0,0 +1,116 @@ +apiVersion: kyverno.io/v1 +kind: Policy +metadata: + name: enforce-topology-spread + namespace: rossum + annotations: + policies.kyverno.io/title: Spread pods based on zone topology + policies.kyverno.io/subject: Pod +spec: + # Depending on the desired outcome for the applications, there could be multiple solutions + # to the problem, naturally, the most complex and difficult was chosen in order to: + # 1. Preserve existing topology spread if defined + # 2. Inject topology spread if not defined + # 3. Override any settings of topology spread based on zones + # + # see https://github.com/kyverno/kyverno/issues/10655 for details why it cannot + # be achieved using the merge patch + rules: + # Check if existing zone topology spread is defined and overwrite it + - name: enforce-zone-topology-spread-configuration + match: + any: + - resources: + kinds: + - Deployment + - StatefulSet + selector: + matchLabels: + # Additional validation policies would be needed to ensure the label is present on every resource + app.kubernetes.io/name: "*" + operations: + - CREATE + - UPDATE + mutate: + foreach: + - list: "request.object.spec.template.spec.topologySpreadConstraints || []" + # Use precondition to mutate + preconditions: + any: + - key: "{{ element.topologyKey }}" + operator: Equals + value: topology.kubernetes.io/zone + patchesJson6902: |- + - path: /spec/template/spec/topologySpreadConstraints/{{elementIndex}} + op: replace + value: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: '{{request.object.spec.selector.matchLabels."app.kubernetes.io/name"}}' + # Check if the zone topology spread is not defined and inject it + - name: inject-zone-topology-spread + match: + any: + - resources: + kinds: + - Deployment + - StatefulSet + selector: + matchLabels: + # Additional validation policies would be needed to ensure the label is present on every resource + app.kubernetes.io/name: "*" + operations: + - CREATE + - UPDATE + mutate: + foreach: + - list: "request.object.spec.template.spec.topologySpreadConstraints || []" + # Use precondition to mutate + preconditions: + any: + - key: "{{ request.object.spec.template.spec.topologySpreadConstraints[].topologyKey }}" + operator: AllNotIn + value: + - topology.kubernetes.io/zone + patchesJson6902: |- + - path: /spec/template/spec/topologySpreadConstraints/0 + op: add + value: + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: '{{request.object.spec.selector.matchLabels."app.kubernetes.io/name"}}' + # Create topology spread if it does not exist + - name: create-topology-spread + match: + any: + - resources: + kinds: + - Deployment + - StatefulSet + selector: + matchLabels: + # Additional validation policies would be needed to ensure the label is present on every resource + app.kubernetes.io/name: "*" + operations: + - CREATE + - UPDATE + mutate: + patchStrategicMerge: + spec: + template: + spec: + # Ensure the zone topology spread is present if undefined + +(topologySpreadConstraints): + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + # Depending on the workload and requirements, ScheduleAnyway or DoNotSchedule might be chosen + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: '{{request.object.spec.selector.matchLabels."app.kubernetes.io/name"}}' diff --git a/anydatacenter/30-policy-demo/main.tf b/anydatacenter/30-policy-demo/main.tf new file mode 100644 index 0000000..592a74b --- /dev/null +++ b/anydatacenter/30-policy-demo/main.tf @@ -0,0 +1,13 @@ +terraform { + backend "local" { + path = "30-policy-demo.tfstate" + } +} + +variable "kubectx" { + type = string +} + +provider "kubernetes" { + config_context = var.kubectx +} diff --git a/anydatacenter/30-policy-demo/postPolicy.tf b/anydatacenter/30-policy-demo/postPolicy.tf new file mode 100644 index 0000000..e310161 --- /dev/null +++ b/anydatacenter/30-policy-demo/postPolicy.tf @@ -0,0 +1,55 @@ +resource "kubernetes_deployment" "post_policy_sleeper" { + metadata { + name = "post-policy-sleeper" + namespace = kubernetes_namespace.rossum.metadata[0].name + labels = { + "app.kubernetes.io/name" = "post-policy-sleeper" + "app.kubernetes.io/version" = "v5" + } + } + + spec { + replicas = 3 + + selector { + match_labels = { + "app.kubernetes.io/name" = "post-policy-sleeper" + } + } + + template { + metadata { + labels = { + "app.kubernetes.io/name" = "post-policy-sleeper" + } + } + + spec { + container { + name = "sleepy" + image = "busybox" + command = [ + "sh", + "-c", + "while true; do sleep 60; done" + ] + } + + security_context { + run_as_user = 1000 + run_as_group = 1000 + } + } + } + } + + lifecycle { + ignore_changes = [ + # Injected by kyverno policy on create + spec[0].template[0].spec[0].topology_spread_constraint + ] + } + + # Execute after the kyverno policy is in place + depends_on = [kubernetes_manifest.kyverno_policy_topology_spread] +} diff --git a/anydatacenter/30-policy-demo/prePolicy.tf b/anydatacenter/30-policy-demo/prePolicy.tf new file mode 100644 index 0000000..5d8a166 --- /dev/null +++ b/anydatacenter/30-policy-demo/prePolicy.tf @@ -0,0 +1,110 @@ +# Deployment will be added before the kyverno policy is created +resource "kubernetes_deployment" "pre_policy_sleeper" { + metadata { + name = "pre-policy-sleeper" + namespace = kubernetes_namespace.rossum.metadata[0].name + labels = { + "app.kubernetes.io/name" = "pre-policy-sleeper" + "app.kubernetes.io/version" = "v3" + } + } + + spec { + replicas = 3 + + selector { + match_labels = { + "app.kubernetes.io/name" = "pre-policy-sleeper" + } + } + + template { + metadata { + labels = { + "app.kubernetes.io/name" = "pre-policy-sleeper" + } + } + + spec { + topology_spread_constraint { + max_skew = 1 + topology_key = "topology.kubernetes.io/hostname" + when_unsatisfiable = "ScheduleAnyway" + label_selector { + match_labels = { + "app.kubernetes.io/name" = "pre-policy-sleeper" + } + } + } + + container { + name = "sleepy" + image = "busybox" + command = [ + "sh", + "-c", + "while true; do sleep 60; done" + ] + } + + security_context { + run_as_user = 1000 + run_as_group = 1000 + } + } + } + } +} + +resource "kubernetes_deployment" "pre_policy_sleeper_without_topology_spread" { + metadata { + name = "pre-policy-sleeper-without-topology-spread" + namespace = kubernetes_namespace.rossum.metadata[0].name + labels = { + "app.kubernetes.io/name" = "pre-policy-sleeper-without-topology-spread" + "app.kubernetes.io/version" = "v2" + } + } + + spec { + replicas = 3 + + selector { + match_labels = { + "app.kubernetes.io/name" = "pre-policy-sleeper-without-topology-spread" + } + } + + template { + metadata { + labels = { + "app.kubernetes.io/name" = "pre-policy-sleeper-without-topology-spread" + } + } + + spec { + container { + name = "sleepy" + image = "busybox" + command = [ + "sh", + "-c", + "while true; do sleep 60; done" + ] + } + + security_context { + run_as_user = 1000 + run_as_group = 1000 + } + } + } + } + + lifecycle { + ignore_changes = [ + # Injected by kyverno policy on update + spec[0].template[0].spec[0].topology_spread_constraint + ] + } +} diff --git a/anydatacenter/30-policy-demo/rossum.tf b/anydatacenter/30-policy-demo/rossum.tf new file mode 100644 index 0000000..6b68c2e --- /dev/null +++ b/anydatacenter/30-policy-demo/rossum.tf @@ -0,0 +1,5 @@ +resource "kubernetes_namespace" "rossum" { + metadata { + name = "rossum" + } +} diff --git a/anydatacenter/30-policy-demo/versions.tf b/anydatacenter/30-policy-demo/versions.tf new file mode 100644 index 0000000..adc4ef7 --- /dev/null +++ b/anydatacenter/30-policy-demo/versions.tf @@ -0,0 +1,3 @@ +module "versions" { + source = "../../modules/versions" +}