Un Talos européen de qualité - Part II - Infra physique

1
terraform {
2
  required_version = ">= 1.13.0"
3
  required_providers {
4
    hcloud = {
5
      source  = "hetznercloud/hcloud"
6
      version = ">= 1.52.0"
7
    }
8
    talos = {
9
      source  = "siderolabs/talos"
10
      version = ">= 0.10.0"
11
    }
12
  }
13
}
14

15
provider "hcloud" {
16
  token = var.hcloud_token
17
}

1
variable "hcloud_token" {
2
  type      = string
3
  sensitive = true
4
}

1
terraform {
2
  required_version = ">= 1.13.0"
3

4
  backend "s3" {
5
    endpoints = {
6
      s3 = "https://s3.gra.io.cloud.ovh.net"
7
    }
8
    skip_credentials_validation = true
9
    skip_region_validation      = true
10
    skip_requesting_account_id  = true
11
    skip_s3_checksum            = true
12
    region                      = "gra"
13
    bucket                      = "ohmytalos-dev"
14
    key                         = "terraform/hcloud.tfstate"
15
    encrypt                     = true
16
  }
17
}

1
variable "hcloud_token" {
2
  type      = string
3
  sensitive = true
4
}
5

6
variable "ts_auth_key" {
7
  type      = string
8
  sensitive = true
9
}
10

11
variable "volume_encryption_passphrase" {
12
  type      = string
13
  sensitive = true
14
}

1
module "hcloud_talos" {
2
  source = "../../modules/hcloud"
3

4
  hcloud_token = var.hcloud_token
5
}

1
module "hcloud_talos" {
2
  source = "../../modules/hcloud"
3

4
  hcloud_token = var.hcloud_token
5

6
  talos_version = "v1.12.2"
7
  talos_snapshots = [
8
    {
9
      name         = "cp",
10
      schematic_id = "4a0d65c669d46663f377e7161e50cfd570c401f26fd9e7bda34a0216b6f1922b"
11
    },
12
    {
13
      name         = "wk",
14
      schematic_id = "613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245"
15
    },
16
  ]
17

18
  kubernetes_version = "v1.35.0"
19
  cluster_name       = "ohmytalos-dev"
20
  network_zone       = "eu-central"
21
  network_ipv4_cidr  = "10.0.0.0/10"
22

23
  config_patches = [
24
    file("${path.module}/patches/volume-state.yaml"),
25
    templatefile("${path.module}/patches/volume-ephemeral.yaml", {
26
      volume_encryption_passphrase = var.volume_encryption_passphrase
27
    }),
28
    file("${path.module}/patches/config-cluster.yaml"),
29
  ]
30

31
  control_planes_ipv4_cidr  = "10.0.0.0/24"
32
  control_planes_image_name = "cp"
33
  control_planes_config_patches = [
34
    templatefile("${path.module}/patches/extension-tailscale.yaml", {
35
      ts_auth_key = var.ts_auth_key
36
    }),
37
    yamlencode({
38
      cluster = {
39
        extraManifests = [
40
          "https://raw.githubusercontent.com/alex1989hu/kubelet-serving-cert-approver/main/deploy/standalone-install.yaml"
41
        ]
42
      }
43
    })
44
  ]
45

46
  control_planes = [
47
    {
48
      name        = "nbg1"
49
      server_type = "cpx22"
50
      location    = "nbg1"
51
      ip          = "10.0.0.2"
52
    },
53
    {
54
      name        = "fsn1"
55
      server_type = "cpx22"
56
      location    = "fsn1"
57
      ip          = "10.0.0.3"
58
    },
59
    {
60
      name        = "hel1"
61
      server_type = "cpx22"
62
      location    = "hel1"
63
      ip          = "10.0.0.4"
64
    }
65
  ]
66

67
  worker_nodepools = [
68
    {
69
      name            = "worker"
70
      server_type     = "cpx32"
71
      location        = "nbg1"
72
      image_name      = "wk"
73
      placement_group = "workers"
74
      ipv4_cidr       = "10.0.1.0/24"
75
      nodes = [
76
        { name = "ndk", ip = "10.0.1.1" },
77
        { name = "opb", ip = "10.0.1.2" },
78
        { name = "ozi", ip = "10.0.1.3" }
79
      ]
80
      config_patches = [
81
        file("${path.module}/patches/config-machine-worker.yaml")
82
      ]
83
    },
84
    {
85
      name            = "storage"
86
      server_type     = "cpx32"
87
      location        = "nbg1"
88
      image_name      = "wk"
89
      placement_group = "storages"
90
      ipv4_cidr       = "10.0.2.0/24"
91
      nodes = [
92
        { name = "vsh", ip = "10.0.2.1" },
93
        { name = "wty", ip = "10.0.2.2" }
94
      ]
95
      volumes = [
96
        { name = "longhorn", size = 80 }
97
      ]
98
      config_patches = [
99
        file("${path.module}/patches/config-machine-storage.yaml")
100
      ]
101
      config_patches_apply = [
102
        templatefile("${path.module}/patches/volume-longhorn.yaml", {
103
          volume_encryption_passphrase = var.volume_encryption_passphrase
104
        })
105
      ]
106
    }
107
  ]
108
}

1
cluster:
2
  externalCloudProvider:
3
    enabled: true
4
  network:
5
    cni:
6
      name: none
7
    podSubnets:
8
      - 10.42.0.0/16
9
    serviceSubnets:
10
      - 10.43.0.0/16
11
  proxy:
12
    disabled: true
13
machine:
14
  kubelet:
15
    extraConfig:
16
      imageGCLowThresholdPercent: 50
17
      imageGCHighThresholdPercent: 55

1
machine:
2
  nodeLabels:
3
    node.longhorn.io/create-default-disk: config
4
  nodeAnnotations:
5
    node.longhorn.io/default-disks-config: '[{"allowScheduling":true,"name":"system","path":"/var/lib/longhorn","tags":["local"]}]'
6
    node.longhorn.io/default-node-tags: '["worker"]'

1
machine:
2
  nodeLabels:
3
    node.kubernetes.io/exclude-from-external-load-balancers: "true"
4
    node.kubernetes.io/role: storage
5
    node.longhorn.io/create-default-disk: config
6
  nodeAnnotations:
7
    node.longhorn.io/default-disks-config: '[{"allowScheduling":true,"name":"system","path":"/var/lib/longhorn","tags":["local"]},{"allowScheduling":true,"name":"volume","path":"/var/mnt/longhorn","tags":["volume"]}]'
8
    node.longhorn.io/default-node-tags: '["storage"]'
9
  kubelet:
10
    extraConfig:
11
      registerWithTaints:
12
        - key: node-role.kubernetes.io/storage
13
          effect: NoSchedule

1
apiVersion: v1alpha1
2
kind: VolumeConfig
3
name: STATE
4
encryption:
5
  provider: luks2
6
  keys:
7
    - slot: 0
8
      nodeID: {}

1
apiVersion: v1alpha1
2
kind: VolumeConfig
3
name: EPHEMERAL
4
encryption:
5
  provider: luks2
6
  keys:
7
    - slot: 0
8
      static:
9
        passphrase: ${volume_encryption_passphrase}
10
      lockToState: true

1
apiVersion: v1alpha1
2
kind: UserVolumeConfig
3
name: longhorn
4
provisioning:
5
  diskSelector:
6
    match: disk.dev_path == '/dev/sdb'
7
  grow: true
8
  minSize: 40Gi
9
encryption:
10
  provider: luks2
11
  keys:
12
    - slot: 0
13
      static:
14
        passphrase: ${volume_longhorn_passphrase}
15
      lockToState: true

1
apiVersion: v1alpha1
2
kind: ExtensionServiceConfig
3
name: tailscale
4
environment:
5
  - TS_AUTHKEY=${ts_auth_key}

1
locals {
2
  talos_snapshots = { for s in var.talos_snapshots : s.name => s.schematic_id }
3
  talos_endpoints = length(var.talos_endpoints) > 0 ? var.talos_endpoints : [for s in local.control_planes : "${var.cluster_name}-${s.name}"]
4

5
  machine_base_config = {
6
    kubelet = {
7
      nodeIP = {
8
        validSubnets = concat(
9
          [
10
            var.control_planes_ipv4_cidr,
11
          ],
12
          [
13
            for s in hcloud_network_subnet.worker : s.ip_range
14
          ]
15
        )
16
      }
17
      extraArgs = {
18
        "rotate-server-certificates" = true
19
      }
20
    }
21
    features = {
22
      hostDNS = {
23
        enabled              = true
24
        forwardKubeDNSToHost = true
25
        resolveMemberNames   = true
26
      }
27
    }
28
    time = {
29
      servers = [
30
        "ntp1.hetzner.de",
31
        "ntp2.hetzner.com",
32
        "ntp3.hetzner.net",
33
        "time.cloudflare.com"
34
      ]
35
    }
36
  }
37

38
  machine_control_plane_config = merge(local.machine_base_config, {
39
    features = merge(local.machine_base_config.features, {
40
      kubernetesTalosAPIAccess = {
41
        enabled = true
42
        allowedRoles = [
43
          "os:reader",
44
          "os:etcd:backup"
45
        ]
46
        allowedKubernetesNamespaces = [
47
          "kube-system",
48
        ]
49
      }
50
    })
51
  })
52

53
  cluster_control_plane_config = {
54
    controllerManager = {
55
      extraArgs = {
56
        "bind-address" = "0.0.0.0"
57
      }
58
    }
59
    etcd = {
60
      advertisedSubnets = [
61
        var.control_planes_ipv4_cidr
62
      ]
63
      extraArgs = {
64
        "listen-metrics-urls" = "http://0.0.0.0:2381"
65
      }
66
    }
67
    scheduler = {
68
      extraArgs = {
69
        "bind-address" = "0.0.0.0"
70
      }
71
    }
72
  }
73

74
  control_planes = [
75
    for i, s in var.control_planes : {
76
      name                 = "control-plane-${s.name}"
77
      server_type          = s.server_type
78
      location             = s.location
79
      machine_type         = "controlplane"
80
      firewall_ids         = [hcloud_firewall.talos_api.id, hcloud_firewall.kube_api.id]
81
      private_ipv4         = s.ip
82
      placement_group_id   = var.control_planes_placement_group != null ? hcloud_placement_group.this[var.control_planes_placement_group].id : null
83
      image_name           = coalesce(var.control_planes_image_name, "default")
84
      machine_config       = local.machine_control_plane_config
85
      cluster_config       = local.cluster_control_plane_config
86
      config_patches       = var.control_planes_config_patches
87
      config_patches_apply = []
88
    }
89
  ]
90
  workers = flatten([
91
    for i, np in var.worker_nodepools : [
92
      for index, n in np.nodes : {
93
        name                 = "${np.name}-${n.name}"
94
        server_type          = coalesce(n.server_type, np.server_type)
95
        location             = coalesce(n.location, np.location)
96
        machine_type         = "worker"
97
        firewall_ids         = [hcloud_firewall.talos_api.id]
98
        private_ipv4         = n.ip
99
        placement_group_id   = np.placement_group != null ? hcloud_placement_group.this[np.placement_group].id : null
100
        image_name           = coalesce(np.image_name, "default")
101
        machine_config       = local.machine_base_config
102
        cluster_config       = {}
103
        config_patches       = coalesce(np.config_patches, [])
104
        config_patches_apply = coalesce(np.config_patches_apply, [])
105
        volumes              = coalesce(np.volumes, [])
106
      }
107
    ]
108
  ])
109

110
  servers = [for s in concat(local.control_planes, local.workers) : merge(s, {
111
    config_patches = concat(
112
      [
113
        yamlencode({
114
          machine = merge(
115
            s.machine_config,
116
            {
117
              install = {
118
                image = "factory.talos.dev/hcloud-installer/${local.talos_snapshots[s.image_name]}:${var.talos_version}"
119
              }
120
            }
121
          )
122
          cluster = s.cluster_config
123
        })
124
      ],
125
      var.config_patches,
126
      coalesce(s.config_patches, [])
127
    )
128
  })]
129

130
  volumes = flatten([
131
    for i, s in local.workers : [
132
      for v in s.volumes : {
133
        server_name = s.name
134
        location    = s.location
135
        name        = "${s.name}-${v.name}"
136
        size        = v.size
137
      }
138
    ]
139
  ])
140
}

1
resource "hcloud_network" "this" {
2
  count    = var.existing_network_id == null ? 1 : 0
3
  name     = var.cluster_name
4
  ip_range = var.network_ipv4_cidr
5
}
6

7
resource "hcloud_network_subnet" "control_plane" {
8
  network_id   = coalesce(var.existing_network_id, hcloud_network.this[0].id)
9
  type         = "cloud"
10
  network_zone = var.network_zone
11
  ip_range     = var.control_planes_ipv4_cidr
12
}
13

14
resource "hcloud_network_subnet" "worker" {
15
  for_each     = { for np in var.worker_nodepools : np.name => np }
16
  network_id   = coalesce(var.existing_network_id, hcloud_network.this[0].id)
17
  type         = "cloud"
18
  network_zone = var.network_zone
19
  ip_range     = each.value.ipv4_cidr
20
}

1
resource "hcloud_firewall" "talos_api" {
2
  name = "${var.cluster_name}-talos-api"
3

4
  rule {
5
    description = "Allow Incoming Talos API Traffic"
6
    direction   = "in"
7
    protocol    = "tcp"
8
    port        = "50000"
9
    source_ips  = var.firewall_talos_api_source
10
  }
11
}
12

13
resource "hcloud_firewall" "kube_api" {
14
  name = "${var.cluster_name}-kube-api"
15

16
  rule {
17
    description = "Allow Incoming Requests to Kube API Server"
18
    direction   = "in"
19
    protocol    = "tcp"
20
    port        = "6443"
21
    source_ips  = var.firewall_kube_api_source
22
  }
23
}

1
data "hcloud_image" "talos_x86_snapshot" {
2
  for_each          = local.talos_snapshots
3
  with_selector     = "version=${var.talos_version},name=${each.key}"
4
  with_architecture = "x86"
5
  most_recent       = true
6
}
7

8
data "hcloud_image" "talos_arm_snapshot" {
9
  for_each          = local.talos_snapshots
10
  with_selector     = "version=${var.talos_version},name=${each.key}"
11
  with_architecture = "arm"
12
  most_recent       = true
13
}
14

15
resource "hcloud_server" "this" {
16
  for_each           = { for s in local.servers : s.name => s }
17
  name               = "${var.cluster_name}-${each.key}"
18
  server_type        = each.value.server_type
19
  location           = each.value.location
20
  image              = substr(each.value.server_type, 0, 3) == "cax" ? data.hcloud_image.talos_arm_snapshot[each.value.image_name].id : data.hcloud_image.talos_x86_snapshot[each.value.image_name].id
21

22
  placement_group_id = each.value.placement_group_id
23
  firewall_ids       = each.value.firewall_ids
24

25
  network {
26
    network_id = coalesce(var.existing_network_id, hcloud_network.this[0].id)
27
    ip         = each.value.private_ipv4
28
    alias_ips  = []
29
  }
30

31
  user_data = data.talos_machine_configuration.this[each.value.name].machine_configuration
32

33
  labels = {
34
    type = each.value.machine_type
35
  }
36

37
  depends_on = [
38
    hcloud_network_subnet.control_plane,
39
    hcloud_network_subnet.worker
40
  ]
41

42
  lifecycle {
43
    ignore_changes = [
44
      user_data,
45
      image
46
    ]
47
  }
48
}
49

50
resource "hcloud_placement_group" "this" {
51
  for_each = { for pg in distinct(concat(
52
    compact([
53
      var.control_planes_placement_group
54
    ]),
55
    [
56
      for s in var.worker_nodepools : s.placement_group if s.placement_group != null
57
    ])
58
  ) : pg => pg }
59
  name = "${var.cluster_name}-${each.value}"
60
  type = "spread"
61
}