Un Talos européen de qualité - Part V - Ingress

1
module "kube_ingress" {
2
  source = "../../modules/kube/ingress"
3

4
  scw_dns_access_key = var.scw_dns_username
5
  scw_dns_secret_key = var.scw_dns_password
6

7
  acme_email  = "me@ohmytalos.io"
8
}

1
variable "scw_dns_username" {
2
  type = string
3
}
4

5
variable "scw_dns_password" {
6
  type      = string
7
  sensitive = true
8
}

# ...

export TF_VAR_scw_dns_username=$(bw_field username scw_dns)
export TF_VAR_scw_dns_password=$(bw_field password scw_dns)

1
variable "scw_dns_access_key" {
2
  description = "The access key for the Scaleway DNS API"
3
  type        = string
4
}
5

6
variable "scw_dns_secret_key" {
7
  description = "The secret key for the Scaleway DNS API"
8
  type        = string
9
  sensitive   = true
10
}

1
resource "helm_release" "cert_manager_webhook_scaleway" {
2
  repository = "https://helm.scw.cloud"
3
  chart      = "scaleway-certmanager-webhook"
4
  version    = "0.4.1"
5

6
  name        = "scw"
7
  namespace   = "cert-manager"
8
  max_history = 2
9

10
  set = [
11
    {
12
      name  = "secret.accessKey"
13
      value = var.scw_dns_access_key
14
    }
15
  ]
16

17
  set_sensitive = [
18
    {
19
      name  = "secret.secretKey"
20
      value = var.scw_dns_secret_key
21
    }
22
  ]
23
}
24

25
resource "kubernetes_manifest" "cluster_issuer_letsencrypt_production" {
26
  manifest = {
27
    apiVersion = "cert-manager.io/v1"
28
    kind       = "ClusterIssuer"
29
    metadata = {
30
      name = "letsencrypt-production"
31
    }
32
    spec = {
33
      acme = {
34
        email = var.acme_email
35
        privateKeySecretRef = {
36
          name = "letsencrypt-production"
37
        }
38
        server = "https://acme-v02.api.letsencrypt.org/directory"
39
        solvers = [
40
          {
41
            dns01 = {
42
              webhook = {
43
                groupName  = "acme.scaleway.com"
44
                solverName = "scaleway"
45
              }
46
            }
47
          }
48
        ]
49
      }
50
    }
51
  }
52
  depends_on = [helm_release.cert_manager_webhook_scaleway]
53
}

1
module "kube_ingress" {
2
  // ...
3

4
  crowdsec_bouncer_lapi_key = var.crowdsec_bouncer_lapi_key
5

6
  traefik_http_basic_auth_username = var.traefik_internal_basic_auth_username
7
  traefik_http_basic_auth_password = var.traefik_internal_basic_auth_password
8

9
  traefik_service_annotations = {
10
    for key, value in {
11
      name                  = "${local.cluster_name}-traefik"
12
      type                  = "lb11"
13
      location              = "nbg1"
14
      use-private-ip        = "true"
15
      private-ipv4          = "10.0.1.100"
16
      uses-proxyprotocol    = "true"
17
      health-check-interval = "15s"
18
      health-check-timeout  = "10s"
19
      health-check-retries  = "3"
20
    } :
21
    "load-balancer.hetzner.cloud/${key}" => value
22
  }
23

24
  common_name = "ohmytalos.io"
25
  dns_names = [
26
    "ohmytalos.io",
27
    "*.ohmytalos.io",
28
    "*.dev.ohmytalos.io"
29
  ]
30
  internal_domain = local.internal_domain
31
}

1
// ...
2

3
variable "traefik_internal_basic_auth_username" {
4
  type = string
5
}
6

7
variable "traefik_internal_basic_auth_password" {
8
  type      = string
9
  sensitive = true
10
}
11

12
variable "crowdsec_bouncer_lapi_key" {
13
  type      = string
14
  sensitive = true
15
}

# ...

export TF_VAR_traefik_internal_basic_auth_username=$(bw_field username traefik_internal_basic_auth)
export TF_VAR_traefik_internal_basic_auth_password=$(bw_field password traefik_internal_basic_auth)
export TF_VAR_crowdsec_bouncer_lapi_key=$(bw_field password crowdsec_bouncer_lapi_key)

1
// ...
2

3
variable "common_name" {
4
  description = "The main domain name to use for the certificate"
5
  type        = string
6
}
7

8
variable "dns_names" {
9
  description = "The list of DNS names to use for the certificate"
10
  type        = list(string)
11
}
12

13
variable "traefik_http_basic_auth_username" {
14
  description = "The username for the basic auth"
15
  sensitive   = true
16
}
17

18
variable "traefik_http_basic_auth_password" {
19
  description = "The password for the basic auth"
20
  sensitive   = true
21
}
22

23
variable "internal_domain" {
24
  description = "The internal domain name to use for the private network"
25
  type        = string
26
}
27

28
variable "traefik_service_annotations" {
29
  description = "The annotations to add to the traefik ingress"
30
  type        = map(string)
31
  default     = {}
32
}
33

34
variable "crowdsec_bouncer_lapi_key" {
35
  description = "The API key for bouncer for the crowdsec local API"
36
  type        = string
37
  sensitive   = true
38
}

1
resource "kubernetes_namespace_v1" "traefik" {
2
  metadata {
3
    name = "traefik"
4
  }
5
}
6

7
resource "kubernetes_manifest" "certificate_default_certificate" {
8
  manifest = {
9
    apiVersion = "cert-manager.io/v1"
10
    kind       = "Certificate"
11
    metadata = {
12
      name      = "default-certificate"
13
      namespace = kubernetes_namespace_v1.traefik.metadata[0].name
14
    }
15
    spec = {
16
      commonName = var.common_name
17
      dnsNames   = var.dns_names
18
      issuerRef = {
19
        kind = kubernetes_manifest.cluster_issuer_letsencrypt_production.manifest.kind
20
        name = kubernetes_manifest.cluster_issuer_letsencrypt_production.manifest.metadata.name
21
      }
22
      secretName = "tls-default-certificate"
23
    }
24
  }
25
}
26

27
resource "kubernetes_secret_v1" "internal_basic_auth" {
28
  metadata {
29
    name      = "internal-basic-auth"
30
    namespace = kubernetes_namespace_v1.traefik.metadata[0].name
31
  }
32
  type = "kubernetes.io/basic-auth"
33

34
  data = {
35
    username = var.traefik_http_basic_auth_username
36
    password = var.traefik_http_basic_auth_password
37
  }
38
}
39

40
resource "kubernetes_manifest" "traefik_middleware_internal_basic_auth" {
41
  manifest = {
42
    apiVersion = "traefik.io/v1alpha1"
43
    kind       = "Middleware"
44
    metadata = {
45
      name      = "internal-basic-auth"
46
      namespace = kubernetes_namespace_v1.traefik.metadata[0].name
47
    }
48
    spec = {
49
      basicAuth = {
50
        secret = kubernetes_secret_v1.internal_basic_auth.metadata[0].name
51
      }
52
    }
53
  }
54
}
55

56
resource "kubernetes_manifest" "traefik_middleware_internal_ips" {
57
  manifest = {
58
    apiVersion = "traefik.io/v1alpha1"
59
    kind       = "Middleware"
60
    metadata = {
61
      name      = "internal-ips"
62
      namespace = kubernetes_namespace_v1.traefik.metadata[0].name
63
    }
64
    spec = {
65
      ipWhiteList = {
66
        sourceRange = [
67
          "127.0.0.1/32",
68
          "100.64.0.0/10",
69
        ]
70
      }
71
    }
72
  }
73
}
74

75
resource "kubernetes_manifest" "traefik_middleware_crowdsec_bouncer" {
76
  manifest = {
77
    apiVersion = "traefik.io/v1alpha1"
78
    kind       = "Middleware"
79
    metadata = {
80
      name      = "crowdsec-bouncer"
81
      namespace = kubernetes_namespace_v1.traefik.metadata[0].name
82
    }
83
    spec = {
84
      plugin = {
85
        bouncer = {
86
          enabled                        = true
87
          crowdsecMode                   = "appsec"
88
          crowdsecAppsecEnabled          = true
89
          crowdsecAppsecHost             = "crowdsec-appsec-service.crowdsec:7422"
90
          crowdsecLapiScheme             = "http"
91
          crowdsecLapiHost               = "crowdsec-service.crowdsec:8080"
92
          crowdsecLapiKey                = var.crowdsec_bouncer_lapi_key
93
          crowdsecAppsecUnreachableBlock = false
94
        }
95
      }
96
    }
97
  }
98
}
99

100
resource "kubernetes_manifest" "traefik_middleware_compress" {
101
  manifest = {
102
    apiVersion = "traefik.io/v1alpha1"
103
    kind       = "Middleware"
104
    metadata = {
105
      name      = "compress"
106
      namespace = kubernetes_namespace_v1.traefik.metadata[0].name
107
    }
108
    spec = {
109
      compress = {}
110
    }
111
  }
112
}
113

114
resource "helm_release" "traefik" {
115
  repository = "https://traefik.github.io/charts"
116
  chart      = "traefik"
117
  version    = "39.0.0"
118

119
  name        = "traefik"
120
  namespace   = kubernetes_namespace_v1.traefik.metadata[0].name
121
  max_history = 2
122

123
  set = [
124
    {
125
      name  = "deployment.kind"
126
      value = "DaemonSet"
127
    },
128
    {
129
      name  = "providers.kubernetesCRD.allowCrossNamespace"
130
      value = "true"
131
    },
132
    {
133
      name  = "ingressRoute.dashboard.enabled"
134
      value = "true"
135
    },
136
    {
137
      name  = "ingressRoute.dashboard.matchRule"
138
      value = "Host(`traefik.${var.internal_domain}`)"
139
    },
140
    {
141
      name  = "ingressRoute.dashboard.middlewares[0].name"
142
      value = kubernetes_manifest.traefik_middleware_internal_basic_auth.manifest.metadata.name
143
    },
144
    {
145
      name  = "tlsStore.default.defaultCertificate.secretName"
146
      value = kubernetes_manifest.certificate_default_certificate.manifest.spec.secretName
147
    },
148
    {
149
      name  = "logs.general.level"
150
      value = "FATAL"
151
    },
152
    {
153
      name  = "logs.access.enabled"
154
      value = "true"
155
    },
156
    {
157
      name  = "logs.access.format"
158
      value = "json"
159
    },
160
    {
161
      name  = "experimental.plugins.bouncer.moduleName"
162
      value = "github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin"
163
    },
164
    {
165
      name  = "experimental.plugins.bouncer.version"
166
      value = "v1.5.0"
167
    },
168
    {
169
      name  = "metrics.prometheus.serviceMonitor.enabled"
170
      value = "true"
171
    },
172
    {
173
      name  = "tracing.addInternals"
174
      value = "true"
175
    },
176
    {
177
      name  = "tracing.otlp.enabled"
178
      value = "true"
179
    },
180
    {
181
      name  = "tracing.otlp.http.enabled"
182
      value = "true"
183
    },
184
    {
185
      name  = "tracing.otlp.http.endpoint"
186
      value = "http://alloy.telemetry:4318/v1/traces"
187
    },
188
    {
189
      name  = "ports.web.http.redirections.entryPoint.to"
190
      value = "websecure"
191
    },
192
    {
193
      name  = "ports.web.http.redirections.entryPoint.scheme"
194
      value = "https"
195
    },
196
    {
197
      name  = "ports.web.http.redirections.entryPoint.permanent"
198
      value = "true"
199
    },
200
    {
201
      name  = "ports.websecure.asDefault"
202
      value = "true"
203
    },
204
    {
205
      name  = "ports.websecure.transport.respondingTimeouts.readTimeout"
206
      value = "300s"
207
    },
208
    {
209
      name  = "ports.ssh.port"
210
      value = "2222"
211
    },
212
    {
213
      name  = "ports.ssh.exposedPort"
214
      value = "22"
215
    },
216
    {
217
      name  = "ports.ssh.expose.default"
218
      value = "true"
219
    },
220
    {
221
      name  = "ports.internal.port"
222
      value = "9443"
223
    },
224
    {
225
      name  = "ports.internal.exposedPort"
226
      value = "443"
227
    },
228
    {
229
      name  = "ports.internal.expose.internal"
230
      value = "true"
231
    },
232
    {
233
      name  = "ports.internal.http.tls.enabled"
234
      value = "true"
235
    },
236
    {
237
      name  = "service.type"
238
      value = length(var.traefik_service_annotations) == 0 ? "ClusterIP" : "LoadBalancer"
239
    },
240
    {
241
      name  = "service.additionalServices.internal.type"
242
      value = "ClusterIP"
243
    }
244
  ]
245

246
  set_list = concat(
247
    [
248
      {
249
        name  = "ports.internal.http.middlewares"
250
        value = ["traefik-${kubernetes_manifest.traefik_middleware_internal_ips.manifest.metadata.name}@kubernetescrd"]
251
      },
252
      {
253
        name  = "ingressRoute.dashboard.entryPoints"
254
        value = ["internal"]
255
      },
256
      {
257
        name = "ports.websecure.http.middlewares"
258
        value = [
259
          "traefik-${kubernetes_manifest.traefik_middleware_crowdsec_bouncer.manifest.metadata.name}@kubernetescrd",
260
          "traefik-${kubernetes_manifest.traefik_middleware_compress.manifest.metadata.name}@kubernetescrd"
261
        ]
262
      }
263
    ],
264
    [
265
      for entry_point in ["ssh", "web", "websecure", "internal"] : {
266
        name  = "ports.${entry_point}.proxyProtocol.trustedIPs"
267
        value = ["127.0.0.1/32", "10.0.0.0/8"]
268
      }
269
    ],
270
    [
271
      for entry_point in ["ssh", "web", "websecure", "internal"] : {
272
        name  = "ports.${entry_point}.forwardedHeaders.trustedIPs"
273
        value = ["127.0.0.1/32", "10.0.0.0/8"]
274
      }
275
    ]
276
  )
277

278
  values = [
279
    yamlencode({
280
      service = {
281
        annotations = var.traefik_service_annotations
282
      }
283
    })
284
  ]
285
}

1
terraform {
2
  required_providers {
3
    hcloud = {
4
      source  = "hetznercloud/hcloud"
5
      version = ">= 1.43.0"
6
    }
7
  }
8

9
  // ...
10
}
11

12
// ...
13

14
provider "hcloud" {
15
  token = var.hcloud_token
16
}

1
data "hcloud_servers" "workers" {
2
  with_selector = "type=worker"
3
}
4

5
data "hcloud_servers" "control_planes" {
6
  with_selector = "type=controlplane"
7
}
8

9
module "kube_ingress" {
10
  // ...
11

12
  crowdsec_enroll_key           = var.crowdsec_enroll_key
13
  crowdsec_enroll_instance_name = local.cluster_name
14

15
  crowdsec_whitelist_ips = concat(
16
    [for s in data.hcloud_servers.control_planes.servers : s.ipv4_address],
17
    [for s in data.hcloud_servers.workers.servers : s.ipv4_address],
18
    [for s in data.hcloud_servers.control_planes.servers : s.ipv6_address],
19
    [for s in data.hcloud_servers.workers.servers : s.ipv6_address]
20
  )
21

22
  crowdsec_whitelist_rule_ids = [
23
    911100,
24
    920420,
25
    920450,
26
  ]
27
}

1
// ...
2
variable "crowdsec_enroll_key" {
3
  type      = string
4
  sensitive = true
5
}

# ...

export TF_VAR_crowdsec_bouncer_lapi_key=$(bw_field password crowdsec_bouncer_lapi_key)

1
// ...
2

3
variable "crowdsec_enroll_key" {
4
  description = "The enroll key for the crowdsec agent"
5
  type        = string
6
  sensitive   = true
7
}
8

9
variable "crowdsec_enroll_instance_name" {
10
  description = "The instance name to use for the crowdsec agent enrollment"
11
  type        = string
12
}
13

14
variable "crowdsec_whitelist_ips" {
15
  description = "The public IPs to whitelist in crowdsec"
16
  type        = list(string)
17
  default     = []
18
}
19

20
variable "crowdsec_whitelist_rule_ids" {
21
  description = "The IDs of the crowdsec rules to whitelist"
22
  type        = list(number)
23
  default     = []
24
}

1
resource "kubernetes_namespace_v1" "crowdsec" {
2
  metadata {
3
    name = "crowdsec"
4
    labels = {
5
      "pod-security.kubernetes.io/enforce" = "privileged"
6
    }
7
  }
8
}
9

10
resource "helm_release" "crowdsec" {
11
  repository = "https://crowdsecurity.github.io/helm-charts"
12
  chart      = "crowdsec"
13
  version    = "0.21.1"
14

15
  name        = "crowdsec"
16
  namespace   = kubernetes_namespace_v1.crowdsec.metadata[0].name
17
  max_history = 2
18

19
  set = [
20
    {
21
      name  = "container_runtime"
22
      value = "containerd"
23
    },
24
    {
25
      name  = "agent.acquisition[0].namespace"
26
      value = "traefik"
27
    },
28
    {
29
      name  = "agent.acquisition[0].podName"
30
      value = "traefik-*"
31
    },
32
    {
33
      name  = "agent.acquisition[0].program"
34
      value = "traefik"
35
    },
36
    {
37
      name  = "agent.acquisition[0].poll_without_inotify"
38
      value = "true"
39
    },
40
    {
41
      name  = "agent.env[0].name"
42
      value = "COLLECTIONS"
43
    },
44
    {
45
      name  = "agent.env[0].value"
46
      value = "crowdsecurity/traefik"
47
    },
48
    {
49
      name  = "agent.env[1].name"
50
      value = "POSTOVERFLOWS"
51
    },
52
    {
53
      name  = "agent.env[1].value"
54
      value = "crowdsecurity/seo-bots-whitelist"
55
    },
56
    {
57
      name  = "lapi.metrics.serviceMonitor.enabled"
58
      value = "true"
59
    },
60
    {
61
      name  = "agent.metrics.serviceMonitor.enabled"
62
      value = "true"
63
    },
64
    {
65
      name  = "lapi.persistentVolume.data.storageClassName"
66
      value = "longhorn-crypto"
67
    },
68
    {
69
      name  = "lapi.persistentVolume.config.storageClassName"
70
      value = "longhorn-crypto"
71
    },
72
    {
73
      name  = "lapi.env[0].name"
74
      value = "BOUNCER_KEY_traefik"
75
    },
76
    {
77
      name  = "lapi.env[0].value"
78
      value = var.crowdsec_bouncer_lapi_key
79
    },
80
    {
81
      name  = "lapi.env[1].name"
82
      value = "ENROLL_KEY"
83
    },
84
    {
85
      name  = "lapi.env[1].value"
86
      value = var.crowdsec_enroll_key
87
    },
88
    {
89
      name  = "lapi.env[2].name"
90
      value = "ENROLL_INSTANCE_NAME"
91
    },
92
    {
93
      name  = "lapi.env[2].value"
94
      value = var.crowdsec_enroll_instance_name
95
    },
96
    {
97
      name = "config.parsers.s02-enrich.01-my-whitelist\\.yaml"
98
      value = yamlencode({
99
        name        = "my/whitelist"
100
        description = "Whitelist events from my IPs"
101
        whitelist = {
102
          reason = "My IPs"
103
          ip     = var.crowdsec_whitelist_ips
104
        }
105
      })
106
    },
107
    {
108
      name  = "appsec.enabled"
109
      value = "true"
110
    },
111
    {
112
      name  = "appsec.env[0].name"
113
      value = "COLLECTIONS"
114
    },
115
    {
116
      name  = "appsec.env[0].value"
117
      value = "crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-crs-inband"
118
    },
119
    {
120
      name = "appsec.configs.my-whitelist-rules\\.yaml"
121
      value = yamlencode({
122
        name = "my/whitelist-rules"
123
        on_load = [
124
          {
125
            apply = [
126
              for id in var.crowdsec_whitelist_rule_ids : "RemoveInBandRuleByID(${id})"
127
            ]
128
          }
129
        ]
130
      })
131
    },
132
  ]
133

134
  values = [
135
    yamlencode({
136
      appsec = {
137
        acquisitions = [
138
          {
139
            source      = "appsec"
140
            listen_addr = "0.0.0.0:7422"
141
            path        = "/"
142
            appsec_configs = [
143
              "crowdsecurity/virtual-patching",
144
              "crowdsecurity/crs-inband",
145
              "my/whitelist-rules"
146
            ]
147
            labels = {
148
              type = "appsec"
149
            }
150
          }
151
        ]
152
      }
153
    })
154
  ]
155
}