diff --git a/charts/apps/templates/ingresscontroller/traefik2.yaml b/charts/apps/templates/ingresscontroller/traefik2.yaml index 61c67479e8b7c163ddcf552c2b660749e4358480..fe0f1622cf4c8c8687a1f1a4738d138a2c3a7a1b 100644 --- a/charts/apps/templates/ingresscontroller/traefik2.yaml +++ b/charts/apps/templates/ingresscontroller/traefik2.yaml @@ -44,6 +44,9 @@ spec: tls: enabled: true certResolver: letsencrypt + {{- if .Values.ingresscontroller.traefik2.ports }} + {{- .Values.ingresscontroller.traefik2.ports | toYaml | nindent 10 }} + {{- end }} additionalArguments: - --providers.kubernetesingress.ingressendpoint.ip={{ .Values.ingresscontroller.publicIP }} {{- if .Values.ingresscontroller.acme }} diff --git a/charts/apps/values.yaml b/charts/apps/values.yaml index ed3fed889dcdbb35c27ace3dee5e26edf817273b..d7a86e90c080f5b6fd983f76081318607624fc87 100644 --- a/charts/apps/values.yaml +++ b/charts/apps/values.yaml @@ -26,6 +26,7 @@ healthmonitor: ingresscontroller: enabled: false + class: traefik2 publicIP: 1.1.1.1 #privateIP: 1.1.1.2 #acme: @@ -35,6 +36,12 @@ ingresscontroller: version: "1.*" traefik2: version: "10.*" + ports: {} + # postgres: + # port: 5432 + # expose: true + # exposedPort: 5432 + # protocol: TCP sealedsecrets: enabled: false diff --git a/terraform/modules/rke2/key.tf b/terraform/modules/rke2/key.tf index e4a0ddff5090d627913c821d875f64a52aa8aa21..7bee3731b52c4aad74c958ed86485b5f905b94c0 100644 --- a/terraform/modules/rke2/key.tf +++ b/terraform/modules/rke2/key.tf @@ -2,7 +2,7 @@ # unique key. resource "openstack_compute_keypair_v2" "key" { #count = 1 #var.openstack_ssh_key == "" ? 0 : 1 - name = var.cluster_name + name = var.cluster_name } # set local variable to hold final key, either created or diff --git a/terraform/modules/rke2/network.tf b/terraform/modules/rke2/network.tf index 321790fad7604868fd0b4fd3cffe8ea5d45622b5..f8d697a3eff2f52f082ae7619d108b5a75f71e92 100644 --- a/terraform/modules/rke2/network.tf +++ b/terraform/modules/rke2/network.tf @@ -70,18 +70,18 @@ resource "openstack_networking_port_v2" "controlplane_ip_public" { # create a port that will be used with the floating ip, this will be associated # with all of the VMs. resource "openstack_networking_port_v2" "floating_ip" { - count = var.floating_ip - depends_on = [ openstack_networking_subnet_v2.cluster_subnet ] - name = format("%s-floating-ip-%02d", var.cluster_name, count.index + 1) - network_id = openstack_networking_network_v2.cluster_net.id + count = var.floating_ip + depends_on = [openstack_networking_subnet_v2.cluster_subnet] + name = format("%s-floating-ip-%02d", var.cluster_name, count.index + 1) + network_id = openstack_networking_network_v2.cluster_net.id } # create floating ip that is associated with a fixed ip resource "openstack_networking_floatingip_v2" "floating_ip" { - count = var.floating_ip + count = var.floating_ip description = format("%s-floating-ip-%02d", var.cluster_name, count.index + 1) - pool = data.openstack_networking_network_v2.ext_net.name - port_id = element(openstack_networking_port_v2.floating_ip.*.id, count.index) + pool = data.openstack_networking_network_v2.ext_net.name + port_id = element(openstack_networking_port_v2.floating_ip.*.id, count.index) } # create worker ip, this can route the ports for the floating ip as diff --git a/terraform/modules/rke2/nodes.tf b/terraform/modules/rke2/nodes.tf index 5f97234a6c5ae772327ca6aa847e42a4d5ed834e..0d775ea0ff89df26856dd5051d0eba4cae5b92ae 100644 --- a/terraform/modules/rke2/nodes.tf +++ b/terraform/modules/rke2/nodes.tf @@ -2,20 +2,20 @@ # control-plane nodes # ---------------------------------------------------------------------- resource "openstack_compute_instance_v2" "controlplane" { - count = var.controlplane_count - depends_on = [ + count = var.controlplane_count + depends_on = [ openstack_networking_secgroup_rule_v2.same_security_group_ingress_tcp, ] - name = format("%s-controlplane-%d", var.cluster_name, count.index + 1) - image_name = var.os - flavor_name = var.controlplane_flavor - key_pair = local.key + name = format("%s-controlplane-%d", var.cluster_name, count.index + 1) + image_name = var.os + flavor_name = var.controlplane_flavor + key_pair = local.key security_groups = [ openstack_networking_secgroup_v2.cluster_security_group.name ] - config_drive = false + config_drive = false - user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", { + user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", { private_key = openstack_compute_keypair_v2.key.private_key project_name = data.openstack_identity_auth_scope_v3.scope.project_name cluster_name = var.cluster_name @@ -54,8 +54,8 @@ resource "openstack_compute_instance_v2" "controlplane" { # ---------------------------------------------------------------------- resource "openstack_compute_instance_v2" "worker" { - count = var.worker_count - depends_on = [ + count = var.worker_count + depends_on = [ openstack_networking_secgroup_rule_v2.same_security_group_ingress_tcp, openstack_networking_port_v2.controlplane_ip ] @@ -63,9 +63,11 @@ resource "openstack_compute_instance_v2" "worker" { flavor_name = var.worker_flavor key_pair = local.key config_drive = false - security_groups = [ openstack_networking_secgroup_v2.cluster_security_group.name ] + security_groups = [ + openstack_networking_secgroup_v2.cluster_security_group.name + ] - user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", { + user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", { private_key = openstack_compute_keypair_v2.key.private_key project_name = data.openstack_identity_auth_scope_v3.scope.project_name cluster_name = var.cluster_name diff --git a/terraform/modules/rke2/outputs.tf b/terraform/modules/rke2/outputs.tf index 6591779b9d8dba10e680aefaaebefa2c8d873a08..4033a2a279d6602346d19c3b61da115d179f6788 100644 --- a/terraform/modules/rke2/outputs.tf +++ b/terraform/modules/rke2/outputs.tf @@ -19,7 +19,7 @@ output "ssh_config" { value = <<-EOT # Automatically created by terraform -%{~ for i, x in openstack_compute_instance_v2.controlplane.* } +%{~for i, x in openstack_compute_instance_v2.controlplane.*} Host ${x.name} HostName ${openstack_networking_floatingip_v2.controlplane_ip[i].address} StrictHostKeyChecking no @@ -27,8 +27,8 @@ Host ${x.name} IdentityFile ${pathexpand("~/.ssh/${var.cluster_name}.pem")} User centos -%{~ endfor } -%{~ for x in openstack_compute_instance_v2.worker.* } +%{~endfor} +%{~for x in openstack_compute_instance_v2.worker.*} Host ${x.name} HostName ${x.network[0].fixed_ip_v4} StrictHostKeyChecking no @@ -37,7 +37,7 @@ Host ${x.name} IdentityFile ${pathexpand("~/.ssh/${var.cluster_name}.pem")} User centos -%{~ endfor } +%{~endfor} EOT } @@ -54,7 +54,7 @@ output "kube_id" { output "floating_ip" { description = "Map for floating ips and associated private ips" - value = [ + value = [ for i, ip in openstack_networking_floatingip_v2.floating_ip.*.address : { private_ip = element(flatten(openstack_networking_port_v2.floating_ip.*.all_fixed_ips), i) public_ip = ip diff --git a/terraform/modules/rke2/rancher.tf b/terraform/modules/rke2/rancher.tf index 2b171599a4514c166dff7becf8183fba59c83b2d..b78b60fb940b73c7b1aa54b86684f7f161126acd 100644 --- a/terraform/modules/rke2/rancher.tf +++ b/terraform/modules/rke2/rancher.tf @@ -3,12 +3,17 @@ # ---------------------------------------------------------------------- resource "rancher2_cluster_v2" "kube" { name = var.cluster_name - kubernetes_version = var.rke2_version default_cluster_role_for_project_members = "user" + kubernetes_version = var.rke2_version + + agent_env_vars { + name = "CATTLE_AGENT_LOGLEVEL" + value = "info" + } rke_config { local_auth_endpoint { - enabled = var.cluster_direct_access + enabled = var.cluster_direct_access } machine_global_config = <<EOF disable: @@ -17,20 +22,75 @@ EOF upgrade_strategy { control_plane_concurrency = 1 control_plane_drain_options { - ignore_daemon_sets = true - delete_empty_dir_data = true - grace_period = 120 + ignore_daemon_sets = true + delete_empty_dir_data = true + grace_period = 120 } worker_concurrency = 1 worker_drain_options { - ignore_daemon_sets = true - delete_empty_dir_data = true - grace_period = 120 + ignore_daemon_sets = true + delete_empty_dir_data = true + grace_period = 120 } } } } +# Create a new rancher2 Cluster Sync for cluster +resource "rancher2_cluster_sync" "kube" { + depends_on = [ openstack_compute_instance_v2.controlplane[0] ] + cluster_id = rancher2_cluster_v2.kube.cluster_v1_id + wait_catalogs = false +} + + +# ---------------------------------------------------------------------- +# applications +# ---------------------------------------------------------------------- +resource "rancher2_app_v2" "monitoring" { + count = var.monitoring_enabled ? 1 : 0 + cluster_id = rancher2_cluster_sync.kube.id + name = "rancher-monitoring" + namespace = "cattle-monitoring-system" + repo_name = "rancher-charts" + chart_name = "rancher-monitoring" + // values = <<EOF + //prometheus: + // resources: + // core: + // limits: + // cpu: "4000m" + // memory: "6144Mi" + //EOF + lifecycle { + ignore_changes = [ + values + ] + } +} + +resource "rancher2_app_v2" "longhorn" { + count = var.longhorn_enabled ? 1 : 0 + cluster_id = rancher2_cluster_v2.kube.cluster_v1_id + name = "longhorn" + namespace = "longhorn-system" + repo_name = "rancher-charts" + chart_name = "longhorn" + values = <<EOF +defaultSettings: + backupTarget: nfs://radiant-nfs.ncsa.illinois.edu:/radiant/projects/${data.openstack_identity_auth_scope_v3.scope.project_name}/${var.cluster_name}/backup + defaultReplicaCount: ${var.longhorn_replicas} +persistence: + defaultClass: false + defaultClassReplicaCount: ${var.longhorn_replicas} +EOF + lifecycle { + ignore_changes = [ + values + ] + } +} + # ---------------------------------------------------------------------- # cluster access # ---------------------------------------------------------------------- diff --git a/terraform/modules/rke2/templates/user_data.tmpl b/terraform/modules/rke2/templates/user_data.tmpl index 70e8fb163f6f6e5f7b827e2968dfa1ed3bb1e516..0916b7e3731acf636864d626afebc5a9a3c6b0d6 100644 --- a/terraform/modules/rke2/templates/user_data.tmpl +++ b/terraform/modules/rke2/templates/user_data.tmpl @@ -6,8 +6,8 @@ ssh: emit_keys_to_console: false # update and upgrade instance -#package_update: true -#package_upgrade: true +package_update: true +package_upgrade: true # files to be created on the system write_files: @@ -32,6 +32,5 @@ write_files: # run this command once the system is booted runcmd: -- echo "${node_command} ${node_options} --node-name ${node_name}" > /kube.sh - ${node_command} ${node_options} --node-name ${node_name} - mount -av diff --git a/terraform/modules/rke2/variables.tf b/terraform/modules/rke2/variables.tf index 32587fc82c2ddc043ff3e6568172d4e9d3310428..31f136d73c985f9dd7e09fe89b2b4c4aa46ab646 100644 --- a/terraform/modules/rke2/variables.tf +++ b/terraform/modules/rke2/variables.tf @@ -34,6 +34,36 @@ variable "rancher_token" { description = "Access token for rancher, clusters are created as this user" } +# get latest version from rancher using: +# curl https://releases.rancher.com/kontainer-driver-metadata/release-v2.6/data.json | jq '.rke2.releases | .[].version' | sort +variable "rke2_version" { + type = string + description = "Version of rke2 to install." + default = "" +} + +# ---------------------------------------------------------------------- +# APPLICATIONS +# ---------------------------------------------------------------------- + +variable "monitoring_enabled" { + type = bool + description = "Enable monitoring in rancher" + default = true +} + +variable "longhorn_enabled" { + type = bool + description = "Enable longhorn storage" + default = true +} + +variable "longhorn_replicas" { + type = string + description = "Number of replicas" + default = 3 +} + # ---------------------------------------------------------------------- # USERS # ---------------------------------------------------------------------- @@ -41,44 +71,25 @@ variable "rancher_token" { variable "admin_users" { type = set(string) description = "List of LDAP users with admin access to cluster." - default = [ ] + default = [] } variable "admin_groups" { type = set(string) description = "List of LDAP groups with admin access to cluster." - default = [ ] + default = [] } variable "member_users" { type = set(string) description = "List of LDAP users with access to cluster." - default = [ ] + default = [] } variable "member_groups" { type = set(string) description = "List of LDAP groups with access to cluster." - default = [ ] -} - -# ---------------------------------------------------------------------- -# RKE2 -# ---------------------------------------------------------------------- - -variable "rke2_secret" { - type = string - sensitive = true - description = "default token to be used, if empty random one is used" - default = "" -} - -# get latest version from rancher using: -# curl https://releases.rancher.com/kontainer-driver-metadata/release-v2.6/data.json | jq '.rke2.releases | .[].version' | sort -variable "rke2_version" { - type = string - description = "Version of rke2 to install." - default = "v1.21.6+rke2r1" + default = [] } # ---------------------------------------------------------------------- diff --git a/terraform/modules/rke2/versions.tf b/terraform/modules/rke2/versions.tf index 1b952653c435a9037b04ee51732e8f00ab2925df..f6e381416ed3cdedd5411193f167355b80620d08 100644 --- a/terraform/modules/rke2/versions.tf +++ b/terraform/modules/rke2/versions.tf @@ -5,11 +5,11 @@ terraform { version = ">= 1.43.0" } rancher2 = { - source = "rancher/rancher2" + source = "rancher/rancher2" version = ">= 1.21.0" } random = { - source = "hashicorp/random" + source = "hashicorp/random" version = ">= 3.1.0" } }