allow to specify more entrypoints

charts/apps/templates/ingresscontroller/traefik2.yaml

@@ -44,6 +44,9 @@ spec:
             tls:
               enabled: true
               certResolver: letsencrypt
+          {{- if .Values.ingresscontroller.traefik2.ports }}
+          {{- .Values.ingresscontroller.traefik2.ports | toYaml | nindent 10 }}
+          {{- end }}
         additionalArguments:
           - --providers.kubernetesingress.ingressendpoint.ip={{ .Values.ingresscontroller.publicIP }}
           {{- if .Values.ingresscontroller.acme }}

charts/apps/values.yaml

@@ -26,6 +26,7 @@ healthmonitor:
 
 ingresscontroller:
   enabled: false
+  class: traefik2
   publicIP: 1.1.1.1
   #privateIP: 1.1.1.2
   #acme:
@@ -35,6 +36,12 @@ ingresscontroller:
     version: "1.*"
   traefik2:
     version: "10.*"
+    ports: {}
+      # postgres:
+      #   port: 5432
+      #   expose: true
+      #   exposedPort: 5432
+      #   protocol: TCP
 
 sealedsecrets:
   enabled: false

terraform/modules/rke2/key.tf

@@ -2,7 +2,7 @@
 # unique key.
 resource "openstack_compute_keypair_v2" "key" {
   #count      = 1 #var.openstack_ssh_key == "" ? 0 : 1
-  name       = var.cluster_name
+  name = var.cluster_name
 }
 
 # set local variable to hold final key, either created or

terraform/modules/rke2/network.tf

@@ -70,18 +70,18 @@ resource "openstack_networking_port_v2" "controlplane_ip_public" {
 # create a port that will be used with the floating ip, this will be associated
 # with all of the VMs.
 resource "openstack_networking_port_v2" "floating_ip" {
-  count       = var.floating_ip
-  depends_on  = [ openstack_networking_subnet_v2.cluster_subnet ]
-  name        = format("%s-floating-ip-%02d", var.cluster_name, count.index + 1)
-  network_id  = openstack_networking_network_v2.cluster_net.id
+  count      = var.floating_ip
+  depends_on = [openstack_networking_subnet_v2.cluster_subnet]
+  name       = format("%s-floating-ip-%02d", var.cluster_name, count.index + 1)
+  network_id = openstack_networking_network_v2.cluster_net.id
 }
 
 # create floating ip that is associated with a fixed ip
 resource "openstack_networking_floatingip_v2" "floating_ip" {
-  count   = var.floating_ip
+  count       = var.floating_ip
   description = format("%s-floating-ip-%02d", var.cluster_name, count.index + 1)
-  pool    = data.openstack_networking_network_v2.ext_net.name
-  port_id = element(openstack_networking_port_v2.floating_ip.*.id, count.index)
+  pool        = data.openstack_networking_network_v2.ext_net.name
+  port_id     = element(openstack_networking_port_v2.floating_ip.*.id, count.index)
 }
 
 # create worker ip, this can route the ports for the floating ip as

terraform/modules/rke2/nodes.tf

@@ -2,20 +2,20 @@
 # control-plane nodes
 # ----------------------------------------------------------------------
 resource "openstack_compute_instance_v2" "controlplane" {
-  count           = var.controlplane_count
-  depends_on      = [
+  count = var.controlplane_count
+  depends_on = [
     openstack_networking_secgroup_rule_v2.same_security_group_ingress_tcp,
   ]
-  name            = format("%s-controlplane-%d", var.cluster_name, count.index + 1)
-  image_name      = var.os
-  flavor_name     = var.controlplane_flavor
-  key_pair        = local.key
+  name        = format("%s-controlplane-%d", var.cluster_name, count.index + 1)
+  image_name  = var.os
+  flavor_name = var.controlplane_flavor
+  key_pair    = local.key
   security_groups = [
     openstack_networking_secgroup_v2.cluster_security_group.name
   ]
-  config_drive    = false
+  config_drive = false
 
-  user_data  = base64encode(templatefile("${path.module}/templates/user_data.tmpl", {
+  user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", {
     private_key  = openstack_compute_keypair_v2.key.private_key
     project_name = data.openstack_identity_auth_scope_v3.scope.project_name
     cluster_name = var.cluster_name
@@ -54,8 +54,8 @@ resource "openstack_compute_instance_v2" "controlplane" {
 # ----------------------------------------------------------------------
 
 resource "openstack_compute_instance_v2" "worker" {
-  count           = var.worker_count
-  depends_on      = [
+  count = var.worker_count
+  depends_on = [
     openstack_networking_secgroup_rule_v2.same_security_group_ingress_tcp,
     openstack_networking_port_v2.controlplane_ip
   ]
@@ -63,9 +63,11 @@ resource "openstack_compute_instance_v2" "worker" {
   flavor_name     = var.worker_flavor
   key_pair        = local.key
   config_drive    = false
-  security_groups = [ openstack_networking_secgroup_v2.cluster_security_group.name ]
+  security_groups = [
+    openstack_networking_secgroup_v2.cluster_security_group.name
+  ]
 
-  user_data  = base64encode(templatefile("${path.module}/templates/user_data.tmpl", {
+  user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", {
     private_key  = openstack_compute_keypair_v2.key.private_key
     project_name = data.openstack_identity_auth_scope_v3.scope.project_name
     cluster_name = var.cluster_name

terraform/modules/rke2/outputs.tf

@@ -19,7 +19,7 @@ output "ssh_config" {
   value       = <<-EOT
 # Automatically created by terraform
 
-%{~ for i, x in openstack_compute_instance_v2.controlplane.* }
+%{~for i, x in openstack_compute_instance_v2.controlplane.*}
 Host ${x.name}
   HostName ${openstack_networking_floatingip_v2.controlplane_ip[i].address}
   StrictHostKeyChecking no
@@ -27,8 +27,8 @@ Host ${x.name}
   IdentityFile ${pathexpand("~/.ssh/${var.cluster_name}.pem")}
   User centos
 
-%{~ endfor }
-%{~ for x in openstack_compute_instance_v2.worker.* }
+%{~endfor}
+%{~for x in openstack_compute_instance_v2.worker.*}
 Host ${x.name}
   HostName ${x.network[0].fixed_ip_v4}
   StrictHostKeyChecking no
@@ -37,7 +37,7 @@ Host ${x.name}
   IdentityFile ${pathexpand("~/.ssh/${var.cluster_name}.pem")}
   User centos
 
-%{~ endfor }
+%{~endfor}
 EOT
 }
 
@@ -54,7 +54,7 @@ output "kube_id" {
 
 output "floating_ip" {
   description = "Map for floating ips and associated private ips"
-  value       = [
+  value = [
     for i, ip in openstack_networking_floatingip_v2.floating_ip.*.address : {
       private_ip = element(flatten(openstack_networking_port_v2.floating_ip.*.all_fixed_ips), i)
       public_ip  = ip

terraform/modules/rke2/rancher.tf

@@ -3,12 +3,17 @@
 # ----------------------------------------------------------------------
 resource "rancher2_cluster_v2" "kube" {
   name                                     = var.cluster_name
-  kubernetes_version                       = var.rke2_version
   default_cluster_role_for_project_members = "user"
+  kubernetes_version                       = var.rke2_version
+
+  agent_env_vars {
+    name  = "CATTLE_AGENT_LOGLEVEL"
+    value = "info"
+  }
 
   rke_config {
     local_auth_endpoint {
-      enabled  = var.cluster_direct_access
+      enabled = var.cluster_direct_access
     }
     machine_global_config = <<EOF
 disable:
@@ -17,20 +22,75 @@ EOF
     upgrade_strategy {
       control_plane_concurrency = 1
       control_plane_drain_options {
-        ignore_daemon_sets = true
-        delete_empty_dir_data  = true
-        grace_period = 120
+        ignore_daemon_sets    = true
+        delete_empty_dir_data = true
+        grace_period          = 120
       }
       worker_concurrency = 1
       worker_drain_options {
-        ignore_daemon_sets = true
-        delete_empty_dir_data  = true
-        grace_period = 120
+        ignore_daemon_sets    = true
+        delete_empty_dir_data = true
+        grace_period          = 120
       }
     }
   }
 }
 
+# Create a new rancher2 Cluster Sync for cluster
+resource "rancher2_cluster_sync" "kube" {
+  depends_on      = [ openstack_compute_instance_v2.controlplane[0] ]
+  cluster_id      = rancher2_cluster_v2.kube.cluster_v1_id
+  wait_catalogs   = false
+}
+
+
+# ----------------------------------------------------------------------
+# applications
+# ----------------------------------------------------------------------
+resource "rancher2_app_v2" "monitoring" {
+  count      = var.monitoring_enabled ? 1 : 0
+  cluster_id = rancher2_cluster_sync.kube.id
+  name       = "rancher-monitoring"
+  namespace  = "cattle-monitoring-system"
+  repo_name  = "rancher-charts"
+  chart_name = "rancher-monitoring"
+  //  values        = <<EOF
+  //prometheus:
+  //  resources:
+  //    core:
+  //      limits:
+  //        cpu: "4000m"
+  //        memory: "6144Mi"
+  //EOF
+  lifecycle {
+    ignore_changes = [
+      values
+    ]
+  }
+}
+
+resource "rancher2_app_v2" "longhorn" {
+  count      = var.longhorn_enabled ? 1 : 0
+  cluster_id = rancher2_cluster_v2.kube.cluster_v1_id
+  name       = "longhorn"
+  namespace  = "longhorn-system"
+  repo_name  = "rancher-charts"
+  chart_name = "longhorn"
+  values     = <<EOF
+defaultSettings:
+  backupTarget: nfs://radiant-nfs.ncsa.illinois.edu:/radiant/projects/${data.openstack_identity_auth_scope_v3.scope.project_name}/${var.cluster_name}/backup
+  defaultReplicaCount: ${var.longhorn_replicas}
+persistence:
+  defaultClass: false
+  defaultClassReplicaCount: ${var.longhorn_replicas}
+EOF
+  lifecycle {
+    ignore_changes = [
+      values
+    ]
+  }
+}
+
 # ----------------------------------------------------------------------
 # cluster access
 # ----------------------------------------------------------------------

terraform/modules/rke2/templates/user_data.tmpl

@@ -6,8 +6,8 @@ ssh:
   emit_keys_to_console: false
 
 # update and upgrade instance
-#package_update: true
-#package_upgrade: true
+package_update: true
+package_upgrade: true
 
 # files to be created on the system
 write_files:
@@ -32,6 +32,5 @@ write_files:
 
 # run this command once the system is booted
 runcmd:
-- echo "${node_command} ${node_options} --node-name ${node_name}" > /kube.sh
 - ${node_command} ${node_options} --node-name ${node_name}
 - mount -av

terraform/modules/rke2/variables.tf

@@ -34,6 +34,36 @@ variable "rancher_token" {
   description = "Access token for rancher, clusters are created as this user"
 }
 
+# get latest version from rancher using:
+# curl https://releases.rancher.com/kontainer-driver-metadata/release-v2.6/data.json | jq '.rke2.releases | .[].version' | sort
+variable "rke2_version" {
+  type        = string
+  description = "Version of rke2 to install."
+  default     = ""
+}
+
+# ----------------------------------------------------------------------
+# APPLICATIONS
+# ----------------------------------------------------------------------
+
+variable "monitoring_enabled" {
+  type        = bool
+  description = "Enable monitoring in rancher"
+  default     = true
+}
+
+variable "longhorn_enabled" {
+  type        = bool
+  description = "Enable longhorn storage"
+  default     = true
+}
+
+variable "longhorn_replicas" {
+  type        = string
+  description = "Number of replicas"
+  default     = 3
+}
+
 # ----------------------------------------------------------------------
 # USERS
 # ----------------------------------------------------------------------
@@ -41,44 +71,25 @@ variable "rancher_token" {
 variable "admin_users" {
   type        = set(string)
   description = "List of LDAP users with admin access to cluster."
-  default     = [ ]
+  default     = []
 }
 
 variable "admin_groups" {
   type        = set(string)
   description = "List of LDAP groups with admin access to cluster."
-  default     = [ ]
+  default     = []
 }
 
 variable "member_users" {
   type        = set(string)
   description = "List of LDAP users with access to cluster."
-  default     = [ ]
+  default     = []
 }
 
 variable "member_groups" {
   type        = set(string)
   description = "List of LDAP groups with access to cluster."
-  default     = [ ]
-}
-
-# ----------------------------------------------------------------------
-# RKE2
-# ----------------------------------------------------------------------
-
-variable "rke2_secret" {
-  type        = string
-  sensitive   = true
-  description = "default token to be used, if empty random one is used"
-  default     = ""
-}
-
-# get latest version from rancher using:
-# curl https://releases.rancher.com/kontainer-driver-metadata/release-v2.6/data.json | jq '.rke2.releases | .[].version' | sort
-variable "rke2_version" {
-  type        = string
-  description = "Version of rke2 to install."
-  default     = "v1.21.6+rke2r1"
+  default     = []
 }
 
 # ----------------------------------------------------------------------

terraform/modules/rke2/versions.tf

@@ -5,11 +5,11 @@ terraform {
       version = ">= 1.43.0"
     }
     rancher2 = {
-      source = "rancher/rancher2"
+      source  = "rancher/rancher2"
       version = ">= 1.21.0"
     }
     random = {
-      source = "hashicorp/random"
+      source  = "hashicorp/random"
       version = ">= 3.1.0"
     }
   }