Skip to content
Snippets Groups Projects
Verified Commit 65a03913 authored by Rob Kooper's avatar Rob Kooper
Browse files

use new way of configuring cluster

parent 7b85f816
No related branches found
Tags v2.0.0
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 174 additions and 386 deletions
CHANGELOG.md
+ 17
− 19
View file @ 65a03913
......@@ -6,27 +6,25 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
## 2.0.0 - 2023-06-28
This is a breaking change. You will need to update your terraform code to use this new version. The cluster is now configured using a json file. The following json file will mimic the original cluster definition:
This is a breaking change. You will need to update your terraform code to use this new version. This is an example of the variable `cluster_machine`.
```json
{
"machines": [
{
"name": "controlplane",
"role": "controlplane",
"count": 3,
"flavor": "gp.medium",
"os": "centos"
},
{
"name": "worker",
"count": 3,
"flavor": "gp.large",
"disk": 40,
"os": "centos"
}
]
}
[
{
"name": "controlplane",
"role": "controlplane",
"count": 3,
"flavor": "gp.medium",
"os": "centos"
},
{
"name": "worker",
"count": 3,
"flavor": "gp.large",
"disk": 40,
"os": "centos"
}
]
```
### Added
......
terraform/modules/rke1/data.tf deleted 100644 → 0
+ 0
− 15
View file @ 7b85f816
# external network
data "openstack_networking_network_v2" "ext_net" {
name = var.openstack_external_net
}
# boot image
data "openstack_images_image_v2" "boot" {
name = var.os
most_recent = true
}
# openstack project name (bbXX)
data "openstack_identity_auth_scope_v3" "scope" {
name = "my_scope"
}
terraform/modules/rke1/key.tf deleted 100644 → 0
+ 0
− 12
View file @ 7b85f816
# Each cluster will either have a shared key, or their own
# unique key.
resource "openstack_compute_keypair_v2" "key" {
#count = 1 #var.openstack_ssh_key == "" ? 0 : 1
name = var.cluster_name
}
# set local variable to hold final key, either created or
# loaded.
locals {
key = var.cluster_name # var.openstack_ssh_key == "" ? var.cluster_name : var.openstack_ssh_key
}
terraform/modules/rke1/machines.tf 0 → 100644
+ 90
− 0
View file @ 65a03913
locals {
images = {
"centos" = "CentOS-7-GenericCloud-Latest",
"ubuntu" = "Ubuntu Jammy (22.04) latest"
}
usernames = {
"centos" = "centos",
"ubuntu" = "ubuntu"
}
node_options = {
"controlplane" = "--address awspublic --internal-address awslocal --controlplane --etcd",
"worker" = "--address awspublic --internal-address awslocal --worker"
}
machines = flatten([
for x in var.cluster_machines : [
for i in range(x.count == null ? 1 : x.count) : {
hostname = format("%s-%s-%02d", var.cluster_name, x.name, (i + 1))
username = lookup(local.usernames, x.os, "centos")
image_name = lookup(local.images, x.os, "CentOS-7-GenericCloud-Latest")
flavor = try(x.flavor, "gp.medium")
image_id = data.openstack_images_image_v2.boot[try(x.os, "centos")].id
disk_size = try(x.disk, 40)
zone = try(x.zone, "nova")
role = try(x.role, "worker")
floating_ip = try(x.floating_ip, can(x.role == "controlplane"))
labels = flatten([x.name, try(x.labels, [])])
}
]
])
jumphost = [for vm in local.machines : vm.hostname if vm.floating_ip][0]
}
data "openstack_images_image_v2" "boot" {
for_each = local.images
name = each.value
most_recent = true
}
data "openstack_identity_auth_scope_v3" "scope" {
name = var.cluster_name
}
resource "openstack_compute_keypair_v2" "key" {
name = var.cluster_name
}
resource "openstack_compute_instance_v2" "machine" {
for_each = { for vm in local.machines : vm.hostname => vm }
name = each.value.hostname
image_name = each.value.image_name
availability_zone = each.value.zone
flavor_name = each.value.flavor
key_pair = openstack_compute_keypair_v2.key.name
config_drive = false
depends_on = [
openstack_networking_secgroup_rule_v2.same_security_group_ingress_tcp,
]
security_groups = [
openstack_networking_secgroup_v2.cluster_security_group.name
]
network {
port = openstack_networking_port_v2.machine_ip[each.key].id
}
block_device {
uuid = each.value.image_id
source_type = "image"
volume_size = each.value.disk_size
destination_type = "volume"
delete_on_termination = true
}
user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", {
private_key = openstack_compute_keypair_v2.key.private_key
project_name = data.openstack_identity_auth_scope_v3.scope.project_name
cluster_name = var.cluster_name
username = each.value.username
node_name = each.value.hostname
node_command = rancher2_cluster.kube.cluster_registration_token.0.node_command
node_options = lookup(local.node_options, each.value.role, "--worker")
node_labels = join(" ", [for l in each.value.labels : format("-l %s", replace(l, " ", "_"))])
}))
}
terraform/modules/rke1/network.tf
+ 19
− 26
View file @ 65a03913
......@@ -2,10 +2,14 @@
# private ip spaces of 192.168.0.0/21. Each of the machines will
# have a fixed ip address in this private IP space.
#
# For the worker nodes, tere will be a set of floating IP addresses
# For the worker machines, there will be a set of floating IP addresses
# that can be given to a load balancer (using for example metallb).
#
data "openstack_networking_network_v2" "ext_net" {
name = var.openstack_external_net
}
# ----------------------------------------------------------------------
# setup network, subnet and router
# ----------------------------------------------------------------------
......@@ -34,25 +38,6 @@ resource "openstack_networking_router_interface_v2" "kube_gateway" {
subnet_id = openstack_networking_subnet_v2.cluster_subnet.id
}
# ----------------------------------------------------------------------
# control plane
# ----------------------------------------------------------------------
resource "openstack_networking_port_v2" "controlplane_ip" {
count = var.controlplane_count
name = local.controlplane[count.index]
network_id = openstack_networking_network_v2.cluster_net.id
security_group_ids = [openstack_networking_secgroup_v2.cluster_security_group.id]
depends_on = [openstack_networking_router_interface_v2.kube_gateway]
}
resource "openstack_networking_floatingip_v2" "controlplane_ip" {
count = var.controlplane_count
description = format("%s-controlplane-%d", var.cluster_name, count.index + 1)
pool = data.openstack_networking_network_v2.ext_net.name
port_id = element(openstack_networking_port_v2.controlplane_ip.*.id, count.index)
}
# ----------------------------------------------------------------------
# floating IP
# ----------------------------------------------------------------------
......@@ -75,17 +60,16 @@ resource "openstack_networking_floatingip_v2" "floating_ip" {
}
# ----------------------------------------------------------------------
# worker nodes
# machines
# ----------------------------------------------------------------------
# create worker ip, this can route the ports for the floating ip as
# well.
resource "openstack_networking_port_v2" "worker_ip" {
count = var.worker_count
name = local.worker[count.index]
resource "openstack_networking_port_v2" "machine_ip" {
for_each = { for vm in local.machines : vm.hostname => vm }
name = each.value.hostname
network_id = openstack_networking_network_v2.cluster_net.id
security_group_ids = [openstack_networking_secgroup_v2.cluster_security_group.id]
depends_on = [openstack_networking_router_interface_v2.kube_gateway]
dynamic "allowed_address_pairs" {
for_each = openstack_networking_port_v2.floating_ip.*.all_fixed_ips.0
content {
......@@ -93,3 +77,12 @@ resource "openstack_networking_port_v2" "worker_ip" {
}
}
}
resource "openstack_networking_floatingip_v2" "machine_ip" {
for_each = { for vm in local.machines : vm.hostname => vm if vm.floating_ip }
description = each.value.hostname
pool = data.openstack_networking_network_v2.ext_net.name
port_id = openstack_networking_port_v2.machine_ip[each.key].id
}
terraform/modules/rke1/nodes.tf deleted 100644 → 0
+ 0
− 114
View file @ 7b85f816
locals {
controlplane = [for l in range(var.controlplane_count) : var.old_hostnames ? format("%s-controlplane-%d", var.cluster_name, l) : format("%s-controlplane-%d", var.cluster_name, l + 1)]
worker = [for l in range(var.worker_count) : var.old_hostnames ? format("%s-worker-%d", var.cluster_name, l) : format("%s-worker-%02d", var.cluster_name, l + 1)]
}
# ----------------------------------------------------------------------
# control-plane nodes
# ----------------------------------------------------------------------
resource "openstack_compute_instance_v2" "controlplane" {
count = var.controlplane_count
name = local.controlplane[count.index]
image_name = var.os
availability_zone = var.openstack_zone
flavor_name = var.controlplane_flavor
key_pair = openstack_compute_keypair_v2.key.name
config_drive = false
depends_on = [
openstack_networking_secgroup_rule_v2.same_security_group_ingress_tcp,
]
security_groups = [
openstack_networking_secgroup_v2.cluster_security_group.name
]
#echo "update hosts"
#%{ for ip in openstack_networking_port_v2.worker_ip[count.index].all_fixed_ips }
#echo "$${ip} $${node_name} $(hostname) $(hostname -f)" >> /etc/hosts
#%{ endfor }
user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", {
private_key = openstack_compute_keypair_v2.key.private_key
project_name = data.openstack_identity_auth_scope_v3.scope.project_name
cluster_name = var.cluster_name
node_name = local.controlplane[count.index]
node_command = rancher2_cluster.kube.cluster_registration_token.0.node_command
node_options = "--address awspublic --internal-address awslocal --controlplane --etcd"
}))
block_device {
uuid = data.openstack_images_image_v2.boot.id
source_type = "image"
volume_size = var.controlplane_disksize
destination_type = "volume"
delete_on_termination = true
}
network {
port = element(openstack_networking_port_v2.controlplane_ip.*.id, count.index)
}
lifecycle {
ignore_changes = [
block_device,
flavor_name,
image_name,
key_pair,
user_data
]
}
}
# ----------------------------------------------------------------------
# worker nodes
# ----------------------------------------------------------------------
resource "openstack_compute_instance_v2" "worker" {
count = var.worker_count
name = local.worker[count.index]
image_name = var.os
availability_zone = var.openstack_zone
flavor_name = var.worker_flavor
key_pair = local.key
config_drive = false
depends_on = [
openstack_networking_secgroup_rule_v2.same_security_group_ingress_tcp
]
security_groups = [
openstack_networking_secgroup_v2.cluster_security_group.name
]
user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", {
private_key = openstack_compute_keypair_v2.key.private_key
project_name = data.openstack_identity_auth_scope_v3.scope.project_name
cluster_name = var.cluster_name
node_name = local.worker[count.index]
node_command = rancher2_cluster.kube.cluster_registration_token.0.node_command
node_options = "--worker"
}))
block_device {
uuid = data.openstack_images_image_v2.boot.id
source_type = "image"
volume_size = var.worker_disksize
destination_type = "volume"
boot_index = 0
delete_on_termination = true
}
network {
port = element(openstack_networking_port_v2.worker_ip.*.id, count.index)
}
lifecycle {
ignore_changes = [
block_device,
flavor_name,
image_name,
key_pair,
user_data
]
}
}
terraform/modules/rke1/outputs.tf
+ 15
− 11
View file @ 65a03913
......@@ -3,6 +3,11 @@ output "project_name" {
value = data.openstack_identity_auth_scope_v3.scope.project_name
}
output "machines" {
description = "List of machines created"
value = local.machines
}
output "node_command" {
description = "Command to join?"
value = rancher2_cluster.kube.cluster_registration_token[0].node_command
......@@ -19,24 +24,23 @@ output "ssh_config" {
value = <<-EOT
# Automatically created by terraform
%{~for i, x in openstack_compute_instance_v2.controlplane.*}
Host ${x.name}
HostName ${openstack_networking_floatingip_v2.controlplane_ip[i].address}
%{~for x in [for m in local.machines : m if m.floating_ip]}
Host ${x.hostname}
HostName ${openstack_networking_floatingip_v2.machine_ip[x.hostname].address}
StrictHostKeyChecking no
UserKnownHostsFile=/dev/null
IdentityFile ${pathexpand("~/.ssh/${var.cluster_name}.pem")}
User centos
User ${x.username}
%{~endfor}
%{~for x in openstack_compute_instance_v2.worker.*}
Host ${x.name}
HostName ${x.network[0].fixed_ip_v4}
%{~for x in [for m in local.machines : m if !m.floating_ip]}
Host ${x.hostname}
ProxyJump ${local.jumphost}
HostName ${openstack_networking_port_v2.machine_ip[x.hostname].all_fixed_ips[0]}
StrictHostKeyChecking no
ProxyJump ${openstack_compute_instance_v2.controlplane[0].name}
UserKnownHostsFile=/dev/null
IdentityFile ${pathexpand("~/.ssh/${var.cluster_name}.pem")}
User centos
User ${x.username}
%{~endfor}
EOT
}
......
terraform/modules/rke1/rancher.tf
+ 8
− 69
View file @ 65a03913
......@@ -32,20 +32,13 @@ resource "rancher2_cluster" "kube" {
}
}
# Create a new rancher2 Cluster Sync for foo-custom cluster
resource "rancher2_cluster_sync" "kube" {
depends_on = [openstack_compute_instance_v2.controlplane[0]]
cluster_id = rancher2_cluster.kube.id
wait_catalogs = false
}
# ----------------------------------------------------------------------
# cluster access
# ----------------------------------------------------------------------
resource "rancher2_cluster_role_template_binding" "admin_users" {
for_each = var.admin_users
name = "${rancher2_cluster_sync.kube.id}-user-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster_sync.kube.id
name = "${rancher2_cluster.kube.id}-user-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster.kube.id
role_template_id = "cluster-owner"
user_principal_id = "openldap_user://uid=${each.value},ou=People,dc=ncsa,dc=illinois,dc=edu"
lifecycle {
......@@ -59,8 +52,8 @@ resource "rancher2_cluster_role_template_binding" "admin_users" {
resource "rancher2_cluster_role_template_binding" "admin_groups" {
for_each = var.admin_groups
name = "${rancher2_cluster_sync.kube.id}-group-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster_sync.kube.id
name = "${rancher2_cluster.kube.id}-group-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster.kube.id
role_template_id = "cluster-owner"
user_principal_id = "openldap_group://cn=${each.value},ou=Groups,dc=ncsa,dc=illinois,dc=edu"
lifecycle {
......@@ -74,8 +67,8 @@ resource "rancher2_cluster_role_template_binding" "admin_groups" {
resource "rancher2_cluster_role_template_binding" "member_users" {
for_each = var.member_users
name = "${rancher2_cluster_sync.kube.id}-user-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster_sync.kube.id
name = "${rancher2_cluster.kube.id}-user-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster.kube.id
role_template_id = "cluster-member"
user_principal_id = "openldap_user://uid=${each.value},ou=People,dc=ncsa,dc=illinois,dc=edu"
lifecycle {
......@@ -89,8 +82,8 @@ resource "rancher2_cluster_role_template_binding" "member_users" {
resource "rancher2_cluster_role_template_binding" "member_groups" {
for_each = var.member_groups
name = "${rancher2_cluster_sync.kube.id}-group-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster_sync.kube.id
name = "${rancher2_cluster.kube.id}-group-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster.kube.id
role_template_id = "cluster-member"
user_principal_id = "openldap_group://cn=${each.value},ou=Groups,dc=ncsa,dc=illinois,dc=edu"
lifecycle {
......@@ -101,57 +94,3 @@ resource "rancher2_cluster_role_template_binding" "member_groups" {
]
}
}
# ----------------------------------------------------------------------
# longhorn storage
# ----------------------------------------------------------------------
resource "rancher2_app_v2" "longhorn-system" {
count = var.longhorn_enabled ? 1 : 0
cluster_id = rancher2_cluster_sync.kube.cluster_id
name = "longhorn"
namespace = "longhorn-system"
repo_name = "rancher-charts"
chart_name = "longhorn"
chart_version = "101.1.0+up1.3.2"
project_id = rancher2_cluster_sync.kube.system_project_id
values = <<EOF
defaultSettings:
backupTarget: nfs://radiant-nfs.ncsa.illinois.edu:/radiant/projects/${data.openstack_identity_auth_scope_v3.scope.project_name}/${var.cluster_name}/backup
defaultReplicaCount: ${var.longhorn_replicas}
persistence:
defaultClass: false
defaultClassReplicaCount: ${var.longhorn_replicas}
EOF
lifecycle {
ignore_changes = [
values
]
}
}
# ----------------------------------------------------------------------
# monitoring
# ----------------------------------------------------------------------
resource "rancher2_app_v2" "monitor" {
count = var.monitoring_enabled ? 1 : 0
cluster_id = rancher2_cluster_sync.kube.cluster_id
name = "rancher-monitoring"
namespace = "cattle-monitoring-system"
repo_name = "rancher-charts"
chart_name = "rancher-monitoring"
chart_version = "101.0.0+up19.0.3"
project_id = rancher2_cluster_sync.kube.system_project_id
// values = <<EOF
//prometheus:
// resources:
// core:
// limits:
// cpu: "4000m"
// memory: "6144Mi"
//EOF
lifecycle {
ignore_changes = [
values
]
}
}
terraform/modules/rke1/security_group.tf
+ 10
− 30
View file @ 65a03913
......@@ -3,26 +3,6 @@ resource "openstack_networking_secgroup_v2" "cluster_security_group" {
description = "${var.cluster_name} kubernetes cluster security group"
}
# ----------------------------------------------------------------------
# Egress
# ----------------------------------------------------------------------
#Egress IPv4 Any Any 0.0.0.0/0 - -
#resource "openstack_networking_secgroup_rule_v2" "egress_ipv4" {
# direction = "egress"
# ethertype = "IPv4"
# security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
# depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
#}
#Egress IPv6 Any Any ::/0 - -
#resource "openstack_networking_secgroup_rule_v2" "egress_ipv6" {
# direction = "egress"
# ethertype = "IPv6"
# security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
# depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
#}
# ----------------------------------------------------------------------
# Ingress
# ----------------------------------------------------------------------
......@@ -86,16 +66,16 @@ resource "openstack_networking_secgroup_rule_v2" "ingress_kubeapi" {
}
# Ingress IPv4 TCP 30000 - 32767 0.0.0.0/0 - nodeport
resource "openstack_networking_secgroup_rule_v2" "ingress_nodeport" {
description = "nodeport"
direction = "ingress"
ethertype = "IPv4"
protocol = "tcp"
port_range_min = 30000
port_range_max = 32767
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
# resource "openstack_networking_secgroup_rule_v2" "ingress_nodeport" {
# description = "nodeport"
# direction = "ingress"
# ethertype = "IPv4"
# protocol = "tcp"
# port_range_min = 30000
# port_range_max = 32767
# security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
# depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
# }
resource "openstack_networking_secgroup_rule_v2" "same_security_group_ingress_tcp" {
direction = "ingress"
......
terraform/modules/rke1/templates/user_data.tmpl
+ 10
− 4
View file @ 65a03913
......@@ -35,9 +35,15 @@ write_files:
content: |
#!/usr/bin/bash
echo "sleeping to wait for network"
while ! ping -c 1 -w 0 mirrorlist.centos.org > /dev/null ; do echo "Sleep 10s"; sleep 10; done
while ! ping -c 1 -w 0 1.1.1.1 > /dev/null ; do echo "Sleep 10s"; sleep 10; done
echo "install iscsi/nfs"
yum -y install iscsi-initiator-utils nfs-utils
if [ -e /usr/bin/yum ]; then
yum -y install iscsi-initiator-utils nfs-utils
elif [ -e /usr/bin/apt ]; then
apt install -y open-iscsi nfs-common
else
echo "Don't know how to install iscsi/nfs"
fi
echo "mounting taiga"
mkdir /taiga
#mount -av
......@@ -45,9 +51,9 @@ write_files:
curl https://releases.rancher.com/install-docker/20.10.sh | sh
systemctl enable docker
systemctl start docker
usermod -aG docker centos
usermod -aG docker ${username}
echo "connect to rancher"
${node_command} ${node_options}
${node_command} ${node_options} ${node_labels}
echo "all done"
# run this command once the system is booted
......
terraform/modules/rke1/variables.tf
+ 5
− 86
View file @ 65a03913
......@@ -18,26 +18,10 @@ variable "cluster_direct_access" {
default = true
}
# ----------------------------------------------------------------------
# APPLICATIONS
# ----------------------------------------------------------------------
variable "monitoring_enabled" {
type = bool
description = "Enable monitoring in rancher"
default = true
}
variable "longhorn_enabled" {
type = bool
description = "Enable longhorn storage"
default = true
}
variable "longhorn_replicas" {
type = string
description = "Number of replicas"
default = 3
variable "cluster_machines" {
type = set(map(any))
description = "machine definition"
default = []
}
# ----------------------------------------------------------------------
......@@ -60,7 +44,6 @@ variable "rancher_token" {
variable "rke1_version" {
type = string
description = "Version of rke1 to install."
default = "v1.21.14-rancher1-1"
}
# ----------------------------------------------------------------------
......@@ -98,7 +81,7 @@ variable "member_groups" {
variable "openstack_url" {
type = string
description = "OpenStack URL"
default = "https://radiant.ncsa.illinois.edu"
default = "https://radiant.ncsa.illinois.edu:5000"
}
variable "openstack_credential_id" {
......@@ -119,76 +102,12 @@ variable "openstack_external_net" {
default = "ext-net"
}
variable "openstack_ssh_key" {
type = string
description = "existing SSH key to use, leave blank for a new one"
default = ""
}
variable "openstack_zone" {
type = string
description = "default zone to use for openstack nodes"
default = "nova"
}
variable "openstack_security_kubernetes" {
type = string
description = "IP address to allow connections to kube api port"
default = "141.142.0.0/16"
}
# ----------------------------------------------------------------------
# OPENSTACK NODES
# ----------------------------------------------------------------------
variable "old_hostnames" {
type = bool
description = "should old hostname be used (base 0)"
default = false
}
variable "os" {
type = string
description = "Base image to use for the OS"
default = "CentOS-7-GenericCloud-Latest"
}
variable "controlplane_count" {
type = string
description = "Desired quantity of control-plane nodes"
default = 1
}
variable "controlplane_flavor" {
type = string
description = "Desired flavor of control-plane nodes"
default = "m1.medium"
}
variable "controlplane_disksize" {
type = string
description = "Desired disksize of control-plane nodes"
default = 40
}
variable "worker_count" {
type = string
description = "Desired quantity of worker nodes"
default = 1
}
variable "worker_flavor" {
type = string
description = "Desired flavor of worker nodes"
default = "m1.large"
}
variable "worker_disksize" {
type = string
description = "Desired disksize of worker nodes"
default = 40
}
# ----------------------------------------------------------------------
# NETWORKING
# ----------------------------------------------------------------------
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Contact help@ncsa.illinois.edu with questions regarding this site. | All rights reserved. ©2024 Board of Trustees of the University of Illinois. | Web Privacy Notice