Skip to content
Snippets Groups Projects
Commit b1535366 authored by Rob Kooper's avatar Rob Kooper
Browse files

rke2 cluster

parent d0c3e93d
No related branches found
No related tags found
No related merge requests found
# external network
data "openstack_networking_network_v2" "ext_net" {
name = var.openstack_external_net
}
# boot image
data "openstack_images_image_v2" "boot" {
name = var.os
most_recent = true
}
# openstack project name (bbXX)
data "openstack_identity_auth_scope_v3" "scope" {
name = "my_scope"
}
# Each cluster will either have a shared key, or their own
# unique key.
resource "openstack_compute_keypair_v2" "key" {
#count = 1 #var.openstack_ssh_key == "" ? 0 : 1
name = var.cluster_name
}
# set local variable to hold final key, either created or
# loaded.
locals {
key = var.cluster_name # var.openstack_ssh_key == "" ? var.cluster_name : var.openstack_ssh_key
}
# Each cluster has their own network. The network will have a
# private ip spaces of 192.168.0.0/21. Each of the machines will
# have a fixed ip address in this private IP space.
#
# For the worker nodes, tere will be a set of floating IP addresses
# that can be given to a load balancer (using for example metallb).
#
# The control plane nodes will have a floating IP address associated,
# until I can figure out how to give them a second IP address that
# works correctly.
# ----------------------------------------------------------------------
# setup network, subnet and router
# ----------------------------------------------------------------------
resource "openstack_networking_network_v2" "cluster_net" {
name = "${var.cluster_name}-net"
admin_state_up = "true"
}
resource "openstack_networking_subnet_v2" "cluster_subnet" {
name = "${var.cluster_name}-subnet"
network_id = openstack_networking_network_v2.cluster_net.id
cidr = var.network_cidr
ip_version = 4
dns_nameservers = var.dns_servers
}
resource "openstack_networking_router_v2" "kube_router" {
name = "${var.cluster_name}-router"
external_network_id = data.openstack_networking_network_v2.ext_net.id
admin_state_up = "true"
}
resource "openstack_networking_router_interface_v2" "kube_gateway" {
router_id = openstack_networking_router_v2.kube_router.id
subnet_id = openstack_networking_subnet_v2.cluster_subnet.id
}
# ----------------------------------------------------------------------
# control plane
# ----------------------------------------------------------------------
resource "openstack_networking_port_v2" "controlplane_ip" {
count = var.controlplane_count
name = format("%s-controlplane-%d", var.cluster_name, count.index + 1)
network_id = openstack_networking_network_v2.cluster_net.id
security_group_ids = [openstack_networking_secgroup_v2.cluster_security_group.id]
depends_on = [openstack_networking_router_interface_v2.kube_gateway]
}
resource "openstack_networking_floatingip_v2" "controlplane_ip" {
count = var.controlplane_count
description = format("%s-controlplane-%d", var.cluster_name, count.index + 1)
pool = data.openstack_networking_network_v2.ext_net.name
port_id = element(openstack_networking_port_v2.controlplane_ip.*.id, count.index)
}
resource "openstack_networking_port_v2" "controlplane_ip_public" {
count = var.controlplane_count
name = format("%s-controlplane-%d", var.cluster_name, count.index + 1)
network_id = data.openstack_networking_network_v2.ext_net.id
security_group_ids = [openstack_networking_secgroup_v2.cluster_security_group.id]
}
# ----------------------------------------------------------------------
# worker nodes
# ----------------------------------------------------------------------
# create a port that will be used with the floating ip, this will be associated
# with all of the VMs.
resource "openstack_networking_port_v2" "floating_ip" {
count = var.floating_ip
depends_on = [ openstack_networking_subnet_v2.cluster_subnet ]
name = format("%s-floating-ip-%02d", var.cluster_name, count.index + 1)
network_id = openstack_networking_network_v2.cluster_net.id
}
# create floating ip that is associated with a fixed ip
resource "openstack_networking_floatingip_v2" "floating_ip" {
count = var.floating_ip
description = format("%s-floating-ip-%02d", var.cluster_name, count.index + 1)
pool = data.openstack_networking_network_v2.ext_net.name
port_id = element(openstack_networking_port_v2.floating_ip.*.id, count.index)
}
# create worker ip, this can route the ports for the floating ip as
# well.
resource "openstack_networking_port_v2" "worker_ip" {
count = var.worker_count
name = format("%s-worker-%02d", var.cluster_name, count.index + 1)
network_id = openstack_networking_network_v2.cluster_net.id
security_group_ids = [openstack_networking_secgroup_v2.cluster_security_group.id]
depends_on = [openstack_networking_router_interface_v2.kube_gateway]
dynamic "allowed_address_pairs" {
for_each = openstack_networking_port_v2.floating_ip.*.all_fixed_ips.0
content {
ip_address = allowed_address_pairs.value
}
}
}
# ----------------------------------------------------------------------
# control-plane nodes
# ----------------------------------------------------------------------
resource "openstack_compute_instance_v2" "controlplane" {
count = var.controlplane_count
depends_on = [
openstack_networking_secgroup_rule_v2.same_security_group_ingress_tcp,
]
name = format("%s-controlplane-%d", var.cluster_name, count.index + 1)
image_name = var.os
flavor_name = var.controlplane_flavor
key_pair = local.key
security_groups = [
openstack_networking_secgroup_v2.cluster_security_group.name
]
config_drive = false
user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", {
private_key = openstack_compute_keypair_v2.key.private_key
project_name = data.openstack_identity_auth_scope_v3.scope.project_name
cluster_name = var.cluster_name
node_name = format("%s-controlplane-%d", var.cluster_name, count.index + 1)
node_command = rancher2_cluster_v2.kube.cluster_registration_token[0].node_command
node_options = "--controlplane --etcd --address awspublic --internal-address awslocal"
}))
block_device {
uuid = data.openstack_images_image_v2.boot.id
source_type = "image"
volume_size = var.controlplane_disksize
destination_type = "volume"
delete_on_termination = true
}
network {
port = element(openstack_networking_port_v2.controlplane_ip.*.id, count.index)
}
# network {
# port = element(openstack_networking_port_v2.controlplane_ip_public.*.id, count.index)
# }
lifecycle {
ignore_changes = [
key_pair,
block_device,
user_data
]
}
}
# ----------------------------------------------------------------------
# worker nodes
# ----------------------------------------------------------------------
resource "openstack_compute_instance_v2" "worker" {
count = var.worker_count
depends_on = [
openstack_networking_secgroup_rule_v2.same_security_group_ingress_tcp,
openstack_networking_port_v2.controlplane_ip
]
name = format("%s-worker-%02d", var.cluster_name, count.index + 1)
flavor_name = var.worker_flavor
key_pair = local.key
config_drive = false
security_groups = [ openstack_networking_secgroup_v2.cluster_security_group.name ]
user_data = base64encode(templatefile("${path.module}/templates/user_data.tmpl", {
private_key = openstack_compute_keypair_v2.key.private_key
project_name = data.openstack_identity_auth_scope_v3.scope.project_name
cluster_name = var.cluster_name
node_name = format("%s-worker-%02d", var.cluster_name, count.index + 1)
node_command = rancher2_cluster_v2.kube.cluster_registration_token[0].node_command
node_options = "--worker --internal-address awslocal"
}))
block_device {
uuid = data.openstack_images_image_v2.boot.id
source_type = "image"
volume_size = var.worker_disksize
destination_type = "volume"
boot_index = 0
delete_on_termination = true
}
network {
port = element(openstack_networking_port_v2.worker_ip.*.id, count.index)
}
lifecycle {
ignore_changes = [
key_pair,
block_device,
user_data
]
}
}
output "project_name" {
description = "OpenStack project name"
value = data.openstack_identity_auth_scope_v3.scope.project_name
}
output "node_command" {
description = "Command to join?"
value = rancher2_cluster_v2.kube.cluster_registration_token[0].node_command
}
output "private_key_ssh" {
description = "Private SSH key"
sensitive = true
value = openstack_compute_keypair_v2.key.private_key
}
output "ssh_config" {
description = "SSH Configuration file for use with ssh/config"
value = <<-EOT
# Automatically created by terraform
%{~ for i, x in openstack_compute_instance_v2.controlplane.* }
Host ${x.name}
HostName ${openstack_networking_floatingip_v2.controlplane_ip[i].address}
StrictHostKeyChecking no
UserKnownHostsFile=/dev/null
IdentityFile ${pathexpand("~/.ssh/${var.cluster_name}.pem")}
User centos
%{~ endfor }
%{~ for x in openstack_compute_instance_v2.worker.* }
Host ${x.name}
HostName ${x.network[0].fixed_ip_v4}
StrictHostKeyChecking no
ProxyJump ${openstack_compute_instance_v2.controlplane[0].name}
UserKnownHostsFile=/dev/null
IdentityFile ${pathexpand("~/.ssh/${var.cluster_name}.pem")}
User centos
%{~ endfor }
EOT
}
output "kubeconfig" {
description = "KUBECONFIG file"
sensitive = true
value = rancher2_cluster_v2.kube.kube_config
}
output "kube_id" {
description = "OpenStack project name"
value = rancher2_cluster_v2.kube.cluster_v1_id
}
output "floating_ip" {
description = "Map for floating ips and associated private ips"
value = [
for i, ip in openstack_networking_floatingip_v2.floating_ip.*.address : {
private_ip = element(flatten(openstack_networking_port_v2.floating_ip.*.all_fixed_ips), i)
public_ip = ip
}
]
}
provider "openstack" {
auth_url = var.openstack_url
region = "RegionOne"
application_credential_id = var.openstack_credential_id
application_credential_secret = var.openstack_credential_secret
}
provider "rancher2" {
api_url = var.rancher_url
token_key = var.rancher_token
}
# ----------------------------------------------------------------------
# cluster definition
# ----------------------------------------------------------------------
resource "rancher2_cluster_v2" "kube" {
name = var.cluster_name
kubernetes_version = var.rke2_version
default_cluster_role_for_project_members = "user"
rke_config {
# chart_values = <<EOF
# rke2-calico:
# calicoctl:
# image: rancher/mirrored-calico-ctl
# tag: v3.19.2
# certs:
# node:
# cert: null
# commonName: null
# key: null
# typha:
# caBundle: null
# cert: null
# commonName: null
# key: null
# felixConfiguration:
# featureDetectOverride: ChecksumOffloadBroken=true
# global:
# clusterCIDRv4: ""
# clusterCIDRv6: ""
# systemDefaultRegistry: ""
# imagePullSecrets: {}
# installation:
# calicoNetwork:
# bgp: Disabled
# ipPools:
# - blockSize: 24
# cidr: 10.42.0.0/16
# encapsulation: VXLAN
# natOutgoing: Enabled
# controlPlaneTolerations:
# - effect: NoSchedule
# key: node-role.kubernetes.io/control-plane
# operator: Exists
# - effect: NoExecute
# key: node-role.kubernetes.io/etcd
# operator: Exists
# enabled: true
# imagePath: rancher
# imagePrefix: mirrored-calico-
# kubernetesProvider: ""
# ipamConfig:
# autoAllocateBlocks: true
# strictAffinity: true
# tigeraOperator:
# image: rancher/mirrored-calico-operator
# registry: docker.io
# version: v1.17.6
# EOF
# etcd {
# snapshot_schedule_cron = "0 */5 * * *"
# snapshot_retention = 5
# }
local_auth_endpoint {
# ca_certs = ""
enabled = var.cluster_direct_access
# fqdn = ""
}
# machine_global_config = <<EOF
# cni: "calico"
# disable-kube-proxy: false
# etcd-expose-metrics: false
# disable:
# - rke2-ingress-nginx
# EOF
machine_global_config = <<EOF
disable:
- rke2-ingress-nginx
EOF
# machinePools: []
# machineSelectorConfig:
# - config:
# protect-kernel-defaults: false
# registries:
# configs: {}
# mirrors: {}
upgrade_strategy {
control_plane_concurrency = 1
control_plane_drain_options {
ignore_daemon_sets = true
delete_empty_dir_data = true
grace_period = 120
}
worker_concurrency = 1
worker_drain_options {
ignore_daemon_sets = true
delete_empty_dir_data = true
grace_period = 120
}
}
}
}
# ----------------------------------------------------------------------
# cluster access
# ----------------------------------------------------------------------
resource "rancher2_cluster_role_template_binding" "admin_users" {
for_each = var.admin_users
name = "admin-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster_v2.kube.cluster_v1_id
role_template_id = "cluster-owner"
user_principal_id = "openldap_user://uid=${each.value},ou=People,dc=ncsa,dc=illinois,dc=edu"
lifecycle {
ignore_changes = [
annotations,
labels,
user_id
]
}
}
resource "rancher2_cluster_role_template_binding" "admin_groups" {
for_each = var.admin_groups
name = "admin-group-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster_v2.kube.cluster_v1_id
role_template_id = "cluster-owner"
user_principal_id = "openldap_group://cn=${each.value},ou=Groups,dc=ncsa,dc=illinois,dc=edu"
lifecycle {
ignore_changes = [
annotations,
labels,
user_id
]
}
}
resource "rancher2_cluster_role_template_binding" "member_users" {
for_each = var.member_users
name = "member-user-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster_v2.kube.cluster_v1_id
role_template_id = "cluster-member"
user_principal_id = "openldap_user://uid=${each.value},ou=People,dc=ncsa,dc=illinois,dc=edu"
lifecycle {
ignore_changes = [
annotations,
labels,
user_id
]
}
}
resource "rancher2_cluster_role_template_binding" "member_groups" {
for_each = var.member_groups
name = "member-group-${replace(each.value, "_", "-")}"
cluster_id = rancher2_cluster_v2.kube.cluster_v1_id
role_template_id = "cluster-member"
user_principal_id = "openldap_group://cn=${each.value},ou=Groups,dc=ncsa,dc=illinois,dc=edu"
lifecycle {
ignore_changes = [
annotations,
labels,
user_id
]
}
}
resource "openstack_networking_secgroup_v2" "cluster_security_group" {
name = var.cluster_name
description = "${var.cluster_name} kubernetes cluster security group"
}
# ----------------------------------------------------------------------
# Egress
# ----------------------------------------------------------------------
#Egress IPv4 Any Any 0.0.0.0/0 - -
#resource "openstack_networking_secgroup_rule_v2" "egress_ipv4" {
# direction = "egress"
# ethertype = "IPv4"
# security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
# depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
#}
#Egress IPv6 Any Any ::/0 - -
#resource "openstack_networking_secgroup_rule_v2" "egress_ipv6" {
# direction = "egress"
# ethertype = "IPv6"
# security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
# depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
#}
# ----------------------------------------------------------------------
# Ingress
# ----------------------------------------------------------------------
# Ingress IPv4 ICMP Any 0.0.0.0/0 - -
resource "openstack_networking_secgroup_rule_v2" "ingress_icmp" {
direction = "ingress"
ethertype = "IPv4"
protocol = "icmp"
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
# Ingress IPv4 TCP 22 (SSH) 0.0.0.0/0 - -
resource "openstack_networking_secgroup_rule_v2" "ingress_ssh" {
description = "ssh"
direction = "ingress"
ethertype = "IPv4"
protocol = "tcp"
port_range_min = 22
port_range_max = 22
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
# Ingress IPv4 TCP 80 (HTTP) 0.0.0.0/0 - -
resource "openstack_networking_secgroup_rule_v2" "ingress_http" {
description = "http"
direction = "ingress"
ethertype = "IPv4"
protocol = "tcp"
port_range_min = 80
port_range_max = 80
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
# Ingress IPv4 TCP 443 (HTTPS) 0.0.0.0/0 - -
resource "openstack_networking_secgroup_rule_v2" "ingress_https" {
description = "https"
direction = "ingress"
ethertype = "IPv4"
protocol = "tcp"
port_range_min = 443
port_range_max = 443
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
# Ingress IPv4 TCP 6443 141.142.0.0/16 - kube api
resource "openstack_networking_secgroup_rule_v2" "ingress_kubeapi" {
description = "kubeapi"
direction = "ingress"
ethertype = "IPv4"
protocol = "tcp"
port_range_min = 6443
port_range_max = 6443
remote_ip_prefix = "141.142.0.0/16"
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
# Ingress IPv4 TCP 9345 141.142.0.0/16 - rke2 api
resource "openstack_networking_secgroup_rule_v2" "ingress_rke2api" {
description = "rke2api"
direction = "ingress"
ethertype = "IPv4"
protocol = "tcp"
port_range_min = 9345
port_range_max = 9345
remote_ip_prefix = "141.142.0.0/16"
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
# Ingress IPv4 TCP 30000 - 32767 0.0.0.0/0 - nodeport
resource "openstack_networking_secgroup_rule_v2" "ingress_nodeport" {
description = "nodeport"
direction = "ingress"
ethertype = "IPv4"
protocol = "tcp"
port_range_min = 30000
port_range_max = 32767
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
resource "openstack_networking_secgroup_rule_v2" "same_security_group_ingress_tcp" {
direction = "ingress"
ethertype = "IPv4"
protocol = "tcp"
remote_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
resource "openstack_networking_secgroup_rule_v2" "same_security_group_ingress_udp" {
direction = "ingress"
ethertype = "IPv4"
protocol = "udp"
remote_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
security_group_id = openstack_networking_secgroup_v2.cluster_security_group.id
depends_on = [openstack_networking_secgroup_v2.cluster_security_group]
}
#cloud-config
# SSH config
no_ssh_fingerprints: false
ssh:
emit_keys_to_console: false
# update and upgrade instance
#package_update: true
#package_upgrade: true
# files to be created on the system
write_files:
- path: /etc/fstab
permissions: "0644"
owner: root:root
content: |
radiant-nfs.ncsa.illinois.edu:/radiant/projects/${project_name}/${cluster_name} /condo nfs defaults 0 0
append: true
- path: /etc/profile.d/kubectl.sh
permissions: "0644"
owner: root:root
content: |
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
export PATH=$${PATH}:/var/lib/rancher/rke2/bin
- path: /etc/NetworkManager/conf.d/50-rke2.conf
permissions: "0644"
owner: root:root
content: |
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:flannel*
# run this command once the system is booted
runcmd:
- echo "${node_command} ${node_options} --node-name ${node_name}" > /kube.sh
- ${node_command} ${node_options} --node-name ${node_name}
- mount -av
# ----------------------------------------------------------------------
# CLUSTER INFO
# ----------------------------------------------------------------------
variable "cluster_name" {
type = string
description = "Desired name of new cluster"
}
variable "cluster_description" {
type = string
description = "Description of new cluster"
default = ""
}
variable "cluster_direct_access" {
type = bool
description = "Allow for direct access"
default = true
}
# ----------------------------------------------------------------------
# RANCHER
# ----------------------------------------------------------------------
variable "rancher_url" {
type = string
description = "URL where rancher runs"
default = "https://gonzo-rancher.ncsa.illinois.edu"
}
variable "rancher_token" {
type = string
sensitive = true
description = "Access token for rancher, clusters are created as this user"
}
# ----------------------------------------------------------------------
# USERS
# ----------------------------------------------------------------------
variable "admin_users" {
type = set(string)
description = "List of LDAP users with admin access to cluster."
default = [ ]
}
variable "admin_groups" {
type = set(string)
description = "List of LDAP groups with admin access to cluster."
default = [ ]
}
variable "member_users" {
type = set(string)
description = "List of LDAP users with access to cluster."
default = [ ]
}
variable "member_groups" {
type = set(string)
description = "List of LDAP groups with access to cluster."
default = [ ]
}
# ----------------------------------------------------------------------
# RKE2
# ----------------------------------------------------------------------
variable "rke2_secret" {
type = string
sensitive = true
description = "default token to be used, if empty random one is used"
default = ""
}
# get latest version from rancher using:
# curl https://releases.rancher.com/kontainer-driver-metadata/release-v2.6/data.json | jq '.rke2.releases | .[].version' | sort
variable "rke2_version" {
type = string
description = "Version of rke2 to install."
default = "v1.21.6+rke2r1"
}
# ----------------------------------------------------------------------
# OPENSTACK
# ----------------------------------------------------------------------
variable "openstack_url" {
type = string
description = "OpenStack URL"
default = "https://radiant.ncsa.illinois.edu"
}
variable "openstack_credential_id" {
type = string
sensitive = true
description = "Openstack credentials"
}
variable "openstack_credential_secret" {
type = string
sensitive = true
description = "Openstack credentials"
}
variable "openstack_external_net" {
type = string
description = "OpenStack external network"
default = "ext-net"
}
variable "openstack_ssh_key" {
type = string
description = "existing SSH key to use, leave blank for a new one"
default = ""
}
# ----------------------------------------------------------------------
# OPENSTACK KUBERNETES
# ----------------------------------------------------------------------
variable "os" {
type = string
description = "Base image to use for the OS"
default = "CentOS-7-GenericCloud-Latest"
}
variable "controlplane_count" {
type = string
description = "Desired quantity of control-plane nodes"
default = 1
}
variable "controlplane_flavor" {
type = string
description = "Desired flavor of control-plane nodes"
default = "m1.medium"
}
variable "controlplane_disksize" {
type = string
description = "Desired disksize of control-plane nodes"
default = 40
}
variable "worker_count" {
type = string
description = "Desired quantity of worker nodes"
default = 1
}
variable "worker_flavor" {
type = string
description = "Desired flavor of worker nodes"
default = "m1.large"
}
variable "worker_disksize" {
type = string
description = "Desired disksize of worker nodes"
default = 40
}
# ----------------------------------------------------------------------
# NETWORKING
# ----------------------------------------------------------------------
variable "network_cidr" {
type = string
description = "CIDR to be used for internal network"
default = "192.168.0.0/21"
}
variable "dns_servers" {
type = set(string)
description = "DNS Servers"
default = ["141.142.2.2", "141.142.230.144"]
}
variable "floating_ip" {
type = string
description = "Number of floating IP addresses available for loadbalancers"
default = 2
}
terraform {
required_providers {
openstack = {
source = "terraform-provider-openstack/openstack"
version = ">= 1.43.0"
}
rancher2 = {
source = "rancher/rancher2"
version = ">= 1.21.0"
}
random = {
source = "hashicorp/random"
version = ">= 3.1.0"
}
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment