initial commit of radiant cluster code

.gitignore

+.idea
+
+.terraform
+.terraform.lock.hcl

argocd/.gitignore

+secrets-argocd.yaml
+argocd-ingress.yaml

argocd/README.md

+This only needs to be installed once on the argocd cluster.

argocd/argocd-project.yaml

+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: argocd
+  namespace: argocd
+spec:
+  description: argocd server project
+  destinations:
+    - namespace: argocd
+      server: https://kubernetes.default.svc
+  sourceRepos:
+    - '*'
+  clusterResourceWhitelist:
+    - group: '*'
+      kind: '*'
+  roles:
+    - name: admin
+      description: Admin privileges to argocd
+      policies:
+        - p, proj:argocd:admin, applications, *, in-cluster/*, allow
+      groups:
+        - isda_admin
+    - name: user
+      description: Read-only privileges to argocd
+      policies:
+        - p, proj:argocd:read-only, applications, get, in-cluster/*, allow
+      groups: []

argocd/install.sh

+#!/bin/bash
+
+#ARGOCD_URL=argocd.141.142.223.4.nip.io
+ARGOCD_URL=argocd.kooper.net
+
+# ----------------------------------------------------------------------
+# install argocd-server
+sed "s/@ARGOCD_URL@/${ARGOCD_URL}/g" values-argocd-server.yaml > sed-argocd-server.yaml
+helm upgrade --namespace argocd --install --create-namespace argocd argo/argo-cd --values sed-argocd-server.yaml
+
+# create project to hold arogcd apps
+kubectl apply -n argocd -f argocd-project.yaml
+
+# create app to update argocd-server
+cat templ-argocd-server.yaml > app-argocd-server.yaml
+sed 's/^/        /' sed-argocd-server.yaml >> app-argocd-server.yaml
+rm sed-argocd-server.yaml
+kubectl apply -f app-argocd-server.yaml
+rm app-argocd-server.yaml
+
+# ----------------------------------------------------------------------
+# install argocd-notifications
+sed "s/@ARGOCD_URL@/${ARGOCD_URL}/g" values-argocd-notifications.yaml > sed-argocd-notifications.yaml
+curl -s https://raw.githubusercontent.com/argoproj-labs/argocd-notifications/v1.1.1/catalog/install.yaml | egrep '^  (trigger.|template.|  )'  >> sed-argocd-notifications.yaml
+helm upgrade --namespace argocd --install --create-namespace argocd-notifications argo/argocd-notifications  --values sed-argocd-notifications.yaml
+
+# create appp to update argocd-notifications
+cat templ-argocd-notifications.yaml > app-argocd-notifications.yaml
+sed 's/^/        /' sed-argocd-notifications.yaml >> app-argocd-notifications.yaml
+rm sed-argocd-notifications.yaml
+kubectl apply -f app-argocd-notifications.yaml
+rm app-argocd-notifications.yaml
+
+# ----------------------------------------------------------------------
+# install ingress route for traefik v2
+sed "s/@ARGOCD_URL@/${ARGOCD_URL}/g" templ-argocd-ingress.yaml > argocd-ingress.yaml
+kubectl apply -f argocd-ingress.yaml || echo "Could not install ingress routes, no traefik v2 installed"
+
+# ----------------------------------------------------------------------
+# install secrets for git repo
+kubectl apply -f secrets-argocd.yaml

argocd/templ-argocd-applicationset.yaml

+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-applicationset
+  namespace: argocd
+  annotations:
+    notifications.argoproj.io/subscribe.slack:sd-argocd
+spec:
+  project: argocd
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+      automated:
+        prune: false
+        selfHeal: false
+  source:
+    repoURL: https://argoproj.github.io/argo-helm
+    chart: argocd-applicationset
+    targetRevision: "1.*"
+    helm:
+      version: v3
+      releaseName: argocd-applicationset
+      values: |

argocd/templ-argocd-ingress.yaml

+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: argocd-server
+  namespace: argocd
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - kind: Rule
+      match: Host(`@ARGOCD_URL@`)
+      priority: 10
+      services:
+        - name: argocd-server
+          port: 80
+    - kind: Rule
+      match: Host(`@ARGOCD_URL@`) && Headers(`Content-Type`, `application/grpc`)
+      priority: 11
+      services:
+        - name: argocd-server
+          port: 80
+          scheme: h2c
+  tls:
+    certResolver: letsencrypt
+    domains:
+      - main: @ARGOCD_URL@

argocd/templ-argocd-notifications.yaml

+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-notifications
+  namespace: argocd
+  annotations:
+    notifications.argoproj.io/subscribe.slack: sd-argocd
+spec:
+  project: argocd
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+      automated:
+        prune: false
+        selfHeal: false
+  source:
+    repoURL: https://argoproj.github.io/argo-helm
+    chart: argocd-notifications
+    targetRevision: "1.*"
+    helm:
+      version: v3
+      releaseName: argocd-notifications
+      values: |

argocd/templ-argocd-server.yaml

+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-server
+  namespace: argocd
+  annotations:
+    notifications.argoproj.io/subscribe.slack: sd-argocd
+spec:
+  project: argocd
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+      automated:
+        prune: false
+        selfHeal: false
+  source:
+    repoURL: https://argoproj.github.io/argo-helm
+    chart: argo-cd
+    targetRevision: "3.*"
+    helm:
+      version: v3
+      releaseName: argocd
+      values: |

argocd/values-argocd-applicationset.yaml

+# nothing

argocd/values-argocd-notifications.yaml

+argocdUrl: https://@ARGOCD_URL@
+
+secret:
+  create: false
+
+notifiers:
+  service.slack: |
+    token: $ncsa-slack-token
+    signingSecret: $ncsa-slack-signing
+
+triggers:
+  defaultTriggers.slack: |
+    - on-deployed
+    - on-health-degraded
+    - on-sync-failed
+    - on-sync-status-unknown
+    - on-sync-succeeded
+    - on-sync-needs-update
+
+#bots:
+#  enabled: true
+
+# templates/triggers inserted at the end
+templates:
+  template.app-sync-needs-update: |	
+    email:	
+      subject: Application {{.app.metadata.name}} is out of sync.	
+    message: |	
+      {{if eq .serviceType "slack"}}:warning:{{end}} Application {{.app.metadata.name}} is out of sync.	
+      Application details: {{.context.argocdUrl}}/applications/{{.app.metadata.name}}.	
+      {{if ne .serviceType "slack"}}	
+      {{range $c := .app.status.conditions}}	
+          * {{$c.message}}	
+      {{end}}	
+      {{end}}	
+    slack:	
+      attachments: |	
+        [{	
+          "title": "{{ .app.metadata.name}}",	
+          "title_link":"{{.context.argocdUrl}}/applications/{{.app.metadata.name}}",	
+          "color": "#F4C030",	
+          "fields": [	
+          {	
+            "title": "Sync Status",	
+            "value": "{{.app.status.sync.status}}",	
+            "short": true	
+          },	
+          {	
+            "title": "Repository",	
+            "value": "{{.app.spec.source.repoURL}}",	
+            "short": true	
+          }	
+          {{range $index, $c := .app.status.conditions}}	
+          {{if not $index}},{{end}}	
+          {{if $index}},{{end}}	
+          {	
+            "title": "{{$c.type}}",	
+            "value": "{{$c.message}}",	
+            "short": true	
+          }	
+          {{end}}	
+          ]	
+        }]	
+    teams:	
+      facts: |	
+        [{	
+          "name": "Sync Status",	
+          "value": "{{.app.status.sync.status}}"	
+        },	
+        {	
+          "name": "Repository",	
+          "value": "{{.app.spec.source.repoURL}}"	
+        }	
+        {{range $index, $c := .app.status.conditions}}	
+          {{if not $index}},{{end}}	
+          {{if $index}},{{end}}	
+          {	
+            "name": "{{$c.type}}",	
+            "value": "{{$c.message}}",	
+          }	
+        {{end}}	
+        ]	
+      potentialAction: |-	
+        [{	
+          "@type":"OpenUri",	
+          "name":"Open Application",	
+          "targets":[{	
+            "os":"default",	
+            "uri":"{{.context.argocdUrl}}/applications/{{.app.metadata.name}}"	
+          }]	
+        },	
+        {	
+          "@type":"OpenUri",	
+          "name":"Open Repository",	
+          "targets":[{	
+            "os":"default",	
+            "uri":"{{.app.spec.source.repoURL | call .repo.RepoURLToHTTPS}}"	
+          }]	
+        }]	
+      title: Application {{.app.metadata.name}} is out of sync.
+  trigger.on-sync-needs-update: |	
+    - description: Application is out of sync	
+      send:	
+      - app-sync-needs-update	
+      when: app.status.sync.status == 'OutOfSync'	

argocd/values-argocd-server.yaml

+# helm upgrade --namespace argocd --install --create-namespace argocd argo/argo-cd --values values-argocd.yaml
+
+server:
+  extraArgs:
+    - --insecure
+
+  ingress:
+    enabled: false
+    hosts:
+      - @ARGOCD_URL@
+
+  rbacConfig:
+    scopes: '[groups, email]'
+    policy.csv: |
+      g, isda_admin, role:admin
+
+  config:
+    # Argo CD's externally facing base URL (optional). Required when configuring SSO
+    url: https://@ARGOCD_URL@
+
+    # Enables application status badge feature
+    statusbadge.enabled: "true"
+
+    # Enables anonymous user access. The anonymous users get default role permissions specified argocd-rbac-cm.yaml.
+    #users.anonymous.enabled: "true"
+    # Specifies token expiration duration
+    users.session.duration: "24h"
+
+    repositories: |
+      - url: https://git.ncsa.illinois.edu/kooper/radiant-cluster.git
+        type: git
+        name: radiant-cluster
+        usernameSecret:
+          name: radiant-cluster
+          key: username
+        passwordSecret:
+          name: radiant-cluster
+          key: password
+
+    # A dex connector configuration (optional). See SSO configuration documentation:
+    # https://github.com/argoproj/argo-cd/blob/master/docs/operator-manual/sso
+    # https://dexidp.io/docs/connectors/
+    dex.config: |
+      connectors:
+      - type: ldap
+        name: ncsa
+        id: ncsa
+        config:
+          # Ldap server address
+          host: ldap1.ncsa.illinois.edu:636
+          insecureNoSSL: false
+          insecureSkipVerify: false
+          # Variable name stores ldap bindDN in argocd-secret
+          #bindDN: "$dex.ldap.bindDN"
+          # Variable name stores ldap bind password in argocd-secret
+          #bindPW: "$dex.ldap.bindPW"
+          usernamePrompt: Username
+          # Ldap user serch attributes
+          userSearch:
+            baseDN: "ou=People,dc=ncsa,dc=illinois,dc=edu"
+            filter: "(&(objectclass=inetOrgPerson)(memberOf=cn=all_users,ou=groups,dc=ncsa,dc=illinois,dc=edu))"
+            username: uid
+            idAttr: DN
+            emailAttr: mail
+            nameAttr: uid
+          # Ldap group serch attributes
+          groupSearch:
+            baseDN: "ou=Groups,dc=ncsa,dc=illinois,dc=edu"
+            filter: "(&(objectclass=groupOfUniqueNames)(|(cn=isda_*)(cn=sd_*)(cn=org_software)(cn=all_disabled_usr)))"
+            userMatchers:
+            - userAttr: DN
+              groupAttr: uniqueMember
+            nameAttr: cn
+

charts/apps/Chart.yaml

+apiVersion: v2
+name: applications
+description: Applications
+type: application
+version: 0.1.0
+appVersion: 1.0.0

charts/apps/templates/healthmonitor.yaml

+{{- if and .Values.healthmonitor .Values.healthmonitor.enabled }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ .Values.cluster.name }}-healthmonitor
+  labels:
+    cluster: {{ .Values.cluster.name | quote }}
+    app: healthmonitor
+  namespace: argocd
+  annotations:
+    {{- toYaml .Values.notifications | nindent 4 }}
+spec:
+  project: {{ .Values.cluster.name }}
+  destination:
+    server: {{ .Values.cluster.url }}
+    namespace: healthmonitor
+  syncPolicy:
+    {{- if .Values.sync }}
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    {{- end }}
+    syncOptions:
+      - CreateNamespace=true
+  source:
+    repoURL: https://git.ncsa.illinois.edu/kooper/radiant-cluster.git
+    path: charts/healthmonitor
+    targetRevision: {{ .Values.healthmonitor.targetRevision | quote }}
+    helm:
+      version: v3
+      releaseName: healthmonitor
+      values: |
+        nfs:
+          enabled: {{ .Values.healthmonitor.nfs }}
+          storageClass: nfs-condo
+        notifiers:
+          {{- toYaml .Values.healthmonitor.notifiers | nindent 10 }}  
+{{- end }}

charts/apps/templates/ingresscontroller/traefik1.yaml

+{{- if and .Values.ingresscontroller.enabled (eq .Values.ingresscontroller.class "traefik1") }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ .Values.cluster.name }}-traefik
+  labels:
+    cluster: {{ .Values.cluster.name | quote }}
+    app: traefik
+  namespace: argocd
+  annotations:
+    {{- toYaml .Values.notifications | nindent 4 }}
+spec:
+  project: {{ .Values.cluster.name }}
+  destination:
+    server: {{ .Values.cluster.url }}
+    namespace: traefik
+  syncPolicy:
+    {{- if .Values.sync }}
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    {{- end }}
+    syncOptions:
+      - CreateNamespace=true
+  source:
+    repoURL: https://charts.helm.sh/stable
+    chart: traefik
+    targetRevision: {{ .Values.ingresscontroller.traefik1.version | quote }}
+    helm:
+      version: v3
+      releaseName: traefik
+      values: |
+        loadBalancerIP: {{ .Values.ingresscontroller.privateIP | default .Values.ingresscontroller.publicIP }}
+        externalIP: {{ .Values.ingresscontroller.publicIP }}
+        externalTrafficPolicy: Local
+        kubernetes:
+          ingressEndpoint:
+            ip: {{ .Values.ingresscontroller.publicIP }}
+        rbac:
+          enabled: true
+        dashboard:
+          enabled: {{ .Values.ingresscontroller.dashboard }}
+          domain: traefik.{{ .Values.ingresscontroller.publicIP }}.xip.io
+          ingress:
+            annotations:
+        acme:
+          enabled: {{ .Values.ingresscontroller.acme }}
+          {{- if .Values.ingresscontroller.acme }}
+          challengeType: http-01
+          email: {{ .Values.ingresscontroller.acme.email }}
+          staging: {{ .Values.ingresscontroller.acme.staging }}
+          logging: true
+          persistence:
+            enabled: true
+          {{- end }}
+        ssl:
+          enabled: true
+          enforced: true
+          insecureSkipVerify: true
+          tlsMinVersion: VersionTLS12
+          cipherSuites:
+            - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+            - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+            - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+            - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+            - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+            - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
+{{- end }}

charts/apps/templates/ingresscontroller/traefik2.yaml

+{{- if and .Values.ingresscontroller.enabled (eq .Values.ingresscontroller.class "traefik2") }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ .Values.cluster.name }}-traefik
+  labels:
+    cluster: {{ .Values.cluster.name | quote }}
+    app: traefik
+  namespace: argocd
+  annotations:
+    {{- toYaml .Values.notifications | nindent 4 }}
+spec:
+  project: {{ .Values.cluster.name }}
+  destination:
+    server: {{ .Values.cluster.url }}
+    namespace: traefik
+  syncPolicy:
+    {{- if .Values.sync }}
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    {{- end }}
+    syncOptions:
+      - CreateNamespace=true
+  source:
+    repoURL: https://helm.traefik.io/traefik
+    chart: traefik
+    targetRevision: {{ .Values.ingresscontroller.traefik2.version | quote }}
+    helm:
+      version: v3
+      releaseName: traefik
+      values: |
+        service:
+          externalIP:
+            - {{ .Values.ingresscontroller.publicIP }}
+          spec:
+            externalTrafficPolicy: Local
+            loadBalancerIP: {{ .Values.ingresscontroller.privateIP | default .Values.ingresscontroller.publicIP }}
+        ports:
+          web:
+            redirectTo: websecure
+          websecure:
+            tls:
+              enabled: true
+              certResolver: letsencrypt
+        additionalArguments:
+          - --providers.kubernetesingress.ingressendpoint.ip={{ .Values.ingresscontroller.publicIP }}
+          {{- if .Values.ingresscontroller.acme }}
+          - --certificatesresolvers.letsencrypt.acme.caserver={{ .Values.ingresscontroller.acme.server | default "https://acme-v02.api.letsencrypt.org/directory" }}
+          - --certificatesresolvers.letsencrypt.acme.email={{ .Values.ingresscontroller.acme.email }}
+          - --certificatesresolvers.letsencrypt.acme.storage=/data/acme.json
+          - --certificatesresolvers.letsencrypt.acme.tlschallenge=true
+          {{- end }}
+        persistence:
+          enabled: true
+        deployment:
+          initContainers:
+          # The "volume-permissions" init container is required if you run into permission issues.
+          # Related issue: https://github.com/traefik/traefik/issues/6972
+            - name: volume-permissions
+              image: busybox:1.31.1
+              command: ["sh", "-c", "if [ -e /data/acme.json ]; then chmod -Rv 600 /data/*; fi"]
+              volumeMounts:
+                - name: data
+                  mountPath: /data
+        ingressRoute:
+          dashboard:
+            enabled: false
+        tlsOptions:
+          default:
+            minVersion: VersionTLS12
+            cipherSuites:
+              - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+              - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+              - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+              - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+              - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+              - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
+{{- end }}

charts/apps/templates/metallb.yaml

+{{ if .Values.metallb.enabled }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ .Values.cluster.name }}-metallb
+  labels:
+    cluster: {{ .Values.cluster.name | quote }}
+    app: metallb
+  namespace: argocd
+  annotations:
+    {{- toYaml .Values.notifications | nindent 4 }}
+spec:
+  project: {{ .Values.cluster.name }}
+  destination:
+    server: {{ .Values.cluster.url }}
+    namespace: metallb-system
+  syncPolicy:
+    {{- if .Values.sync }}
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    {{- end }}
+    syncOptions:
+      - CreateNamespace=true
+  source:
+    repoURL: https://metallb.github.io/metallb
+    chart: metallb
+    targetRevision: {{ .Values.metallb.version | quote }}
+    helm:
+      version: v3
+      releaseName: metallb
+      values: |
+        configInline:
+          address-pools:
+            - name: default
+              protocol: layer2
+              addresses:
+                {{- toYaml .Values.metallb.addresses | nindent 16 }}
+{{- end }}

charts/apps/templates/storage/cinder.yaml

+{{- if and .Values.cinder .Values.cinder.enabled }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ .Values.cluster.name }}-cinder
+  labels:
+    cluster: {{ .Values.cluster.name | quote }}
+    app: cinder
+  namespace: argocd
+  annotations:
+  {{- toYaml .Values.notifications | nindent 4 }}
+spec:
+  project: {{ .Values.cluster.name }}
+  destination:
+    server: {{ .Values.cluster.url }}
+    namespace: cinder-csi
+  syncPolicy:
+    {{- if .Values.sync }}
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    {{- end }}
+    syncOptions:
+      - CreateNamespace=true
+  source:
+    repoURL: https://kubernetes.github.io/cloud-provider-openstack
+    chart: openstack-cinder-csi
+    targetRevision: {{ .Values.cinder.version | quote }}
+    helm:
+      version: v3
+      releaseName: cinder-csi
+      values: |
+        secret:
+          enabled: true
+          create: true
+          name: cinder-csi-cloud-config
+          data:
+            cloud-config: |-
+              [Global]
+              auth-url={{ .Values.openstack.auth_url }}
+              region=RegionOne
+              application-credential-id={{ .Values.openstack.credential_id }}
+              application-credential-secret={{ .Values.openstack.credential_secret }}
+{{- end }}

charts/apps/templates/storage/longhorn.yaml

+{{- if and .Values.longhorn .Values.longhorn.enabled }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ .Values.cluster.name }}-longhorn
+  labels:
+    cluster: {{ .Values.cluster.name | quote }}
+    app: longhorn
+  namespace: argocd
+  annotations:
+    {{- toYaml .Values.notifications | nindent 4 }}
+spec:
+  project: {{ .Values.cluster.name }}
+  destination:
+    server: {{ .Values.cluster.url }}
+    namespace: longhorn-system
+  syncPolicy:
+    {{- if .Values.sync }}
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    {{- end }}
+    syncOptions:
+      - CreateNamespace=true
+  source:
+    repoURL: https://charts.longhorn.io
+    chart: longhorn
+    targetRevision: {{ .Values.longhorn.version | quote }}
+    helm:
+      version: v3
+      releaseName: longhorn-system
+      values: |
+        defaultSettings:
+          backupTarget: nfs://radiant-nfs.ncsa.illinois.edu:/radiant/projects/{{ .Values.openstack.project }}/{{ .Values.cluster.name }}/backup
+          defaultReplicaCount: {{ .Values.longhorn.replicas }}
+        global:
+          cattle:
+            clusterId: {{ .Values.cluster.rancher.id }}
+            clusterName: {{ .Values.cluster.name }}
+            systemDefaultRegistry: ""
+          systemDefaultRegistry: ""
+        persistence:
+          defaultClass: false
+          defaultClassReplicaCount: {{ .Values.longhorn.replicas }}
+{{- end }}