diff --git a/group_vars/stage_dev/argocd.yml b/group_vars/all/argocd.yml similarity index 87% rename from group_vars/stage_dev/argocd.yml rename to group_vars/all/argocd.yml index 2587e61..d64cfba 100644 --- a/group_vars/stage_dev/argocd.yml +++ b/group_vars/all/argocd.yml @@ -1,10 +1,13 @@ --- -argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}" +k8s_argocd_with_keycloak: false + k8s_argocd_helm__name: "argo-cd" k8s_argocd_helm__release_namespace: "argo-cd" -#k8s_argocd_with_keycloak: False +argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}" + +k8s_argocd_helm__chart_version: 5.19.0 # https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd k8s_argocd_helm__release_values: @@ -149,6 +152,22 @@ k8s_argocd_helm__release_values: https service: sessionAffinity: ClientIP + ingress: + enabled: true + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + cert-manager.io/issue-temporary-certificate: "true" + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist ) | join(',') }}" + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + hosts: + - "{{ shared_service_kube_argocd_hostname }}" + tls: + - secretName: "{{ stage }}-kube-argocd-cert" + hosts: + - "{{ shared_service_kube_argocd_hostname }}" dex: enabled: false applicationSet: diff --git a/group_vars/stage_prodwork01/kubespray.yml b/group_vars/all/kubespray.yml similarity index 75% rename from group_vars/stage_prodwork01/kubespray.yml rename to group_vars/all/kubespray.yml index 1ef3699..c6887be 100644 --- a/group_vars/stage_prodwork01/kubespray.yml +++ b/group_vars/all/kubespray.yml @@ -1,2 +1,2 @@ -kube_version: v1.23.7 +--- kube_image_repo: "prodnso-harbor-01.smardigo.digital/k8sgcrio-proxy" diff --git a/group_vars/k8s_cluster/plain.yml b/group_vars/k8s_cluster/plain.yml index b456d73..bf0cbb9 100644 --- a/group_vars/k8s_cluster/plain.yml +++ b/group_vars/k8s_cluster/plain.yml @@ -4,8 +4,6 @@ ip: "{{ stage_private_server_ip | default('### use dynamic inventory ###') }}" ### parameters used by kubespray ### -kube_image_repo: "{{ stage }}-harbor-01.smardigo.digital/k8sgcrio-proxy" - kube_version: v1.23.16 cloud_provider: external diff --git a/group_vars/kube_control_plane/plain.yml b/group_vars/kube_control_plane/plain.yml index d5fa32d..6c9cae1 100644 --- a/group_vars/kube_control_plane/plain.yml +++ b/group_vars/kube_control_plane/plain.yml @@ -1,6 +1,6 @@ --- -hetzner_server_type: "{{ hetzner_server_type_kube_master | default('cpx21') }}" +hetzner_server_type: "{{ hetzner_server_type_kube_cpl | default('cpx21') }}" hetzner_server_labels: "stage={{ stage }} service=kube_control_plane" docker_enabled: false diff --git a/group_vars/stage_dev/kubernetes.yml b/group_vars/stage_dev/kubernetes.yml new file mode 100644 index 0000000..2af52c9 --- /dev/null +++ b/group_vars/stage_dev/kubernetes.yml @@ -0,0 +1,5 @@ +--- + +kubernetes_with_certmanager: true +kubernetes_with_externaldns: true +kubernetes_with_ingress: true diff --git a/group_vars/stage_devscr/argocd.yml b/group_vars/stage_devscr/argocd.yml deleted file mode 100644 index ac7b5fe..0000000 --- a/group_vars/stage_devscr/argocd.yml +++ /dev/null @@ -1,179 +0,0 @@ ---- -k8s_argocd_helm__name: "argo-cd" -k8s_argocd_helm__release_namespace: "argo-cd" - -k8s_argocd_with_keycloak: False - -k8s_argocd_helm__domain: &argourl "{{ stage }}-argocd.{{ domain }}" - -# https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd -k8s_argocd_helm__release_values: - controller: - logLevel: warn - logFormat: json - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: "{{ k8s_argocd_helm__release_namespace }}" - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" - repoServer: - logLevel: warn - logFormat: json - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: "{{ k8s_argocd_helm__release_namespace }}" - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" - env: - - name: ARGOCD_MAX_CONCURRENT_LOGIN_REQUESTS_COUNT - value: "0" - - name: ARGOCD_EXEC_TIMEOUT - value: "300s" - - name: XDG_CONFIG_HOME - value: /.config - - name: GNUPGHOME - value: /home/argocd/.gnupg - - name: HELM_PLUGINS - value: /custom-tools/helm-plugins/ - - name: HELM_SECRETS_HELM_PATH - value: /usr/local/bin/helm - - name: HELM_SECRETS_SOPS_PATH - value: /custom-tools/sops - - name: HELM_SECRETS_KUBECTL_PATH - value: /custom-tools/kubectl - - name: HELM_SECRETS_CURL_PATH - value: /custom-tools/curl - # https://github.com/jkroepke/helm-secrets/wiki/Security-in-shared-environments - - name: HELM_SECRETS_VALUES_ALLOW_SYMLINKS - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_ABSOLUTE_PATH - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_PATH_TRAVERSAL - value: "false" - - name: HELM_SECRETS_KEY_LOCATION_PREFIX - value: "/sops-gpg/" - volumes: - - name: custom-tools - emptyDir: {} - - name: gnupg-home - emptyDir: {} - - name: sops-gpg - secret: - secretName: sops-gpg - volumeMounts: - - mountPath: /home/argocd/.gnupg - name: gnupg-home - subPath: .gnupg - - mountPath: /usr/local/bin/kustomize - name: custom-tools - subPath: kustomize - # Verify this matches a XDG_CONFIG_HOME=/.config env variable - - mountPath: /.config/kustomize/plugin/viaduct.ai/v1/ksops/ksops - name: custom-tools - subPath: ksops - initContainers: - - name: 1-install-ksops - image: viaductoss/ksops:v3.0.1 - command: ["/bin/sh", "-c"] - args: - - echo "Installing KSOPS..."; - mv ksops /custom-tools/; - mv $GOPATH/bin/kustomize /custom-tools/; - echo "Done."; - volumeMounts: - - mountPath: /custom-tools - name: custom-tools - - name: 2-download-tools - image: alpine:latest - command: ["/bin/sh", "-ec"] - env: - - name: HELM_SECRETS_VERSION - value: "3.12.0" - - name: SOPS_VERSION - value: "3.7.1" - - name: KUBECTL_VERSION - value: "1.22.0" - args: - - | - mkdir -p /custom-tools/helm-plugins - wget -qO- https://github.com/jkroepke/helm-secrets/releases/download/v${HELM_SECRETS_VERSION}/helm-secrets.tar.gz | tar -C /custom-tools/helm-plugins -xzf-; - - wget -qO /custom-tools/sops https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux - wget -qO /custom-tools/kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl - wget -qO /custom-tools/curl https://github.com/moparisthebest/static-curl/releases/latest/download/curl-amd64 \ - - chmod +x /custom-tools/* - volumeMounts: - - mountPath: /custom-tools - name: custom-tools - - name: 3-import-gpg-key - image: argoproj/argocd:v2.2.5 - command: ["gpg", "--import","/sops-gpg/gpg_key_smardigo_automation__private"] - env: - - name: GNUPGHOME - value: /gnupg-home/.gnupg - volumeMounts: - - mountPath: /sops-gpg - name: sops-gpg - - mountPath: /gnupg-home - name: gnupg-home - server: - logLevel: warn - logFormat: json - config: - url: 'https://{{ k8s_argocd_helm__domain }}' - helm.valuesFileSchemes: >- - secrets+gpg-import, secrets+gpg-import-kubernetes, - secrets+age-import, secrets+age-import-kubernetes, - secrets, - https - kustomize.buildOptions: "--enable-alpha-plugins" - rbacConfig: - policy.default: role:readonly - policy.csv: | - g, {{ argo_realm_group }}, role:admin - g, admin, role:admin - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: "{{ k8s_argocd_helm__release_namespace }}" - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" - service: - sessionAffinity: ClientIP - ingress: - enabled: true - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - cert-manager.io/issue-temporary-certificate: "true" - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist ) | join(',') }}" - nginx.ingress.kubernetes.io/force-ssl-redirect: "false" - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - hosts: - - "{{ k8s_argocd_helm__domain }}" - tls: - - secretName: "{{ stage }}-argocd-cert" - hosts: - - "{{ k8s_argocd_helm__domain }}" - redis: - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: "{{ k8s_argocd_helm__release_namespace }}" - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" - dex: - enabled: false - applicationSet: - enabled: false - configs: - secret: - argocdServerAdminPassword: '{{ argocd_server_admin_password | password_hash("bcrypt") }}' diff --git a/group_vars/stage_devscr/bootstrap.yml b/group_vars/stage_devscr/bootstrap.yml new file mode 100644 index 0000000..55c9db4 --- /dev/null +++ b/group_vars/stage_devscr/bootstrap.yml @@ -0,0 +1,19 @@ +--- + +argocd_bootstrap_infrastructure: true + +harbor_bootstrap_helm_url: "prodnso-harbor-01.smardigo.digital/infrastructure" +harbor_bootstrap_helm_name: "infrastructure" +harbor_bootstrap_username: "{{ harbor_bootstrap_username_vault }}" +harbor_bootstrap_password: "{{ harbor_bootstrap_password_vault }}" + +gitea_bootstrap_username: "{{ gitea_admin_username }}" +gitea_bootstrap_password: "{{ gitea_admin_password }}" +gitea_bootstrap_url: "https://{{ stage_kube }}-gitea.smardigo.digital/{{ stage }}/{{ stage }}-argocd" + +custom_ip_whitelist: + - '94.130.225.244' + - '78.47.103.109' + - '167.235.66.68' + - '94.130.177.76' + - '167.235.69.85' \ No newline at end of file diff --git a/group_vars/stage_devscr/kubernetes.yml b/group_vars/stage_devscr/kubernetes.yml new file mode 100644 index 0000000..1a2f942 --- /dev/null +++ b/group_vars/stage_devscr/kubernetes.yml @@ -0,0 +1,6 @@ +--- + +kubernetes_with_certmanager: true +kubernetes_with_externaldns: true +kubernetes_with_ingress: true +kubernetes_with_gitea: true diff --git a/group_vars/stage_devscr/kubespray.yml b/group_vars/stage_devscr/kubespray.yml deleted file mode 100644 index 9b5a85b..0000000 --- a/group_vars/stage_devscr/kubespray.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -helm_enabled: true -kube_version: v1.23.7 diff --git a/group_vars/stage_devscr/plain.yml b/group_vars/stage_devscr/plain.yml index 3aac3b9..de2dfe3 100644 --- a/group_vars/stage_devscr/plain.yml +++ b/group_vars/stage_devscr/plain.yml @@ -2,17 +2,14 @@ stage: "devscr" -default_plattform_users: - - 'claus.paetow' - - 'friedrich.goerz' - - 'sven.ketelsen' - - 'michael.haehnel' - - 'hoan.to' - - '{{ awx_ansible_user_name }}' - - '{{ gitlab_ansible_user_name }}' +hetzner_server_type_kube_cpl: cpx21 +hetzner_server_type_kube_node: cpx41 + +custom_stage_plattform_users: - 'daniel.risse' - 'esther.fuhrmann' - 'philipp.eichhorn' + - 'hp.wissenbach' # TODO read configuration with hetzner rest api shared_service_network: "10.1.0.0/16" @@ -25,8 +22,8 @@ netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}" # push mirror: https://{{ stage }}-gitea-01.smardigo.digital/communication-keys.git gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}' -kubernetes_with_awx: False -kubernetes_with_gitea: True - -shared_service_hostname_harbor: "{{ stage }}-harbor.{{ domain }}" +gitea_admin_username: "gitea-admin" +gitea_admin_password: "{{ gitea_admin_password_vault }}" +gitea_postgres_username: "gitea-postgres" +gitea_postgres_password: "{{ gitea_postgres_password_vault }}" \ No newline at end of file diff --git a/group_vars/stage_devscr/vault.yml b/group_vars/stage_devscr/vault.yml index 9b6e4db..b000972 100644 --- a/group_vars/stage_devscr/vault.yml +++ b/group_vars/stage_devscr/vault.yml @@ -1,478 +1,482 @@ $ANSIBLE_VAULT;1.1;AES256 -32373530646437623564353664653131393162616632666237323836386433633865653533383438 -6334343735303230643837306361393839303163323861320a653832636538623431376638316336 -39633332323166653737626630343266616664373130323537376139646565666536303830303137 -3032323738646230320a363964393234383762633862303030313436326561626533373165313738 -37376236333832313233346539383762363232383132633733346132346638386232383366333763 -33383433613266306139666265393731626631373466353230626632643637643637383531653639 -61643033626439663933363937363961646636346333623866303165663835393934633262313238 -38356433313663663032386165313162393836636432646265623533336664643861363136636363 -36303562353465383464326131646462393533376533386664313436373338306430376662333439 -31656233633963376130386332343634313064373230633632366164323139303037363638396635 -39616633623030306633323439373539313635356332663732643139613134643162633334663033 -33356635626235353031623738396439663132323034653064323563336564633134613331313538 -36396338613362333839346339366533376437383034306439383030323635393130313234356133 -38343433396138316261333364343466386238303464323434316361323937636565613231633636 -65383238626232383239663938326339356236333239623865633766396438323766386232323366 -36353065363537363637636231343561613961636534646236326263376437373434653237646334 -62623964393135366431303636646539336335343061336261663661333163636433343736633435 -37313033323563313930613139303062303862363732393034636533636364623466366534316234 -63656338366636336362626462656665313365313362366565373937623565396234373135663534 -64613730336564616238303839396464343765306461306537363036333564646238313565663562 -36636432323231393465323431316235636633346161363762373762326438336436343061663564 -38653966633231646366643533633930383063356237383761646363323761623638303634653235 -31633764336431656132633730383331306363333566393062393133333135346437646363663535 -37396337323538663363326534306561333161333136313637633563633734323037363436353738 -33383333386164666563373162333638653132633533663233333762383234643465656331633232 -32653530383835666663323564346663373835313034353730323261653434666331633637666539 -63346631653138666336346231323130643166343731363161383739653134633532313037656530 -39633931623233623530653235663363336137353165613939353964376139643739353039633730 -36396637393131313563303237333335363064623334383865326430633134323963363634616130 -39633133303930303336303064653331653363306662383134353361333535306634623330323133 -33373236303463616530326364316439306530386664346531393766643235343735383563356463 -35316564623561363265326139373263313362353933393731396134316365353836396261626231 -64313138303933313161313435306239356632316364303332623861386537396435366332613239 -36396566386139626461313461386332313534653065663030323233373337383831626532643134 -37366666396338333538333431323162396263393039383530623631313032316339653766666361 -65666638383265623661613830323233376230663630373135393565666437303432353031623737 -38613931653162646262326136346431646632316230646537656438323132633837383235623332 -37303264343038646666336632626339333036396663303162366238653530373162663138666331 -61313763643735316533323761626563316430653538363730363336636538313562376236366265 -61383565346633313937613839616564656365343839383664616439666635353239343339393230 -35663231326631643237643366376261366165343432633533306231386563643031343062623063 -38373833363731353935343366613134373930613937303635626161303632383261656565363639 -35663938356534633531653131313039386135316230616638306636613535343936663934623438 -37366665303333633630383062326331356165643930643966346138366564363231306639643862 -31643038303961366530313433653937373563393761303465363564343130333534396333633233 -66613561303662663033643031613237396430343464353838363733353737323765383532313630 -33303965333365353838316337356466643865343338333932343566393035356662653935383932 -32316463323962313461663465343334613434353063636166386335336432306138353935346561 -32326131656231636263323563356231313564613030366232376331643164336231336535313564 -32393930346630646335663636633963316333623362373363323839336665666163633436363630 -32383737656438643564366334313135626332656162656261346535336362373963353436306261 -61666430306139616361656537303064333833353634336434623765326539373666633535623565 -34633861393830663831663434353166346631343262333431616365326537386164616537636662 -65326639363963383931666636653364313234636431646133353761306532396339326565356338 -39386632343437323462303335333664313636343538343865343331366131643034383662363532 -37376236623361386432316637346666613735633535386363373837353339316138653039666534 -31643039393665303863353038393430366265316430623838396634613630303263663032633330 -35396532383333363236373638383661386630353432326637613266326264303164613233643863 -66353931376230396536643631633635326562616233356561343139393738326137643739623463 -38313161623164326261363365303737663766353230626364336636613037353162633863343164 -32653432646466306162393862616562313537353261616465636638613638383330633463306531 -65666533636636306437366566323866626639326536383938663936326365346463343162343062 -65353435613333396234633637313064386333323537636563326461643834356566363765303532 -30633935623632646332333537636565333766396362356361386666363962643233316464383036 -62646462666631626365663730626366653632303130623836653732623233356365656433376638 -65376366356435343264356432393161323361623466393561636137643631653965343766666362 -64303931623235383235626266383936373431616330366434393633656338363735386362353533 -62336364646361623565616533613861343039643066613330333565633433626362663463323832 -35653738623266633162613165626464343664663635323362333861643066623564353431616535 -31306238336535393037313264616533346237393964303164666236616631313038356365396162 -39393661323066643430616331363263633964356631303936313863343061303234663732383462 -32326263333238653030643164333037613432303266326335343335633163366162656362613734 -62356665316136646133316436353632633233393463323830383365386234646534353061313139 -32313637303739623263333633613838346336353165396362363564366561653730653536353536 -64343263616231646365643065343939663164643165613235396263343465303564613034663263 -63643734663832303665303861306336303433346336366331396238346136636262336264653465 -37616239653866306234393735626333326130623036396531366637633639393936653737343637 -65633966363534313665663735646131656136626337393865663332393264643639613763626137 -38663361393961346633306638383937373135323439333064623666316535633162623930373138 -39343332366137623338623834323434393035356335306236303838373136356265646336666233 -35653262393031396237316332616132613030333664633231346430346539346634366265663632 -63303863303065316633373730653539393464636463373164303835333132653334313636396539 -62326332303263626362306336333631643261383637663533373030323137343734316136363134 -31616638323835613965653465306632633261633937356662396261643065303532333936666435 -32383437313031373164393664633364303939623835626639313365353566373933313961643265 -64353539363735616233653438383731323263633633383639353938383535343730663362383933 -37333066376638633866643334333031383438626162373762323436633161313564386335646634 -62396636383833636338303937303835373137653730386364393032343965326564653539333532 -31303637623439663736313564383538626132663864616631653263323662333738353064333364 -64373264393961373164366537613137303239383733653037383535323533343136316166633134 -39313134373935656461636531656433616339326566356363376564323635643639633563663461 -31383630343963343165663061393739613766376236343836616237326530313064613366336633 -34633938386364363437633162303336366366393234356365366338636338376232636132343161 -36356165333963393632616335306434623737353661663863313136336530613261643663303930 -31623239666531356463613361666634366337393832643931633733616361626632306633333932 -34303435346161636464316461616162346264313639656135336133346432643134356365333239 -37623931396162623638656132393437656538323836653236326561633463333631393331663139 -35323264386236363131393433326161663838306361353332646463306437393565336635333339 -61346434613439646137643663376362646139373736663636333032333635613632666631643665 -62633639643763623030373830323164373066346463313435306134303539303561376662653064 -62653135373065613331636564363037353333643961396332393663643430316462373038323731 -38396633653836313638353032636363653866623532663532383562336139636234373338646162 -38663863643538343161623531373537343062333262363439666563626262326338636438346338 -62383662333739643739616461336663343639643439616663363635366566383537666564343930 -63643465356366306430366430313331613666336130623735313539663730663732323131323266 -33643365656632623562323732653031316435303963323830636531343736323239626263636334 -64346435616431333535653132653466356564623463343633303938613831356132656135356335 -31383438633136613135386131643438303233653232363636653933313334643735393737306239 -62396437363636623561623532333230646439333937656264393931643934393339373831333462 -38333163383139313736343733316538616266386136356237353236643638356432356164383033 -34313164353837656236663431653065666638373435316333666430306265343462323162353738 -63383730623966356161633935613730623736393562326664373037616637306463316261626339 -62396532373334313139653630643866366163363539646638396530353166656132663632303635 -65613334623463616533633637346633346238363536656664623835333630666335353965613564 -61613934326434323532343631333565633538323138393736383030373165643135663736343332 -30346339633362363139613938663762633739656461366437613961616333396163383162346665 -34383037326164393164333931656131336633316237343565306234313664323134613164646264 -65336562366662363966373661356237363539303036363534643136366434303261353061333731 -61663065626232636432663036373533366266343364366334663436653261323161613638376436 -64306562323639653534363336366532653539653566306633343162343862343633646433633866 -39363838633734636462613262343764663331356139633662363934306133363639643166356131 -30623339376635653962666661616261323361386564313061633663336161356631613565373761 -63326631323363313264616561306135636461323166323862306166643538326262656261373866 -34383161303162643636616534663938393663666335336639643063663435616138333265663164 -37303936633964613735616639656461623030353832303337623133326439363564653061323038 -64303534396263626531323966393836356663383966343132353037316166646664393761356532 -30316330626530633166663934356665343033633761326535316233656638623564326162373664 -63326463363339653837633835616430653166363739313531623536383738396635333830343532 -35383164366161663836646364303166643031346261613431366437343466343030643738306231 -61343835663165623137333166666134613138316634663763373861336133363764373533393662 -65396630616165323330383266333065326566613661343232353264653962613532663638623361 -37653462336239376263613461323430323032323566363732626665373433353064346564306465 -66306662626364316164326662333336336134613066636466343565623163386361333265383362 -39626562346263303832616439613961313334346530393331633535343065613935393635356234 -33336562333034393233346636393830333635326637333862373336633438316261636463656434 -35326564323736623537666338323330366566666665663964363633393365333332323233313838 -63373830333436323237656137366335303165323731663833636136653937386564663638346464 -64363862353264623033666661353066636639383833353437663661383361633931306636653731 -63323831303466363961643264366538363338386131663334646130613431373363353338616362 -61303565346431393965323439343737356164636336653663366166623837613732333531326163 -66623031363334393164383664633231626338656538666436643230393336663939643734656331 -31643462303761303237643737386234643165396333316237636434333031646461323461313134 -34336263366632316437663933666539636332376362386638313535393266626661366564363439 -30306230333235646163666564663566383532323339323730616266306436643138323661386534 -62326632373431336131623464356533663962666465343230663638363137386538643339363038 -31386433383739623334356165383036613261633136666562663638643236316532343663396661 -39656539356230383839383061636566666234393834306665313537343564386435346638663866 -37663635646531383739623339613736623530383132316363643136383638636434623235343233 -63333865346361656666373938353533333030643261316336663362663061396261373239323663 -31636463306239373539326332633164326430616630626565306434653864333738643638656163 -35363035333133666263346334393963626539663464376638303432366662393661303064663130 -38366235373938353337326134383965343761666431613766373337366135666235346433343538 -33373564373861663964396262663235316637333533366461353036623433623662653263313530 -63643639616136646333383036636632353934646564343564653830393332303139366339643432 -39346362646233313537376263626137316433366163346236643839386565616230326432333638 -32363937383037323738313932633034303232393365326234376436356662393830356136623161 -36316539343462633865666562303730646663303731333431623437633534623662313938343931 -37643930346631666465646264313463623432633966633339346363616636636538353564666336 -34626335313661373630366531313263636638393639333131303166366566376362653263643364 -33393239646637653631393261633838343333323933613135656233613133323638353037353766 -39353431663136643234336634386535646266636434363864666566333834316465303564663934 -34613839383236363437316163363738353832646438666133366134363465336666363133653063 -66623562306533386431623663356164636563346166383432643762343138363162376337626330 -62363562326133653034383036396663646665356366363935346334383362396663356439656264 -36663665353936633765623864326139643538336363333530333763396533333562313664353261 -33633133666264633466303634306264313165363336643666313738303034383132313037636534 -63313538653035653637373033333632633661383861316366393631393865303931643263316361 -39636434623361373865633435646136366639643866363637363461303862666466643736373461 -65636530356165653837646662333839363734343462343064313433346362636665373461346162 -39313863323431386566336332356237663362383335326364646635656435336162343263643464 -35646461323837336333643335396663346466323237386164656461653236623565643263363234 -65636633343339613833326265396464346461623964336361303966383236346661303931363231 -66646165366164366464363130323065386138323138376432346238653837353939633534623036 -62356330613538353536326531316339396565613334613462376632383839636266616530376363 -65363038306562383138356234363937373139333432373762346134333462343030316434323934 -36353433346531653862393439653062373662363462356562623763646566626139663362343935 -61636565373634633937323763646534643631356639336337323939373364343330656366643036 -39623934313866623034656534373233306566373163396533306632316438623164383138633532 -34336663363639333739333338343939663334646262373038653465396630303430376631356165 -64623537376530386139633461326635373663306464396138333062666233386435383530633333 -34643436353737663835643830666231393830303235306631343061393130383039643432656362 -35663566313134383764626237393162333637353666623131303437656463666539393437353362 -64323532613138666166393664636433653166326365643636333834353863343435303935653561 -64623638326438323933306633353763656638376138663531366530663039663237363965376564 -36646136383965323264383865656164363732626232333631616436313931646465343763363837 -32353738643033353761386335316564633138306135663639396666316139386564656266333262 -30353461383038623032633136613536373236626363346265383065336138356632373261623665 -65303337353432373037316537643938303839653835323062393764373034356430396230616332 -64313832336663333939663732313761326135326631316538316232643638386361643738626132 -37343234663965626232356238313966656330643538363164346637616338633163633132393132 -62643235643837663564666131653737346431636665656630653166636665323538373336313034 -30666634363662333864653736333234383631366638623739353835376638643764303536326334 -63333139313263623862653335393334666364663366363933663065343565333761353165383932 -37666136643430356362663339343137333037336436663964626134643234303338626139353132 -36636636343963333238383566646262396162323834643432626465333430346131663662376463 -34353137376366323739653432303339393437613130333534373537353135393939613864616263 -63643833656364383333343263306364643231653036376265313237393935303832306133666266 -65373532386363363037333066363435323330326538303938313464363633633839346135333530 -31633062623063613235353738333036316239316438346261326332396366653931643133393537 -34623137613133393439616461663562343165636536326135646163343836666464366237333664 -31333630303332383839613735316564666465363230353063333365633430386165306164653565 -63633532663865636334343762373063643262633537616139613563663264383832343836343162 -37356434333563623835313361653465353139646562316533613462313232636565663439616234 -30323561393935356337343031666431633963616330303733383130663135653563366564643366 -33343136313730626564623331613731303539346238303666663933643836643566626466303035 -64633264356433353037383137663036323939666233303637306265353066316631326630653536 -33616465366238663435663632663563376363316665336436663132336435346531333835326438 -33663466363063656332313365363031343364636134626663613130653637643466643634393732 -65383336356135396439313731663362303137623333353137313961643336663364666266303761 -31666538386234623930323435663130346339356561363764656530663134396330623830303039 -36643832333739663130393930633734326136376366653262346538653166623764626466623964 -32373663346562376230323031313665396237353839393135333039623638663239363635643563 -38396633376366636665386236313063336566363031646664363137626639393334616535343231 -63363834353335333339646366633661393062303831663035396635373935316339316263383266 -66303635323736303333393633386233366362356465636538363136383864653431613133643464 -36333931363732373439376330633733366366643933313339343065383634306661663464353464 -35323039653033343235636365323939303539363237356136373963653363353837313662313631 -39623038343139336264353866333637663132616631346232326133653933383061663963663830 -64663737323831613961383737343232333232616133306161646463323062653130313366383533 -33303163336535353636363163643634333461636531373265653431616264343832393761663237 -61333536363466313834646632626164316161343636313561643539393030356437656238626537 -66353636333232373835353861666336396236376230666635646636363035653138633735656566 -39313361393438663333346263613633336164356234653461323763306564613462366563386332 -62323961383531333135313233303934643132386662396161653736363062616137353939613365 -64383465343034616633383533623533626539343835316265316334386462623935383835336238 -31396238663236326663313037313063653261643636613032353235383936356565663733613539 -33656661366563646237343536633730383662346333343736393739663462376466336138356431 -35316565363761343861383963613735303639373564393063336335343563613665336231363963 -30376263366365303664383464636164333335643965636161613835323062616461333230366136 -39313262373938313362326235333232356130346635373531636434366535336236313531336265 -34666565343136623539396634333737306336666437383838343539646237643061363739613638 -38363463646265306164626439353131333866623238396465646235353034333531363433613232 -31343566643564623337613564613465353131313634396532336133393362396539333033376139 -62373934313437386161633336393134363263336131313435306261323933663463316630616437 -33316230656263376137383161386134316565663535396466346631613636663865393864306430 -38626265633431333730326635343538393931626339396130343535386138396630316137626436 -30613336653966363830646530323461366538663962633731313638666361393065323536373633 -33336339336633363965316565336531653663333837363833636263633439653064353132663164 -35626636643137376638373561656338313131346537383061393933666535643764316336373363 -32373562396138353831633837653832336265333331666137613064623064383137343166613932 -64343439396230376362346532383438393166343031393666663537336334313566666634633737 -36666138323862656535356537323538656135383639646236363365336636666432356466666562 -66333630633965353131316237356633303331643036343531383964663539333439633330656131 -34363766333234323031656231343031396332633736373531636236626431666431626364366162 -62313132316134393761306639643465396561643834326636303736303138623736646236333261 -62626138373466663539646535373632633631336463336635633933393233663135343436336637 -38383065333733616435323839643832396431313864663861383464373532626637383336366432 -66373532343436333736316262333037386138393762646634643035633631323664353831326536 -33616432633433636565313866343239623163363365663236633866376338343964643735646138 -64323566313237386161633731616631303666633130663333643137636230383537653261303465 -61623432653537343639303134333561633465323365316635653030666461346163653161313462 -65386133346233336163633663303236623330343532333864303837326638613238333530626130 -34346530623063646636336233376138643333663066613461616633333531613734383336323564 -63373162366465616136646330333938306231666530636462343231653032636536386335366363 -36323632333463353533316138663232393464346364383161636632643134363361326530663765 -30323537623235333466363431353866343632303462316361313362336366646330393062333562 -35623039656663666333656237343562363262353734353636343163613664326337663763653065 -37373631303464626562663133626361623665646265393735363732343238623732386437653561 -39633061376635636436333463353037336133326132653236323663306332376537353764616565 -34663332643433646430363931343037393238656534386333376231393739333839343735383134 -31306338343539393136366639393231626464303236363066313666316663383233383862333838 -38346434356166613264666632393833356535663262616333363635313332343166316565316134 -34386564343164376534343435656633333534643938373562613533366237363134333632336363 -38626266626333393836373534636265646266653966363637393632323735656261373638353834 -61613362393162613733663636333362663937393163636635343135613534343363366565623038 -31366165313562396562613861316231386133323137333535626435383333313934386137643066 -38346237343034336564333261626536346430373866373430636537386263643064313131346230 -65643233306430643030646431343130376262623764393961663833383033643066663835646661 -34626131393932333636643263633231646238376432653537323963343439393361303163313130 -32353036363032623138353235376335353362653765333834663038643963396364613036336161 -63343530643232663839633939363765393038373366613430636337373531636464346365626137 -32653838303832383765393534613739633762376335303362666664656237323039373563363162 -64373837376366393830636266383366666161366231306461653932303034633965666336356332 -33306662356566343135306465323464643964653265616362353431353333653038336161386262 -36613138653036666532396533633037383862653834373361363965643637623836306530643361 -62363834666632623863623765616135346362393831653965383833326534346665353938653964 -30373539316664666636333865346465346562366166396139346331303466323530623434376138 -33393130633330343366633335333439396134313933373539373964373063666436333330656530 -64323336343431393330346432396330333237326265323438323138653835616561386531373966 -34383831663061373261303939303836383463346430613136313834386531623766303530633562 -32323337633232306337633839616565663736323334633033303561386436373538356462663831 -34633261643361373864303538363561333534663032666464356633376337376337393639626539 -63366330316439336534343337376266376664303832386335653236323433626231663239313935 -38353662356436313839666664393839633731323734643166613335653137353962376534353634 -66396431303633376334323436613032386237653130383138366234363636613464663136313836 -37313931653737636132663364373637383133643330653435643031623763316339303431383963 -35346139363636306536343332666230373435373562333466383339663461633136656261323336 -34636463386333316235333634336439623934313632373938326435626434303033666366363963 -61663935363364626235356130386462326562666462323432626338666435383162316533373035 -39363734383062633436333336623764616135666436613238643965643262313363303038303037 -36373438393366343435316239613266623132343838303635663639346634376538656335326135 -63626338316230363066663032336238663061366339366139666634653461613233323462336336 -66383832346666386636313166386161363833326332363565333065666338303032613337356639 -33323866653635633036326536636636633334623733333334626430323065343339653833316562 -63653861663534656265656439306532623361363139303231383863653466613262343832313130 -65353734306164643061353530666563306531336561663339633437353565613966393331383930 -39383433633261376264663030346231366235613033653930313165653763386637373266633266 -32663564343063343630303062346138613035653262366434393465323536323433346163646538 -32383438643236663134396635363930383534626164646539343264656662356563646231623266 -30373762626130386436373461313039343034386462393665323837356264633531393632623235 -30663730336232643735653166343938663639653335623933663030323164306266646633636266 -62653032346262343465616561306639386135316134363161303831336137623235353736396437 -35336239333761396162313035613565343231333761343665663562356530376162303864663933 -31656134633133626632383332363139386135326439373731623861326238383937333631383263 -39643131626539386331353834333539343264633462323436326635653336363966353564666365 -31366536306238356433306535363135313161336330313764633331386665643364353162373864 -31353465623833653430316135303634353061303734333166336236643636353937646263323632 -64323232646131653633323264366630306339656437633465613836636565373531303436663535 -65366264363361373565353865316662623663613663653361303761376463306565343836373766 -33666230303034373038336633376161306134336639363966613239613366316161323639346463 -31666461646332376436653433306233643361636337373838306363633964303565383434666639 -63323865336230333334366262626536323034333162613832356664303234343238346131633933 -37323564363139353762363861303333323564386533313534633736643766323230373866633431 -64633461623461646565386634666561633462396338386332336239373363613732623166636365 -34333465313335356265333036323465306265653063326230303335316366386135316163386632 -35666135643765613134353966636430626464623731366331396538306239393962636237653436 -62623133336638333061333264336138633536396230313235663362633532666237353132373066 -33323138383561626138366630646135663638623134323336323437363039306166666265313936 -63396132313833356661623032663439363662323938666264373861393061303865386666623562 -35646563396134656439633462633531373964343363613432323931303031313735363632623831 -61343034336233333263643334656531303462356639323962666465653864653633383266643765 -32346630333964656339343564613031646261303433356137356538666166376563336665343261 -65313366666637343465326362383535393238626363383233383632363239383434653439373961 -30346361643238303336653265353562626630646539393337323036623639626462626463653035 -30353562626632653334373534623466656565633365303437326463326564303034366530393265 -63613266623531323331343762663766626265376330323339306236616134386232646163636162 -34306139623532633461393964626538396432663163306266346639323436613864666233616138 -30623264366236316162656365373365656436646337616262313830326535323635306330373865 -63376533646139303762366531613035316465646631303361333437646635383236636665353566 -30303531333638616464363165636662386461396234373936373139626339396564636365303639 -64636662343334613637316136643666396561376165333631366363383934663433636464323936 -32366330316230313836393131323966376532656163366439633136323233643434356563343161 -66363165383133383435303364353637666430356538623763353061626636383135656663303536 -66303231626261633236613362376435316262316134303332336233383862376563353535663632 -66626332653664376137323864633933653732333832323361636239343237653339623563326233 -35666465666631363361653164356562313932653135366134633935383961323834393866633364 -36386165663163353736633865383237313161613065366363613265393539333035663433653662 -37326535356164616536363762363136636139386665303264636636373965343366313764313133 -61613632386366346364323539333562316636306338396464633830343564326537373139303433 -34303937373230336436646263623766643730303734663031396530323762383837386235343666 -35393532353366633132366638656637343966386231613532343531373333613836356361356434 -34353864353461663638626337303463336563376661633635333639663034393262616132636465 -65343033666339666562633234633831313264613665333863663334663464373330613461316237 -38373531363639383234396263643631316237623539336666623862383436666566613932343132 -63383937353233323734623837333162333062613337346265303932656433653935363161636336 -63306135373237336363376639333263363166626333393163633637353535343033376332363930 -38373764383538323065633137316632633631383536326236393330343437623437353134653034 -36323631303537336132383031333761643432386239623630373865646332343638373661396237 -62363730626135336236336637666137633834653266623936386333626335656633633635313133 -37386330383036636339313233383831333665323037636334376532346330373966376335356235 -65653962666435363764343133306539633161643662613462306166663638613735386230336466 -63646632313537363562643135656237386565383332306464336133396534316138343963313131 -62373633303337343233646635666437376132636333623062656537633862383135656137663632 -38366131353134613665353764626231636130323934343463613365623230383932646561383431 -33356238626135366231356631393234343136663064336136646337343064393238616236383962 -34333466396535663638613965646661363832343130386434396465393533346361363939663761 -34313738303461396331363331616530383636316130353166636231643465633230643065663539 -61336664336663633134633963313266616431663038333765363333623335636139666339313437 -64306133393963653565663962356565643030373833653461653334393434663430653835363635 -38356466323339343665363463316633623239313336393238643431313732343538313038323832 -64366339323837303962306365343331326164303931653865353766626436633030383032653936 -33623561306433393361353562386132366561396435386431613566323637323333343166333666 -65663834323834656537336566323532653434383966633763366262393431353437393139653164 -38363562383134623239363463366264326231383230376436373365313937623066636266333036 -35643062633132366232303339636537613636323532656630336536316263333139613537373733 -33366464363361373939663032313439373438386432666433373737376537326438633237623164 -34326165306265666463616333356136643565666339396537656239333763383532343661636266 -39633566613361366537306130306464396139313666363839376531626366333835666235626132 -39633238373431333231646162353664396539636264396563313966393837663833643934363061 -31316139333561373061333561356537346438356231306462363632383166643135353366353433 -32643033313439306534326335353833393433653230633061643835303564333036626531613430 -32623632326166396263386133313731333639616531306233336333616633343339386631343065 -32373434303539383837663639313536333662346131303931663330666131363461626266336536 -66623465326564353238663531613831313036333462393264343035313338306465303638333834 -33323134613534383166663965613937356132623535656538613132663534636534326663663530 -66633436643236313334306463333963316431383736356430643534663634383432366336316232 -31653836613638656135626261386230383937393536353963326236373833396439663364343034 -33383432306433356339393736373961613130306634333833376234373462633139616139386463 -32343134666231336233333732626232353065393831353336353838636234373964626565616265 -36633739643930326137623937376536353634373131373361366466646235646330663664623139 -34616564336436346465396662343437333338643266616134633864386133303235376564346264 -39633362333939326335343834643834653732653733613135326435643365333338393035633036 -32386635613230656263626364633837386361353032326635356166343235666364613036333539 -35366465353137636536666662386439343331383533303661656430333036396462353866626437 -62306132663162383134643563616433323264643038303664346362316466656531336162653236 -62396263376533326535363937346665623738633830663161626430313162343036613237663066 -65633065626635623165386661643366306230643335636262336263313461356136626461663832 -66356166366334666464383737643334326434313863633735316238666566343238663262666536 -64353762303838623162633731376662373061616463623139633839313835353339663464623130 -30346231626635666162643264616436313932383766323033373562363862353663666662646633 -64633335386538356162383739353530383034333033343132333238373034313863376132383734 -31313061303830313030366461633237653232656361303265323933383363663137346631663933 -35326536353035613036663433313830653135393863303261623563343661623930623365616365 -37653962343036303862313633343433663666383034633866333766313635653436393234653162 -61346361353935366332303366343765643532313763643331333531653132306133653064366238 -62346132616263303138386464383738633639346165623361363166336337653062353065376163 -34313564343338303465323864356662386562613739633565386239333739663036383131373834 -33666636383936613836363762383361383039373065653737306336363036376363623538646133 -33303734623939663836663736663765373363613730353737643362636361336537386339373539 -61393061656334656337636233623239356531303939353133356230326230643830303236386434 -63626231376663333136366135633332323363366565653538303731383338323734636639343463 -35663637633965373432326662643036343135386635613464383835373561393065303238376538 -32643463356438393335636133373965653662616131633239643138633839323535626463336139 -35626238316435616165333166356630643631613161303038333865383832366663643461323765 -30633333323162646663363632656465323062396538653635376163306162373930643934653863 -31383665666535303966633866386233343837363339623137333366383138643664626532376562 -62643431393835303833396134366163643963386666386436393562323763623563343761666664 -37313539636136336366316661633333653264653633363138633930326365303230636536363263 -35336336343534636139353166396237316631336335383730306538393864306530383836326263 -66383062336364623833303730636636383835303739336137306338383439663864613764653863 -63326535303361393532393562333336363637343534656361373033626433306566353235633263 -32633962396239336538383738376139356437303336356437316661623064623939346365653235 -61386534326164313337316365313132316261333733313063623739306264616666616337656464 -64643861633461316538646266313162613663356335656462643633303235653066623162316465 -30663065313336333963386364356233313437383834653661393837663666613634313764313934 -35353062336464303665663733323935313435353032353238656436353065326239646638613964 -33663166633264666530393063383439396137643239333664373764626138336335666566663333 -37313634323937383862313838616363663633343866366662323261366666323930353037353262 -39363530313463653131643066353831383164346364376639313438363966623238643061373134 -62356163303563643562333436623434613831653835323662363839383535613763313736333566 -39303063633438643764393332613031356632393930623662323236376664313130346435653933 -31663536353538373230393165356631353437613162646362383066373830316135643438666134 -61636431663336326631613135363733663232626562366635353939396539616365356565356665 -37313936363137343339323336393739323963343462313162653235396262636133613962666130 -32623565316635353637613965633361356165363563633761386131303630356438633739393639 -62636432326362366464386533306165386361336561636631643066386339323339303036373233 -37653033303539306230303961393432343866306564616239373961306336306564363533356665 -33616433613564663865356531356233343362316137356464333832343262336336323737363561 -61646239616336383935636531376232333332366631653336333564316262333066323130363665 -30623532633631616534393935353462633863386262306434663165363733643862636431386537 -64333964666663343134613066373662646534323939366437623331653864656533393366613564 -35346236323336636532636236346538303230363036636634666136623634356664633532373762 -35363466326263613730616663633331613234313761616533643761633034623238376563316137 -36386363366231343638363261366264326265343965633266373462356139633162323061663834 -63396635613339623638633537363565646264363764396466373533653066376635646333653735 -35653564633339666166346536366364386530653035323835373932333730653135643130373762 -38326664323161363864616263353439393366363236613437313766623132303830396636633930 -66336166343639373266303733666564643033346161623961623835393465306365613664373461 -30353262663364376663353236633236633834333939333665653333623265353233663339646637 -65626437386366353338333066633635626465393431666330396566613533616337626366616135 -63306464646239313435333538376663353335396431316238346233363065623266626134326233 -64633964363065656133666664623261373937393436656134643630373864376235666635663563 -36363462393534616263303431396437373637656363633738363230613237316663303736316632 -64393234653832333734626564633434626233323738376239383565373436323161653830663231 -30343539303862343031373935383961376664326139373830653635383132323033613238323238 -39363636376265363637383934613538323764396136393633636431613463616234373964383533 -32343937663835333737313339666436616665663965396336383262656634383631613631396434 -66633630396263636564383634323837666137666264386161663964616637323165343432653637 -35396663643938346639316566316161663065653438383837323965636233336139666132656664 -36366265383262393065376337386338636335313538313233333763393762386630626466663433 -35646566373566663664343536303739633066366235386530316463363239346533623263323365 -63653533373964343836313233376366333062653831313561343166643533393035366561613734 -31333361343965633034663631323466376163643535323439356633613330343039663330326432 -63613662613661666431396236303565366134323131386366636363633162613730613934663637 -31353665353631663933353632366531663463663336373235383231396538326136336365363330 -32663262373161623964333837306662363632303032333965616134613131353839666536613134 -62366334663461363162633834663135636365356261613134396365666662626131326565353435 -31626338333133616366306465386565376332376338623334383262323365386536653436636562 -64643734643233316565333237656236323534323338323035316164646131383862643638636232 -63316135623264346332306631396236306664643666333639326236353736666562633635363866 -35616562363434333136633437653032306139633237363631643536336331666364623032313834 -34393430663837343532313066633639323066313563613336363631616261666139363163653964 -38343133333131326663333434346336333130666533326231323430373065623062616465613436 -36373931633039623566386232383035353138623761366463653438396436623036636631626137 -66376136656366623965656363623363303732383834363332376230363430653564366630383266 -61303661653934633063326335333661333764346666313662336565313462313463356662646233 -37333835343466376461383863303265383538376332646539353037383638336362316639656365 -33623161653363616564643135346539656433396536646166366361656639643664393033616439 -37636663363766653039333737326336373135633264666366383264613136313736363831336565 -32623663383164656637343836376233636331363038663937663061393463383861393766303665 -65633864333837346563396161376537303530613766326237356264363966336532393735386263 -38633837316364626366393430393161306331376639613737636362353137363061663362613163 -32346538623061613165636339336333326235366664386138373361633730663063613336316235 -37363736373536666464613834346164646538383233363239346565303033646231363134376439 -39346361383737613865383938396634666161356263626334346462386666613363653062313938 -38303262343232643034333961343533613031366539333335306133663136663831393764343130 -3331 +66663865646337663866393130636464626438316632623439326261623730663466613838396135 +3936393237346564343265653235323263376664653135340a666566356137333462363933393664 +32383432306134373638306162646434323836653633343963633738363230396162633236386333 +6237366539616465310a303638343438653731376661323932616437386264333664613331626634 +64316630653437326235306661626539663732643462343232323464643132393439616235373838 +34303039393461383039323231626663373563396331633565616266666133623237616166306538 +33373336336532656538336364613965313536623935373034323034356330316130353632633039 +34366432306235343335363435306439383339343237353339346336643066633361373735396335 +34636538376164336264303662336230353863393263616262346330663761343730333238313565 +37646235396262636335623234326161383431306530643830646262373562613035393437633635 +64613464323063613437626330353336656633373130343932616466663864393963613061636136 +63376432613834633034336332316430613835373763636333653961643037393837336164666637 +62303932313061346130666136353630356336633136346231653634643033363730336164303863 +31396636343939336632383033313765383664353231376163643732333736626631613036363663 +35326264353362383933366463623433346539326662396565346562653061373034376137393766 +30376164306437636564306364626436666637363639303162363463376665663630636466386162 +33333263353862393365376430313730633535643639643435663734383937356337643236373265 +38383061643261636263353362306333656664633064633132636539396261306638363436373434 +31646333343133346536633931633132643838373637636566313534343664303730653832383735 +30396161323964376235633133623939306433373430643839653566376532646133613135373739 +62336264626236633833663963613331646436393966333838363165666631313534346461346364 +36646137663330666262316631313337383766613330336233383538353333646162353066393538 +37313765386566366364643130643262346264653461623336623337366233343135306538663736 +65366336383332613135356537316134633263646264333761343865356632323731333366353865 +31646664333235656164626262666566346162386461636634616436636338663861663436353134 +62343838323738323032636439326538633337363531303632323837383837326331383164353965 +30653433343934373233316233386563333832326332363131376538626430306164343065306263 +62626138643038396466656537666635616661376237666334396531663534323737636534626637 +62613236656565633337383265376162396632666261363362613337303864353833666565396339 +30303834396665306563316361626137623433653737653064333161386335323163386436366135 +37613132643136646663303162303132383161383661633665633365313066653730373862333830 +32393030346339393139323436353030633930363634643936633333613036663138633839313334 +37383063643637343432373034306638613038313266623032633630393736396664396434333832 +63653039346165373864613937343139373034313635393031303865326638633739373235633761 +39343861323338646234666362333838616532343263633766346138396230313735393065653561 +34636533663765363234303166343933373136323738666532333838333064656239376537323932 +35633937393461373936663964303530363435356664643332353739343039356630313533323761 +34326238373365613138383639323664626230656637643766316333346437386131303031393732 +36346330613866393263366637313834383738643534633034626266356235316561383466616534 +36623033323065383639396563626165376436366237646265373465643066613265373936333262 +65386238383665353563626663343662613932383163636632326462663739656338303938306262 +33373238376261333739313237353736306262313531353032623361613432623533336539316163 +32366362376462363264326237323961633834366566313935636435363339316164633635326334 +35383264666331326261613030663764626265396531316335396335636139653630343766336461 +65343934376662636132376536643536363235386365646366633930333166626163333265373862 +66666264656236343063633033376237323439616333323964373635366234646232366431363535 +32363031373337336135643466303134616466636163393162653161666633613436326438626563 +37356665366639636339363665663562663965386430383931303966623834383764623831333730 +37353239616262353331663166373161316563663432323131386638386263346139383936636635 +39643030316563626534383432356430646234303234356461396661626339313931656234643232 +31306661313062303466363930613164366433373735633836343135323266636435613432316338 +64333231363832383732663635313063633835373036306562613632356137633033323935343261 +64653462656631663231656537373535623762326431363631663432393433363863366231303363 +30633436316638336663303130316333653966623636396430653637613337343866346664636461 +39306264613738616665643739396361336162646463316364636362633133633262643165383966 +64353839393339333736636530373765343437376161313733353661346330313435623336346138 +34373933363039633731623536393732653131353366653933363361633264303265653332356662 +63356339313564326263623863643537343661323964373538643539303463643763353762366234 +35353661363064613363373764386337353437366261646339346538613337373363633032316139 +39646237623137663538333063633035646530623336373135623035363635666164663239356534 +33363637323061363336326362613961393239623735656165353035323936633161353738343164 +63613965663932346562396636663937303533646262393136323133376634383066663630306266 +61626535393234376631666362366136353834383539656131646162313465613832373033326137 +66303662643362643434616262353839616435363361663935333461336364306439366263336230 +64626431343238386531353335336565373862393338333030653539356238393333643963363865 +38633466643938616437646364336661386665633131313636613339633836653163313031643461 +65333931393964636336636135336130383766636439346537396539366234353366346137316330 +61646564373534356133323633353838343039313139346434336232313130653335343638333132 +64623539376163323462326637643765616134663662313830306331303335663361626632363532 +37326633373431353165366138373730323562653338313638633065663439343166633632353863 +36313333383438363237333761623764646661613335643231303538396238666531653234313831 +35316361343839633836653363333262323232623437613732383038623565646633653838313562 +35323130616364616463306236636562323063623365383761376262626664346134313931303333 +35303862356631623637373963353035633765666335663432343330663564383937656562613931 +34323731633637333834323937373235313439323533353263643136626565643336396336626364 +62653062313837633930623934653466383362343763323930623334656439396664636639623530 +62303730313535336165633833343134633263353939313237663862663635613931336431353935 +65353936386633356262383435356365613663623138363561666536613632383734636138613337 +64323461633939316535613633613761393965366230396230613865623361313535386237376636 +37313837663736373161373363383762346535303034366637653839346365373138653162303366 +61336662336366646266643433393134646237323437343735333632306139376539353434373064 +38316538346531636134303461336533376336336164333431343361373438343166653563323264 +36663966356538303863616363633331616535616466383037626232396164633132613161653233 +61333034623935663234343231393266336461386133393937343638366533303934323232383239 +36633437643433613563363366323230313236626662326665326663623530363035353636633965 +65393338646334396236383130393663623139626135656330663039303761396430396632393737 +64666163363066356462343433633739366339396536623539623331393961316164356634383335 +32343635633933643633383865623333636436346663333365636666393365363837323230613265 +31623136653737646330626666386130393538376436353365393638613063613132613039643964 +33366464643563366130613137653963306437323462346561376462616337636239383834326431 +32376134393265316537323337353732613436356230343637613735643131643931646266396235 +62336165356633343334363935386237333964626366323839656632663334373961663736616366 +35343134646462323635323331623662356264383262313137363961383134363365333533376337 +30393030636266623061656261613436656666353866356231393230666339633834646430336434 +34396132353532343335653239636631313466336266633163383366363032646537623138633062 +37396665343039383038626362396136643933633238363765646232376339316136356131363236 +34636366303639313365306538653938663735373965353762666165306638616261376235396330 +39326533333131313637626532366431666634633165343837306537643132373936356139663334 +32313933383461343065306466333166646135643464623539623539653664613530353865376236 +37373865326331666562666236636563356331366635616666336237636635636334316131316162 +34623435393465616237353137356232323266313235306632313238336538323663653836636434 +32623434613439393238333564653765623165343761333634313562393338663039303532626563 +66386432323263666433656164356339383437376665323532383631383234653366366533646331 +34363931323037336332353835383332636562356639653031643965653565356338656163616532 +35643431316239383331616361323265323831663638316561386134633034613866383261353039 +34386130653965373937646439643336333662616235393938326533643462633361633530363730 +65633732633565626239386261396437376630303066666462306466363535353730616435373733 +36666432363730633662613330366631663039343836333761333735336665346134383431363332 +66623963396537383637363030303265383862373839306136316162653338616566643635396164 +65656364323238346435633937313931613231666161316135643835316361323663633937373133 +38633531393866303166343161396361643739343733343964303236313037336465663433316130 +32353536613866666531343935313334366232313333343939343435313666663930343463333435 +66376232353539623963383735616430363736393438653936303534383961356231663034393530 +31383633653439343838393933333433646161666139653539326663343638323363363761306530 +65663230386538616233386437323538313462663034333838346538343865373465633662643938 +36393761386339613533653236356366353463646437373435316466356637346532373539356432 +30303738616231393535313034653735353538353466376364656436303630373765346463396635 +64653739303439316361613238336330343534633738653936616631376539346432656532313061 +38333639383464313164356664663663393861613539396537313966666438353066613934383936 +64643934353764306161313361623932306336386531346530663633666631656565363262653661 +65633761383334346563653638643036343137626161653430663862326636323262616631613533 +32323161313934356465393035636230313536633366656231326330343138303533396130336630 +61373133613037636434393637383637353032373533386561376536383764626665373238313737 +39616161383037376566633364613064336161353838663932306139376165333766643965366530 +64373261336261336563313966626365383435303234336239376163326233303132663639323331 +65313234303462663663346238376231316136326265663135616531656638663335623038653635 +64643266646366303463333230313039613033333164613962623837303865353339666662393633 +36326435306662383761643733393639356265633433386131323035303231303461613863303164 +34636161633430393332313838623633656138626138646132646334666636656336656231303637 +38663833643661666361323463386261613663356230373236616131643935346331393961366164 +65653965653863313563326435323130396262393838663263356366356331336366313961376264 +35353264356336366534336565643433663831306639376631326362346133613663383336663334 +39346239643436353961653337306464306639376635333631326637303932313161646136366534 +66336638366534353862653235303566356265313333353362656637616233366261653332623335 +35613162653266353136653333333135613439633330353235333436613465396564663664616136 +35346536366438373937323334613161386234333361633134633765666332373265623763376535 +33393062306566343734343264363432386432353465363937626331386531643565356134313235 +65383335633638303566353138643166396231353637643030386531313964326666373664336430 +30633936653035393464353561373031653462323737396238316162373762356566633133353266 +33333732396633613630333430613863353137363335613134623932393764366430623039613334 +62663435353964346232636630643337356532663131346661323839336331316230326563363030 +34613031346137613932626638303638373838376230316565393764396364363734373939613437 +65366331353834323163653464326563633562626234376364306333353863396666613364633133 +31633165646133356566383532613965306438383733643034363434303139333234366337333634 +39386535346239663933316635623966393430383265653737616265353132616261666337343239 +31313639663466393966316638633561343965383366653431643935356662653037633231313163 +37303665326632616236633365343764386633333763633966643266303365333034323062663566 +33363134663037643263646561383662326435393463393661396137656563383036336138613962 +62383861613831643761333433636336376261363835623535636237626634313861336634613137 +31383864613633666537333636383338333838653435336639623463633837346335633032653933 +31316131373861343935653663353737666465623464376438303937353439663563376161643062 +36313838663733643539663263306235623333383637663265373262356530646430373561643838 +30343165326535336462666461336532363432666532633636393063646566653731303862626164 +65383139626538653361636165336238346533616239613537343634316562633239396563343664 +36396334353362376633333662633238323739333434373061313565626332353363636238383833 +39393839326231633932326434626236336161623938303431636536633339343362333030663130 +33393536366432353430616636643865383434636634643462306365356238393163666632363734 +63303139343761656336636463643533363138623466343332643936393130343535366261343436 +39303635353265363834353262323734366133613361633937373432653862353162393862393531 +30316565373035376562383664383063633938336262373363643233373136663063613738313165 +31303230306330636136326231653935376433623864303866353866396339653137393334653530 +62336237386561336538366364343334616161663034336262316532393562613433623763376333 +36336364313937356164623139643437316230313266336334653963373839353963393535343639 +37636463303932633265356533623662396166396230616134643431633736333264633064336130 +34386237366566323438313761623335356466316432376133663865303665363934666535303764 +34366631376664366631643439383234396130333565383439626261346661613134326232303266 +34663362643235626562376564386535343161626161373662386635363161636636353636626333 +61383731356431646436326337653265653830326365346366353632326330353363316239343730 +33323266393038356163383432626431646264353132303562336430363261303130393031343939 +31613532643632383366336665363364356132353463386235663161393063656337623965643837 +33393861393437303536616237616464333535386461623937303434393565653562613466343436 +35393964663236333739346262643862316135306330393635616332666638353135323233343435 +39393764633862313762326262613838366236643665376533353139313365316535633938353666 +36623266313938643337626565646433636166366335626637396337643939306265326465663135 +36393963393337333233353962326634653434643036633537636134623639613738346133383164 +32663239613462643463616564393433373361346635613135393062356666616161656633393933 +38353764386166303639643630393466326464396165656630626437643364613530383663366133 +32323065303265306234336665663533393131373430623961636530356635343535386164303537 +31306634303030373531666630386637303134643865616364393433636639396666353639633634 +63323164663233396166663834336362346261343662356265623433333330393164306634316332 +64636230383564366264313066643434656636333565386164306631303937626232303661646166 +39386161333330663534663530636136343663303866623262623862643165663861353633313464 +62323338636562353666643538313563383036343630376362383763626338663339636562343964 +32643565336332666166336436646137383037323163323337373235326531336437626361326537 +66666463326333363530393636666365656138636337343834333664396133346534303636396139 +32613066333665643236643137643763366636623361663462666564396337363436626237336230 +65626431633034396239386466623237363930313565353730333066636164373363633165373833 +37346137663061303266626162626663323530363532303231386563663538376562623863393264 +38643332626337353161666630323138346538303434353137613961653333616434353562326666 +31353033346463386365636263373463636236356432613334386566373735616561366662613864 +34643036373838663164363261396133626538383934646533656637343632356664316362616161 +61356161373136376665303934623731303663376436663030343363333661336131363563393064 +38323431336135623132626235663935306438653434316430663339353264666538643135643333 +66656437316162396561386363633335623239613966396431386139643135333234326637633662 +37633564383031383933636338383837306364306462333662336136346533313564353934653136 +36323539626562326136623838666632626263376564363232363631323932626235313339636438 +38633931356339386465666536636331356130356135363064313066333035396366306437633763 +65646535313333306637306137333732313037333062366130396638663132383766363164393132 +64393033666535333133633330323339326137383336346338343635643234613861326664666366 +61633766313032663662626163653965356138383438353535303762313337666531343730636466 +64353932396131346534373166363763383363623938643737306535633165326566373131353932 +61393963386436303634623730646164356365653637346435653236386239373462313030306637 +64393964633963353632656165386639306332306362616163376665383066616635656137366638 +61633565656530353661396162616566313938373866386630353764376433303134353566656562 +66633831653262393662633234613363313261626638356164326165373138366334333437623363 +39326637353937353431653637343031313331633265373532613361353465333335623462326332 +34343761386438383963626534333666666362393862393530663935373234333063303634666565 +30663237313634373130396165373161373561383138333831653238363332346364336636323834 +37376230613130353530353633636637386361633437326233666463363934636139663733643039 +63316564383463383331393438633462313763353466613733653362376130326134323131306136 +30396431323232336131333166623139393664646465363430373335306230396331653665373663 +61386133343566306264376134316163396236383736303531653331306536393364303531336565 +32343237323266366666653837323764303362663362343137376439313237633334363537373636 +61373231646239626636633633656130316630643131356234373464613730643039626633323161 +35313465626566346666633163623930633635393039623063636162376264363365633264393938 +38366534616464663733396461653039386264306132623637643465613763363339323965353634 +38643830376339323835623564323538636463336231633138616435373366373138653933333337 +36366331666234303432656366373261643039383863326137306539333738323431363834373431 +35623435653037393866653965636366326239613131383138616235306265613131616135643839 +64613130353237613838376238383130343530313461336366346533643361663035396430623066 +62323138323338303763613535643663333663653231356434623333663034356535356534663962 +32303639663432626662616264333532643433336163646531626132303331326665323163316538 +66363938393466623334626161393763336162303166366364643432663766313035386431386661 +37396634316332303563643261383938376537323831663866616632653463323036633363306636 +66623738633539663561643332636265643431343031373430666237343664393236373333383537 +33353834323832373235626435646530366163396335396535626462306464333332313564303939 +62643437613461393433623764353036373036363862373263376366643738306133333065333166 +35653463666461336132656531366530633131376464346137666563653462303734353561313637 +66393134326332303838343139636437303062646333663265346437323937303164636462623463 +38303033336563356334643031323230626234323764666139383832323166303839646133306536 +64383534623036373163636438653637303039626637623736376564313739626336653966623236 +63333234313537353132346433383164356430643538633939323862343266343531636464646465 +37316136616332663239323362613233346465323630623861626534653063363438663236666566 +63383430666330353730353037633666373066313666336238613164633163303933373465643133 +64376135333564316338313666633365646463363938363465313430343562623232663764356363 +34313431623035356336346365316635326363396163356239646538373230623761646131333534 +64666338376638366236363338366138646466313465346534303437373939626637333935663564 +61643163616633333363623536323765366634323462646362386531393932623337663061663664 +31343535613264636431376631636239303630643037336335323831636637353738663832393961 +61383238376537353430643232656534646561656266353461633165353965653431313030363238 +32356661633835326236636664633365616432613662383762366239383633626462346435313962 +30666162323234656162346662353238353163613364313038663236646131616465396464386132 +32353834653965626661646235323165326535616138343666643665316437323932633133343438 +36303863383237643737656139653366343035663239656136663132643337306664613030633235 +61313833303633316235353934656537366535616435666166323837383630653539383538316432 +33633331353066623937373566626262353032346438613437373136656435646333383234646234 +31393137663262623633653537393439393638613763366662666332353339643963633661393034 +36373036663030393037346636623534663037633233316264393839366434613761316666366461 +36333639306161376139626331343266393430363134373335613934393130616335373034303538 +63366263343433653263646337343761366431656135383464323662303836626539303532333432 +62333132326362366537376566383834656666353131376539333834303161323532633431333263 +39613166313933393830313237633538363766623031623833306466366464616135666633363637 +66303161306236326163366333306639336430303637663762316166653530633862653561373737 +62613031386630613638343236346531616532313633356431376237383166633363363934616564 +36313365313539313530636466303231383462306134313731666434353863376236663465656162 +32383739303036646465653165353762646438366139333537376230663239626561633935626139 +39633338646333393239396230306530323534663938386465643531313665373265313566313234 +38643233373034373262346138306463363935323033353837613064356638396337333830386163 +65303230653639363939363535663864626134386237336565343930616138346566393139353032 +35356561643734383364383265363538643031633964376664623063396232363335646633373262 +35323062313966306233333531653030666165633734303331333133306365366234333238623937 +30363230653135366533623133376334623537633039326334643035313135326135393837383765 +63383261376562663932653330333766663261343332353736393439306566333038313432616532 +33626464333234633536333735363730333461333966653064626237626338613862353937613261 +64643833396231383738613334366139313839353034323563333566633666313237363635613231 +63396430363134353537363133626238366431353838383339663364613262353839656431663334 +65383031333564366538616332626631633439346265356538346235646263396631663065653633 +39626136326466336138666261373539356133343135323138386635353632323862626238323561 +39646563656666383334653562653362393835333233363530393437623933383636346663633465 +36633831383633636236666635363564656536396336303732663836303431353263323132306436 +36306133363431333733366166333235613130356231646435396638356161346638323231323563 +65363664316231653361303936343264396234653833623365323939613131323638613864326461 +35356462666539653566363138636465333138326437336163306363376364323261353163303366 +32623636666337323161643536656233376361313930663764343266386539386132313631653537 +66346538653561626663303030346334376234356231646665383462373966343538313531373034 +66636166356231653966313937316365346162613963383765313764343465356238326238333433 +65343164633035646338346236323130613439626466623433623739343466633936323865376365 +64356164636431663132643934623433393531373434396431363161346636353433363765396666 +65343138643963373737333831363038653065343736613262363935663766643534636335336435 +63306262343031633539316165303965626466373035663832353534316232323232306133383362 +65643563353730313966313564616432646566323736383436393435393664633166663232663934 +62386362373533626532376132656263386437633566326163396663623365343233636633616239 +64393134316433363262303738666537303531613431623232626462333538643431353032653764 +39393666353061636366333766386133303666396636656532316564373230323662323834326431 +62326162343536363765653262343937633539383437316233323035353934613734356330666336 +36343339373638616363323532333035323230393834336130626437343963316533323561363436 +35633931373430646262656336633561613233666633353137643065653730303632316630393039 +64333935393063393238373565333265343766393435663961313735613866333933613537386161 +32333666633834666131616439303662623961386234643962396463313636393263626131363236 +65313033323932396434323164636163383338343936343530623732353732616331396638323233 +30643663393466613932306665323832616262323566633863326636323437326364333431336566 +66663837336337623935363065306131383935383330666332396331346435373261613930653538 +38666336623336336264313234323535366132386233623330613538656166613161353733616230 +30356363616466316235393363396239346366333133623136393236623737636662346238336561 +30333465343732636561373932616533326466356165656330316337623861313663666433366534 +63323764306365343434653136616132356331316436663132666136643665613735666332333235 +65383932366462643437333535346563353664643035616461643762653261636165326339613037 +32633232646132663939623730396632386133396262373736623230323363613138303230646430 +33663331386338333432366665336239353262323331633966343832353633346266356433356433 +38363030656133393131613865643039626365323738633734646137616361636266646630613830 +38636163633633626232656336383831353735653865623437396464366465306139323631376632 +30653830363336643730653536303139663436623334653263356561386164376639306532376439 +31613738643034626162306332306336306466633537636130623663616231396366393061646164 +62353761313032303638636266373565323635376363663865393630653938346635663639636639 +34616435323062396461383030656162656266643463393835656237363531616334303964646466 +31643339333564643633363938353736656133363432646461326261386565343034663832343434 +35613836666463636665366136376430386662643234323736623239653766656166613230633037 +65616163353466636437343239343765653065333439383836653430326264373562376465643238 +38636463646433616363363036343836646635316230326435393031306333303737353535653037 +31313930653162303432626137353962363430663937613931376339373735386336656136353333 +30626466393632653163353830386666316661386531643738313834653339396236666233646166 +32376136663932356262373232373330343933653565306433303530396530303939653335316332 +61643531663135396532336136313339343732623338396163643238643563323563393163363133 +34613362323233346533333031343934633934353266333861303465373532616165343934613934 +63363038643963373638656362383661663331656232303630623861316562393930333362663331 +61333930346231613461663238366662643330333563376261353663666638623362643433383561 +66303733343335343435663430376365623961373337613635313530363963626235346133346337 +38343261373835366661616662356166393130656536393234333834343562346537323130663437 +62336430383361326332393132366631653833666138353034363334623664623863656266383861 +33633066653866336665366233653562373062623330643337643230373238323435393762353537 +39356339303737336261383131343061343235396636383035393630343164393635386561613361 +62336536333134663465373165303234626165626236303032656664323265633133336337313533 +30613361613239636637646566383231633866356636346635643433666339343862336265366461 +61386437323933643539663837616538353334656430653162343861623539316638663064313030 +64336631393532636539316632663236643962633663333461393831623066343132613436386562 +66623936366663363234386439626465373266633362363261373764333033396364656166616430 +38663632633036633364613032376636303037383262653134366237376632303235396438326436 +66313732656337663065663737643639623065386261623734343234316366656163386236323735 +65336162316236383339613436643332646562376164616439656532356135613737303338366132 +64613930316464613065366437353333386464306335623137356335653266303039366563623237 +62373164643037316439363166313838633736353762366361303361646135333563386266633533 +66653034313161646138623165613465313462613337656562653362353561626566353236643663 +38306430353539666239353436363533663865316230343131393764336631613730333537616462 +63666338313739333362326432363636633963653662363936366534303763373239633537613834 +36663631386563343164396637386635336631376331353732386166333564363734393135303036 +62383063623037383238663538333962653235306134313861353365363263313137313032623766 +32626564343365333032653039623636316362623864363434376433316635336562633133396265 +64366433626630386538383132346633613233336331623938343733653932383663323238306163 +35346464313434323062623530633838666434303136353531383630663266613830643134633066 +34663865396166346338613964323865376138643235313632656664666337623565336565316238 +37626565626164636463333466343337393535666137303034363537623336373364343139386534 +32633636613364313836336662343534346532373062323663636637643234346463633135336162 +61633765366537626166343066396636313036653439643463393339333536393866303434353261 +64353938323634333765643232343733633262656434333738656133303535633265326439333766 +63343261306265653034393565333032343435616434353861313538616365346563643361333730 +39616238353736623962663065633039373032613334626437386637636163323935643733303738 +30663163643535373763326364353332626139373561663961376365393564666230646633313039 +33653838643933613039303836353331393632306633656230666431633731313530633863623661 +63363933636265353363623131333364376134306337373138383064626663336438633630363865 +61613337313637353063616632616466326164353930386231393130653766646362393933643564 +63313336306563663466366433303865656332613161636535353764373865373630363635643036 +32663865636166373963343664363835376535636634633333343263613662303037663538636530 +64333364346436303164326532373134343836353435363435646237333136303232333766333834 +65373538653062616166653962346564346533363330303437663462613662303438643737626530 +37636132333364313463303934333766366166306164393032323064313332373030643966343334 +38613863363631346466636332386235343431313433633430666661646265336262636331613037 +31323065626661393130386338613766356539326631646361383663636634613531306366626464 +65666137383436613036346664626438393037393132313866616339623264303835373533333832 +38636664393130383161356433383564326638333562613633666564336431626438306364356237 +36376166386465376562343134656339386535393035313761653735616334363161313863323865 +37663161393436386536396130386432343435393737396264373038383436373530633636643033 +65646437366434626538323737653731316638623130613836336462653834346331396563386362 +34343665643065656336376666613161363939623236383430323535343938396338646131666639 +38353232396664376461663239666338316665373739353264363637306430353162396666366134 +63346532383265313634386161333062346565313237383038623835633964363463383935633439 +31346432633639663230366264626334393731306536336131363864646636306163333935616661 +34643132333938633035396231633937356364623531626134663063316362623866396565343537 +31363935623765353733386339633633386366333333356361333962386466383536663930356432 +61613363346430636133356239646362636334393533383437633261613666393661623934396436 +66366238376465613236633235326637313035353364626239303164643763656166613365376138 +32633163613237656539323036313836663336613934316639313965316465613131373062636161 +64616538353432646161323631663561383762343131653063626137306431383639393931333863 +33616566633032343237373735663732623163656536386263653330336330383965356331343639 +33393062663633333832363364366235623966363333616637323262383361616234366235393934 +32343838663862306333336364323631663235633933343336613666343036313761633333353562 +34616166656266363065613766366133323833363763613738343762326639326666376463363464 +36393833393261626633306630376138383666653034386366386532323437303435616334313739 +30653133643663326232306436323834313161623432373733383134663130383430373230366533 +32666263326130323931316132353066313337636361643965353832343337353032656237326563 +38623566313165376664356262626133363235323066323036363064643337376433613135653536 +39626631613838356235343134643761373136326564303432383835303261663366353932616431 +31343038643465333762393361613735313131633739343464343739623836363738363132343136 +35643134366665633134353764343966383936393538353363306434386336663839373732393365 +32316232626462653038313538323939336533313735623763653436333434326138623738626132 +65353031613964303836373832613561373834376561303063383361396531633234383238386335 +32323364303636363061326639326630383163313435346637633630396531363733353261326437 +32363235393335633566363732613039373765636432636633626434343338613966336465326164 +35636134633633343732366562656362366262623661393632663937396165613637383632626332 +36396530633933623131626430623466343730653365643338616132383838393736383335636461 +33336165313038333635333838383163336234393138663132336463303362363932393161636537 +63623139646461633139373532663832303637353332376166653331356137313761666530666430 +63356532393166616330653263643131376639643930306635666663383265653832303532626630 +36376235343265373233333665303731613933626231366537363635323766323530303133383963 +66333330626439366532643937323565636539623835643032316633306630356130623731383535 +65313532353533393635383430656237626564656535356538333030373962643035363837383435 +64366330653462616535396665373830643139323264373537303030353737373963646462393231 +37613933316130343730373730326365383738633839346665663166306236653262383831653236 +36616536343862313337626430366331383332313530303363646262306462643532303366623362 +38613465326364383033363065333062313830666663643335323734303566666563363930656264 +34373736313339653536656362633231613365396365316531626362366233343862366330343663 +66316337383933376530393061313562393338623231613466313163373861663236633363336562 +39376330613366653538383535353030646634353039633038366338303231653362316135333039 +66666636623237396537393830316634343630366637333163353766383632303465383531623761 +62313132623834613536376462626165343636636565393831613137343663306561663034313165 +30636536383965383834383533363166326664643264653430316433373435353964646332633837 +32313766336461323463663066626233333739663939333764336333636637373139383063653532 +36323131323432383735316337633833343964633237613934376532326162326265313263333063 +65386332373538356561323635343234306263653935323866613966323438366637333133363135 +64353034356661666333386563343436666166323035616639336137646266366335623531643066 +38613932303861393033386263626163636532346131316339393663616631613863643035613965 +64616465333765333362633462343130663538633938653164336438376537626262373961636134 +66336464663731633762336530393339616530646461343235316165376530663036316234383764 +30656533666665393630336665646664623138623837613933663433346361663466363963623936 +37376132646265343836313461323265316364306463323932313665326137653039393265663838 +38636332643538313264336366666365363663356539366337613965646663656637373137356464 +30393463313161643665636264303535666463373834393262393034623137353466343761626130 +33303463623961393931373134303464383634326335376566343564356262393865316233373035 +64653865376435633863323833663838323961383165323532393762316534623264303066346539 +33303536333430656636646133386232386638346639613137383064653132376133373164626665 +63333337393132616361353937363464383662363164393038373636343538653338326565376463 +33326330383037646163326538343430666534653337366265393632613863633962653330666264 +31626638376463393936303162343566376531633431636663653631376462616562326336373063 +65326131326362613736623334396238303734393838313339303332306335616561656632613936 +64653638663733643433643134396531633662336531326364393664393565356335383039383563 +35663763313532313465343937396362653636343638313464313963323834396531393461333763 +38316530623265346532386137633332646664333734336665666639636363396363346139386330 +31646666636166353437393066393736383132653565366336666163336333646664653139643033 +38616534656239386131396430663934366266636139303738633665336663353163646562366530 +34396466633830653664656139626431313237346639663633613565386538636363303961633638 +64383537663338623739316233313837623661386263633636386133386531303135626665353164 +66633062643265313839336461386261653830306261633038343561356330373761663466383036 +38386166653461396236346537613736346533636564643634656465613230386130623938633937 +35613066633566613239363065613131393034333732623539346631636663626165656566336465 +36373832333237633739353662616532373730363766653138633065346265323338656265643938 +65363632353865626430623137393464333133616163343230373062373038323962336464366637 +31343733636464646538663430663635393366396339316330613061663239333163363436663361 +35326166646137613430323931326162613963393335653431633839326335323431306332653234 +65306137366233656534343464636432393632316636643833313065353739636535313938663464 +63336230343736623738646364393636363161393238353036346565376434643363653838356134 +32616662383537373136643033306332346133613835326535303838626338323530393163623230 +33613564393162616262353265333264613737616437656565616364373234616238643533303264 +64333063613030626462616237393262346132663937303966343938333530366532346164313535 +36306233333337623531643431653966386330653461636362626662636334663563376331616461 +33303631316339623330373532373261333665663535633265643161633038653561646631633133 +64363534376535313835366236346332336239346638343335613030333035626436646264323530 +37626333373537366164376134383035313738383039396366326534643132666631626133303838 +62303939323939646538643764363562633934373765633037306237333264303231663061376537 +33303761386366653233303362623061326539383534303165633137393961326263353539363665 +37646233376162396362366332623637643364623266343936353161373632653930343037623534 +39333934633433646435303538643736316666366136393336353335356332633031396338633935 +64323736333937656133303233623832636432323531663164323137326533396230386434663862 +31386631303361346235376566336664643330353533383264353532366635653534663732303734 +62363061363438646361343637303263653061303333616365386430633264656161663334316365 +37643330653962623636613731373763393938633430316161643139613966356136666361353866 +38366334386537663366646238346666396130346238633363353133343565393837643239313231 +37336235623064643039663136393162396431383434626533373332616366653962646334383135 +66383034353937636566323836353165323333653565336566646631313737323333396639386162 +33626365656537613538633633353437613831653438633265653563623934616162656638653231 +38333830616337373930626530356639616634353165323062393263346435303662373163646233 +62336133316434373962656234333231343665323732363532326133616135316533626565323830 +62316334343261663366666437336564613364663066343564633061373535336137313165373433 +37626136303035336666303637353463376430646565373635366535613336656132613738636635 +61633733356338313336626333643530643966383438343237363836363334306264393236363033 +38353137333639613764396665363231333665653132363361643435353665636530383235616333 +39303662653066303934303237393934373762646434633535303334636434663833653866366637 +64643036643339666530613130353966363230646161643532323339623162663531646561633535 +39356135373861333464393563313565386562346238616666643564333562643065363832313039 +30326561626630623435393165643631313562356230343939656639633439633063613764383433 +32313562643731323833623834383361386464646332366263323762343464323235633137316633 +33353731636163313735306565303765303633303761313236373139616433653930613734373337 +34346433363665343838346563663438323531363237633430316639363031393561653436336636 +38383338326438393562396531666331386632343337633538616566313839383366633932353830 +65376633363931613931323831623661316531353062633864656463313536633133626532613666 +64646237313930393330326237656239346261316566303266613533373932626263363637363936 +33333361373538656434373030643631346332336636636533666338636536663839383038323836 +33666430373932333235306336306433393538373139643534653333313630316531616563356133 +35386337383133663333653138633839663932623861373566613264346664356639326262376264 +32303331316637626537666561343933323664613238616330353138633236326466613530666662 +61633636313333643338633232303065643835363830623631643532313564393261353161353863 +32636234656666633664306335303634616332636230386266636632643931643364313336666261 +3065 diff --git a/group_vars/stage_prodnso/argocd.yml b/group_vars/stage_prodnso/argocd.yml deleted file mode 100644 index 6b7b31c..0000000 --- a/group_vars/stage_prodnso/argocd.yml +++ /dev/null @@ -1,158 +0,0 @@ ---- - -argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}" -k8s_argocd_helm__name: "argo-cd" -k8s_argocd_helm__release_namespace: "argo-cd" - -k8s_argocd_with_keycloak: False - -# https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd -k8s_argocd_helm__release_values: - repoServer: - serviceAccount: - create: true - name: argo-cd-argocd-repo-server - rbac: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - logLevel: warn - logFormat: json - env: - - name: ARGOCD_MAX_CONCURRENT_LOGIN_REQUESTS_COUNT - value: "0" - - name: ARGOCD_EXEC_TIMEOUT - value: "300s" - - name: XDG_CONFIG_HOME - value: /.config - - name: GNUPGHOME - value: /home/argocd/.gnupg - - name: HELM_PLUGINS - value: /custom-tools/helm-plugins/ - - name: HELM_SECRETS_SOPS_PATH - value: /custom-tools/sops - - name: HELM_SECRETS_VALS_PATH - value: /custom-tools/vals - - name: HELM_SECRETS_KUBECTL_PATH - value: /custom-tools/kubectl - - name: HELM_SECRETS_CURL_PATH - value: /custom-tools/curl - # https://github.com/jkroepke/helm-secrets/wiki/Security-in-shared-environments - - name: HELM_SECRETS_KEY_LOCATION_PREFIX - value: "/sops-gpg/" - - name: HELM_SECRETS_VALUES_ALLOW_SYMLINKS - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_ABSOLUTE_PATH - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_PATH_TRAVERSAL - value: "false" - volumes: - - name: custom-tools - emptyDir: {} - - name: custom-tools-helm - emptyDir: {} - - name: gnupg-home - emptyDir: {} - - name: sops-gpg - secret: - secretName: sops-gpg - volumeMounts: - - mountPath: /home/argocd/.gnupg - name: gnupg-home - subPath: .gnupg - - mountPath: /usr/local/bin/kustomize - name: custom-tools - subPath: kustomize - # Verify this matches a XDG_CONFIG_HOME=/.config env variable - - mountPath: /.config/kustomize/plugin/viaduct.ai/v1/ksops/ksops - name: custom-tools - subPath: ksops - - mountPath: /custom-tools/helm-plugins - name: custom-tools-helm - subPath: helm-plugins - - mountPath: /custom-tools/kubectl - name: custom-tools-helm - subPath: kubectl - - mountPath: /custom-tools/sops - name: custom-tools-helm - subPath: sops - - mountPath: /custom-tools/vals - name: custom-tools-helm - subPath: vals - initContainers: - - name: 1-install-ksops - image: viaductoss/ksops:v3.0.1 - command: ["/bin/sh", "-c"] - args: - - echo "Installing KSOPS..."; - mv ksops /custom-tools/; - mv $GOPATH/bin/kustomize /custom-tools/; - echo "Done."; - volumeMounts: - - mountPath: /custom-tools - name: custom-tools - - name: 2-download-tools - image: alpine:latest - command: [sh, -ec] - env: - - name: HELM_SECRETS_VERSION - value: "3.12.0" - - name: KUBECTL_VERSION - value: "1.24.3" - - name: VALS_VERSION - value: "0.18.0" - - name: SOPS_VERSION - value: "3.7.3" - args: - - | - echo "Installing helm secrets..."; - mkdir -p /custom-tools/helm-plugins - wget -qO- https://github.com/jkroepke/helm-secrets/releases/download/v${HELM_SECRETS_VERSION}/helm-secrets.tar.gz | tar -C /custom-tools/helm-plugins -xzf-; - echo "Done."; - - echo "Downloading SOPS=${SOPS_VERSION} and kubectl ..."; - wget -qO /custom-tools/sops https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux - wget -qO /custom-tools/kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl - echo "Done."; - - echo "Downloading vals..."; - wget -qO- https://github.com/variantdev/vals/releases/download/v${VALS_VERSION}/vals_${VALS_VERSION}_linux_amd64.tar.gz | tar -xzf- -C /custom-tools/ vals; - echo "Done."; - - chmod +x /custom-tools/*; - volumeMounts: - - mountPath: /custom-tools - name: custom-tools-helm - - name: 3-import-gpg-key - image: argoproj/argocd:v2.2.5 - command: ["gpg", "--import","/sops-gpg/gpg_key_smardigo_automation__private"] - env: - - name: GNUPGHOME - value: /gnupg-home/.gnupg - volumeMounts: - - mountPath: /sops-gpg - name: sops-gpg - - mountPath: /gnupg-home - name: gnupg-home - server: - logLevel: warn - logFormat: json - config: - kustomize.buildOptions: "--enable-alpha-plugins" - helm.valuesFileSchemes: >- - secrets+gpg-import, secrets+gpg-import-kubernetes, - secrets+age-import, secrets+age-import-kubernetes, - secrets,secrets+literal, - https - service: - sessionAffinity: ClientIP - dex: - enabled: false - applicationSet: - enabled: false - configs: - secret: - argocdServerAdminPassword: '{{ argocd_server_admin_password | password_hash("bcrypt") }}' diff --git a/group_vars/stage_prodnso/kubernetes.yml b/group_vars/stage_prodnso/kubernetes.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/group_vars/stage_prodnso/kubernetes.yml @@ -0,0 +1 @@ +--- diff --git a/group_vars/stage_prodnso/plain.yml b/group_vars/stage_prodnso/plain.yml index 99c2093..6b2d237 100644 --- a/group_vars/stage_prodnso/plain.yml +++ b/group_vars/stage_prodnso/plain.yml @@ -105,10 +105,3 @@ management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}" # https://git.dev-at.de/smardigo-hetzner/communication-keys/ # push mirror: https://{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys/ gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}' - -kubernetes_with_prometheus: False -cert_manager_dplmt: False -kubernetes_with_certmanager: False -kubernetes_with_extdns: False -kubernetes_with_ingress: False -kubernetes_with_gitea: False diff --git a/group_vars/stage_prodwork01/argocd.yml b/group_vars/stage_prodwork01/argocd.yml deleted file mode 100644 index 6b7b31c..0000000 --- a/group_vars/stage_prodwork01/argocd.yml +++ /dev/null @@ -1,158 +0,0 @@ ---- - -argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}" -k8s_argocd_helm__name: "argo-cd" -k8s_argocd_helm__release_namespace: "argo-cd" - -k8s_argocd_with_keycloak: False - -# https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd -k8s_argocd_helm__release_values: - repoServer: - serviceAccount: - create: true - name: argo-cd-argocd-repo-server - rbac: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - logLevel: warn - logFormat: json - env: - - name: ARGOCD_MAX_CONCURRENT_LOGIN_REQUESTS_COUNT - value: "0" - - name: ARGOCD_EXEC_TIMEOUT - value: "300s" - - name: XDG_CONFIG_HOME - value: /.config - - name: GNUPGHOME - value: /home/argocd/.gnupg - - name: HELM_PLUGINS - value: /custom-tools/helm-plugins/ - - name: HELM_SECRETS_SOPS_PATH - value: /custom-tools/sops - - name: HELM_SECRETS_VALS_PATH - value: /custom-tools/vals - - name: HELM_SECRETS_KUBECTL_PATH - value: /custom-tools/kubectl - - name: HELM_SECRETS_CURL_PATH - value: /custom-tools/curl - # https://github.com/jkroepke/helm-secrets/wiki/Security-in-shared-environments - - name: HELM_SECRETS_KEY_LOCATION_PREFIX - value: "/sops-gpg/" - - name: HELM_SECRETS_VALUES_ALLOW_SYMLINKS - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_ABSOLUTE_PATH - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_PATH_TRAVERSAL - value: "false" - volumes: - - name: custom-tools - emptyDir: {} - - name: custom-tools-helm - emptyDir: {} - - name: gnupg-home - emptyDir: {} - - name: sops-gpg - secret: - secretName: sops-gpg - volumeMounts: - - mountPath: /home/argocd/.gnupg - name: gnupg-home - subPath: .gnupg - - mountPath: /usr/local/bin/kustomize - name: custom-tools - subPath: kustomize - # Verify this matches a XDG_CONFIG_HOME=/.config env variable - - mountPath: /.config/kustomize/plugin/viaduct.ai/v1/ksops/ksops - name: custom-tools - subPath: ksops - - mountPath: /custom-tools/helm-plugins - name: custom-tools-helm - subPath: helm-plugins - - mountPath: /custom-tools/kubectl - name: custom-tools-helm - subPath: kubectl - - mountPath: /custom-tools/sops - name: custom-tools-helm - subPath: sops - - mountPath: /custom-tools/vals - name: custom-tools-helm - subPath: vals - initContainers: - - name: 1-install-ksops - image: viaductoss/ksops:v3.0.1 - command: ["/bin/sh", "-c"] - args: - - echo "Installing KSOPS..."; - mv ksops /custom-tools/; - mv $GOPATH/bin/kustomize /custom-tools/; - echo "Done."; - volumeMounts: - - mountPath: /custom-tools - name: custom-tools - - name: 2-download-tools - image: alpine:latest - command: [sh, -ec] - env: - - name: HELM_SECRETS_VERSION - value: "3.12.0" - - name: KUBECTL_VERSION - value: "1.24.3" - - name: VALS_VERSION - value: "0.18.0" - - name: SOPS_VERSION - value: "3.7.3" - args: - - | - echo "Installing helm secrets..."; - mkdir -p /custom-tools/helm-plugins - wget -qO- https://github.com/jkroepke/helm-secrets/releases/download/v${HELM_SECRETS_VERSION}/helm-secrets.tar.gz | tar -C /custom-tools/helm-plugins -xzf-; - echo "Done."; - - echo "Downloading SOPS=${SOPS_VERSION} and kubectl ..."; - wget -qO /custom-tools/sops https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux - wget -qO /custom-tools/kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl - echo "Done."; - - echo "Downloading vals..."; - wget -qO- https://github.com/variantdev/vals/releases/download/v${VALS_VERSION}/vals_${VALS_VERSION}_linux_amd64.tar.gz | tar -xzf- -C /custom-tools/ vals; - echo "Done."; - - chmod +x /custom-tools/*; - volumeMounts: - - mountPath: /custom-tools - name: custom-tools-helm - - name: 3-import-gpg-key - image: argoproj/argocd:v2.2.5 - command: ["gpg", "--import","/sops-gpg/gpg_key_smardigo_automation__private"] - env: - - name: GNUPGHOME - value: /gnupg-home/.gnupg - volumeMounts: - - mountPath: /sops-gpg - name: sops-gpg - - mountPath: /gnupg-home - name: gnupg-home - server: - logLevel: warn - logFormat: json - config: - kustomize.buildOptions: "--enable-alpha-plugins" - helm.valuesFileSchemes: >- - secrets+gpg-import, secrets+gpg-import-kubernetes, - secrets+age-import, secrets+age-import-kubernetes, - secrets,secrets+literal, - https - service: - sessionAffinity: ClientIP - dex: - enabled: false - applicationSet: - enabled: false - configs: - secret: - argocdServerAdminPassword: '{{ argocd_server_admin_password | password_hash("bcrypt") }}' diff --git a/group_vars/stage_prodwork01/kubernetes.yml b/group_vars/stage_prodwork01/kubernetes.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/group_vars/stage_prodwork01/kubernetes.yml @@ -0,0 +1 @@ +--- diff --git a/group_vars/stage_prodwork01/plain.yml b/group_vars/stage_prodwork01/plain.yml index adf49d9..3966f8c 100644 --- a/group_vars/stage_prodwork01/plain.yml +++ b/group_vars/stage_prodwork01/plain.yml @@ -24,11 +24,3 @@ netgo_msteams_hook_alerting: "{{ netgo_msteams_hook_alerting_vault }}" # https://git.dev-at.de/smardigo-hetzner/communication-keys/ # push mirror: https://{{ stage }}-gitea-01.smardigo.digital/communication-keys.git gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}' - -kubernetes_with_prometheus: False -cert_manager_dplmt: False -kubernetes_with_certmanager: False -kubernetes_with_extdns: False -kubernetes_with_ingress: False -kubernetes_with_awx: False -kubernetes_with_gitea: False diff --git a/group_vars/stage_qa/argocd.yml b/group_vars/stage_qa/argocd.yml deleted file mode 100644 index 6b7b31c..0000000 --- a/group_vars/stage_qa/argocd.yml +++ /dev/null @@ -1,158 +0,0 @@ ---- - -argocd_server_admin_password: "{{ argocd_server_admin_password_vault }}" -k8s_argocd_helm__name: "argo-cd" -k8s_argocd_helm__release_namespace: "argo-cd" - -k8s_argocd_with_keycloak: False - -# https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd -k8s_argocd_helm__release_values: - repoServer: - serviceAccount: - create: true - name: argo-cd-argocd-repo-server - rbac: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - logLevel: warn - logFormat: json - env: - - name: ARGOCD_MAX_CONCURRENT_LOGIN_REQUESTS_COUNT - value: "0" - - name: ARGOCD_EXEC_TIMEOUT - value: "300s" - - name: XDG_CONFIG_HOME - value: /.config - - name: GNUPGHOME - value: /home/argocd/.gnupg - - name: HELM_PLUGINS - value: /custom-tools/helm-plugins/ - - name: HELM_SECRETS_SOPS_PATH - value: /custom-tools/sops - - name: HELM_SECRETS_VALS_PATH - value: /custom-tools/vals - - name: HELM_SECRETS_KUBECTL_PATH - value: /custom-tools/kubectl - - name: HELM_SECRETS_CURL_PATH - value: /custom-tools/curl - # https://github.com/jkroepke/helm-secrets/wiki/Security-in-shared-environments - - name: HELM_SECRETS_KEY_LOCATION_PREFIX - value: "/sops-gpg/" - - name: HELM_SECRETS_VALUES_ALLOW_SYMLINKS - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_ABSOLUTE_PATH - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_PATH_TRAVERSAL - value: "false" - volumes: - - name: custom-tools - emptyDir: {} - - name: custom-tools-helm - emptyDir: {} - - name: gnupg-home - emptyDir: {} - - name: sops-gpg - secret: - secretName: sops-gpg - volumeMounts: - - mountPath: /home/argocd/.gnupg - name: gnupg-home - subPath: .gnupg - - mountPath: /usr/local/bin/kustomize - name: custom-tools - subPath: kustomize - # Verify this matches a XDG_CONFIG_HOME=/.config env variable - - mountPath: /.config/kustomize/plugin/viaduct.ai/v1/ksops/ksops - name: custom-tools - subPath: ksops - - mountPath: /custom-tools/helm-plugins - name: custom-tools-helm - subPath: helm-plugins - - mountPath: /custom-tools/kubectl - name: custom-tools-helm - subPath: kubectl - - mountPath: /custom-tools/sops - name: custom-tools-helm - subPath: sops - - mountPath: /custom-tools/vals - name: custom-tools-helm - subPath: vals - initContainers: - - name: 1-install-ksops - image: viaductoss/ksops:v3.0.1 - command: ["/bin/sh", "-c"] - args: - - echo "Installing KSOPS..."; - mv ksops /custom-tools/; - mv $GOPATH/bin/kustomize /custom-tools/; - echo "Done."; - volumeMounts: - - mountPath: /custom-tools - name: custom-tools - - name: 2-download-tools - image: alpine:latest - command: [sh, -ec] - env: - - name: HELM_SECRETS_VERSION - value: "3.12.0" - - name: KUBECTL_VERSION - value: "1.24.3" - - name: VALS_VERSION - value: "0.18.0" - - name: SOPS_VERSION - value: "3.7.3" - args: - - | - echo "Installing helm secrets..."; - mkdir -p /custom-tools/helm-plugins - wget -qO- https://github.com/jkroepke/helm-secrets/releases/download/v${HELM_SECRETS_VERSION}/helm-secrets.tar.gz | tar -C /custom-tools/helm-plugins -xzf-; - echo "Done."; - - echo "Downloading SOPS=${SOPS_VERSION} and kubectl ..."; - wget -qO /custom-tools/sops https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux - wget -qO /custom-tools/kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl - echo "Done."; - - echo "Downloading vals..."; - wget -qO- https://github.com/variantdev/vals/releases/download/v${VALS_VERSION}/vals_${VALS_VERSION}_linux_amd64.tar.gz | tar -xzf- -C /custom-tools/ vals; - echo "Done."; - - chmod +x /custom-tools/*; - volumeMounts: - - mountPath: /custom-tools - name: custom-tools-helm - - name: 3-import-gpg-key - image: argoproj/argocd:v2.2.5 - command: ["gpg", "--import","/sops-gpg/gpg_key_smardigo_automation__private"] - env: - - name: GNUPGHOME - value: /gnupg-home/.gnupg - volumeMounts: - - mountPath: /sops-gpg - name: sops-gpg - - mountPath: /gnupg-home - name: gnupg-home - server: - logLevel: warn - logFormat: json - config: - kustomize.buildOptions: "--enable-alpha-plugins" - helm.valuesFileSchemes: >- - secrets+gpg-import, secrets+gpg-import-kubernetes, - secrets+age-import, secrets+age-import-kubernetes, - secrets,secrets+literal, - https - service: - sessionAffinity: ClientIP - dex: - enabled: false - applicationSet: - enabled: false - configs: - secret: - argocdServerAdminPassword: '{{ argocd_server_admin_password | password_hash("bcrypt") }}' diff --git a/group_vars/stage_qa/kubernetes.yml b/group_vars/stage_qa/kubernetes.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/group_vars/stage_qa/kubernetes.yml @@ -0,0 +1 @@ +--- diff --git a/group_vars/stage_qa/plain.yml b/group_vars/stage_qa/plain.yml index 9ef0396..bcf2561 100644 --- a/group_vars/stage_qa/plain.yml +++ b/group_vars/stage_qa/plain.yml @@ -105,10 +105,3 @@ management_oidc_client_secret: "{{ management_oidc_client_secret_vault }}" # https://git.dev-at.de/smardigo-hetzner/communication-keys/ # push mirror: https://{{ stage }}-gitea-01.smardigo.digital/gitea-admin/communication-keys/ gpg_key_smardigo_automation__private: '{{ gpg_key_smardigo_automation__private__vault }}' - -kubernetes_with_prometheus: False -cert_manager_dplmt: False -kubernetes_with_certmanager: False -kubernetes_with_extdns: False -kubernetes_with_ingress: False -kubernetes_with_gitea: False diff --git a/host_vars/devscr-kube-node-04.yml b/host_vars/devscr-kube-node-04.yml deleted file mode 100644 index bd9530e..0000000 --- a/host_vars/devscr-kube-node-04.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -hetzner_server_type: cpx41 diff --git a/host_vars/devscr-kube-node-05.yml b/host_vars/devscr-kube-node-05.yml deleted file mode 100644 index bd9530e..0000000 --- a/host_vars/devscr-kube-node-05.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -hetzner_server_type: cpx41 diff --git a/host_vars/devscr-kube-node-06.yml b/host_vars/devscr-kube-node-06.yml deleted file mode 100644 index bd9530e..0000000 --- a/host_vars/devscr-kube-node-06.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -hetzner_server_type: cpx41 diff --git a/kubernetes.yml b/kubernetes.yml index 5c7d844..6a804c6 100644 --- a/kubernetes.yml +++ b/kubernetes.yml @@ -23,35 +23,47 @@ roles: - { role: kubernetes/base } -# - { role: kubernetes/namespace } - role: kubernetes/cloud_controller_manager when: kubernetes_with_ccm | default(true) tags: - ccm - - { role: kubernetes/container_storage_interface } + + - role: kubernetes/container_storage_interface + when: kubernetes_with_csi | default(true) + tags: + - csi + + - role: kubernetes/external_dns + when: kubernetes_with_externaldns | default(false) + tags: + - external-dns + + - role: kubernetes/cert_manager + when: kubernetes_with_certmanager | default(false) + tags: + - cert-manager + + - role: kubernetes/ingress_controller + when: kubernetes_with_ingress | default(false) + tags: + - ingress + + - role: kubernetes/gitea + when: kubernetes_with_gitea | default(false) + tags: + - gitea - role: kubernetes/argocd when: kubernetes_with_argocd | default(true) tags: - argocd -# - role: kubernetes/prometheus -# tags: -# - prometheus -# when: kubernetes_with_prometheus | default(True) -# - role: kubernetes/cert_manager -# when: kubernetes_with_certmanager | default(True) -# - role: kubernetes/external_dns -# when: kubernetes_with_extdns | default(True) -# tags: -# - external-dns -# - role: kubernetes/ingress_controller -# when: kubernetes_with_ingress | default(True) + - role: kubernetes/bootstrap + when: kubernetes_with_bootstrap | default(true) + tags: + - bootstrap + - role: kubernetes/awx - when: kubernetes_with_awx | default(True) + when: kubernetes_with_awx | default(false) tags: - awx -# - role: kubernetes/gitea -# when: kubernetes_with_gitea | default(False) -# tags: -# - gitea diff --git a/kubespray b/kubespray index 00550ba..0634be4 160000 --- a/kubespray +++ b/kubespray @@ -1 +1 @@ -Subproject commit 00550ba832aa5d4f59bce03ead09d9e940e3a672 +Subproject commit 0634be4c8819cbb78afd6e53fc99cb001edba8c0 diff --git a/roles/kubernetes/argocd/defaults/main.yml b/roles/kubernetes/argocd/defaults/main.yml index a82ecc9..c2bd5a9 100644 --- a/roles/kubernetes/argocd/defaults/main.yml +++ b/roles/kubernetes/argocd/defaults/main.yml @@ -1,25 +1,20 @@ --- -k8s_argocd_helm__name: "argo-cd" -k8s_argocd_helm__release_namespace: "argo-cd" - -k8s_argocd_with_keycloak: True argo_realm_name: &argoname 'argocd' argo_realm_display_name: *argoname -k8s_argocd_helm__domain: &argourl "{{ stage_kube }}-argocd.{{ domain }}" argo_realm_group: argoadmins # shouldn't be 'admin' due to default adminuser called 'admin' in argo argo_keycloak_clientscope_protocol: openid-connect argo_keycloak_clientscope_name: groups argo_client_id: *argoname -argo_client_root_url: 'https://{{ k8s_argocd_helm__domain }}' +argo_client_root_url: 'https://{{ shared_service_kube_argocd_hostname }}' argo_client_redirect_uris: - - 'https://{{ k8s_argocd_helm__domain }}/auth/callback' + - 'https://{{ shared_service_kube_argocd_hostname }}/auth/callback' argo_client_base_url: '/applications' -argo_client_admin_url: 'https://{{ k8s_argocd_helm__domain }}' +argo_client_admin_url: 'https://{{ shared_service_kube_argocd_hostname }}' argo_client_web_origins: - - 'https://{{ k8s_argocd_helm__domain }}' + - 'https://{{ shared_service_kube_argocd_hostname }}' argo_realm_users: [ { @@ -28,283 +23,3 @@ argo_realm_users: [ "requiredActions": [] } ] -argocd_server_admin_password: "{{ argocd_server_admin_password_vault | default( lookup('community.general.random_string', length=20) ) }}" - -k8s_argocd_helm__chart_version: 5.19.0 - -# https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd -k8s_argocd_helm__release_values: - controller: - logLevel: info - logFormat: json - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: "{{ k8s_argocd_helm__release_namespace }}" - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" - repoServer: - serviceAccount: - create: true - name: argo-cd-argocd-repo-server - rbac: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - logLevel: info - logFormat: json - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: "{{ k8s_argocd_helm__release_namespace }}" - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" - env: - - name: ARGOCD_MAX_CONCURRENT_LOGIN_REQUESTS_COUNT - value: "0" - - name: ARGOCD_EXEC_TIMEOUT - value: "300s" - - name: XDG_CONFIG_HOME - value: /.config - - name: HELM_CONFIG_HOME - value: /.config - - name: GNUPGHOME - value: /home/argocd/.gnupg - - name: HELM_PLUGINS - value: /custom-tools/helm-plugins/ - - name: HELM_SECRETS_SOPS_PATH - value: /custom-tools/sops - - name: HELM_SECRETS_VALS_PATH - value: /custom-tools/vals - - name: HELM_SECRETS_KUBECTL_PATH - value: /custom-tools/kubectl - - name: HELM_SECRETS_CURL_PATH - value: /custom-tools/curl - # https://github.com/jkroepke/helm-secrets/wiki/Security-in-shared-environments - - name: HELM_SECRETS_KEY_LOCATION_PREFIX - value: "/sops-gpg/" - - name: HELM_SECRETS_VALUES_ALLOW_SYMLINKS - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_ABSOLUTE_PATH - value: "false" - - name: HELM_SECRETS_VALUES_ALLOW_PATH_TRAVERSAL - value: "false" - volumes: - - name: custom-tools - emptyDir: {} - - name: custom-tools-helm - emptyDir: {} - - name: gnupg-home - emptyDir: {} - - name: sops-gpg - secret: - secretName: sops-gpg - volumeMounts: - - mountPath: /home/argocd/.gnupg - name: gnupg-home - subPath: .gnupg - - mountPath: /usr/local/bin/kustomize - name: custom-tools - subPath: kustomize - # Verify this matches a XDG_CONFIG_HOME=/.config env variable - - mountPath: /.config/kustomize/plugin/viaduct.ai/v1/ksops/ksops - name: custom-tools - subPath: ksops - - mountPath: /custom-tools/helm-plugins - name: custom-tools-helm - subPath: helm-plugins - - mountPath: /custom-tools/kubectl - name: custom-tools-helm - subPath: kubectl - - mountPath: /custom-tools/sops - name: custom-tools-helm - subPath: sops - - mountPath: /custom-tools/vals - name: custom-tools-helm - subPath: vals - initContainers: - - name: 1-install-ksops - image: viaductoss/ksops:v3.0.1 - command: ["/bin/sh", "-c"] - args: - - echo "Installing KSOPS..."; - mv ksops /custom-tools/; - mv $GOPATH/bin/kustomize /custom-tools/; - echo "Done."; - volumeMounts: - - mountPath: /custom-tools - name: custom-tools - - name: 2-download-tools - image: alpine:latest - command: [sh, -ec] - env: - - name: HELM_SECRETS_VERSION - value: "3.12.0" - - name: KUBECTL_VERSION - value: "1.24.3" - - name: VALS_VERSION - value: "0.18.0" - - name: SOPS_VERSION - value: "3.7.3" - args: - - | - echo "Installing helm secrets..."; - mkdir -p /custom-tools/helm-plugins - wget -qO- https://github.com/jkroepke/helm-secrets/releases/download/v${HELM_SECRETS_VERSION}/helm-secrets.tar.gz | tar -C /custom-tools/helm-plugins -xzf-; - echo "Done."; - - echo "Downloading SOPS=${SOPS_VERSION} and kubectl ..."; - wget -qO /custom-tools/sops https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux - wget -qO /custom-tools/kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl - echo "Done."; - - echo "Downloading vals..."; - wget -qO- https://github.com/variantdev/vals/releases/download/v${VALS_VERSION}/vals_${VALS_VERSION}_linux_amd64.tar.gz | tar -xzf- -C /custom-tools/ vals; - echo "Done."; - - chmod +x /custom-tools/*; - volumeMounts: - - mountPath: /custom-tools - name: custom-tools-helm - - name: 3-import-gpg-key - image: argoproj/argocd:v2.5.7 - command: ["gpg", "--import","/sops-gpg/gpg_key_smardigo_automation__private"] - env: - - name: GNUPGHOME - value: /gnupg-home/.gnupg - volumeMounts: - - mountPath: /sops-gpg - name: sops-gpg - - mountPath: /gnupg-home - name: gnupg-home - server: - logLevel: info - logFormat: json - config: - oidc.config: | - name: Keycloak - issuer: '{{ shared_service_url_keycloak }}/auth/realms/argocd' - clientID: '{{ argo_client_id }}' - clientSecret: $oidc.keycloak.clientSecret - requestedScopes: ["openid", "profile", "email", "{{ argo_keycloak_clientscope_name }}"] - url: 'https://{{ k8s_argocd_helm__domain }}' - kustomize.buildOptions: "--enable-alpha-plugins" - helm.valuesFileSchemes: >- - secrets+gpg-import, secrets+gpg-import-kubernetes, - secrets+age-import, secrets+age-import-kubernetes, - secrets,secrets+literal, - https - rbacConfig: - policy.default: role:readonly - policy.csv: | - g, {{ argo_realm_group }}, role:admin - g, admin, role:admin - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: "{{ k8s_argocd_helm__release_namespace }}" - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" - service: - sessionAffinity: ClientIP - ingress: - enabled: true - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - cert-manager.io/issue-temporary-certificate: "true" - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist ) | join(',') }}" - nginx.ingress.kubernetes.io/force-ssl-redirect: "false" - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - hosts: - - "{{ k8s_argocd_helm__domain }}" - tls: - - secretName: "{{ stage }}-kube-argocd-cert" - hosts: - - "{{ k8s_argocd_helm__domain }}" - additionalProjects: - - name: bootstrap - namespace: '{{ k8s_argocd_helm__release_namespace }}' - additionalLabels: {} - additionalAnnotations: {} - description: application declarations for bootstraping k8s cluster with argo-cd - sourceRepos: - - '*' - destinations: - - namespace: '*' - server: https://kubernetes.default.svc - clusterResourceWhitelist: - - group: '*' - kind: '*' - orphanedResources: - warn: false - - name: kube-system - namespace: '{{ k8s_argocd_helm__release_namespace }}' - additionalLabels: {} - additionalAnnotations: {} - description: applications for kube-system namespace - sourceRepos: - - '*' - destinations: - - namespace: kube-system - server: https://kubernetes.default.svc - clusterResourceWhitelist: - - group: '*' - kind: '*' - orphanedResources: - warn: false - - name: infrastructure - namespace: '{{ k8s_argocd_helm__release_namespace }}' - additionalLabels: {} - additionalAnnotations: {} - description: infrastructure applications - sourceRepos: - - '*' - destinations: - - namespace: '*' - server: https://kubernetes.default.svc - clusterResourceWhitelist: - - group: '*' - kind: '*' - orphanedResources: - warn: false - additionalApplications: - - - name: bootstrap - namespace: '{{ k8s_argocd_helm__release_namespace }}' - destination: - namespace: bootstrap - server: https://kubernetes.default.svc - project: bootstrap - source: - path: config/default - repoURL: https://{{ shared_service_gitea_hostname }}/argocd/argocd.git - targetRevision: '{{ awx_smardigo_revision | default(stage) }}' - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - redis: - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: "{{ k8s_argocd_helm__release_namespace }}" - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" - dex: - enabled: false - applicationSet: - enabled: false - configs: - secret: - argocdServerAdminPassword: '{{ argocd_server_admin_password | password_hash("bcrypt") }}' diff --git a/roles/kubernetes/argocd/tasks/main.yml b/roles/kubernetes/argocd/tasks/main.yml index 74ab357..f564aa5 100644 --- a/roles/kubernetes/argocd/tasks/main.yml +++ b/roles/kubernetes/argocd/tasks/main.yml @@ -266,58 +266,14 @@ - name: Deploy argo-cd inside argo-cd namespace become: yes kubernetes.core.helm: + create_namespace: yes name: "{{ k8s_argocd_helm__name }}" chart_ref: "{{ k8s_argocd_helm__chart_ref | default('argo-cd') }}" chart_repo_url: "{{ k8s_argocd_helm__chart_repo_url | default('https://argoproj.github.io/argo-helm') }}" - release_namespace: "{{ k8s_argocd_helm__release_namespace }}" chart_version: "{{ k8s_argocd_helm__chart_version }}" - create_namespace: yes + release_namespace: "{{ k8s_argocd_helm__release_namespace }}" release_values: "{{ combined_helm__release_values }}" when: - inventory_hostname == groups['kube_control_plane'][0] tags: - argo-cd - -- name: Setup gitea Secret - become: yes - kubernetes.core.k8s: - state: present - template: 'gitea-secret.j2' - when: - - argocd_bootstrap_infrastructure - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - argo-cd - -- name: Setup Harbor Secret - become: yes - kubernetes.core.k8s: - state: present - template: 'harbor-secret.j2' - when: - - argocd_bootstrap_infrastructure - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - argo-cd - -- name: Setup argo-cd application for bootstrap - become: yes - kubernetes.core.k8s: - state: present - template: 'bootstrap-application.j2' - when: - - argocd_bootstrap_infrastructure - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - argo-cd - -- name: Setup argo-cd infrastructure project - become: yes - kubernetes.core.k8s: - state: present - template: 'project-infrastructure.j2' - when: - - argocd_bootstrap_infrastructure - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - argo-cd diff --git a/roles/kubernetes/bootstrap/defaults/main.yml b/roles/kubernetes/bootstrap/defaults/main.yml new file mode 100644 index 0000000..19a7490 --- /dev/null +++ b/roles/kubernetes/bootstrap/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +# TODO SKEN move to groups/all +k8s_argocd_helm__release_namespace: "argo-cd" \ No newline at end of file diff --git a/roles/kubernetes/bootstrap/tasks/main.yml b/roles/kubernetes/bootstrap/tasks/main.yml new file mode 100644 index 0000000..a572eb4 --- /dev/null +++ b/roles/kubernetes/bootstrap/tasks/main.yml @@ -0,0 +1,45 @@ +--- + +- name: Setup gitea Secret + become: yes + kubernetes.core.k8s: + state: present + template: 'gitea-secret.j2' + when: + - argocd_bootstrap_infrastructure + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - argo-cd + +- name: Setup Harbor Secret + become: yes + kubernetes.core.k8s: + state: present + template: 'harbor-secret.j2' + when: + - argocd_bootstrap_infrastructure + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - argo-cd + +- name: Setup argocd application for bootstrap + become: yes + kubernetes.core.k8s: + state: present + template: 'bootstrap-application.j2' + when: + - argocd_bootstrap_infrastructure + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - argo-cd + +- name: Setup argocd infrastructure project + become: yes + kubernetes.core.k8s: + state: present + template: 'project-infrastructure.j2' + when: + - argocd_bootstrap_infrastructure + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - argo-cd diff --git a/roles/kubernetes/argocd/templates/bootstrap-application.j2 b/roles/kubernetes/bootstrap/templates/bootstrap-application.j2 similarity index 100% rename from roles/kubernetes/argocd/templates/bootstrap-application.j2 rename to roles/kubernetes/bootstrap/templates/bootstrap-application.j2 diff --git a/roles/kubernetes/argocd/templates/gitea-secret.j2 b/roles/kubernetes/bootstrap/templates/gitea-secret.j2 similarity index 100% rename from roles/kubernetes/argocd/templates/gitea-secret.j2 rename to roles/kubernetes/bootstrap/templates/gitea-secret.j2 diff --git a/roles/kubernetes/argocd/templates/harbor-secret.j2 b/roles/kubernetes/bootstrap/templates/harbor-secret.j2 similarity index 100% rename from roles/kubernetes/argocd/templates/harbor-secret.j2 rename to roles/kubernetes/bootstrap/templates/harbor-secret.j2 diff --git a/roles/kubernetes/argocd/templates/project-infrastructure.j2 b/roles/kubernetes/bootstrap/templates/project-infrastructure.j2 similarity index 100% rename from roles/kubernetes/argocd/templates/project-infrastructure.j2 rename to roles/kubernetes/bootstrap/templates/project-infrastructure.j2 diff --git a/roles/kubernetes/cert_manager/defaults/main.yml b/roles/kubernetes/cert_manager/defaults/main.yml index 67273ca..1e26144 100644 --- a/roles/kubernetes/cert_manager/defaults/main.yml +++ b/roles/kubernetes/cert_manager/defaults/main.yml @@ -1,9 +1,5 @@ --- -k8s_prometheus_helm__name: "prometheus" - -cert_manager_dplmt: True - k8s_certmanager_helm__chart_ref: cert-manager k8s_certmanager_helm__chart_repo_url: https://charts.jetstack.io k8s_certmanager_helm__release_namespace: cert-manager @@ -11,15 +7,16 @@ k8s_certmanager_helm__release_namespace: cert-manager k8s_certmanager_helm__release_values: installCRDs: true webhook.timeoutSeconds: 4 - prometheus: - enabled: true - servicemonitor: - enabled: true - namespace: cert-manager - labels: - release: "{{ k8s_prometheus_helm__name }}" -k8s_certmanager_helm__cluster_issuers: +k8s_certmanager_helm__cluster_issuers_http: + prod-http: + email: "{{ lets_encrypt_email }}" + server: https://acme-v02.api.letsencrypt.org/directory + staging-http: + email: "{{ lets_encrypt_email }}" + server: https://acme-staging-v02.api.letsencrypt.org/directory + +k8s_certmanager_helm__cluster_issuers_dns01: prod: email: "{{ lets_encrypt_email }}" server: https://acme-v02.api.letsencrypt.org/directory diff --git a/roles/kubernetes/cert_manager/tasks/main.yml b/roles/kubernetes/cert_manager/tasks/main.yml index 36be783..c2554d9 100644 --- a/roles/kubernetes/cert_manager/tasks/main.yml +++ b/roles/kubernetes/cert_manager/tasks/main.yml @@ -38,14 +38,15 @@ - name: Install cert-manager via helm become: yes kubernetes.core.helm: - name: cert-manager + create_namespace: yes + name: "{{ k8s_certmanager_helm__chart_ref }}" chart_ref: "{{ k8s_certmanager_helm__chart_ref }}" chart_repo_url: "{{ k8s_certmanager_helm__chart_repo_url }}" + chart_version: v1.9.1 release_namespace: "{{ k8s_certmanager_helm__release_namespace }}" - create_namespace: yes release_values: "{{ k8s_certmanager_helm__release_values }}" - - name: Create ClusterIssuer for letsencrypt (prod/staging) + - name: Create ClusterIssuer for letsencrypt (prod/staging) with dns challenge become: yes kubernetes.core.k8s: definition: @@ -68,11 +69,30 @@ selector: dnsZones: - 'smardigo.digital' - loop: "{{ k8s_certmanager_helm__cluster_issuers | dict2items }}" + loop: "{{ k8s_certmanager_helm__cluster_issuers_dns01 | dict2items }}" + + - name: Create ClusterIssuer for letsencrypt (prod/staging) with http challenge + become: yes + kubernetes.core.k8s: + definition: + api_version: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: "letsencrypt-{{ item.key }}" + spec: + acme: + email: "{{ item.value.email }}" + server: "{{ item.value.server }}" + privateKeySecretRef: + name: issuer-account-key + solvers: + - http01: + ingress: + class: nginx + loop: "{{ k8s_certmanager_helm__cluster_issuers_http | dict2items }}" # end of block statement when: - inventory_hostname == groups['kube_control_plane'][0] - - cert_manager_dplmt tags: - cert-manager diff --git a/roles/kubernetes/external_dns/defaults/main.yml b/roles/kubernetes/external_dns/defaults/main.yml index f9fafc5..0ab6a27 100644 --- a/roles/kubernetes/external_dns/defaults/main.yml +++ b/roles/kubernetes/external_dns/defaults/main.yml @@ -1,7 +1,5 @@ --- -k8s_prometheus_helm__name: "prometheus" - k8s_externaldns_helm__chart_ref: external-dns k8s_externaldns_helm__chart_repo_url: https://kubernetes-sigs.github.io/external-dns/ k8s_externaldns_helm__release_namespace: external-dns @@ -23,7 +21,3 @@ k8s_externaldns_helm__release_values: ] txtOwnerId: "{{ stage }}-external-dns" txtPrefix: "{{ stage }}" - serviceMonitor: - enabled: true - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" diff --git a/roles/kubernetes/external_dns/tasks/main.yml b/roles/kubernetes/external_dns/tasks/main.yml index c0e1729..ebc6a97 100644 --- a/roles/kubernetes/external_dns/tasks/main.yml +++ b/roles/kubernetes/external_dns/tasks/main.yml @@ -6,11 +6,12 @@ - name: Install external-dns via helm become: yes kubernetes.core.helm: - name: external-dns + create_namespace: yes + name: "{{ k8s_externaldns_helm__chart_ref }}" chart_ref: "{{ k8s_externaldns_helm__chart_ref }}" chart_repo_url: "{{ k8s_externaldns_helm__chart_repo_url }}" + chart_version: 1.9.0 release_namespace: "{{ k8s_externaldns_helm__release_namespace }}" - create_namespace: yes release_values: "{{ k8s_externaldns_helm__release_values }}" when: - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/kubernetes/gitea/defaults/main.yml b/roles/kubernetes/gitea/defaults/main.yml index 1f54512..4fe1ede 100644 --- a/roles/kubernetes/gitea/defaults/main.yml +++ b/roles/kubernetes/gitea/defaults/main.yml @@ -1,14 +1,11 @@ --- + k8s_gitea_helm__release_values: gitea: admin: - username: gitea_admin - password: '{{ k8s_gitea_helm__gitea_admin_password_vault }}' + username: '{{ gitea_admin_username }}' + password: '{{ gitea_admin_password }}' email: '{{ devops_email_address }}' - metrics: - enabled: true - serviceMonitor: - enabled: true config: cache: ENABLED: false @@ -29,7 +26,7 @@ k8s_gitea_helm__release_values: cert-manager.io/issue-temporary-certificate: "true" kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist ) | join(',') }}" + nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist + ( custom_ip_whitelist | default([]) )) | join(',') }}" hosts: - host: "{{ stage }}-gitea.{{ domain }}" paths: @@ -45,5 +42,7 @@ k8s_gitea_helm__release_values: postgresql: global: postgresql: - postgresqlUsername: gitti - postgresqlPassword: '{{ k8s_gitea_helm__postgresql_global_postgresql_postgresqlPassword_vault }}' + auth: + # if "username" is configured, a randomized password will break each update + password: '{{ gitea_postgres_password }}' + postgresPassword: '{{ gitea_postgres_password }}' diff --git a/roles/kubernetes/gitea/tasks/main.yml b/roles/kubernetes/gitea/tasks/main.yml index 11638d2..9080615 100644 --- a/roles/kubernetes/gitea/tasks/main.yml +++ b/roles/kubernetes/gitea/tasks/main.yml @@ -1,4 +1,5 @@ --- + - name: DEBUG copy: dest: /tmp/gitea_values.yaml @@ -15,11 +16,12 @@ - name: Deploy gitea inside namespace become: yes kubernetes.core.helm: + create_namespace: yes name: "gitea" chart_ref: "{{ k8s_gitea_helm__chart_ref | default('gitea') }}" chart_repo_url: "{{ k8s_gitea_helm__chart_repo_url | default('https://dl.gitea.io/charts/') }}" - release_namespace: "{{ k8s_gitea_helm__release_namespace | default('infrastructure') }}" - create_namespace: yes + chart_version: 8.0.2 + release_namespace: "{{ k8s_gitea_helm__release_namespace | default('gitea') }}" release_values: "{{ k8s_gitea_helm__release_values }}" when: - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/kubernetes/ingress_controller/defaults/main.yml b/roles/kubernetes/ingress_controller/defaults/main.yml index 6f9dc42..4157eb6 100644 --- a/roles/kubernetes/ingress_controller/defaults/main.yml +++ b/roles/kubernetes/ingress_controller/defaults/main.yml @@ -1,6 +1,5 @@ --- -k8s_prometheus_helm__name: "prometheus" k8s_ingress_helm__release_namespace: "ingress" k8s_ingress_helm__release_values: @@ -36,8 +35,8 @@ k8s_ingress_helm__release_values: # see https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations load-balancer.hetzner.cloud/type: "lb11" load-balancer.hetzner.cloud/location: nbg1 - load-balancer.hetzner.cloud/name: "{{ stage }}-ingress" - load-balancer.hetzner.cloud/hostname: "{{ stage }}-ingress" + load-balancer.hetzner.cloud/name: "{{ stage_kube }}-ingress" + load-balancer.hetzner.cloud/hostname: "{{ stage_kube }}-ingress" load-balancer.hetzner.cloud/disable-public-network: false load-balancer.hetzner.cloud/disable-private-ingress: true load-balancer.hetzner.cloud/use-private-ip: true @@ -47,13 +46,6 @@ k8s_ingress_helm__release_values: load-balancer.hetzner.cloud/health-check-retries: 3 load-balancer.hetzner.cloud/health-check-protocol: "tcp" load-balancer.hetzner.cloud/health-check-port: *httpnodeport - metrics: - enabled: true - serviceMonitor: - enabled: true - namespace: "{{ k8s_ingress_helm__release_namespace }}" - additionalLabels: - release: "{{ k8s_prometheus_helm__name }}" defaultBackend: enabled: true diff --git a/roles/kubernetes/ingress_controller/tasks/main.yml b/roles/kubernetes/ingress_controller/tasks/main.yml index 2f5dc58..3d27812 100644 --- a/roles/kubernetes/ingress_controller/tasks/main.yml +++ b/roles/kubernetes/ingress_controller/tasks/main.yml @@ -6,13 +6,13 @@ - name: Install ingress via helm become: yes kubernetes.core.helm: - name: ingress - chart_repo_url: "{{ k8s_ingress_helm__chart_repo_url | default('https://kubernetes.github.io/ingress-nginx') }}" - chart_ref: "{{ k8s_ingress_helm__chart_ref | default('ingress-nginx') }}" - release_namespace: "{{ k8s_ingress_helm__release_namespace }}" - chart_version: 4.4.2 create_namespace: yes + name: "ingress-nginx" + chart_ref: "ingress-nginx" + chart_repo_url: "{{ k8s_ingress_helm__chart_repo_url | default('https://kubernetes.github.io/ingress-nginx') }}" + chart_version: 4.2.5 release_values: "{{ k8s_ingress_helm__release_values }}" + release_namespace: "{{ k8s_ingress_helm__release_namespace }}" when: - inventory_hostname == groups['kube_control_plane'][0] tags: diff --git a/roles/kubernetes/namespace/defaults/main.yml b/roles/kubernetes/namespace/defaults/main.yml deleted file mode 100644 index 74b395a..0000000 --- a/roles/kubernetes/namespace/defaults/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- - -k8s_docker_registry_key: "harbor-pull-secret-key" diff --git a/roles/kubernetes/namespace/tasks/main.yml b/roles/kubernetes/namespace/tasks/main.yml deleted file mode 100644 index 405952f..0000000 --- a/roles/kubernetes/namespace/tasks/main.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- - -### tags: -### namespace - -- name: "Create k8s namespace <{{ k8s_namespace }}>" - become: yes - kubernetes.core.k8s: - name: "{{ k8s_namespace }}" - api_version: v1 - kind: Namespace - state: present - when: - - k8s_namespace is defined - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - namespace - -- name: "Create docker registry secret for namespace <{{ k8s_namespace }}" - become: yes - kubernetes.core.k8s: - state: present - merge_type: merge - definition: - apiVersion: v1 - data: - .dockerconfigjson: "{{ lookup('template', 'docker-secret.json.j2') | to_json | b64encode }}" - kind: Secret - metadata: - name: "{{ k8s_docker_registry_key }}" - namespace: "{{ k8s_namespace }}" - type: kubernetes.io/dockerconfigjson - when: - - k8s_namespace is defined - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - namespace - -- name: "Create secrets for <{{ k8s_namespace }}>" - become: yes - kubernetes.core.k8s: - definition: - api_version: v1 - kind: Secret - metadata: - namespace: "{{ k8s_namespace }}" - name: "{{ item.name }}" - type: Opaque - data: "{{ item.data }}" - loop: "{{ k8s_secrets | default([]) }}" - when: - - k8s_namespace is defined - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - namespace diff --git a/roles/kubernetes/namespace/templates/docker-secret.json.j2 b/roles/kubernetes/namespace/templates/docker-secret.json.j2 deleted file mode 100644 index c38fb65..0000000 --- a/roles/kubernetes/namespace/templates/docker-secret.json.j2 +++ /dev/null @@ -1,7 +0,0 @@ -{ - "auths": { - "{{ shared_service_hostname_harbor }}": { - "auth": "{{ [harbor_username, harbor_token] | join(":") | string | b64encode }}" - } - } -} \ No newline at end of file diff --git a/roles/kubernetes/prometheus/defaults/main.yml b/roles/kubernetes/prometheus/defaults/main.yml deleted file mode 100644 index c4786b5..0000000 --- a/roles/kubernetes/prometheus/defaults/main.yml +++ /dev/null @@ -1,91 +0,0 @@ ---- - -k8s_prometheus_helm__name: "prometheus" -k8s_prometheus_helm__release_namespace: "monitoring" - -k8s_prometheus_basic_auth_secret_name: "prometheus-basic-auth" -k8s_alertmanager_basic_auth_secret_name: "alertmanager-basic-auth" - -# https://github.com/grafana/helm-charts -# https://github.com/prometheus-community/helm-charts -k8s_prometheus_helm__release_values: - prometheus: - ingress: - enabled: true - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - cert-manager.io/issue-temporary-certificate: "true" - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist ) | join(',') }}" - nginx.ingress.kubernetes.io/auth-type: "basic" - nginx.ingress.kubernetes.io/auth-secret: "{{ k8s_prometheus_basic_auth_secret_name }}" - nginx.ingress.kubernetes.io/auth-realm: "Authentication Required" - hosts: - - "{{ stage }}-kube-prometheus.{{ domain }}" - tls: - - secretName: "{{ stage }}-kube-prometheus-cert" - hosts: - - "{{ stage }}-kube-prometheus.{{ domain }}" - prometheusSpec: - # TODO Using PersistentVolumeClaim - storageSpec: {} - serviceMonitorSelectorNilUsesHelmValues: false - podMonitorSelectorNilUsesHelmValues: false - externalLabels: - stage: "{{ stage }}" - deploymentStrategy: - type: Recreate - alertmanager: - ingress: - enabled: true - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - cert-manager.io/issue-temporary-certificate: "true" - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist ) | join(',') }}" - nginx.ingress.kubernetes.io/auth-type: "basic" - nginx.ingress.kubernetes.io/auth-secret: "{{ k8s_alertmanager_basic_auth_secret_name }}" - nginx.ingress.kubernetes.io/auth-realm: "Authentication Required" - hosts: - - "{{ stage }}-kube-alertmanager.{{ domain }}" - tls: - - secretName: "{{ stage }}-kube-alertmanager-cert" - hosts: - - "{{ stage }}-kube-alertmanager.{{ domain }}" - deploymentStrategy: - type: Recreate - grafana: - adminUser: "{{ grafana_admin_username }}" - adminPassword: "{{ grafana_admin_password }}" - ingress: - enabled: true - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - cert-manager.io/issue-temporary-certificate: "true" - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/whitelist-source-range: "{{ ( ip_whitelist ) | join(',') }}" - hosts: - - "{{ stage }}-kube-grafana.{{ domain }}" - tls: - - secretName: "{{ stage }}-kube-grafana-cert" - hosts: - - "{{ stage }}-kube-grafana.{{ domain }}" - persistence: - enabled: true - size: 10Gi - deploymentStrategy: - type: Recreate - kubeControllerManager: - service: - port: 10257 - targetPort: 10257 - serviceMonitor: - https: true - insecureSkipVerify: true - kube-state-metrics: - metricLabelsAllowlist: - - pods=[*] - - deployments=[app.kubernetes.io/name,app.kubernetes.io/component,app.kubernetes.io/instance] diff --git a/roles/kubernetes/prometheus/tasks/_create_auth_secret.yml b/roles/kubernetes/prometheus/tasks/_create_auth_secret.yml deleted file mode 100644 index cfdd34b..0000000 --- a/roles/kubernetes/prometheus/tasks/_create_auth_secret.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- - -- name: "Create empty htpswd file" - file: - path: "{{ htpasswd_file_path }}" - state: touch - mode: '0600' - -- name: "Install latest passlib with pip" - pip: name=passlib - -- name: "Add a user and password to empty htpswd file" - community.general.htpasswd: - path: "{{ htpasswd_file_path }}" - name: "{{ basic_auth_username }}" - password: "{{ basic_auth_password }}" - mode: '0600' - -- name: "Read credentials out of htpasswd file" - ansible.builtin.slurp: - src: "{{ htpasswd_file_path }}" - register: credentials - -- name: "Create prometheus secrets" - become: yes - kubernetes.core.k8s: - definition: - api_version: v1 - kind: Secret - metadata: - namespace: "{{ namespace }}" - name: "{{ basic_auth_secret_name }}" - type: Opaque - data: - auth: "{{ credentials['content'] }}" - -- name: "Delete htpasswd file" - become: yes - file: - path: "{{ htpasswd_file_path }}" - state: absent diff --git a/roles/kubernetes/prometheus/tasks/main.yml b/roles/kubernetes/prometheus/tasks/main.yml deleted file mode 100644 index dfe4d68..0000000 --- a/roles/kubernetes/prometheus/tasks/main.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- - -### tags: -### prometheus - -- name: "Create Prometheus Basic Auth Secret" - include_tasks: _create_auth_secret.yml - vars: - htpasswd_file_path: "/tmp/prometheus-auth" - basic_auth_username: "{{ prometheus_admin_username }}" - basic_auth_password: "{{ prometheus_admin_password }}" - basic_auth_secret_name: "{{ k8s_prometheus_basic_auth_secret_name }}" - namespace: "{{ k8s_prometheus_helm__release_namespace }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - args: - apply: - tags: - - prometheus - tags: - - prometheus - -- name: "Create Alertmanager Basic Auth Secret" - include_tasks: _create_auth_secret.yml - vars: - htpasswd_file_path: "/tmp/alertmanager-auth" - basic_auth_username: "{{ alertmanager_admin_username }}" - basic_auth_password: "{{ alertmanager_admin_password }}" - basic_auth_secret_name: "{{ k8s_alertmanager_basic_auth_secret_name }}" - namespace: "{{ k8s_prometheus_helm__release_namespace }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - args: - apply: - tags: - - prometheus - tags: - - prometheus - -- name: Deploy kube-prometheus-stack inside monitoring namespace - become: yes - kubernetes.core.helm: - name: "{{ k8s_prometheus_helm__name }}" - chart_repo_url: "{{ k8s_prometheus_helm__chart_repo_url | default('https://prometheus-community.github.io/helm-charts') }}" - chart_ref: "{{ k8s_prometheus_helm__chart_ref | default('kube-prometheus-stack') }}" - chart_version: 40.1.0 - release_namespace: "{{ k8s_prometheus_helm__release_namespace }}" - create_namespace: yes - release_values: "{{ k8s_prometheus_helm__release_values }}" - when: - - inventory_hostname == groups['kube_control_plane'][0] - tags: - - prometheus diff --git a/stage-devscr b/stage-devscr index dd9d487..72f5b03 100644 --- a/stage-devscr +++ b/stage-devscr @@ -14,7 +14,6 @@ devscr-kube-node-02 devscr-kube-node-03 devscr-kube-node-04 devscr-kube-node-05 -devscr-kube-node-06 [k8s_cluster:children] kube_control_plane diff --git a/stage-devscr-netgo-hcloud.yml b/stage-devscr-netgo-hcloud.yml index e21c5cd..e8c9022 100644 --- a/stage-devscr-netgo-hcloud.yml +++ b/stage-devscr-netgo-hcloud.yml @@ -12,6 +12,7 @@ plugin: netgo-hcloud stage: "devscr" +stage_kube: "devscr" label_selector: "stage=devscr" # jinja isn't available here api_token: !vault |