====== Harbor ====== ===== Instalando o harbor via helm ===== ==== Baixando o repositório ==== $ helm repo add harbor https://helm.goharbor.io $ helm repo update ==== Baixando e editando os values ==== $ helm show values harbor/harbor > values.yaml expose: ingress: hosts: core: hub.juntotelecom.com.br notary: notary.juntotelecom.com.br className: "nginx" externalURL: https://hub.juntotelecom.com.br persistence: persistentVolumeClaim: registry: storageClass: "nfs-client" size: 32Gi chartmuseum: storageClass: "nfs-client" jobservice: storageClass: "nfs-client" database: storageClass: "nfs-client" size: 4Gi redis: storageClass: "nfs-client" trivy: storageClass: "nfs-client" ==== Fazendo o deployment ==== $ kubectl create namespace harbor-system namespace/harbor-system created $ helm install harbor -f values.yaml harbor/harbor -n harbor-system ==== Checando o status ==== $ kubectl get all -n harbor-system NAME READY STATUS RESTARTS AGE pod/harbor-chartmuseum-64db8b6499-ggmrw 1/1 Running 0 4m26s pod/harbor-core-78ffd79c47-65ml4 1/1 Running 0 4m26s pod/harbor-database-0 1/1 Running 0 4m26s pod/harbor-jobservice-bd8f54684-z6gwb 1/1 Running 1 (2m28s ago) 4m26s pod/harbor-notary-server-fb67ffbc6-27bl8 1/1 Running 2 (3m10s ago) 4m26s pod/harbor-notary-signer-96f9d78d4-jdl4s 1/1 Running 2 (2m51s ago) 4m26s pod/harbor-portal-97fcbbd96-r5zrq 1/1 Running 0 4m26s pod/harbor-redis-0 1/1 Running 0 4m26s pod/harbor-registry-6fd6c8567c-dmjlw 2/2 Running 0 4m26s pod/harbor-trivy-0 1/1 Running 0 4m26s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/harbor-chartmuseum ClusterIP 10.96.85.185 80/TCP 4m26s service/harbor-core ClusterIP 10.96.94.100 80/TCP 4m26s service/harbor-database ClusterIP 10.96.10.67 5432/TCP 4m26s service/harbor-jobservice ClusterIP 10.96.9.18 80/TCP 4m26s service/harbor-notary-server ClusterIP 10.96.181.185 4443/TCP 4m26s service/harbor-notary-signer ClusterIP 10.96.140.165 7899/TCP 4m26s service/harbor-portal ClusterIP 10.96.169.229 80/TCP 4m26s service/harbor-redis ClusterIP 10.96.248.13 6379/TCP 4m26s service/harbor-registry ClusterIP 10.96.5.124 5000/TCP,8080/TCP 4m26s service/harbor-trivy ClusterIP 10.96.16.130 8080/TCP 4m26s NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/harbor-chartmuseum 1/1 1 1 4m26s deployment.apps/harbor-core 1/1 1 1 4m26s deployment.apps/harbor-jobservice 1/1 1 1 4m26s deployment.apps/harbor-notary-server 1/1 1 1 4m26s deployment.apps/harbor-notary-signer 1/1 1 1 4m26s deployment.apps/harbor-portal 1/1 1 1 4m26s deployment.apps/harbor-registry 1/1 1 1 4m26s NAME DESIRED CURRENT READY AGE replicaset.apps/harbor-chartmuseum-64db8b6499 1 1 1 4m26s replicaset.apps/harbor-core-78ffd79c47 1 1 1 4m26s replicaset.apps/harbor-jobservice-bd8f54684 1 1 1 4m26s replicaset.apps/harbor-notary-server-fb67ffbc6 1 1 1 4m26s replicaset.apps/harbor-notary-signer-96f9d78d4 1 1 1 4m26s replicaset.apps/harbor-portal-97fcbbd96 1 1 1 4m26s replicaset.apps/harbor-registry-6fd6c8567c 1 1 1 4m26s NAME READY AGE statefulset.apps/harbor-database 1/1 4m26s statefulset.apps/harbor-redis 1/1 4m26s statefulset.apps/harbor-trivy 1/1 4m26s $ kubectl get pvc -n harbor-system NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE data-harbor-redis-0 Bound pvc-9a3e4d9d-ff5e-4e35-8f61-21f10a03aa04 1Gi RWO nfs-client 128m data-harbor-trivy-0 Bound pvc-31d6501a-5883-4aa6-83fa-703075736a9e 5Gi RWO nfs-client 128m database-data-harbor-database-0 Bound pvc-cde696e7-fe55-4ae4-8de1-3cb3a14ad229 4Gi RWO nfs-client 128m harbor-chartmuseum Bound pvc-8c263e11-10b9-4fa8-a35a-d4c68d87c8c0 5Gi RWO nfs-client 128m harbor-jobservice Bound pvc-4086cd1e-d4de-4b1e-8576-bf6241e51612 1Gi RWO nfs-client 128m harbor-registry Bound pvc-9b9388ac-28c1-4cdc-be8c-415f834f9756 32Gi RWO nfs-client 128m $ kubectl get pods -n harbor-system -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES harbor-chartmuseum-bc8d6988b-7cgx2 1/1 Running 0 5m16s 10.244.213.154 kube-worker-02.juntotelecom.com.br harbor-core-858469fffc-mqmc2 1/1 Running 0 5m16s 10.244.213.155 kube-worker-02.juntotelecom.com.br harbor-database-0 1/1 Running 0 25m 10.244.213.153 kube-worker-02.juntotelecom.com.br harbor-jobservice-6b9f555f74-c9ql9 1/1 Running 0 5m16s 10.244.213.157 kube-worker-02.juntotelecom.com.br harbor-notary-server-57c64f449-m6rdk 0/1 Running 0 5m16s 10.244.101.111 kube-worker-01.juntotelecom.com.br harbor-notary-server-fb67ffbc6-27bl8 1/1 Running 3 (3m57s ago) 25m 10.244.101.106 kube-worker-01.juntotelecom.com.br harbor-notary-signer-6fb6bbf88-klhfj 1/1 Running 0 5m16s 10.244.213.156 kube-worker-02.juntotelecom.com.br harbor-portal-97fcbbd96-r5zrq 1/1 Running 0 25m 10.244.213.149 kube-worker-02.juntotelecom.com.br harbor-redis-0 1/1 Running 0 25m 10.244.213.152 kube-worker-02.juntotelecom.com.br harbor-registry-77545c67ff-bfp4g 2/2 Running 0 5m16s 10.244.101.112 kube-worker-01.juntotelecom.com.br harbor-trivy-0 1/1 Running 0 25m 10.244.101.110 kube-worker-01.juntotelecom.com.br $ kubectl get ingress -n harbor-system NAME CLASS HOSTS ADDRESS PORTS AGE harbor-ingress nginx hub.juntotelecom.com.br 172.28.128.99 80, 443 24m harbor-ingress-notary nginx notary.juntotelecom.com.br 172.28.128.99 80, 443 24m $ kubectl describe ingress harbor-ingress -n harbor-system Name: harbor-ingress Labels: app=harbor app.kubernetes.io/managed-by=Helm chart=harbor heritage=Helm release=harbor Namespace: harbor-system Address: 172.28.128.99 Default backend: default-http-backend:80 () TLS: harbor-ingress terminates hub.juntotelecom.com.br Rules: Host Path Backends ---- ---- -------- hub.juntotelecom.com.br /api/ harbor-core:80 (10.244.213.155:8080) /service/ harbor-core:80 (10.244.213.155:8080) /v2 harbor-core:80 (10.244.213.155:8080) /chartrepo/ harbor-core:80 (10.244.213.155:8080) /c/ harbor-core:80 (10.244.213.155:8080) / harbor-portal:80 (10.244.213.149:8080) Annotations: ingress.kubernetes.io/proxy-body-size: 0 ingress.kubernetes.io/ssl-redirect: true meta.helm.sh/release-name: harbor meta.helm.sh/release-namespace: harbor-system nginx.ingress.kubernetes.io/proxy-body-size: 0 nginx.ingress.kubernetes.io/ssl-redirect: true Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Sync 3m55s (x2 over 4m37s) nginx-ingress-controller Scheduled for sync $ kubectl describe ingress harbor-ingress-notary -n harbor-system Name: harbor-ingress-notary Labels: app=harbor app.kubernetes.io/managed-by=Helm chart=harbor heritage=Helm release=harbor Namespace: harbor-system Address: 172.28.128.99 Default backend: default-http-backend:80 () TLS: harbor-ingress terminates notary.juntotelecom.com.br Rules: Host Path Backends ---- ---- -------- notary.juntotelecom.com.br / harbor-notary-server:4443 (10.244.101.111:4443) Annotations: ingress.kubernetes.io/proxy-body-size: 0 ingress.kubernetes.io/ssl-redirect: true meta.helm.sh/release-name: harbor meta.helm.sh/release-namespace: harbor-system nginx.ingress.kubernetes.io/proxy-body-size: 0 nginx.ingress.kubernetes.io/ssl-redirect: true Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Sync 5m36s (x2 over 6m18s) nginx-ingress-controller Scheduled for sync $ kubectl get configmap -n harbor-system NAME DATA AGE harbor-chartmuseum 22 27m harbor-core 35 27m harbor-jobservice 1 27m harbor-jobservice-env 8 27m harbor-portal 1 27m harbor-registry 2 27m harbor-registryctl 0 27m kube-root-ca.crt 1 28m $ kubectl get secret -n harbor-system NAME TYPE DATA AGE default-token-f8gzd kubernetes.io/service-account-token 3 28m harbor-chartmuseum Opaque 1 28m harbor-core Opaque 8 28m harbor-database Opaque 1 28m harbor-ingress kubernetes.io/tls 3 28m harbor-jobservice Opaque 2 28m harbor-notary-server Opaque 5 28m harbor-registry Opaque 2 28m harbor-registry-htpasswd Opaque 1 28m harbor-registryctl Opaque 0 28m harbor-trivy Opaque 2 28m sh.helm.release.v1.harbor.v1 helm.sh/release.v1 1 28m sh.helm.release.v1.harbor.v2 helm.sh/release.v1 1 7m31s ===== Senha do usuáriodo admin ===== $ kubectl -n harbor-system get secrets harbor-core -o jsonpath="{.data.HARBOR_ADMIN_PASSWORD}" | base64 --decode ===== Sertificado ===== $ kubectl -n harbor-system get secrets harbor-ingress -o jsonpath="{.data['ca\.crt']}" | base64 --decode > harbor-ca.crt ===== Usando o hub no docker ===== $ docker login hub.juntotelecom.com.br Username: admin Password: Error response from daemon: Get "https://hub.juntotelecom.com.br/v2/": x509: certificate signed by unknown authority ==== Copiando o certificado para a vm do docker ==== $ cp harbor-ca.crt /tmp/ $ scp [2804:694:4c00:4007::98]:/tmp/harbor-ca.crt . suporte@2804:694:4c00:4007::98's password: harbor-ca.crt $ sudo mkdir -p /etc/docker/certs.d/hub.juntotelecom.com.br $ sudo cp harbor-ca.crt /etc/docker/certs.d/hub.juntotelecom.com.br/ $ sudo systemctl restart docker $ docker login hub.juntotelecom.com.br Username: admin Password: WARNING! Your password will be stored unencrypted in /home/suporte/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/engine/reference/commandline/login/#credentials-store Login Succeeded $ docker pull busybox Using default tag: latest latest: Pulling from library/busybox 50e8d59317eb: Pull complete Digest: sha256:d2b53584f580310186df7a2055ce3ff83cc0df6caacf1e3489bff8cf5d0af5d8 Status: Downloaded newer image for busybox:latest docker.io/library/busybox:latest $ docker tag busybox hub.juntotelecom.com.br/library/busybox:latest $ docker push hub.juntotelecom.com.br/library/busybox:latest The push refers to repository [hub.juntotelecom.com.br/library/busybox] eb6b01329ebe: Pushed latest: digest: sha256:52f431d980baa76878329b68ddb69cb124c25efa6e206d8b0bd797a828f0528e size: 527 {{ :captura_hub.png |}} ===== Usando o Kubernetes ===== $ kubectl create secret docker-registry harbor \ --docker-server=hub.juntotelecom.com.br \ --docker-email=noc@juntotelecom.com.br \ --docker-username=admin \ --docker-password='SGFyYm9yMTIzNDU=' **Inserir em ambos os nodes** $ sudo vim /etc/containers/registries.conf [...] # # An array of host[:port] registries to try when pulling an unqualified image, in order. unqualified-search-registries = ["docker.io", "quay.io"] # [[registry]] location = "hub.juntotelecom.com.br" insecure = true [...] $ sudo systemctl restart crio kubelet ==== Teste de deploy ==== $ docker tag nginx:latest hub.juntotelecom.com.br/library/nginx:latest $ docker push hub.juntotelecom.com.br/library/nginx:latest The push refers to repository [hub.juntotelecom.com.br/library/nginx] a059c9abe376: Pushed 09be960dcde4: Pushed 18be1897f940: Pushed dfe7577521f0: Pushed d253f69cb991: Pushed fd95118eade9: Pushed latest: digest: sha256:b495f952df67472c3598b260f4b2e2ba9b5a8b0af837575cf4369c95c8d8a215 size: 1570 $ kubectl create deploy nginx --image=hub.juntotelecom.com.br/library/nginx deployment.apps/nginx created $ kubectl get deploy NAME READY UP-TO-DATE AVAILABLE AGE nginx 1/1 1 1 37m $ kubectl describe deploy nginx Name: nginx Namespace: default CreationTimestamp: Thu, 19 May 2022 15:37:50 -0300 Labels: app=nginx Annotations: deployment.kubernetes.io/revision: 1 Selector: app=nginx Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable StrategyType: RollingUpdate MinReadySeconds: 0 RollingUpdateStrategy: 25% max unavailable, 25% max surge Pod Template: Labels: app=nginx Containers: nginx: Image: hub.juntotelecom.com.br/library/nginx Port: Host Port: Environment: Mounts: Volumes: Conditions: Type Status Reason ---- ------ ------ Available True MinimumReplicasAvailable Progressing True NewReplicaSetAvailable OldReplicaSets: NewReplicaSet: nginx-574c97cb8 (1/1 replicas created) Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal ScalingReplicaSet 37m deployment-controller Scaled up replica set nginx-574c97cb8 to 1 $ kubectl get deploy nginx -o yaml apiVersion: apps/v1 kind: Deployment metadata: annotations: deployment.kubernetes.io/revision: "1" creationTimestamp: "2022-05-19T18:37:50Z" generation: 1 labels: app: nginx name: nginx namespace: default resourceVersion: "430148" uid: d9190efc-48f3-4b2f-bf7b-9b0ca41c4672 spec: progressDeadlineSeconds: 600 replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: app: nginx strategy: rollingUpdate: maxSurge: 25% maxUnavailable: 25% type: RollingUpdate template: metadata: creationTimestamp: null labels: app: nginx spec: containers: - image: hub.juntotelecom.com.br/library/nginx imagePullPolicy: Always name: nginx resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler securityContext: {} terminationGracePeriodSeconds: 30 status: availableReplicas: 1 conditions: - lastTransitionTime: "2022-05-19T18:38:01Z" lastUpdateTime: "2022-05-19T18:38:01Z" message: Deployment has minimum availability. reason: MinimumReplicasAvailable status: "True" type: Available - lastTransitionTime: "2022-05-19T18:37:50Z" lastUpdateTime: "2022-05-19T18:38:01Z" message: ReplicaSet "nginx-574c97cb8" has successfully progressed. reason: NewReplicaSetAvailable status: "True" type: Progressing observedGeneration: 1 readyReplicas: 1 replicas: 1 updatedReplicas: 1 ===== Referências ===== - [[https://artifacthub.io/packages/helm/harbor/harbor|Harbor]] - [[https://loft.sh/blog/harbor-kubernetes-self-hosted-container-registry/|Harbor + Kubernetes = Self-Hosted Container Registry]] - [[https://tanzu.vmware.com/developer/guides/harbor-gs/|Installing Harbor on Kubernetes with Project Contour, Cert Manager, and Let’s Encrypt]] - [[https://geko.cloud/en/harbor-private-registry-kubernetes/|Harbor: Private docker registry in Kubernetes]] - [[https://dewble.com/kubernetes/kubernetes-registry-install-harbor/|[Kubernetes](Registry)Install harbor with helm chart in Kubernetes]] - [[https://computingforgeeks.com/install-harbor-image-registry-on-kubernetes-openshift-with-helm-chart/|Install Harbor Image Registry on Kubernetes/OpenShift with Helm Chart]] - [[https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/|Pull an Image from a Private Registry]] - [[https://www.redhat.com/sysadmin/manage-container-registries|How to manage Linux container registries]]