【DevOps】Harbor私有镜像仓库并利用cert-manager自签名证书开启HTTPS（helm方式安装）

前提条件

cert-manager
ingress-nginx
Helm 2.8.0+
Kubernetes cluster 1.10+

1 2	# 命名空间 kubectl create ns devops

自签名证书

apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: selfsigned
spec:
  selfSigned: {}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: devops-ca
  namespace: cert-manager
spec:
  isCA: true
  commonName: "*.basepoint.net"
  secretName: devops-selfsigned-secret
  duration: 876000h
  renewBefore: 8760h
  subject:
    countries:
    - China
    localities:
    - NanJing
    organizations:
    - basepoint
    organizationalUnits:
    - devops
  privateKey:
    algorithm: ECDSA
    size: 256
  issuerRef:
    name: selfsigned
    kind: ClusterIssuer
    group: cert-manager.io
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: devops-issuer
spec:
  ca:
    secretName: devops-selfsigned-secret
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: harbor-ca
  namespace: devops
  #namespace: cert-manager
spec:
  isCA: false
  usages:
    - server auth
    - client auth
  dnsNames:
  - basepoint.net
  - harbor.basepoint.net
  commonName: "harbor.basepoint.net"
  secretName: harbor-selfsigned-secret
  duration: 876000h
  renewBefore: 8760h
  subject:
    countries:
    - China
    localities:
    - NanJing
    organizations:
    - basepoint
    organizationalUnits:
    - harbor
  privateKey:
    algorithm: ECDSA
    size: 256
  issuerRef:
    name: devops-issuer
    kind: ClusterIssuer
    group: cert-manager.io

详解，原文链接：使用cert-manager 生成自签名证书 - Elijah Blog (sreok.cn )

查看tls文件

1
2
3

[root@k8s-master01 cert-manager]# kubectl get secrets -n devops harbor-selfsigned-secret
NAME                       TYPE                DATA   AGE
harbor-selfsigned-secret   kubernetes.io/tls   3      18s

安装Harbor

1 2	helm repo add harbor https://helm.goharbor.io helm pull harbor/harbor --untar

修改values.yaml

1 2	cd ./harbor vim values.yaml

expose:
  type: ingress
  tls:
    enabled: true
    # 改成secret，使用刚才生成的自签名证书
    certSource: secret
    auto:
      commonName: ""
    secret:
      # 改为自签名证书secret名
      secretName: "harbor-selfsigned-secret"
  ingress:
    hosts:
      # harbor域名
      core: harbor.basepoint.net
    controller: default
    kubeVersionOverride: ""
    className: "nginx"
    annotations:nginx lines below
      ingress.kubernetes.io/ssl-redirect: "true"
      ingress.kubernetes.io/proxy-body-size: "0"
      nginx.ingress.kubernetes.io/ssl-redirect: "true"
      nginx.ingress.kubernetes.io/proxy-body-size: "0"
    labels: {}
  clusterIP:
    name: harbor
    staticClusterIP: ""
    ports:
      # The service port Harbor listens on when serving HTTP
      httpPort: 80
      # The service port Harbor listens on when serving HTTPS
      httpsPort: 443
    # Annotations on the ClusterIP service
    annotations: {}
    # ClusterIP-specific labels
    labels: {}
  nodePort:
    # The name of NodePort service
    name: harbor
    ports:
      http:
        # The service port Harbor listens on when serving HTTP
        port: 80
        # The node port Harbor listens on when serving HTTP
        nodePort: 30002
      https:
        # The service port Harbor listens on when serving HTTPS
        port: 443
        # The node port Harbor listens on when serving HTTPS
        nodePort: 30003
    # Annotations on the nodePort service
    annotations: {}
    # nodePort-specific labels
    labels: {}
  loadBalancer:
    # The name of LoadBalancer service
    name: harbor
    # Set the IP if the LoadBalancer supports assigning IP
    IP: ""
    ports:
      # The service port Harbor listens on when serving HTTP
      httpPort: 80
      # The service port Harbor listens on when serving HTTPS
      httpsPort: 443
    # Annotations on the loadBalancer service
    annotations: {}
    # loadBalancer-specific labels
    labels: {}
    sourceRanges: []

# The external URL for Harbor core service. It is used to
# 1) populate the docker/helm commands showed on portal
# 2) populate the token service URL returned to docker client
#
# Format: protocol://domain[:port]. Usually:
# 1) if "expose.type" is "ingress", the "domain" should be
# the value of "expose.ingress.hosts.core"
# 2) if "expose.type" is "clusterIP", the "domain" should be
# the value of "expose.clusterIP.name"
# 3) if "expose.type" is "nodePort", the "domain" should be
# the IP address of k8s node
#
# If Harbor is deployed behind the proxy, set it as the URL of proxy
# 改为
externalURL: https://harbor.basepoint.net

# The persistence is enabled by default and a default StorageClass
# is needed in the k8s cluster to provision volumes dynamically.
# Specify another StorageClass in the "storageClass" or set "existingClaim"
# if you already have existing persistent volumes to use
#
# For storing images and charts, you can also use "azure", "gcs", "s3",
# "swift" or "oss". Set it in the "imageChartStorage" section
persistence:
  enabled: true
  # Setting it to "keep" to avoid removing PVCs during a helm delete
  # operation. Leaving it empty will delete PVCs after the chart deleted
  # (this does not apply for PVCs that are created for internal database
  # and redis components, i.e. they are never deleted automatically)
  resourcePolicy: "keep"
  persistentVolumeClaim:
    registry:
      existingClaim: ""
      # storageClass根据实际更改，不设置代表使用默认的storageClass
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      # 大小根据需求更改
      size: 5Gi
      annotations: {}
    jobservice:
      jobLog:
        existingClaim: ""
        storageClass: ""
        subPath: ""
        accessMode: ReadWriteOnce
        size: 1Gi
        annotations: {}
    # If external database is used, the following settings for database will
    # be ignored
    database:
      existingClaim: ""
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      size: 1Gi
      annotations: {}
    # If external Redis is used, the following settings for Redis will
    # be ignored
    redis:
      existingClaim: ""
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      size: 1Gi
      annotations: {}
    trivy:
      existingClaim: ""
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      size: 5Gi
      annotations: {}
  imageChartStorage:
    type: filesystem
    filesystem:
      rootdirectory: /storage
      #maxthreads: 100
    azure:
      accountname: accountname
      accountkey: base64encodedaccountkey
      container: containername
      #realm: core.windows.net
      # To use existing secret, the key must be AZURE_STORAGE_ACCESS_KEY
      existingSecret: ""
    gcs:
      bucket: bucketname
      # The base64 encoded json file which contains the key
      encodedkey: base64-encoded-json-key-file
      #rootdirectory: /gcs/object/name/prefix
      #chunksize: "5242880"
      # To use existing secret, the key must be GCS_KEY_DATA
      existingSecret: ""
      useWorkloadIdentity: false
    s3:
      # Set an existing secret for S3 accesskey and secretkey
      # keys in the secret should be REGISTRY_STORAGE_S3_ACCESSKEY and REGISTRY_STORAGE_S3_SECRETKEY for registry
      #existingSecret: ""
      region: us-west-1
      bucket: bucketname
      #accesskey: awsaccesskey
      #secretkey: awssecretkey
      #regionendpoint: http://myobjects.local
      #encrypt: false
      #keyid: mykeyid
      #secure: true
      #skipverify: false
      #v4auth: true
      #chunksize: "5242880"
      #rootdirectory: /s3/object/name/prefix
      #storageclass: STANDARD
      #multipartcopychunksize: "33554432"
      #multipartcopymaxconcurrency: 100
      #multipartcopythresholdsize: "33554432"
    swift:
      authurl: https://storage.myprovider.com/v3/auth
      username: username
      password: password
      container: containername
      # keys in existing secret must be REGISTRY_STORAGE_SWIFT_PASSWORD, REGISTRY_STORAGE_SWIFT_SECRETKEY, REGISTRY_STORAGE_SWIFT_ACCESSKEY
      existingSecret: ""
      #region: fr
      #tenant: tenantname
      #tenantid: tenantid
      #domain: domainname
      #domainid: domainid
      #trustid: trustid
      #insecureskipverify: false
      #chunksize: 5M
      #prefix:
      #secretkey: secretkey
      #accesskey: accesskey
      #authversion: 3
      #endpointtype: public
      #tempurlcontainerkey: false
      #tempurlmethods:
    oss:
      accesskeyid: accesskeyid
      accesskeysecret: accesskeysecret
      region: regionname
      bucket: bucketname
      # key in existingSecret must be REGISTRY_STORAGE_OSS_ACCESSKEYSECRET
      existingSecret: ""
      #endpoint: endpoint
      #internal: false
      #encrypt: false
      #secure: true
      #chunksize: 10M
      #rootdirectory: rootdirectory


existingSecretAdminPasswordKey: HARBOR_ADMIN_PASSWORD
# 默认密码，按需修改
harborAdminPassword: "Harbor12345"

...
# 下面的配置按需求修改，我这里没改动就不贴了，太长了。

中文注释都是改动的位置

安装

1	helm upgrade --install harbor -n devops .

[root@k8s-h3c-master01 ~]# kubectl get pod -n devops
NAME                                READY   STATUS    RESTARTS        AGE
harbor-core-659b9b95fc-7l69q        1/1     Running   0               21h
harbor-database-0                   1/1     Running   0               3d17h
harbor-jobservice-8b5f47bd5-lcdhw   1/1     Running   0               21h
harbor-portal-67749b88d8-tcjqd      1/1     Running   0               3d17h
harbor-redis-0                      1/1     Running   0               3d17h
harbor-registry-687cc8d459-5w9l8    2/2     Running   0               21h
harbor-trivy-0                      1/1     Running   0               3d17h

查看ingress

1
2
3

[root@k8s-h3c-master01 ~]# kubectl get ingress -n devops
NAME             CLASS   HOSTS                  ADDRESS        PORTS     AGE
harbor-ingress   nginx   harbor.basepoint.net   172.25.2.204   80, 443

添加hosts

系统添加，略过。

查看证书

系统颁发证书

1 2	# basse64解码并保存到电脑 echo "connect in ca.crt" \| base64 -d

MacOS 导入证书

Linux 导入证书（全部k8s节点操作）

安装ca-certificates，一般系统是自带的

1	yum install ca-certificates

将证书放置以下目录

1	/etc/pki/ca-trust/source/anchors/

执行信任此证书

1	update-ca-trust

此操作会更新以下文件，而不是只是tls-ca-bundle.pem
1
2
3
4
/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt
/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
/etc/pki/ca-trust/extracted/pem/email-ca-bundle.pem
/etc/pki/ca-trust/extracted/pem/objsign-ca-bundle.pem  
删除时在/etc/pki/ca-trust/source/anchors/文件中删掉对应的ca证书，重新执行update-ca-trust即可。

创建imagePullSecrets

方式一：在命令行上提供凭据来创建 Secret

官方文档：从私有仓库拉取镜像 | Kubernetes

kubectl create secret docker-registry regcred \
--docker-server=harbor.basepoint.net \
--docker-username=admin \
--docker-password=Harbor12345 \
[email protected]
# -n 命名空间

方式二：创建一个基于现有凭据的 Secret

官方文档：从私有仓库拉取镜像 | Kubernetes

选择其中一个节点安装docker

1	yum -y install docker-ce

登录docker

1	docker login harbor.basepoint.net -u admin -p Harbor12345

登录成功后会创建一个文件：~/.docker/config.json

kubectl create secret generic harbor-admin \
--from-file=.dockerconfigjson=~/.docker/config.json \
--type=kubernetes.io/dockerconfigjson
# -n 命名空间

测试拉取镜像

将library设置为私有仓库

拉取私有仓库镜像

cat > nginx.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: harbor.basepoint.net/library/nginx:1.25.0
        # 每次启动都拉取
        imagePullPolicy: Always
      # 注意命名空间，如果要安装在其他命名空间，就不存在这个seccret，则需要重新创建
      imagePullSecrets:
      - name: harbor-admin
EOF

1	kubectl apply -f nginx.yaml

成功即可

1
2
3

[root@k8s-h3c-master01 ~]# kubectl get pod
NAME                         READY   STATUS    RESTARTS   AGE
nginx-67d8d49fb6-4vwk9       1/1     Running   0          59m