高可用harbor集群搭建

一、安装部署

1.节点角色

角色	数量	名称	备注
harbor主节点	2	harbor-1 harbor-2	双主模式
haproxy	2	HA-1 HA-2	需要通过keepalived绑定vip
Redis	高可用集群	redis	harbor组件，本地启动或远程配置
postgresql	高可用集群	postgres	harbor组件，本地启动或远程配置
ceph	高可用集群	ceph	harbor组件

2.离线安装

1、下载离线源码包解压至harbor-1 harbor-2 节点 /data/harbor目录下，修改启动脚本harbor.cfg各项配置参数如下



## Configuration file of Harbor

#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!

_version = 1.7.0

#The IP address or hostname to access admin UI and registry service.

#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.

#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail.

hostname = harbor.xxx.xxx.com //配置域名信息

#The protocol for accessing the UI and token/notification service, by default it is http.

#It can be set to https if ssl is enabled on nginx.

ui_url_protocol = https //指定为https协议

#Maximum number of job workers in job service

max_job_workers = 10

#Determine whether or not to generate certificate for the registry's token.

#If the value is on, the prepare script creates new root cert and private key

#for generating token to access the registry. If the value is off the default key/cert will be used.

#This flag also controls the creation of the notary signer's cert.

customize_crt = on

#The path of cert and key files for nginx, they are applied only the protocol is set to https

ssl_cert = /data/harbor/cert/uat.cmft.com.pem

ssl_cert_key = /data/harbor/cert/uat.cmft.com.pem

#The path of secretkey storage

secretkey_path = /data

#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone

admiral_url = NA

#Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.

log_rotate_count = 50

#Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.

#If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G

#are all valid.

log_rotate_size = 200M

#Config http proxy for Clair, e.g. http://my.proxy.com:3128

#Clair doesn't need to connect to harbor internal components via http proxy.

http_proxy =

https_proxy =

no_proxy = 127.0.0.1,localhost,core,registry

#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES

#only take effect in the first boot, the subsequent changes of these properties

#should be performed on web ui

#************************BEGIN INITIAL PROPERTIES************************

#Email account settings for sending out password resetting emails.

#Email server uses the given username and password to authenticate on TLS connections to host and act as identity.

#Identity left blank to act as username.

email_identity =

email_server = smtp.mydomain.com

email_server_port = 25

email_username = sample_admin@mydomain.com

email_password = abc

email_from = admin <sample_admin@mydomain.com>

email_ssl = false

email_insecure = false

##The initial password of Harbor admin, only works for the first time when Harbor starts.

#It has no effect after the first launch of Harbor.

#Change the admin password from UI after launching Harbor.

harbor_admin_password = yourpassword //配置harboradmin账户密码，建议修改

##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.

#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.

auth_mode = db_auth

#The url for an ldap endpoint.

ldap_url = ldaps://ldap.mydomain.com

#A user's DN who has the permission to search the LDAP/AD server.

#If your LDAP/AD server does not support anonymous search, you should configure this DN and ldap_search_pwd.

#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com

#the password of the ldap_searchdn

#ldap_search_pwd = password

#The base DN from which to look up a user in LDAP/AD

ldap_basedn = ou=people,dc=mydomain,dc=com

#Search filter for LDAP/AD, make sure the syntax of the filter is correct.

#ldap_filter = (objectClass=person)

# The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD

ldap_uid = uid

#the scope to search for users, 0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE

ldap_scope = 2

#Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds.

ldap_timeout = 5

#Verify certificate from LDAP server

ldap_verify_cert = true

#The base dn from which to lookup a group in LDAP/AD

ldap_group_basedn = ou=group,dc=mydomain,dc=com

#filter to search LDAP/AD group

ldap_group_filter = objectclass=group

#The attribute used to name a LDAP/AD group, it could be cn, name

ldap_group_gid = cn

#The scope to search for ldap groups. 0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE

ldap_group_scope = 2

#Turn on or off the self-registration feature

self_registration = on

#The expiration time (in minute) of token created by token service, default is 30 minutes

token_expiration = 30

#The flag to control what users have permission to create projects

#The default value "everyone" allows everyone to creates a project.

#Set to "adminonly" so that only admin user can create project.

project_creation_restriction = everyone

#************************END INITIAL PROPERTIES************************

#######Harbor DB configuration section#######

#The address of the Harbor database. Only need to change when using external db.

db_host = 100.12.13.10

#The password for the root user of Harbor DB. Change this before any production use.

db_password = yourdbpassword

#The port of Harbor database host

db_port = 7810

#The user name of Harbor database

db_user = harbordata

#若采用本地配置，则采用如下配置形式，指定db为本地启动。

#db_host = postgresql

#db_password = yourpassword

#db_port = 5432

#db_user = postgres

##### End of Harbor DB configuration#######

##########Redis server configuration.############

#Redis connection address

redis_host = 100.101.21.3

#Redis connection port

redis_port = 4410

#Redis connection password

redis_password = 123456

#Redis connection db index

#db_index 1,2,3 is for registry, jobservice and chartmuseum.

#db_index 0 is for UI, it's unchangeable

redis_db_index = 1,2,3

########## End of Redis server configuration ############

##########Clair DB configuration############

#Clair DB host address. Only change it when using an exteral DB.

clair_db_host = 100.101.12.3 //clair 使用与db相同的数据库，注意使用新的database，防止数据冲突。

#The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair.

#Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database.

clair_db_password = 123456

#Clair DB connect port

clair_db_port = 7810

#Clair DB username

clair_db_username = harbordata

#Clair default database

clair_db = template1 //由于与主数据库公用 不可与db指定相同的database

#clair_db_host = postgresql

#clair_db_password = root123

#clair_db_port = 5432

#clair_db_username = postgres

#clair_db = postgres

#The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.

clair_updaters_interval = 12

##########End of Clair DB configuration############

#The following attributes only need to be set when auth mode is uaa_auth

uaa_endpoint = uaa.mydomain.org

uaa_clientid = id

uaa_clientsecret = secret

uaa_verify_cert = true

uaa_ca_cert = /path/to/ca.pem

### Harbor Storage settings ###

#若采用本地ceph 指定 registry_storage_provider_name = filesystem 且忽略registry_storage_provider_config

#默认申请ceph具有如下形式

#bucket region accesskey secretkey regionendpoint参数

#Please be aware that the following storage settings will be applied to both docker registry and helm chart repository.

#registry_storage_provider can be: filesystem, s3, gcs, azure, etc.

registry_storage_provider_name = s3//使用s3协议连接ceph

#registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2".

#To avoid duplicated configurations, both docker registry and chart repository follow the same storage configuration specifications of docker registry.

#Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration.

registry_storage_provider_config = bucket: registry, region: default, accesskey: 123456, secretkey: 123456, regionendpoint: http://100.xx.xxx.xxx:7480, rootdirectory: /harbor-registry/st

#ceph建议使用域名配置 若使用ip有可能出现 401 signatureMisMatch的错误。

#registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore

#of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signeed certificate.

registry_custom_ca_bundle =

#If reload_config=true, all settings which present in harbor.cfg take effect after prepare and restart harbor, it overwrites exsiting settings.

#reload_config=false

#Regular expression to match skipped environment variables

#skip_reload_env_pattern=(^EMAIL.*)|(^LDAP.*)

3.初始化与证书生成

1.harbor-1节点/data/harbor目录下执行命令: ./install.sh --with-notary --with-clair --with-chartmuseum #初始化运行harbor watch 'docker ps -a' #监测各组件运行情况 2.拷贝证书： /data/harbor/common/config/core/private_key.pem /data/harbor/common/config/registry/root.crt 到/tmp/certfile目录下同时拷贝到harbor-2节点的相同目录下

4、指定生成证书



#关闭harbor-1节点与相关组件：

docker-compose down -v --remove-orphans

1.修改启动脚本install.sh

 if [ -f harbor*.tar.gz ]

 then

 h2 "[Step $item]: loading Harbor images ..."; let item+=1

 #docker load -i ./harbor*.tar.gz

 fi

 echo "" #关闭重复加载image

2.（下列修改操作同时针对 harbor-1 与 harbor-2）

if customize_crt == 'on' and openssl_installed():

print("creating cert-----------------------------")

shell_stat = subprocess.check_call(["which", "openssl"], stdout=FNULL, stderr=subprocess.STDOUT)

empty_subj = "/"

private_key_pem = os.path.join(config_dir, "core", "private_key.pem")

root_crt = os.path.join(config_dir, "registry", "root.crt")

create_root_cert(empty_subj, key_path=private_key_pem, cert_path=root_crt)

mark_file(private_key_pem)

mark_file(root_crt)

else:

print("Copied configuration file: %s" % os.path.join("/tmp", "certfile", "private_key.pem"))

shutil.copyfile(os.path.join("/tmp", "certfile", "private_key.pem"), os.path.join(core_config_dir, "private_key.pem"))

print("Copied configuration file: %s" % os.path.join("/tmp", "certfile", "root.crt"))

shutil.copyfile(os.path.join("/tmp", "certfile", "root.crt"), os.path.join(registry_config_dir, "root.crt"))

#此处将harbor自动生成证书关闭，将证书拷贝到外部路径并直接指定，否则将导致出现两个节点鉴权失败的问题。

修改install.sh 添加参数 customize_crt == off  #关闭自生成证书

5、启动harbor

分别进入harbor-1 harbor-2 执行启动命令

 ./install.sh --with-notary --with-clair --with-chartmuseum

 watch 'docker ps -a' #监测各组件运行情况

6、配置haproxy/keep-alived

在HA-1 HA-2节点安装ha/keepalived 修改配置文件如下

harproxy(haproxy.cfg)：

#---------------------------------------------------------------------

# Global settings

#---------------------------------------------------------------------

global

# to have these messages end up in /var/log/haproxy.log you will

# need to:

#

# 1) configure syslog to accept network log events. This is done

# by adding the '-r' option to the SYSLOGD_OPTIONS in

# /etc/sysconfig/syslog

#

# 2) configure local2 events to go to the /var/log/haproxy.log

# file. A line like the following can be added to

# /etc/sysconfig/syslog

#

# local2.* /var/log/haproxy.log

#

log 127.0.0.1 local2

chroot /var/lib/haproxy

pidfile /var/run/haproxy.pid

maxconn 20000

user haproxy

group haproxy

daemon

tune.bufsize 131072

tune.ssl.default-dh-param 2048

# turn on stats unix socket

stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------

# common defaults that all the 'listen' and 'backend' sections will

# use if not designated in their block

#---------------------------------------------------------------------

defaults

mode http

log global

option httplog

option dontlog-normal

option http-server-close

option forwardfor except 127.0.0.0/8

option redispatch

retries 3

timeout http-request 10s

timeout queue 1m

timeout connect 10s

timeout client 1m

timeout server 30m

timeout http-keep-alive 10s

timeout check 10s

maxconn 10000

#---------------------------------------------------------------------

# static backend for serving up images, stylesheets and such

#---------------------------------------------------------------------

#---------------------------------------------------------------------

# round robin balancing between the various backends

#---------------------------------------------------------------------

frontend app

bind *:443

balance roundrobin

mode tcp

server s1 100.65.22.2:443 weight 1  #绑定harbor-1 harbor-2 的443端口

#server s2 100.65.22.3:443 weight 1

#---------------------------------------------------------------------

# round robin balancing between the various backends

#---------------------------------------------------------------------

frontend ceph

bind *:80

balance roundrobin

server app1 100.xx.xx.xx:7480 check  #ceph 可选配置

keepalived (keepalived.conf):



vrrp_script chk_http_haproxy {

script "/app/haproxy/bin/check_haproxy.sh"

interval 10

weight 2

}

global_defs {

notification_email {

admin@admin.com

}

notification_email_from admin@admin.com

smtp_server mail.admin.com

smtp_connect_timeout 30

router_id CMRH_WEB

}

vrrp_instance VI_1 {

state MASTER

interface eth0

virtual_router_id 6

priority 101

advert_int 1

track_script {

#run the script

chk_http_haproxy

}

virtual_ipaddress {

100.70.42.6

}

}

    /app/haproxy/bin/check_haproxy.sh

#! /bin/bash

num=`ps -C haproxy --no-header |wc -l`

if [ $num -eq 0 ]; then

/app/haproxy/sbin/haproxy -f /app/haproxy/conf/haproxy.cfg

sleep 3

if [ `ps -C haproxy --no-header |wc -l` -eq 0 ]; then

killall keepalived

fi

fi

#! /bin/bash

num=`ps -C haproxy --no-header |wc -l`

if [ $num -eq 0 ]; then

/app/haproxy/sbin/haproxy -f /app/haproxy/conf/haproxy.cfg

sleep 3

if [ `ps -C haproxy --no-header |wc -l` -eq 0 ]; then

killall keepalived

fi

fi

配置完成后启动在HA-1 HA-2 上启动haproxy

systemcl start keepalived

systemctl start haproxy

7、访问测试

1、访问ha-vip页面 443端口测试前端访问

2、pull/push测试

3、关闭harbor-1 harbor-2节点：高可用测试

8、常见问题：

1、页面可正常访问，但经常性显示错误，账号登出

haproxy/keepalive配置错误，确认配置

2、harbor启动日志查看/问题排查：/var/log/harbor/下存在的log文件

3、401 SignatureMisMatch ：

ceph地址配置错误，建议使用域名进行配置

4、harbor仓库内部数据访问异常：

此处可能是harbor迁移时，使用的新数据库内部有脏数据导致，可清除template1数据库内所有表数据重置。

5、notary数据库为本地：待解决

6、日志刷新时有概率出现无数据问题：交替访问harbor节点导致，待解决

7、unkown blob问题

When setting up Harbor behind an nginx proxy or elastic load balancing, look for the line below, in common/templates/nginx/nginx.http.conf and remove it from the sections if the proxy already has similar settings: location /, location /v2/ and location /service/.

proxy_set_header X-Forwarded-Proto $scheme;

巴特西

harbor高可用集群搭建