部分网段访问不了harbor问题排查

故障说明

habror地址：10.0.2.3

• 192.168.10无法访问10.0.3.4

排查

• 查看ip地址

# ifconfig | grep 192
        inet 192.168.0.1  netmask 255.255.240.0  broadcast 192.168.15.255

排查发现harbor多了一个192.168.0的网段的接口

• 查看路由

[root@harbor ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.0.2.1        0.0.0.0         UG    0      0        0 eth0
10.0.2.0        0.0.0.0         255.255.255.0   U     0      0        0 eth0
169.254.169.254 10.0.2.2        255.255.255.255 UGH   0      0        0 eth0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
172.20.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-f93ff97579e2
172.21.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-be7b013827bd
172.23.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-331bc475d728
172.30.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-b5e8013343a6
172.31.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-9ba1fff0d823
192.168.0.0     0.0.0.0         255.255.240.0   U     0      0        0 br-43310448c144

发现192.168.0为一个harbor的桥接接口

• 查看docker的网络设置

[root@harbor ~]# docker network ls
NETWORK ID          NAME                        DRIVER              SCOPE
9b60e3c7937a        bridge                      bridge              local
f93ff97579e2        gitlab-runner_default       bridge              local
b5e8013343a6        harbor_harbor               bridge              local
43310448c144        harbor_harbor-chartmuseum   bridge              local
9ba1fff0d823        harbor_harbor-clair         bridge              local
be7b013827bd        harbor_harbor-notary        bridge              local
331bc475d728        harbor_notary-sig           bridge              local
a838e680d2be        host                        host                local
0eaf2acc3d37        none                        null                local
[root@harbor ~]# docker network inspect 43310448c144
[
    {
        "Name": "harbor_harbor-chartmuseum",
        "Id": "43310448c144c4a929d3846ab1282aeab1ac4a467a4cc8fbff1d080ff07f83ac",
        "Created": "2019-12-09T01:17:11.039036489Z",
        "Scope": "local",
        "Driver": "bridge",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": null,
            "Config": [
                {
                    "Subnet": "192.168.0.0/20",
                    "Gateway": "192.168.0.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": true,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": {
            "5237caf42d22efaa4430f6fc6cd56ed068482db557bfd60ce5be7f35b46f750c": {
                "Name": "redis",
                "EndpointID": "d437d56ffe9c8525607dfad54ac169570f3263e0fdaba2de729416589a01c29d",
                "MacAddress": "02:42:c0:a8:00:02",
                "IPv4Address": "192.168.0.2/20",
                "IPv6Address": ""
            },
            "d0a15ee865bd53c1a7b988f05e2b283931ddbbcafc238abdfdf6864b960b9d75": {
                "Name": "harbor-core",
                "EndpointID": "d01bf44bc7bb44dc48f75a0478bdfa5d5e69181a8792f5bfab2b6d7b5a0dc22e",
                "MacAddress": "02:42:c0:a8:00:03",
                "IPv4Address": "192.168.0.3/20",
                "IPv6Address": ""
            },
            "f8f8955378e974017d17cfaf885d62c2a9ddadcf5a8032e6631e5f19251d625a": {
                "Name": "chartmuseum",
                "EndpointID": "a5e28706db88ede8ed618e25d67a57f30711916da4ff6cd16ec233b4fe8f29e7",
                "MacAddress": "02:42:c0:a8:00:04",
                "IPv4Address": "192.168.0.4/20",
                "IPv6Address": ""
            }
        },
        "Options": {},
        "Labels": {
            "com.docker.compose.network": "harbor-chartmuseum",
            "com.docker.compose.project": "harbor",
            "com.docker.compose.version": "1.23.2"
        }
    }
]

发现改网络为harbor的网络。这里和物理网络冲突了。

解决

永久方案

修改harbor的网络配置

停止harbor
#docker-compose -f docker-compose down
备份harbor
#cp -rf harbor harbor.bak
#cp -rf /data/database ./harbor.bak/
修改网络地址，修改docker-compose文件
networks:
  harbor:
    driver: bridge
    ipam:
      config:
      - subnet: 172.22.0.0/16
        gateway: 172.22.0.1
  harbor-notary:
    driver: bridge
    ipam:
      config:
      - subnet: 172.23.0.0/16
        gateway: 172.23.0.1
  notary-sig:
    driver: bridge
    ipam:
      config:
      - subnet: 172.24.0.0/16
        gateway: 172.24.0.1
  harbor-chartmuseum:
    driver: bridge
    ipam:
      config:
      - subnet: 172.25.0.0/16
        gateway: 172.25.0.1
重启harbor
docker-compose -f docker-compose up -d
查看网络是否符合预期
# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.0.7.1        0.0.0.0         UG    0      0        0 eth0
10.0.7.0        0.0.0.0         255.255.255.0   U     0      0        0 eth0
169.254.0.0     0.0.0.0         255.255.0.0     U     1002   0        0 eth0
169.254.169.254 10.0.7.2        255.255.255.255 UGH   0      0        0 eth0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
172.20.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-a2ddedff457b
172.21.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-7b0af7ada27d
172.22.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-070d36b5e586
172.23.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-5f4831e6f306
172.24.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-c45b8c41f4dc
172.25.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-77c5311cb2ee

临时方案

添加一条到目的地址的路由。如果目的地址和harbor内部地址重合，改方案无效

route add -net 192.168.10.0 netmask 255.255.254.0 gw 10.0.2.1