I have been trying to get Gitlab 16.4 to work with external nginx (all on docker swarm). I am stuck in trying to get the container registry to work.
Configuration:
Docker swarm on ubuntu 22.
Stack 1 - Lab - This is gitlab and GitLab runner
Stack 2 - Nginx - Central nginx
Network - overlay network shared between the 2
Certificates - wild card certificates installed only on nginx. No certificate is moved to gitlab instance
Gitlab Config:
external_url 'https://lab.domain.com'
registry_external_url 'https://registry.domain.com'
web_server['external_users'] = ['nginx', 'gitlab-www']
gitlab_workhorse['listen_network'] = "tcp"
gitlab_workhorse['listen_addr'] = "0.0.0.0:8181"
gitlab_rails['trusted_proxies'] = [ '10.X.Y.Z', '10.A.V.C' ]
registry['enable'] = true
registry['log_level'] = "debug"
registry['registry_http_addr'] = "0.0.0.0:5000"
# Configure gitlab
gitlab_rails['registry_enabled'] = true
gitlab_rails['registry_api_url'] = "http://0.0.0.0:5000"
gitlab_rails['registry_host'] = "registry.domain.com"
nginx['enable'] = false
registry_nginx['enable'] = false
Nginx Config:
upstream registry_upstream {
server lab_gitlab:5000 fail_timeout=0;
}
server {
server_name registry.domain.com;
listen 80;
listen [::]:80;
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /etc/nginx/certs/domain.com.crt;
ssl_certificate_key /etc/nginx/certs/domain.com.key;
ssl_stapling on;
ssl_stapling_verify on;
ssl_trusted_certificate /etc/nginx/certs/domain.com_ca.crt;
add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload" always;
# HTTP -> HTTPS redirect
include /etc/nginx/conf.d/https-redirect.conf;
access_log /var/log/nginx/$server_name/access-$log_date.log main;
error_log /var/log/nginx/registry.domain.com/error.log;
root /webroot/$server_name;
location /v2/ {
add_header 'Docker-Distribution-Api-Version' 'registry/2.0' always;
proxy_set_header Host $http_host; # required for docker client's sake
proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 900;
proxy_pass http://$upstream_name;
}
}
Issue: When I try to login to registry using
docker login registry.domain.com -u user -p pass
I get error response from daemon:
login attempt to https://registry.domain.com/v2/ failed with status: 400 Bad Request
I have tried with both password and PAT with no difference.
I would like to avoid using bundled nginx.
NOTE - If I try logging in with incorrect credentials, it throws error saying access denied, provided password is incorrect etc etc. This tells me the system is working otherwise.
EDIT - Some logs I could pull out:
production_json.log
{
"method": "GET",
"path": "/jwt/auth",
"format": "html",
"controller": "JwtController",
"action": "auth",
"status": 200,
"time": "2023-09-30T10:15:22.537Z",
"params": [
{
"key": "account",
"value": "root"
},
{
"key": "client_id",
"value": "docker"
},
{
"key": "offline_token",
"value": "[FILTERED]"
},
{
"key": "service",
"value": "container_registry"
}
],
"correlation_id": "01HBJSBV2B6FP6EYMZT98CHFVW",
"meta.caller_id": "JwtController#auth",
"meta.remote_ip": "SERVER_IP",
"meta.feature_category": "system_access",
"meta.user": "root",
"meta.user_id": 1,
"meta.client_id": "user/1",
"remote_ip": "SERVER_IP",
"user_id": 1,
"username": "root",
"ua": "docker/24.0.6 go/go1.20.7 git-commit/1a79695 kernel/6.2.0-1012-azure os/linux arch/amd64 UpstreamClient(Docker-Client/24.0.6 \\(linux\\))",
"queue_duration_s": 0.006021,
"request_urgency": "low",
"target_duration_s": 5,
"db_count": 3,
"db_write_count": 0,
"db_cached_count": 0,
"db_replica_count": 0,
"db_primary_count": 3,
"db_main_count": 3,
"db_ci_count": 0,
"db_main_replica_count": 0,
"db_ci_replica_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 0,
"db_main_cached_count": 0,
"db_ci_cached_count": 0,
"db_main_replica_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_main_wal_count": 0,
"db_ci_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_ci_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_ci_replica_wal_cached_count": 0,
"db_replica_duration_s": 0,
"db_primary_duration_s": 0.005,
"db_main_duration_s": 0.005,
"db_ci_duration_s": 0,
"db_main_replica_duration_s": 0,
"db_ci_replica_duration_s": 0,
"cpu_s": 0.028255,
"mem_objects": 10716,
"mem_bytes": 1737696,
"mem_mallocs": 4089,
"mem_total_bytes": 2166336,
"pid": 650,
"worker_id": "puma_2",
"rate_limiting_gates": [],
"db_duration_s": 0.00518,
"view_duration_s": 0.00015,
"duration_s": 0.08678
}
gitlab-workhorse
{
"content_type": "application/json; charset=utf-8",
"correlation_id": "01HBJSBV2B6FP6EYMZT98CHFVW",
"duration_ms": 95,
"host": "lab.domain.com",
"level": "info",
"method": "GET",
"msg": "access",
"proto": "HTTP/1.1",
"referrer": "",
"remote_addr": "SERVER_IP:38228",
"remote_ip": "SERVER_IP",
"route": "",
"status": 200,
"system": "http",
"time": "2023-09-30T10:15:22Z",
"ttfb_ms": 95,
"uri": "/jwt/auth?account=root&client_id=docker&offline_token=[FILTERED]&service=container_registry",
"user_agent": "docker/24.0.6 go/go1.20.7 git-commit/1a79695 kernel/6.2.0-1012-azure os/linux arch/amd64 UpstreamClient(Docker-Client/24.0.6 \\(linux\\))",
"written_bytes": 2524
}