From 61e44c63abf00d9c3d4029e27366a140fc17c07b Mon Sep 17 00:00:00 2001 From: "soroush.asadi" Date: Thu, 28 May 2026 14:35:55 +0330 Subject: [PATCH] refactor(mirror): replace 3 services with single Nexus Repository Manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidates BaGet + Verdaccio + registry:2 into one Sonatype Nexus OSS instance with a REST API provisioning script. docker-compose.mirror.yml: single nexus service, ports 8081 (UI/NuGet/npm) and 8083 (Docker Hub pull-through proxy) mirrors/nexus/provision.sh: idempotent setup — changes admin password, enables anonymous access, creates nuget-proxy / npm-proxy / docker-hub-proxy nuget.mirror.config: updated source URL to Nexus NuGet proxy endpoint ci-cd.yml: updated npm --registry to Nexus npm proxy endpoint Run once on server: docker compose -f docker-compose.mirror.yml up -d then: ./mirrors/nexus/provision.sh Co-Authored-By: Claude Sonnet 4.6 --- .gitea/workflows/ci-cd.yml | 16 ++-- docker-compose.mirror.yml | 103 ++++++---------------- mirrors/nexus/provision.sh | 170 +++++++++++++++++++++++++++++++++++++ nuget.mirror.config | 8 +- 4 files changed, 210 insertions(+), 87 deletions(-) create mode 100644 mirrors/nexus/provision.sh diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 5c67eae..6142641 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -24,11 +24,11 @@ concurrency: # node:20-alpine → no git → checkout fails # Fix: plain shell git clone via http.extraheader (token never in process list). # -# Local mirrors (started via docker-compose.mirror.yml): +# Local mirrors — Nexus Repository Manager (docker-compose.mirror.yml): # "mirror" hostname → host-gateway (docker bridge IP 172.17.0.1) -# NuGet → http://mirror:5101 (BaGet — nuget.mirror.config) -# npm → http://mirror:4873 (Verdaccio — --registry flag) -# Docker Hub → configured in /etc/docker/daemon.json on the host +# NuGet → http://mirror:8081/repository/nuget-proxy/ (nuget.mirror.config) +# npm → http://mirror:8081/repository/npm-proxy/ (--registry flag) +# Docker → http://mirror:8083 (daemon.json registry-mirrors) # ───────────────────────────────────────────────────────────────────────────── jobs: @@ -136,7 +136,7 @@ jobs: - name: Install dependencies working-directory: web/dashboard - run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:4873 + run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:8081/repository/npm-proxy/ - name: TypeScript check working-directory: web/dashboard @@ -168,7 +168,7 @@ jobs: - name: Install dependencies working-directory: web/admin - run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:4873 + run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:8081/repository/npm-proxy/ - name: TypeScript check working-directory: web/admin @@ -200,7 +200,7 @@ jobs: - name: Install dependencies working-directory: web/website - run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:4873 + run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:8081/repository/npm-proxy/ - name: TypeScript check working-directory: web/website @@ -232,7 +232,7 @@ jobs: - name: Install dependencies working-directory: web/finder - run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:4873 + run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:8081/repository/npm-proxy/ - name: TypeScript check working-directory: web/finder diff --git a/docker-compose.mirror.yml b/docker-compose.mirror.yml index 7068fbe..a24dc7d 100644 --- a/docker-compose.mirror.yml +++ b/docker-compose.mirror.yml @@ -1,90 +1,43 @@ # ───────────────────────────────────────────────────────────────────────────── -# Local pull-through mirrors +# Nexus Repository Manager OSS — single pull-through mirror for everything # ───────────────────────────────────────────────────────────────────────────── -# Start: docker compose -f docker-compose.mirror.yml up -d -# Stop: docker compose -f docker-compose.mirror.yml down +# FIRST-TIME SETUP (run once after starting): +# docker compose -f docker-compose.mirror.yml up -d +# ./mirrors/nexus/provision.sh # creates all proxy repos + enables anon access # -# Endpoints (reachable from CI containers via host-gateway as "mirror"): -# NuGet → http://SERVER_IP:5101/v3/index.json -# npm → http://SERVER_IP:4873 -# Docker → http://SERVER_IP:5100 (add to /etc/docker/daemon.json) +# Endpoints (after provisioning): +# UI → http://SERVER_IP:8081 (admin / see provision.sh output) +# NuGet → http://SERVER_IP:8081/repository/nuget-proxy/index.json +# npm → http://SERVER_IP:8081/repository/npm-proxy/ +# Docker → http://SERVER_IP:8083 (add to /etc/docker/daemon.json) # -# First request for any package fetches from upstream and caches locally. -# Subsequent requests are served from disk — no upstream needed. +# Memory: needs ~2 GB JVM heap — recommended on a server with 4 GB+ total RAM. +# Adjust INSTALL4J_ADD_VM_PARAMS below if your server has more/less RAM. # ───────────────────────────────────────────────────────────────────────────── services: - - # ── NuGet mirror (BaGet) ──────────────────────────────────────────────────── - # Proxies → https://api.nuget.org/v3/index.json - # CI usage: dotnet restore --configfile nuget.mirror.config - baget: - image: loicsharma/baget:latest + nexus: + image: sonatype/nexus3:latest + container_name: meezi-mirror-nexus restart: unless-stopped environment: - ApiKey: "ci-mirror-key" # only needed for package *publish*; reads are open - Storage__Type: FileSystem - Storage__Path: /var/baget/packages - Database__Type: Sqlite - Database__ConnectionString: "Data Source=/var/baget/db/baget.db" - Mirror__Enabled: "true" - Mirror__PackageSource: "https://api.nuget.org/v3/index.json" + # Heap: Xmx = max Java heap. MaxDirectMemorySize = off-heap (blob cache). + # Total Nexus RAM ≈ Xmx + MaxDirectMemorySize + ~512 MB OS/JVM overhead. + # 4 GB server: values below (2 GB heap + 1 GB off-heap + 512 MB overhead ≈ 3.5 GB) + # 8 GB server: -Xms1g -Xmx4g -XX:MaxDirectMemorySize=2g + INSTALL4J_ADD_VM_PARAMS: "-Xms512m -Xmx2g -XX:MaxDirectMemorySize=1g -Djava.util.prefs.userRoot=/nexus-data/javaprefs" volumes: - - baget-packages:/var/baget/packages - - baget-db:/var/baget/db + - nexus-data:/nexus-data ports: - - "5101:80" + - "8081:8081" # Web UI + NuGet + npm REST API + - "8083:8083" # Docker Hub pull-through proxy (dedicated port required by Docker protocol) healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost/health"] + test: ["CMD", "curl", "-sf", "http://localhost:8081/service/rest/v1/status"] interval: 30s - timeout: 10s - retries: 3 - - # ── npm mirror (Verdaccio) ────────────────────────────────────────────────── - # Proxies → https://registry.npmjs.org - # CI usage: npm install --registry http://mirror:4873 - verdaccio: - image: verdaccio/verdaccio:latest - restart: unless-stopped - volumes: - - verdaccio-storage:/verdaccio/storage - - ./mirrors/verdaccio/config.yaml:/verdaccio/conf/config.yaml:ro - ports: - - "4873:4873" - healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:4873/-/ping"] - interval: 30s - timeout: 10s - retries: 3 - - # ── Docker Hub pull-through cache ─────────────────────────────────────────── - # Proxies → https://registry-1.docker.io (Docker Hub only) - # Activate by adding to /etc/docker/daemon.json on the server: - # { "registry-mirrors": ["http://localhost:5100"] } - # then: systemctl restart docker - registry: - image: registry:2 - restart: unless-stopped - environment: - REGISTRY_PROXY_REMOTEURL: "https://registry-1.docker.io" - REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /data - REGISTRY_PROXY_TTL: "168h" # cache pulled layers for 7 days - volumes: - - registry-data:/data - ports: - - "5100:5000" - healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:5000/v2/"] - interval: 30s - timeout: 10s - retries: 3 + timeout: 15s + retries: 10 + start_period: 120s # Nexus JVM startup takes ~2 min on first boot volumes: - baget-packages: - name: meezi-mirror-baget-packages - baget-db: - name: meezi-mirror-baget-db - verdaccio-storage: - name: meezi-mirror-verdaccio - registry-data: - name: meezi-mirror-registry + nexus-data: + name: meezi-mirror-nexus-data diff --git a/mirrors/nexus/provision.sh b/mirrors/nexus/provision.sh new file mode 100644 index 0000000..272fb95 --- /dev/null +++ b/mirrors/nexus/provision.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# Nexus first-time provisioning +# Run once on the server after: docker compose -f docker-compose.mirror.yml up -d +# +# What this does: +# 1. Waits for Nexus to finish starting up +# 2. Reads the one-time admin password, changes it to NEXUS_ADMIN_PASS +# 3. Enables anonymous (unauthenticated) read access — needed for CI +# 4. Creates proxy repos: nuget-proxy, npm-proxy, docker-hub-proxy +# +# Usage: +# ./mirrors/nexus/provision.sh +# NEXUS_ADMIN_PASS=MySecret ./mirrors/nexus/provision.sh +# ───────────────────────────────────────────────────────────────────────────── +set -euo pipefail + +NEXUS_URL="http://localhost:8081" +NEW_PASS="${NEXUS_ADMIN_PASS:-Mirror@2024!}" # change this or set env var +CONTAINER="meezi-mirror-nexus" + +# ── 1. Wait for Nexus ──────────────────────────────────────────────────────── +echo "⏳ Waiting for Nexus (can take 2-3 min on first boot)..." +until curl -sf "$NEXUS_URL/service/rest/v1/status" | grep -q '"edition"'; do + printf "." + sleep 6 +done +echo "" +echo "✅ Nexus is up" + +# ── 2. Resolve admin password ──────────────────────────────────────────────── +INIT_PASS_FILE=$(docker exec "$CONTAINER" sh -c 'cat /nexus-data/admin.password 2>/dev/null || true') + +if [ -n "$INIT_PASS_FILE" ]; then + echo "🔐 First-time password found — changing to configured password..." + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + -u "admin:$INIT_PASS_FILE" -X PUT \ + "$NEXUS_URL/service/rest/v1/security/users/admin/change-password" \ + -H "Content-Type: text/plain" \ + -d "$NEW_PASS") + if [ "$HTTP_CODE" = "204" ]; then + echo "✅ Password updated" + else + echo "⚠️ Password change returned HTTP $HTTP_CODE — continuing with original password" + NEW_PASS="$INIT_PASS_FILE" + fi + ADMIN_PASS="$NEW_PASS" +else + echo "ℹ️ admin.password not present — using NEXUS_ADMIN_PASS (already provisioned?)" + ADMIN_PASS="$NEW_PASS" +fi + +AUTH="-u admin:$ADMIN_PASS" + +# ── 3. Enable anonymous read access ───────────────────────────────────────── +echo "🔓 Enabling anonymous access..." +curl -sf $AUTH -X PUT "$NEXUS_URL/service/rest/v1/security/anonymous" \ + -H "Content-Type: application/json" \ + -d '{"enabled":true,"userId":"anonymous","realmName":"NexusAuthorizingRealm"}' \ + && echo "✅ Anonymous access enabled" + +# Enable NuGet + npm token realms +curl -sf $AUTH -X PUT "$NEXUS_URL/service/rest/v1/security/realms/active" \ + -H "Content-Type: application/json" \ + -d '["NexusAuthenticatingRealm","NexusAuthorizingRealm","NuGetApiKey","NpmToken"]' \ + && echo "✅ Realms configured (NuGet, npm)" + +# ── Helper: create repo (skip if already exists) ──────────────────────────── +create_repo() { + local TYPE="$1" + local JSON="$2" + local NAME + NAME=$(echo "$JSON" | grep -o '"name":"[^"]*"' | head -1 | cut -d'"' -f4) + HTTP=$(curl -s -o /dev/null -w "%{http_code}" $AUTH \ + -X POST "$NEXUS_URL/service/rest/v1/repositories/$TYPE" \ + -H "Content-Type: application/json" \ + -d "$JSON") + case "$HTTP" in + 201) echo "✅ $NAME created" ;; + 400) echo "⚠️ $NAME already exists (skipped)" ;; + *) echo "❌ $NAME failed — HTTP $HTTP" ;; + esac +} + +# ── 4. NuGet proxy ────────────────────────────────────────────────────────── +echo "" +echo "📦 Creating NuGet proxy → nuget.org ..." +create_repo "nuget/proxy" '{ + "name": "nuget-proxy", + "online": true, + "storage": { + "blobStoreName": "default", + "strictContentTypeValidation": true + }, + "proxy": { + "remoteUrl": "https://api.nuget.org/v3/index.json", + "contentMaxAge": 1440, + "metadataMaxAge": 1440 + }, + "negativeCache": { "enabled": true, "timeToLive": 1440 }, + "httpClient": { "blocked": false, "autoBlock": true } +}' + +# ── 5. npm proxy ───────────────────────────────────────────────────────────── +echo "📦 Creating npm proxy → registry.npmjs.org ..." +create_repo "npm/proxy" '{ + "name": "npm-proxy", + "online": true, + "storage": { + "blobStoreName": "default", + "strictContentTypeValidation": false + }, + "proxy": { + "remoteUrl": "https://registry.npmjs.org", + "contentMaxAge": 1440, + "metadataMaxAge": 1440 + }, + "negativeCache": { "enabled": true, "timeToLive": 1440 }, + "httpClient": { "blocked": false, "autoBlock": true } +}' + +# ── 6. Docker Hub proxy (port 8083) ───────────────────────────────────────── +echo "🐳 Creating Docker Hub proxy → registry-1.docker.io (port 8083) ..." +create_repo "docker/proxy" '{ + "name": "docker-hub-proxy", + "online": true, + "storage": { + "blobStoreName": "default", + "strictContentTypeValidation": true + }, + "proxy": { + "remoteUrl": "https://registry-1.docker.io", + "contentMaxAge": 1440, + "metadataMaxAge": 1440 + }, + "negativeCache": { "enabled": true, "timeToLive": 1440 }, + "httpClient": { "blocked": false, "autoBlock": true }, + "docker": { + "v1Enabled": false, + "forceBasicAuth": false, + "httpPort": 8083 + }, + "dockerProxy": { + "indexType": "HUB", + "cacheForeignLayers": false + } +}' + +# ── Done ───────────────────────────────────────────────────────────────────── +echo "" +echo "═══════════════════════════════════════════════════════" +echo "🎉 Nexus provisioned successfully!" +echo "═══════════════════════════════════════════════════════" +echo "" +echo " UI → http://SERVER_IP:8081" +echo " admin / $ADMIN_PASS" +echo "" +echo " NuGet → http://SERVER_IP:8081/repository/nuget-proxy/index.json" +echo " npm → http://SERVER_IP:8081/repository/npm-proxy/" +echo " Docker → http://SERVER_IP:8083" +echo "" +echo "To activate Docker Hub mirror — edit /etc/docker/daemon.json:" +cat << 'EOF' + { + "insecure-registries": ["SERVER_IP:8083"], + "registry-mirrors": ["http://SERVER_IP:8083"] + } +EOF +echo " then: systemctl restart docker" +echo "" diff --git a/nuget.mirror.config b/nuget.mirror.config index c1aae20..1825e29 100644 --- a/nuget.mirror.config +++ b/nuget.mirror.config @@ -1,12 +1,12 @@ - + Nexus fetches from nuget.org on first request, then caches locally. + DO NOT use for local development (mirror must be running on the server). --> - +