From 6f85cfe4d394ac6ba5a4458e544458f98a918827 Mon Sep 17 00:00:00 2001 From: "soroush.asadi" Date: Thu, 28 May 2026 14:31:12 +0330 Subject: [PATCH] feat(infra): add local pull-through mirrors for NuGet, npm, Docker Hub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docker-compose.mirror.yml: - BaGet (port 5101) → proxies nuget.org - Verdaccio (port 4873) → proxies npmjs.com - registry:2 (port 5100) → proxies Docker Hub nuget.mirror.config: points dotnet restore at http://mirror:5101 mirrors/verdaccio/config.yaml: open reads, upstream npmjs fallback CI workflow: - All container jobs: --add-host=mirror:host-gateway - dotnet restore --configfile nuget.mirror.config - npm install --registry http://mirror:4873 First run: packages fetched from upstream through the VPS. All subsequent runs: served from local disk, no CDN needed. Co-Authored-By: Claude Sonnet 4.6 --- .gitea/workflows/ci-cd.yml | 62 +++++++++++++----------- docker-compose.mirror.yml | 90 +++++++++++++++++++++++++++++++++++ mirrors/verdaccio/config.yaml | 44 +++++++++++++++++ nuget.mirror.config | 15 ++++++ 4 files changed, 182 insertions(+), 29 deletions(-) create mode 100644 docker-compose.mirror.yml create mode 100644 mirrors/verdaccio/config.yaml create mode 100644 nuget.mirror.config diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 222f928..5c67eae 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -18,23 +18,17 @@ concurrency: # ubuntu-latest:docker://node:20-alpine ← CI jobs run in real Docker containers # self-hosted:host ← deploy runs directly on the server # -# With docker:// labels: -# - container: image: overrides the base image for the job ✅ -# - services: creates sidecar containers on the same network ✅ -# - workspace is properly mounted into the container ✅ -# # WHY we don't use actions/checkout@v4 in container jobs: -# actions/checkout is a JavaScript action — the runner executes it with -# `node index.js` INSIDE the job container. +# actions/checkout is a JS action — needs `node` in the container. +# mcr.microsoft.com/dotnet/sdk → no Node.js → exit 127 +# node:20-alpine → no git → checkout fails +# Fix: plain shell git clone via http.extraheader (token never in process list). # -# mcr.microsoft.com/dotnet/sdk → no Node.js → exit 127 -# node:20-alpine → no git → checkout fails -# -# Fix: plain shell git clone (TOKEN via http.extraheader so it never -# appears in the process list or git log). -# -# deploy (self-hosted:host) runs on the runner itself which HAS node+git, -# so actions/checkout@v4 works there normally. +# Local mirrors (started via docker-compose.mirror.yml): +# "mirror" hostname → host-gateway (docker bridge IP 172.17.0.1) +# NuGet → http://mirror:5101 (BaGet — nuget.mirror.config) +# npm → http://mirror:4873 (Verdaccio — --registry flag) +# Docker Hub → configured in /etc/docker/daemon.json on the host # ───────────────────────────────────────────────────────────────────────────── jobs: @@ -45,9 +39,9 @@ jobs: runs-on: ubuntu-latest container: image: mcr.microsoft.com/dotnet/sdk:10.0 - # host-gateway → docker bridge IP (172.17.0.1); Gitea port 3000 is - # published there, so 'gitea' hostname resolves from inside the job container - options: --add-host=gitea:host-gateway + options: >- + --add-host=gitea:host-gateway + --add-host=mirror:host-gateway services: postgres: image: postgres:16-alpine @@ -80,7 +74,7 @@ jobs: git checkout FETCH_HEAD - name: Restore - run: dotnet restore src/Meezi.API/Meezi.API.csproj + run: dotnet restore src/Meezi.API/Meezi.API.csproj --configfile nuget.mirror.config - name: Build run: dotnet build src/Meezi.API/Meezi.API.csproj --no-restore -c Release @@ -97,7 +91,9 @@ jobs: runs-on: ubuntu-latest container: image: mcr.microsoft.com/dotnet/sdk:10.0 - options: --add-host=gitea:host-gateway + options: >- + --add-host=gitea:host-gateway + --add-host=mirror:host-gateway steps: - name: Checkout env: @@ -111,7 +107,7 @@ jobs: git checkout FETCH_HEAD - name: Restore - run: dotnet restore src/Meezi.Admin.API/Meezi.Admin.API.csproj + run: dotnet restore src/Meezi.Admin.API/Meezi.Admin.API.csproj --configfile nuget.mirror.config - name: Build run: dotnet build src/Meezi.Admin.API/Meezi.Admin.API.csproj --no-restore -c Release @@ -122,7 +118,9 @@ jobs: runs-on: ubuntu-latest container: image: node:20-alpine - options: --add-host=gitea:host-gateway + options: >- + --add-host=gitea:host-gateway + --add-host=mirror:host-gateway steps: - name: Checkout env: @@ -138,7 +136,7 @@ jobs: - name: Install dependencies working-directory: web/dashboard - run: npm install --legacy-peer-deps --ignore-scripts + run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:4873 - name: TypeScript check working-directory: web/dashboard @@ -152,7 +150,9 @@ jobs: runs-on: ubuntu-latest container: image: node:20-alpine - options: --add-host=gitea:host-gateway + options: >- + --add-host=gitea:host-gateway + --add-host=mirror:host-gateway steps: - name: Checkout env: @@ -168,7 +168,7 @@ jobs: - name: Install dependencies working-directory: web/admin - run: npm install --legacy-peer-deps --ignore-scripts + run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:4873 - name: TypeScript check working-directory: web/admin @@ -182,7 +182,9 @@ jobs: runs-on: ubuntu-latest container: image: node:20-alpine - options: --add-host=gitea:host-gateway + options: >- + --add-host=gitea:host-gateway + --add-host=mirror:host-gateway steps: - name: Checkout env: @@ -198,7 +200,7 @@ jobs: - name: Install dependencies working-directory: web/website - run: npm install --legacy-peer-deps --ignore-scripts + run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:4873 - name: TypeScript check working-directory: web/website @@ -212,7 +214,9 @@ jobs: runs-on: ubuntu-latest container: image: node:20-alpine - options: --add-host=gitea:host-gateway + options: >- + --add-host=gitea:host-gateway + --add-host=mirror:host-gateway steps: - name: Checkout env: @@ -228,7 +232,7 @@ jobs: - name: Install dependencies working-directory: web/finder - run: npm install --legacy-peer-deps --ignore-scripts + run: npm install --legacy-peer-deps --ignore-scripts --registry http://mirror:4873 - name: TypeScript check working-directory: web/finder diff --git a/docker-compose.mirror.yml b/docker-compose.mirror.yml new file mode 100644 index 0000000..7068fbe --- /dev/null +++ b/docker-compose.mirror.yml @@ -0,0 +1,90 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Local pull-through mirrors +# ───────────────────────────────────────────────────────────────────────────── +# Start: docker compose -f docker-compose.mirror.yml up -d +# Stop: docker compose -f docker-compose.mirror.yml down +# +# Endpoints (reachable from CI containers via host-gateway as "mirror"): +# NuGet → http://SERVER_IP:5101/v3/index.json +# npm → http://SERVER_IP:4873 +# Docker → http://SERVER_IP:5100 (add to /etc/docker/daemon.json) +# +# First request for any package fetches from upstream and caches locally. +# Subsequent requests are served from disk — no upstream needed. +# ───────────────────────────────────────────────────────────────────────────── + +services: + + # ── NuGet mirror (BaGet) ──────────────────────────────────────────────────── + # Proxies → https://api.nuget.org/v3/index.json + # CI usage: dotnet restore --configfile nuget.mirror.config + baget: + image: loicsharma/baget:latest + restart: unless-stopped + environment: + ApiKey: "ci-mirror-key" # only needed for package *publish*; reads are open + Storage__Type: FileSystem + Storage__Path: /var/baget/packages + Database__Type: Sqlite + Database__ConnectionString: "Data Source=/var/baget/db/baget.db" + Mirror__Enabled: "true" + Mirror__PackageSource: "https://api.nuget.org/v3/index.json" + volumes: + - baget-packages:/var/baget/packages + - baget-db:/var/baget/db + ports: + - "5101:80" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost/health"] + interval: 30s + timeout: 10s + retries: 3 + + # ── npm mirror (Verdaccio) ────────────────────────────────────────────────── + # Proxies → https://registry.npmjs.org + # CI usage: npm install --registry http://mirror:4873 + verdaccio: + image: verdaccio/verdaccio:latest + restart: unless-stopped + volumes: + - verdaccio-storage:/verdaccio/storage + - ./mirrors/verdaccio/config.yaml:/verdaccio/conf/config.yaml:ro + ports: + - "4873:4873" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:4873/-/ping"] + interval: 30s + timeout: 10s + retries: 3 + + # ── Docker Hub pull-through cache ─────────────────────────────────────────── + # Proxies → https://registry-1.docker.io (Docker Hub only) + # Activate by adding to /etc/docker/daemon.json on the server: + # { "registry-mirrors": ["http://localhost:5100"] } + # then: systemctl restart docker + registry: + image: registry:2 + restart: unless-stopped + environment: + REGISTRY_PROXY_REMOTEURL: "https://registry-1.docker.io" + REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /data + REGISTRY_PROXY_TTL: "168h" # cache pulled layers for 7 days + volumes: + - registry-data:/data + ports: + - "5100:5000" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:5000/v2/"] + interval: 30s + timeout: 10s + retries: 3 + +volumes: + baget-packages: + name: meezi-mirror-baget-packages + baget-db: + name: meezi-mirror-baget-db + verdaccio-storage: + name: meezi-mirror-verdaccio + registry-data: + name: meezi-mirror-registry diff --git a/mirrors/verdaccio/config.yaml b/mirrors/verdaccio/config.yaml new file mode 100644 index 0000000..ac64234 --- /dev/null +++ b/mirrors/verdaccio/config.yaml @@ -0,0 +1,44 @@ +# Verdaccio — npm pull-through proxy +# All packages are served anonymously (no login needed in CI). +# On first request: fetches from npmjs.org, caches locally. +# On subsequent requests: served from local disk. + +storage: /verdaccio/storage +plugins: /verdaccio/plugins + +# Listen on all interfaces inside the container +listen: 0.0.0.0:4873 + +# No auth required for reading (CI-friendly) +auth: + htpasswd: + file: /verdaccio/conf/htpasswd + max_users: -1 # disable self-registration; reads stay open + +uplinks: + npmjs: + url: https://registry.npmjs.org/ + timeout: 120s + maxage: 10m + max_fails: 3 + fail_timeout: 5m + +packages: + # Scoped packages (@org/package) + "@*/*": + access: $all + publish: $all + proxy: npmjs + + # All other packages + "**": + access: $all + publish: $all + proxy: npmjs + +# Cache settings +publish: + allow_offline: true # serve cached version even if upstream is unreachable + +logs: + - { type: stdout, format: pretty, level: http } diff --git a/nuget.mirror.config b/nuget.mirror.config new file mode 100644 index 0000000..c1aae20 --- /dev/null +++ b/nuget.mirror.config @@ -0,0 +1,15 @@ + + + + + + + + + + + +