From 2162be61d140fcac7b6ac64ff2513c6d0d75822e Mon Sep 17 00:00:00 2001 From: Anonymous <> Date: Sat, 24 Feb 2024 17:54:55 +0800 Subject: [PATCH] makes caddy optional --- .env.example | 3 +- README.md | 61 +++++++++++++++++++++++++++++++--------- caddy/Caddyfile | 8 +----- caddy/Dockerfile | 8 ------ docker-compose-caddy.yml | 48 +++++++++++++++++++++++++++++++ docker-compose.yml | 12 -------- mirrors/Dockerfile | 9 ++++++ mirrors/config.py | 8 ++++-- mirrors/server.py | 42 ++++++++++++++++++++++----- 9 files changed, 146 insertions(+), 53 deletions(-) create mode 100644 docker-compose-caddy.yml diff --git a/.env.example b/.env.example index 5825b57..7e97b0c 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,7 @@ - # for caddy CLOUDFLARE_DNS_API_TOKEN=xxxx -# for caddy and mirrors +SCHEME=http BASE_DOMAIN=local.homeinfra.org # for aria2 and mirrors diff --git a/README.md b/README.md index 0273239..db98c28 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,8 @@ LightMirrors是一个开源的缓存镜像站服务，用于加速软件包下载和镜像拉取。 目前支持**DockerHub**、PyPI、PyTorch、NPM等镜像缓存服务。 - - + [![GitHub](https://img.shields.io/github/stars/NoCLin/LightMirrors?style=social)](https://github.com/NoCLin/LightMirrors) [![GitHub](https://img.shields.io/github/forks/NoCLin/LightMirrors?style=social)](https://github.com/NoCLin/LightMirrors) @@ -23,41 +22,75 @@ LightMirrors是一个开源的缓存镜像站服务，用于加速软件包下 ### Prerequisites - docker + docker-compose. -- 一个域名，设置 `*.local.homeinfra.org` 的A记录指向您的服务器. +- 一个域名，设置 `*.yourdomain` 的A记录指向您服务器的IP. + - `*.local.homeinfra.org` 默认指向 `127.0.0.1`，本地测试可以直接使用。 - 代理服务器（如有必要）. -- 一个Cloudflare账户（非强制，也可以使用其他DNS服务，请自行修改Caddy） + +> 如果需要使用HTTPS，可以在外层新增一个HTTP网关（如Caddy），请参考后续章节。 + +### QuickStart + +```bash + +cp .env.example .env +docker-compose up + +``` ### Deployment 修改 `.env` 文件，设置下列参数： - `BASE_DOMAIN`: 基础域名，如 `local.homeinfra.org`，镜像站将会使用 `*.local.homeinfra.org` 的子域名。 -- `CLOUDFLARE_DNS_API_TOKEN`，Cloudflare的API Token，用于管理DNS申请HTTPS证书。 - `RPC_SECRET`：Aria2的RPC密钥。 - `all_proxy`：代理服务器地址，如有必要。 +- `SCHEME`：`http` 或 `https`。 + +如果您需要HTTPS，请确保docker-compose.yml文件中开放443端口，并使用`cloudflare` 相关的Caddyfile和Dockerfile. ```bash docker-compose up ``` +测试命令： + +```bash +docker pull docker.local.homeinfra.org/alpine +pip3 download -i http://pypi.local.homeinfra.org/simple/ jinja2 --trusted-host pypi.local.homeinfra.org +pip3 download -i http://torch.local.homeinfra.org/whl/ torch --trusted-host torch.local.homeinfra.org +``` + ## Design -LightMirrors依赖于三个组件： +LightMirrors依赖于两个组件： -- aria2 + Aria2Ng : 下载器与管理UI。 -- mirrors: 镜像HTTP服务器。 -- caddy: HTTP网关。 +- aria2 : 下载器与管理UI。 +- mirrors: 镜像HTTP服务器，根据不同域名转发请求到不同模块。 + - Aria2Ng + - PyPI + - DockerHub + - ... ## Test > 假设我们的域名为 local.homeinfra.org -| subdomain | source | test command | -|-----------|---------------------------------|-------------------------------------------------------------------| -| pypi | https://pypi.org | `pip3 download -i https://pypi.local.homeinfra.org/simple jinja2` | -| torch | https://download.pytorch.org | `pip3 download -i https://torch.local.homeinfra.org/whl/ torch` | -| dockerhub | https://registry-1.docker.io/v2 | `docker pull docker.local.homeinfra.org/alpine` | +| subdomain | source | test command | test command (http) | +|-----------|---------------------------------|-------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------| +| pypi | https://pypi.org | `pip3 download -i https://pypi.local.homeinfra.org/simple jinja2` | `pip3 download -i http://pypi.local.homeinfra.org/simple jinja2 --trusted-host pypi.local.homeinfra.org` | +| torch | https://download.pytorch.org | `pip3 download -i https://torch.local.homeinfra.org/whl/ torch` | `pip3 download -i http://torch.local.homeinfra.org/whl/ torch --trusted-host torch.local.homeinfra.org` | +| dockerhub | https://registry-1.docker.io/v2 | `docker pull docker.local.homeinfra.org/alpine` | `docker pull docker.local.homeinfra.org/alpine` | +## HTTPS + +在 .env 中配置 `SCHEME=https` 与 CLOUDFLARE_DNS_API_TOKEN。 +本项目提供了一个基于Cloudflare DNS的Caddyfile和Dockerfile，可以直接使用。如果您希望使用其他DNS Provider，请自行修改。 + +配置完成后，执行下列命令： + +```bash +docker-compose -f docker-compose-caddy.yml up +``` ## Star History diff --git a/caddy/Caddyfile b/caddy/Caddyfile index e4111c2..9fbc0f9 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -4,12 +4,6 @@ dns cloudflare {env.CLOUDFLARE_DNS_API_TOKEN} } - reverse_proxy http://lightmirrors:8080 + reverse_proxy http://lightmirrors:80 - @aria2 host aria2.{$BASE_DOMAIN} - handle @aria2 { - root * /wwwroot - file_server - reverse_proxy /jsonrpc http://aria2:6800 - } } \ No newline at end of file diff --git a/caddy/Dockerfile b/caddy/Dockerfile index bf50fe3..3b820c2 100644 --- a/caddy/Dockerfile +++ b/caddy/Dockerfile @@ -10,14 +10,6 @@ FROM caddy:2.7.6-alpine COPY --from=builder /usr/bin/caddy /usr/bin/caddy -RUN mkdir -p /wwwroot -WORKDIR /wwwroot - -# Optimization for China as the project is aimed at Chinese users -ADD https://hub.gitmirror.com/https://github.com/mayswind/AriaNg/releases/download/1.3.7/AriaNg-1.3.7.zip /wwwroot/ - -RUN unzip AriaNg-1.3.7.zip && rm AriaNg-1.3.7.zip - ENTRYPOINT ["caddy"] CMD ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] \ No newline at end of file diff --git a/docker-compose-caddy.yml b/docker-compose-caddy.yml new file mode 100644 index 0000000..555e875 --- /dev/null +++ b/docker-compose-caddy.yml @@ -0,0 +1,48 @@ +services: + lightmirrors: + image: lightmirrors/mirrors + build: ./mirrors + volumes: + - ./mirrors:/app + - ./data/cache:/app/cache + env_file: + - .env + networks: + - app + restart: unless-stopped + # for linux + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "80:80" + caddy: + image: lightmirrors/caddy + build: + context: caddy + dockerfile: Dockerfile + + ports: + - "443:443" + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile + - ./caddy/caddy_data:/data/caddy + env_file: + - .env + networks: + - app + restart: unless-stopped + aria2: + image: lightmirrors/aria2 + build: ./aria2 + volumes: + - ./aria2/aria2.conf:/aria2.conf + - ./data/cache:/app/cache + - ./data/aria2:/data/ + networks: + - app + env_file: + - .env + restart: unless-stopped +networks: + app: + driver: bridge \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f63372f..ce5f3b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,20 +13,8 @@ services: # for linux extra_hosts: - "host.docker.internal:host-gateway" - caddy: - image: lightmirrors/caddy - build: ./caddy ports: - "80:80" - - "443:443" - volumes: - - ./caddy/Caddyfile:/etc/caddy/Caddyfile - - ./caddy/caddy_data:/data/caddy - env_file: - - .env - networks: - - app - restart: unless-stopped aria2: image: lightmirrors/aria2 build: ./aria2 diff --git a/mirrors/Dockerfile b/mirrors/Dockerfile index aec3db5..530adbc 100644 --- a/mirrors/Dockerfile +++ b/mirrors/Dockerfile @@ -1,5 +1,14 @@ FROM python:3.11-alpine +RUN mkdir -p /wwwroot +WORKDIR /wwwroot + +# Optimization for China as the project is aimed at Chinese users +ADD https://hub.gitmirror.com/https://github.com/mayswind/AriaNg/releases/download/1.3.7/AriaNg-1.3.7.zip /wwwroot/ + +RUN unzip AriaNg-1.3.7.zip && rm AriaNg-1.3.7.zip + + ADD requirements.txt /app/requirements.txt RUN pip install -r /app/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple/ diff --git a/mirrors/config.py b/mirrors/config.py index ef42740..1fb5e9b 100644 --- a/mirrors/config.py +++ b/mirrors/config.py @@ -2,10 +2,12 @@ import os ARIA2_RPC_URL = os.environ.get("ARIA2_RPC_URL", 'http://aria2:6800/jsonrpc') RPC_SECRET = os.environ.get("RPC_SECRET", '') -BASE_DOMAIN = os.environ.get("BASE_DOMAIN", '127.0.0.1.nip.io') +BASE_DOMAIN = os.environ.get("BASE_DOMAIN", 'local.homeinfra.org') PROXY = os.environ.get("PROXY", None) -CACHE_DIR = os.environ.get("CACHE_DIR", "/app/cache/") +SCHEME = os.environ.get("SCHEME", None) +assert SCHEME in ["http", "https"] +CACHE_DIR = os.environ.get("CACHE_DIR", "/app/cache/") EXTERNAL_HOST_ARIA2 = f"aria2." + BASE_DOMAIN -EXTERNAL_URL_ARIA2 = f"https://" + EXTERNAL_HOST_ARIA2 +EXTERNAL_URL_ARIA2 = f"{SCHEME}://{EXTERNAL_HOST_ARIA2}/aria2/index.html" diff --git a/mirrors/server.py b/mirrors/server.py index 5c4a636..6383224 100644 --- a/mirrors/server.py +++ b/mirrors/server.py @@ -2,11 +2,14 @@ import base64 import signal import urllib.parse +import httpx import uvicorn from fastapi import FastAPI from starlette.requests import Request +from starlette.responses import RedirectResponse, Response +from starlette.staticfiles import StaticFiles -from config import BASE_DOMAIN, RPC_SECRET, EXTERNAL_URL_ARIA2, EXTERNAL_HOST_ARIA2 +from config import BASE_DOMAIN, RPC_SECRET, EXTERNAL_URL_ARIA2, EXTERNAL_HOST_ARIA2, SCHEME from sites.docker import docker from sites.npm import npm from sites.pypi import pypi @@ -14,6 +17,26 @@ from sites.torch import torch app = FastAPI() +app.mount("/aria2/", StaticFiles(directory="/wwwroot/"), name="static", ) + + +async def aria2(request: Request, call_next): + if request.url.path == "/": + return RedirectResponse("/aria2/index.html") + if request.url.path == "/jsonrpc": + async with httpx.AsyncClient(mounts={ + "all://": httpx.AsyncHTTPTransport() + }) as client: + data = (await request.body()) + response = await client.request(url="http://aria2:6800/jsonrpc", + method=request.method, + headers=request.headers, content=data) + headers = response.headers + headers.pop("content-length", None) + headers.pop("content-encoding", None) + return Response(content=response.content, status_code=response.status_code, headers=headers) + return await call_next(request) + @app.middleware("http") async def capture_request(request: Request, call_next: callable): @@ -21,6 +44,9 @@ async def capture_request(request: Request, call_next: callable): if not hostname.endswith(f".{BASE_DOMAIN}"): return await call_next(request) + if hostname.startswith("aria2."): + return await aria2(request, call_next) + if hostname.startswith("pypi."): return await pypi(request) if hostname.startswith("torch."): @@ -35,24 +61,26 @@ async def capture_request(request: Request, call_next: callable): if __name__ == '__main__': signal.signal(signal.SIGINT, signal.SIG_DFL) - port = 8080 - print(f"Server started at https://*.{BASE_DOMAIN})") + port = 80 + print(f"Server started at {SCHEME}://*.{BASE_DOMAIN})") for dn in ["pypi", "torch", "docker", "npm"]: - print(f" - https://{dn}.{BASE_DOMAIN}") + print(f" - {SCHEME}://{dn}.{BASE_DOMAIN}") aria2_secret = base64.b64encode(RPC_SECRET.encode()).decode() params = { - 'protocol': 'https', + 'protocol': SCHEME, 'host': EXTERNAL_HOST_ARIA2, - 'port': '443', + 'port': '443' if SCHEME == 'https' else '80', 'interface': 'jsonrpc', 'secret': aria2_secret } query_string = urllib.parse.urlencode(params) - aria2_url_with_auth = EXTERNAL_URL_ARIA2 + "/#!/settings/rpc/set?" + query_string + aria2_url_with_auth = EXTERNAL_URL_ARIA2 + "#!/settings/rpc/set?" + query_string print(f"Download manager (Aria2) at {aria2_url_with_auth}") + # FIXME: only proxy headers if SCHME is https + # reload only in dev mode uvicorn.run(app="server:app", host="0.0.0.0", port=port, reload=True, proxy_headers=True, forwarded_allow_ips="*")