diff --git a/.env.example b/.env.example index 7e97b0c..45ed94f 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,15 @@ BASE_DOMAIN=local.homeinfra.org # for aria2 and mirrors RPC_SECRET=changeit -# for all, if you want set proxy for all services +# in case any other service need the uppercase or lowercase one # aria2 need the lowercase one, FYI https://aria2.github.io/manual/en/html/aria2c.html#environment -# all_proxy= \ No newline at end of file + +no_proxy=lightmirrors,aria2 +NO_PROXY=lightmirrors,aria2 + +# all_proxy= +# ALL_PROXY= +# http_proxy= +# HTTP_PROXY= +# https_proxy= +# HTTPS_PROXY= \ No newline at end of file diff --git a/docker-compose-caddy.yml b/docker-compose-caddy.yml index 02fbacb..24d37f8 100644 --- a/docker-compose-caddy.yml +++ b/docker-compose-caddy.yml @@ -8,21 +8,21 @@ services: - ./data/cache:/app/cache env_file: - .env + environment: + - SSL_SELF_SIGNED=false networks: - app restart: unless-stopped # for linux extra_hosts: - "host.docker.internal:host-gateway" - ports: - - "80:80" caddy: image: lightmirrors/caddy build: context: caddy dockerfile: Dockerfile - ports: + - "80:80" - "443:443" volumes: - ./caddy/Caddyfile:/etc/caddy/Caddyfile diff --git a/docker-compose.yml b/docker-compose.yml index e01a3fa..b64d958 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,8 @@ services: - ./data/cache:/app/cache env_file: - .env + environment: + - SSL_SELF_SIGNED=true networks: - app restart: unless-stopped @@ -16,6 +18,7 @@ services: - "host.docker.internal:host-gateway" ports: - "80:80" + - "443:443" aria2: image: lightmirrors/aria2 build: ./aria2 diff --git a/src/Dockerfile b/src/Dockerfile index 8de2f43..88f731c 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -3,8 +3,7 @@ FROM python:3.11-alpine RUN mkdir -p /wwwroot WORKDIR /wwwroot -# Optimization for China as the project is aimed at Chinese users -ADD https://hub.gitmirror.com/https://github.com/mayswind/AriaNg/releases/download/1.3.7/AriaNg-1.3.7.zip /wwwroot/ +ADD https://github.com/mayswind/AriaNg/releases/download/1.3.7/AriaNg-1.3.7.zip /wwwroot/ RUN unzip AriaNg-1.3.7.zip && rm AriaNg-1.3.7.zip diff --git a/src/certs/certificate.crt b/src/certs/certificate.crt new file mode 100644 index 0000000..f7e63b0 --- /dev/null +++ b/src/certs/certificate.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJjCCAg6gAwIBAgIUPL5HwTzU1jkc1C8mPpvLTmhFreYwDQYJKoZIhvcNAQEL +BQAwIDEeMBwGA1UEAwwVKi5sb2NhbC5ob21laW5mcmEub3JnMB4XDTI0MDcwNjAy +MzMwMloXDTM0MDcwNDAyMzMwMlowIDEeMBwGA1UEAwwVKi5sb2NhbC5ob21laW5m +cmEub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4Ei8SI5+9ZKd +dT4F14sbxSiiVRv7vqNLKcetvsFLZf4pqigkS5YlhAT+QXhdCTu0bumERqU36LdW +8cb6E5DvaTVnRGZIjOCl5Y7G3Cw3n37fyurWhhC8LnDeri5FrU43bqRNLdH1mcoY +7+8aFrdOzoeTx4FEJMHxi/NEmrJvX6t37bRLbLiD+g1rpgnPsCAUpDEn6psSN7Se +vKxFeqncdkHjS5S7Uj1gWlcV21Sia4F2dkLQBrfMg4yL3wu+q6bcTpT9E1C1YBz8 +/l23Cf2rmLT0c8G8SNMbpX9nBB5V7y0QkAU3pSL8L1CRnpLmKfsROsIH1wo3oWUK +ZuWNIUlBDQIDAQABo1gwVjA1BgNVHREELjAsghNsb2NhbC5ob21laW5mcmEub3Jn +ghUqLmxvY2FsLmhvbWVpbmZyYS5vcmcwHQYDVR0OBBYEFPbvlkTvPGfym8Yaz70W +m8XC54kqMA0GCSqGSIb3DQEBCwUAA4IBAQADhsPOxB3jq/w3ZcZdN+ur0oOoRAK1 +rDG+5BeJx3EN3sTHBYYBVYmPNQI4KcSrgNw8kMGZSa4VrHOoiqDT0OgXYcfXsfmM +nGZQBvDmOrTMdEauY1oQLCvZxnu4GYSAnTQIim4j59qV8IfOcXw6l1JEtGxvJEbG +0BUV4aOg52+W155ov4gZh+lnl5Yzc6qpUcj11yC9PNThXAifMoN/AqdtauKOpPmF +NiHiOXNnMJfxh5QsJkWbLsfrFKQ4KMJRt5mdVCdsAKOQpISxu5TzVmgQsXgnGZc1 +xUavylzHxSg7lzJLs7DPFa4H4/xtJB5fwHGQz8QGFCdNeipyxtqA9zJo +-----END CERTIFICATE----- diff --git a/src/certs/certificate.pem b/src/certs/certificate.pem new file mode 100644 index 0000000..f2a7fe5 --- /dev/null +++ b/src/certs/certificate.pem @@ -0,0 +1,47 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDgSLxIjn71kp11 +PgXXixvFKKJVG/u+o0spx62+wUtl/imqKCRLliWEBP5BeF0JO7Ru6YRGpTfot1bx +xvoTkO9pNWdEZkiM4KXljsbcLDefft/K6taGELwucN6uLkWtTjdupE0t0fWZyhjv +7xoWt07Oh5PHgUQkwfGL80Sasm9fq3fttEtsuIP6DWumCc+wIBSkMSfqmxI3tJ68 +rEV6qdx2QeNLlLtSPWBaVxXbVKJrgXZ2QtAGt8yDjIvfC76rptxOlP0TULVgHPz+ +XbcJ/auYtPRzwbxI0xulf2cEHlXvLRCQBTelIvwvUJGekuYp+xE6wgfXCjehZQpm +5Y0hSUENAgMBAAECggEAFUWGE1JDI6Kv1mJ3MWCF1hGAMUGXLcHVAMsUZRwgvN/5 +2aPhqrLnL0897L2wAo6BaBIzs8jltZJLPQvEb2ihRuwutXkHS9KJSW1c2khDT5g7 +53CVv19aSa6gyrT5chy8pdIQE9bETBKz13Bo8VtErn0t/I+mWTTKZkiV6soeRVy+ +EiOzCMVae7vKtCs6OuStfaVMGsoR5lOQPs0AzUUYslwxA6lpfet+ydNDYSVUOlep +h0D2GJTzpYgg3lb70D/JPmjS4N/Qs4MthkqwVQkpyCqR1UelLc01pEn0CpR3/lt0 +S/3hmddz2truT0mIavZeoKPQXoMxNnOtGaBk+5JwDQKBgQD39zKG4iaMe1XQ4oQB +eembxJeO8KZAYq4p+dzFcC6qUY5EzaV5suiLeLj/rEFVkWYxAHxDg7Bn0vtyxLE1 +drbSYl67G/nXVx0YXjMeFSVrZUuw/mT0Dcv/vBIpn6FdKY/avwjTEngAlx5I68iV +z+w+DiD55bAtZekJAQ1eYJLblwKBgQDnjRtmJobLeh7XKHyaw+IwdGaiDCrM2JNb +sOXk9qW6miDUYiFKkta3scxUk/PYkburbUxq0BYdZ2NZ4iP4usStQmJwTKYO+nc/ +0fOoI1ZbW0BTC/l9nDKVXP+PmxSDd4izMkYzoKtFFefah6k524BWXansQLHKrtL+ +2/lFcqIs+wKBgQCg4OkPSjroejy7QbuTmZ1PSsVqIAg9TcjhwxLRpm8gYbmEOkha +JRFrK/AbMY9SDPvYXxBAXXheZCxv0d2HanZDGT+M9iGq7QIbhUJ1ChM1EouBUgv9 +q9XnHgZKqx7uasDCk58OErV47JLhjNifGj+UI2aRu8wKKEyBYwVhbhbaEQKBgQC/ +Twt7K4ulJCHE/LNunzcLd7CwZEMcejfNwus+cDZfZSaIswb0eTukC+dZXeBTZ5zZ +tvUzLveGlbw1MC0uPQlwaO+x/vIYYAIUnZ+lpISd+Fe+BZ+9k0cLtwmDGcsqD6HH +BFuSHkZDTyT/naRIpD11ne9dvdnnLugvH4nlQiAR7wKBgC2N9cRlyng4K9hBfBTU +029T3pGKHaXSWnW6tS30HBDSVlkdb4sDpjHo1IfOcP9C3errP7Xq2j5V4C6k1Z1y +BnVKcB0v17ZbCoRwxCzpnVGS4cyTVIQClTWlBHvdwm5SfBlD9tnLTiq65y+2TLJu +1UswIYVccfrbHGd7X5vxE4yY +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDJjCCAg6gAwIBAgIUPL5HwTzU1jkc1C8mPpvLTmhFreYwDQYJKoZIhvcNAQEL +BQAwIDEeMBwGA1UEAwwVKi5sb2NhbC5ob21laW5mcmEub3JnMB4XDTI0MDcwNjAy +MzMwMloXDTM0MDcwNDAyMzMwMlowIDEeMBwGA1UEAwwVKi5sb2NhbC5ob21laW5m +cmEub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4Ei8SI5+9ZKd +dT4F14sbxSiiVRv7vqNLKcetvsFLZf4pqigkS5YlhAT+QXhdCTu0bumERqU36LdW +8cb6E5DvaTVnRGZIjOCl5Y7G3Cw3n37fyurWhhC8LnDeri5FrU43bqRNLdH1mcoY +7+8aFrdOzoeTx4FEJMHxi/NEmrJvX6t37bRLbLiD+g1rpgnPsCAUpDEn6psSN7Se +vKxFeqncdkHjS5S7Uj1gWlcV21Sia4F2dkLQBrfMg4yL3wu+q6bcTpT9E1C1YBz8 +/l23Cf2rmLT0c8G8SNMbpX9nBB5V7y0QkAU3pSL8L1CRnpLmKfsROsIH1wo3oWUK +ZuWNIUlBDQIDAQABo1gwVjA1BgNVHREELjAsghNsb2NhbC5ob21laW5mcmEub3Jn +ghUqLmxvY2FsLmhvbWVpbmZyYS5vcmcwHQYDVR0OBBYEFPbvlkTvPGfym8Yaz70W +m8XC54kqMA0GCSqGSIb3DQEBCwUAA4IBAQADhsPOxB3jq/w3ZcZdN+ur0oOoRAK1 +rDG+5BeJx3EN3sTHBYYBVYmPNQI4KcSrgNw8kMGZSa4VrHOoiqDT0OgXYcfXsfmM +nGZQBvDmOrTMdEauY1oQLCvZxnu4GYSAnTQIim4j59qV8IfOcXw6l1JEtGxvJEbG +0BUV4aOg52+W155ov4gZh+lnl5Yzc6qpUcj11yC9PNThXAifMoN/AqdtauKOpPmF +NiHiOXNnMJfxh5QsJkWbLsfrFKQ4KMJRt5mdVCdsAKOQpISxu5TzVmgQsXgnGZc1 +xUavylzHxSg7lzJLs7DPFa4H4/xtJB5fwHGQz8QGFCdNeipyxtqA9zJo +-----END CERTIFICATE----- diff --git a/src/certs/csr.pem b/src/certs/csr.pem new file mode 100644 index 0000000..3318715 --- /dev/null +++ b/src/certs/csr.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICZTCCAU0CAQAwIDEeMBwGA1UEAwwVKi5sb2NhbC5ob21laW5mcmEub3JnMIIB +IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzStc22z67EbnI7YuBtcQEtEb +XSOE2YM4RLIqrF7c1M1Eqv+Ekp51uq3NCreBi17T4+8/Iw7aNSmts3Et8AFFq8kh +ehdwvwH0ZcuoVmkGQlFU5KeLH38Sgp/2pCWi8XjX5VVv67tcYGzVy3ATqeGRMU2y +EyJvFMy3Uc1aMKw4hofSBfiGLT0Kb93jarny753QynSuLSzixvQil/+6eenagzOz +VUNeqVBfJUIovGaeDIqMQfNPC0LaWEV+w0Us+TxYCYRa06xz5wq8JGbirx3quU+B +mRiKPnd16p28m+V4LZiu5OWVfGo3+X7Xxizke0hcCyc1bP671xssA383EAyR6wID +AQABoAAwDQYJKoZIhvcNAQELBQADggEBAAU7hmM0/R8jyZLlG8D2JMdakzcy62HW +QYBC9nivStBN7R2oZFZ63egRiCBC4sqjL2epr7QAuixLaDwLrlzq7nJczRqyw5/f +X83Qo7+tLs4/4LXfVheWfDYYmKJja7lp72EmDPGBAnO4gb1bFLkYlrJU734gdtA+ +tOzNbwmD9pSLPIBGrI9it7gY4666TnwgtlYs6kIzhW9m3fh4XnhArAFWAmU1Z/Ud +1ewnNCIsuHVa3vzPiL2ncN/IcIfDNdjkuk8kIh0QSFKHGRPk2wB02V8tWvITD4Rw +kXIX/H30cdBQTsgdNDAm7xxXumcO2f2meiyfBWjO1nd1mDVIAvPsX/Q= +-----END CERTIFICATE REQUEST----- diff --git a/src/certs/generate.sh b/src/certs/generate.sh new file mode 100644 index 0000000..d9f56dd --- /dev/null +++ b/src/certs/generate.sh @@ -0,0 +1,4 @@ +openssl genpkey -algorithm RSA -out private.key +openssl req -new -key private.key -out csr.pem -config san.cnf +openssl x509 -req -days 3650 -in csr.pem -signkey private.key -out certificate.crt -extensions v3_req -extfile san.cnf +cat private.key certificate.crt > certificate.pem \ No newline at end of file diff --git a/src/certs/private.key b/src/certs/private.key new file mode 100644 index 0000000..4b41d5d --- /dev/null +++ b/src/certs/private.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDgSLxIjn71kp11 +PgXXixvFKKJVG/u+o0spx62+wUtl/imqKCRLliWEBP5BeF0JO7Ru6YRGpTfot1bx +xvoTkO9pNWdEZkiM4KXljsbcLDefft/K6taGELwucN6uLkWtTjdupE0t0fWZyhjv +7xoWt07Oh5PHgUQkwfGL80Sasm9fq3fttEtsuIP6DWumCc+wIBSkMSfqmxI3tJ68 +rEV6qdx2QeNLlLtSPWBaVxXbVKJrgXZ2QtAGt8yDjIvfC76rptxOlP0TULVgHPz+ +XbcJ/auYtPRzwbxI0xulf2cEHlXvLRCQBTelIvwvUJGekuYp+xE6wgfXCjehZQpm +5Y0hSUENAgMBAAECggEAFUWGE1JDI6Kv1mJ3MWCF1hGAMUGXLcHVAMsUZRwgvN/5 +2aPhqrLnL0897L2wAo6BaBIzs8jltZJLPQvEb2ihRuwutXkHS9KJSW1c2khDT5g7 +53CVv19aSa6gyrT5chy8pdIQE9bETBKz13Bo8VtErn0t/I+mWTTKZkiV6soeRVy+ +EiOzCMVae7vKtCs6OuStfaVMGsoR5lOQPs0AzUUYslwxA6lpfet+ydNDYSVUOlep +h0D2GJTzpYgg3lb70D/JPmjS4N/Qs4MthkqwVQkpyCqR1UelLc01pEn0CpR3/lt0 +S/3hmddz2truT0mIavZeoKPQXoMxNnOtGaBk+5JwDQKBgQD39zKG4iaMe1XQ4oQB +eembxJeO8KZAYq4p+dzFcC6qUY5EzaV5suiLeLj/rEFVkWYxAHxDg7Bn0vtyxLE1 +drbSYl67G/nXVx0YXjMeFSVrZUuw/mT0Dcv/vBIpn6FdKY/avwjTEngAlx5I68iV +z+w+DiD55bAtZekJAQ1eYJLblwKBgQDnjRtmJobLeh7XKHyaw+IwdGaiDCrM2JNb +sOXk9qW6miDUYiFKkta3scxUk/PYkburbUxq0BYdZ2NZ4iP4usStQmJwTKYO+nc/ +0fOoI1ZbW0BTC/l9nDKVXP+PmxSDd4izMkYzoKtFFefah6k524BWXansQLHKrtL+ +2/lFcqIs+wKBgQCg4OkPSjroejy7QbuTmZ1PSsVqIAg9TcjhwxLRpm8gYbmEOkha +JRFrK/AbMY9SDPvYXxBAXXheZCxv0d2HanZDGT+M9iGq7QIbhUJ1ChM1EouBUgv9 +q9XnHgZKqx7uasDCk58OErV47JLhjNifGj+UI2aRu8wKKEyBYwVhbhbaEQKBgQC/ +Twt7K4ulJCHE/LNunzcLd7CwZEMcejfNwus+cDZfZSaIswb0eTukC+dZXeBTZ5zZ +tvUzLveGlbw1MC0uPQlwaO+x/vIYYAIUnZ+lpISd+Fe+BZ+9k0cLtwmDGcsqD6HH +BFuSHkZDTyT/naRIpD11ne9dvdnnLugvH4nlQiAR7wKBgC2N9cRlyng4K9hBfBTU +029T3pGKHaXSWnW6tS30HBDSVlkdb4sDpjHo1IfOcP9C3errP7Xq2j5V4C6k1Z1y +BnVKcB0v17ZbCoRwxCzpnVGS4cyTVIQClTWlBHvdwm5SfBlD9tnLTiq65y+2TLJu +1UswIYVccfrbHGd7X5vxE4yY +-----END PRIVATE KEY----- diff --git a/src/certs/san.cnf b/src/certs/san.cnf new file mode 100644 index 0000000..82ed561 --- /dev/null +++ b/src/certs/san.cnf @@ -0,0 +1,12 @@ +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name + +[req_distinguished_name] + +[v3_req] +subjectAltName = @alt_names + +[alt_names] +DNS.1 = local.homeinfra.org +DNS.2 = *.local.homeinfra.org \ No newline at end of file diff --git a/src/mirrorsrun/config.py b/src/mirrorsrun/config.py index 47446b2..b7fdd27 100644 --- a/src/mirrorsrun/config.py +++ b/src/mirrorsrun/config.py @@ -4,8 +4,9 @@ ARIA2_RPC_URL = os.environ.get("ARIA2_RPC_URL", "http://aria2:6800/jsonrpc") RPC_SECRET = os.environ.get("RPC_SECRET", "") BASE_DOMAIN = os.environ.get("BASE_DOMAIN", "local.homeinfra.org") -SCHEME = os.environ.get("SCHEME", "http").lower() -assert SCHEME in ["http", "https"] +SCHEME = "https" + +SSL_SELF_SIGNED = os.environ.get("SSL_SELF_SIGNED", "true") == "true" CACHE_DIR = os.environ.get("CACHE_DIR", "/app/cache/") EXTERNAL_HOST_ARIA2 = f"aria2.{BASE_DOMAIN}" diff --git a/src/mirrorsrun/server.py b/src/mirrorsrun/server.py index ea8bd95..5705f4b 100644 --- a/src/mirrorsrun/server.py +++ b/src/mirrorsrun/server.py @@ -21,7 +21,7 @@ from mirrorsrun.config import ( RPC_SECRET, EXTERNAL_URL_ARIA2, EXTERNAL_HOST_ARIA2, - SCHEME, + SCHEME, SSL_SELF_SIGNED, ) from mirrorsrun.sites.npm import npm @@ -131,8 +131,10 @@ if __name__ == "__main__": uvicorn.run( app="server:app", host="0.0.0.0", - port=port, + ssl_keyfile='/app/certs/private.key' if SSL_SELF_SIGNED else None, + ssl_certfile='/app/certs/certificate.pem' if SSL_SELF_SIGNED else None, + port=443 if SSL_SELF_SIGNED else 80, reload=True, # TODO: reload only in dev mode - proxy_headers=True, # trust x-forwarded-for etc. + proxy_headers=not SSL_SELF_SIGNED, # trust x-forwarded-for etc. forwarded_allow_ips="*", ) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index eb9ace6..da7a467 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -1,25 +1,25 @@ +version: "3.3" services: - test: - image: lightmirrors/test - build: - context: . - dockerfile: Dockerfile + + docker_test: + image: docker:dind volumes: - - .:/app - - /var/run/docker.sock:/var/run/docker.sock - networks: - - lightmirrors_app - external_links: - - lightmirrors:aria2.local.homeinfra.org - - lightmirrors:docker.local.homeinfra.org - - lightmirrors:pypi.local.homeinfra.org - - lightmirrors:torch.local.homeinfra.org - - lightmirrors:npm.local.homeinfra.org - - lightmirrors:ubuntu.local.homeinfra.org - - lightmirrors:debian.local.homeinfra.org - - lightmirrors:proxy.local.homeinfra.org - - lightmirrors:github.local.homeinfra.org - - lightmirrors:alpine.local.homeinfra.org -networks: - lightmirrors_app: - external: true \ No newline at end of file + - ./scripts:/scripts + command: sh /scripts/docker.sh + network_mode: host + privileged: true + + python_test: + image: python:3-alpine + volumes: + - ./scripts:/scripts + command: sh /scripts/python.sh + network_mode: host + + + golang_test: + image: golang:alpine + volumes: + - ./scripts:/scripts + command: sh /scripts/golang.sh + network_mode: host diff --git a/test/mirrors_test.py b/test/mirrors_test.py deleted file mode 100644 index 82397a9..0000000 --- a/test/mirrors_test.py +++ /dev/null @@ -1,32 +0,0 @@ -import unittest - -from utils import call - -PYPI_HOST = "pypi.local.homeinfra.org" -PYPI_INDEX = f"http://{PYPI_HOST}/simple" -TORCH_HOST = "torch.local.homeinfra.org" -TORCH_INDEX = f"http://{TORCH_HOST}/whl" - - -class TestPypi(unittest.TestCase): - - def test_pypi_http(self): - call(f"pip download -i {PYPI_INDEX} django --trusted-host {PYPI_HOST} --dest /tmp/pypi/") - - def test_torch_http(self): - call(f"pip download -i {TORCH_INDEX} tqdm --trusted-host {TORCH_HOST} --dest /tmp/torch/") - - def test_dockerhub_pull(self): - call(f"docker pull docker.local.homeinfra.org/alpine:3.12") - - def test_ghcr_pull(self): - call(f"docker pull ghcr.local.homeinfra.org/linuxcontainers/alpine") - - def test_quay_pull(self): - call(f"docker pull quay.local.homeinfra.org/quay/busybox") - - def test_k8s_pull(self): - call(f"docker pull k8s.local.homeinfra.org/pause:3.5") - - def test_nvcr_pull(self): - call(f"docker pull nvcr.local.homeinfra.org/nvidia/cuda") diff --git a/test/utils.py b/test/runner.py old mode 100644 new mode 100755 similarity index 52% rename from test/utils.py rename to test/runner.py index 4edc728..e2908fb --- a/test/utils.py +++ b/test/runner.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import os import subprocess from pathlib import Path @@ -9,19 +10,29 @@ root_dir = Path(__file__).parent.parent def call(cmd): print(f">> {cmd}") p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() assert p.returncode == 0, f"Error: {stderr.decode()}" print(">>", stdout.decode()) return stdout.decode(), stderr.decode() -class SetupMirrors(): - def __enter__(self): - os.chdir(root_dir) - call("docker-compose up -d") - return self +services = [ + "python_test", + "docker_test", + "golang_test" +] - def __exit__(self, exc_type, exc_val, exc_tb): - call("docker-compose down") - os.chdir(test_dir) - return False +os.chdir(root_dir) +call("docker-compose up -d --force-recreate --wait") + +os.chdir(test_dir) + +try: + for service in services: + call(f'docker-compose up --force-recreate --exit-code-from {service} {service}') +except Exception as e: + raise e +finally: + os.chdir(root_dir) + call("docker-compose down") diff --git a/test/scripts/certs/certificate.crt b/test/scripts/certs/certificate.crt new file mode 100644 index 0000000..f7e63b0 --- /dev/null +++ b/test/scripts/certs/certificate.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJjCCAg6gAwIBAgIUPL5HwTzU1jkc1C8mPpvLTmhFreYwDQYJKoZIhvcNAQEL +BQAwIDEeMBwGA1UEAwwVKi5sb2NhbC5ob21laW5mcmEub3JnMB4XDTI0MDcwNjAy +MzMwMloXDTM0MDcwNDAyMzMwMlowIDEeMBwGA1UEAwwVKi5sb2NhbC5ob21laW5m +cmEub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4Ei8SI5+9ZKd +dT4F14sbxSiiVRv7vqNLKcetvsFLZf4pqigkS5YlhAT+QXhdCTu0bumERqU36LdW +8cb6E5DvaTVnRGZIjOCl5Y7G3Cw3n37fyurWhhC8LnDeri5FrU43bqRNLdH1mcoY +7+8aFrdOzoeTx4FEJMHxi/NEmrJvX6t37bRLbLiD+g1rpgnPsCAUpDEn6psSN7Se +vKxFeqncdkHjS5S7Uj1gWlcV21Sia4F2dkLQBrfMg4yL3wu+q6bcTpT9E1C1YBz8 +/l23Cf2rmLT0c8G8SNMbpX9nBB5V7y0QkAU3pSL8L1CRnpLmKfsROsIH1wo3oWUK +ZuWNIUlBDQIDAQABo1gwVjA1BgNVHREELjAsghNsb2NhbC5ob21laW5mcmEub3Jn +ghUqLmxvY2FsLmhvbWVpbmZyYS5vcmcwHQYDVR0OBBYEFPbvlkTvPGfym8Yaz70W +m8XC54kqMA0GCSqGSIb3DQEBCwUAA4IBAQADhsPOxB3jq/w3ZcZdN+ur0oOoRAK1 +rDG+5BeJx3EN3sTHBYYBVYmPNQI4KcSrgNw8kMGZSa4VrHOoiqDT0OgXYcfXsfmM +nGZQBvDmOrTMdEauY1oQLCvZxnu4GYSAnTQIim4j59qV8IfOcXw6l1JEtGxvJEbG +0BUV4aOg52+W155ov4gZh+lnl5Yzc6qpUcj11yC9PNThXAifMoN/AqdtauKOpPmF +NiHiOXNnMJfxh5QsJkWbLsfrFKQ4KMJRt5mdVCdsAKOQpISxu5TzVmgQsXgnGZc1 +xUavylzHxSg7lzJLs7DPFa4H4/xtJB5fwHGQz8QGFCdNeipyxtqA9zJo +-----END CERTIFICATE----- diff --git a/test/scripts/docker.sh b/test/scripts/docker.sh new file mode 100644 index 0000000..fb56300 --- /dev/null +++ b/test/scripts/docker.sh @@ -0,0 +1,37 @@ +set -ex +export DOCKER_HOST=unix:///var/run/docker.sock + +cat /scripts/certs/certificate.crt >> /etc/ssl/certs/ca-certificates.crt + +dockerd& + +docker_ready() { + docker version >/dev/null 2>&1 +} + +max_wait_time=5 +elapsed_time=0 + +# Wait for Docker to be ready +while [ true ]; do + if docker_ready; then + echo "Docker is ready!" + break + else + echo "Waiting for Docker to start..." + sleep 1 + elapsed_time=$((elapsed_time + 1)) + if [ $elapsed_time -gt $max_wait_time ]; then + echo "Docker failed to start in $max_wait_time seconds!" + exit 1 + fi + fi +done + +docker pull docker.local.homeinfra.org/busybox +docker pull ghcr.local.homeinfra.org/linuxcontainers/alpine +docker pull quay.local.homeinfra.org/quay/busybox +docker pull k8s.local.homeinfra.org/pause + +# https is required +echo 'FROM docker.local.homeinfra.org/alpine' | docker build - diff --git a/test/scripts/golang.sh b/test/scripts/golang.sh new file mode 100644 index 0000000..f13fda3 --- /dev/null +++ b/test/scripts/golang.sh @@ -0,0 +1,9 @@ +set -ex + +cat /scripts/certs/certificate.crt >> /etc/ssl/certs/ca-certificates.crt + +go env -w GOPROXY=https://goproxy.local.homeinfra.org,direct + +go clean -modcache +go mod init test +go get golang.org/x/sys@v0.22.0 diff --git a/test/scripts/npm.sh b/test/scripts/npm.sh new file mode 100644 index 0000000..e69de29 diff --git a/test/scripts/python.sh b/test/scripts/python.sh new file mode 100644 index 0000000..155da61 --- /dev/null +++ b/test/scripts/python.sh @@ -0,0 +1,12 @@ +set -ex + +cat /scripts/certs/certificate.crt >> /etc/ssl/certs/ca-certificates.crt + +pip config set global.index-url https://pypi.local.homeinfra.org/simple + pip config set global.trusted-host pypi.local.homeinfra.org +pip download jinja2 --dest /tmp/pypi/ + + +pip config set global.index-url https://torch.local.homeinfra.org/whl +pip config set global.trusted-host torch.local.homeinfra.org +pip download tqdm --dest /tmp/torch/ \ No newline at end of file