sashatrask
369 lines
16 KiB
Docker
369 lines
16 KiB
Docker
FROM python:3.12-slim-bookworm@sha256:782412e85d0f0984994c290652577d4018aff08145c85b262bb63dc0c7522254 AS python-base
|
|
|
|
ENV PATH=/usr/local/bin:/usr/bin:/bin:/usr/lib/postgresql/16/bin \
|
|
HOME=/data/home \
|
|
LANG=C.UTF-8 \
|
|
LC_ALL=C.UTF-8 \
|
|
PYTHONDONTWRITEBYTECODE=1
|
|
|
|
RUN /usr/local/bin/python3 -I -S -B -c "import sys; assert sys.version_info[:3] == (3, 12, 14), sys.version"
|
|
|
|
FROM python-base AS lock-generator
|
|
|
|
COPY docker/build-dependencies/requirements.lock /tmp/compiler.lock
|
|
RUN python3 -m pip --isolated install --index-url=https://pypi.org/simple \
|
|
--require-hashes --only-binary=:all: --no-compile --no-cache-dir \
|
|
-r /tmp/compiler.lock \
|
|
&& rm /tmp/compiler.lock
|
|
WORKDIR /src
|
|
COPY app/requirements.txt app/requirements-keycheckers.txt ./app/
|
|
COPY docker/requirements.in docker/requirements.lock docker/requirements-test.in docker/requirements-test.lock docker/requirements-worker.in docker/requirements-worker.lock ./docker/
|
|
COPY docker/build-dependencies/requirements.in docker/build-dependencies/requirements.lock ./docker/build-dependencies/
|
|
ENV CUSTOM_COMPILE_COMMAND="See docker/build-dependencies/README.md for the pinned Python 3.12.14 pip-tools generation command."
|
|
CMD ["python3", "-m", "piptools", "compile", "--generate-hashes", "--allow-unsafe", "--resolver=backtracking", "--strip-extras", "--no-emit-index-url", "--no-emit-trusted-host", "--index-url=https://pypi.org/simple", "--pip-args=--only-binary=:all:", "--output-file=docker/requirements.lock", "docker/requirements.in"]
|
|
|
|
FROM python-base AS trufflehog-download
|
|
|
|
ARG TARGETARCH
|
|
RUN python3 -I -S -B - "$TARGETARCH" <<'PY'
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tarfile
|
|
import urllib.request
|
|
|
|
checksums = {
|
|
"amd64": "dc24007c2f233bd61c05beabeb44aa27ea9b43288166279209abe0458c5ce76b",
|
|
"arm64": "7e65e771d2a247964056aa5edba0f8ae3945895e5dce867fe0ffbc7b0128239a",
|
|
}
|
|
architecture = sys.argv[1]
|
|
if architecture not in checksums:
|
|
raise SystemExit("TruffleHog is pinned only for linux/amd64 and linux/arm64")
|
|
name = f"trufflehog_3.97.4_linux_{architecture}.tar.gz"
|
|
url = "https://github.com/trufflesecurity/trufflehog/releases/download/v3.97.4/" + name
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
with urllib.request.urlopen(url, timeout=60) as response, open("/tmp/trufflehog.tar.gz", "wb") as output:
|
|
while chunk := response.read(1024 * 1024):
|
|
digest.update(chunk)
|
|
size += len(chunk)
|
|
output.write(chunk)
|
|
if digest.hexdigest() != checksums[architecture]:
|
|
raise SystemExit("TruffleHog archive SHA-256 mismatch")
|
|
if architecture == "amd64" and size != 34970205:
|
|
raise SystemExit("TruffleHog amd64 archive length mismatch")
|
|
with tarfile.open("/tmp/trufflehog.tar.gz", "r:gz") as archive:
|
|
member = archive.getmember("trufflehog")
|
|
if not member.isfile():
|
|
raise SystemExit("TruffleHog archive executable must be a regular file")
|
|
with archive.extractfile(member) as source, open("/trufflehog", "wb") as output:
|
|
shutil.copyfileobj(source, output)
|
|
os.chmod("/trufflehog", 0o755)
|
|
os.unlink("/tmp/trufflehog.tar.gz")
|
|
print(f"Verified {name}: {size} bytes, sha256:{digest.hexdigest()}")
|
|
PY
|
|
|
|
FROM python-base AS worker-dependencies
|
|
|
|
COPY docker/requirements-worker.lock /tmp/requirements-worker.lock
|
|
RUN python3 -m pip --isolated install --index-url=https://pypi.org/simple \
|
|
--require-hashes --only-binary=:all: --no-compile --no-cache-dir \
|
|
--target /worker-dependencies -r /tmp/requirements-worker.lock \
|
|
&& PYTHONPATH=/worker-dependencies python3 -I -S -B - <<'PY'
|
|
import sys
|
|
sys.path.insert(0, "/worker-dependencies")
|
|
import requests
|
|
import yaml
|
|
import zstandard
|
|
PY
|
|
RUN rm /tmp/requirements-worker.lock
|
|
|
|
FROM python-base AS worker-native-dependencies
|
|
|
|
RUN <<'SH'
|
|
set -eu
|
|
rm -f /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources
|
|
printf '%s\n' \
|
|
'Types: deb' \
|
|
'URIs: https://snapshot.debian.org/archive/debian/20260914T000000Z/' \
|
|
'Suites: bookworm bookworm-updates' \
|
|
'Components: main' \
|
|
'Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg' \
|
|
'Check-Valid-Until: no' \
|
|
'' \
|
|
'Types: deb' \
|
|
'URIs: https://snapshot.debian.org/archive/debian-security/20260914T000000Z/' \
|
|
'Suites: bookworm-security' \
|
|
'Components: main' \
|
|
'Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg' \
|
|
'Check-Valid-Until: no' \
|
|
> /etc/apt/sources.list.d/debian.sources
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get -o Acquire::Retries=3 -o Acquire::https::Timeout=30 -o APT::Update::Error-Mode=any update
|
|
apt-get install -y --no-install-recommends \
|
|
ca-certificates=20250419~deb12u1 \
|
|
git=1:2.39.5-0+deb12u3 \
|
|
tini=0.19.0-1+b3
|
|
install -d -o 10001 -g 10001 -m 0700 /data /data/home
|
|
/usr/sbin/groupadd --gid 10001 truf
|
|
/usr/sbin/useradd --uid 10001 --gid 10001 --no-create-home --home-dir /data/home --shell /usr/sbin/nologin truf
|
|
install -d -o 0 -g 0 -m 0755 /worker-git/bin /worker-git/libexec /worker-git/share
|
|
cp -aL /usr/bin/git /worker-git/bin/git
|
|
cp -aL /usr/lib/git-core /worker-git/libexec/git-core
|
|
cp -aL /usr/share/git-core /worker-git/share/git-core
|
|
find /worker-git -type d -exec chmod 0755 {} +
|
|
find /worker-git -type f -exec chmod go-w {} +
|
|
test -x /worker-git/bin/git
|
|
test -x /worker-git/libexec/git-core/git-remote-https
|
|
rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /var/log/apt/*
|
|
SH
|
|
|
|
COPY --from=trufflehog-download --chown=0:0 --chmod=0755 /trufflehog /usr/local/bin/trufflehog
|
|
|
|
FROM worker-native-dependencies AS worker-package-build
|
|
|
|
ARG TARGETARCH
|
|
COPY --from=worker-dependencies --chown=0:0 /worker-dependencies /build/app/dependencies
|
|
COPY app/ /build/app/
|
|
COPY docs/remote-worker-quickstart-ru.md /build/README_RU.md
|
|
COPY docs/remote-worker-cheatsheet-windows-ru.md /build/
|
|
COPY docs/remote-worker-cheatsheet-linux-ru.md /build/
|
|
COPY docs/remote-worker-cheatsheet-docker-ru.md /build/
|
|
COPY docker/worker-package-pins.json /build/worker-package-pins.json
|
|
RUN case "$TARGETARCH" in \
|
|
amd64) platform_tag=linux-x86_64 ;; \
|
|
arm64) platform_tag=linux-aarch64 ;; \
|
|
*) echo "unsupported worker architecture" >&2; exit 1 ;; \
|
|
esac \
|
|
&& python3 -u -I -S -B /build/app/worker_package_builder.py assemble \
|
|
--output /opt/truf-worker \
|
|
--source-app /build/app \
|
|
--dependencies /build/app/dependencies \
|
|
--detector-policy /build/app/trufflehog-custom-detectors.yaml \
|
|
--trufflehog /usr/local/bin/trufflehog \
|
|
--git-root /worker-git \
|
|
--git-executable bin/git \
|
|
--platform-tag "$platform_tag" \
|
|
--operator-readme /build/README_RU.md \
|
|
--operator-cheatsheet /build/remote-worker-cheatsheet-windows-ru.md \
|
|
--operator-cheatsheet /build/remote-worker-cheatsheet-linux-ru.md \
|
|
--operator-cheatsheet /build/remote-worker-cheatsheet-docker-ru.md \
|
|
--build-inputs /build/worker-package-pins.json
|
|
|
|
FROM worker-native-dependencies AS worker
|
|
|
|
ENV GIT_EXEC_PATH=/opt/truf-worker/runtime/git/libexec/git-core \
|
|
GIT_TEMPLATE_DIR=/opt/truf-worker/runtime/git/share/git-core/templates
|
|
COPY --from=worker-package-build --chown=0:0 /opt/truf-worker /opt/truf-worker
|
|
RUN chown -R 10001:10001 /opt/truf-worker/app \
|
|
&& find /opt/truf-worker/app -type d -exec chmod 0700 {} + \
|
|
&& find /opt/truf-worker/app -type f -exec chmod 0600 {} + \
|
|
&& chown 10001:10001 /opt/truf-worker/worker-package.json \
|
|
&& chmod 0600 /opt/truf-worker/worker-package.json \
|
|
&& find /opt/truf-worker/bin /opt/truf-worker/runtime -type d -exec chmod 0755 {} + \
|
|
&& find /opt/truf-worker/bin /opt/truf-worker/runtime -type f -exec chmod go-w {} + \
|
|
&& test ! -e /opt/truf-worker/app/keycheck_runner.py \
|
|
&& test ! -d /opt/truf-worker/app/keycheckers \
|
|
&& test ! -e /usr/lib/postgresql \
|
|
&& test ! -e /usr/bin/psql
|
|
USER 10001:10001
|
|
RUN /opt/truf-worker/bin/trufflehog --version >/dev/null \
|
|
&& /opt/truf-worker/runtime/git/bin/git --version >/dev/null \
|
|
&& /usr/local/bin/python3 -I -S -B - <<'PY'
|
|
import sys
|
|
sys.path[:0] = ['/opt/truf-worker/app', '/opt/truf-worker/app/dependencies']
|
|
from importlib.util import find_spec
|
|
from worker_cli import parse_args
|
|
from worker_package import verify_worker_package
|
|
|
|
package = verify_worker_package('/opt/truf-worker/worker-package.json')
|
|
assert package['manifest']['schema'] == 3
|
|
assert package['manifest']['protocol_version'] == 2
|
|
assert {
|
|
(item['source'], item['platform'], item['planning_kind'])
|
|
for item in package['manifest']['capabilities']
|
|
} == {
|
|
('gitlab', 'gitlab', 'exact_git_v1'),
|
|
('dockerhub', 'docker', 'docker_direct_v1'),
|
|
('huggingface', 'huggingface', 'huggingface_space_v1'),
|
|
}
|
|
assert set(package['runtime_trees']) == {'git'}
|
|
assert parse_args(['run', '--server', 'https://worker.example', '--token', 'x' * 32]).command == 'run'
|
|
assert all(find_spec(name) is None for name in ('httpx', 'psycopg', 'starlette', 'streamlit'))
|
|
PY
|
|
WORKDIR /data
|
|
ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/python3", "-u", "-I", "-S", "-B", "/opt/truf-worker/app/remote_worker_bootstrap.py", "--"]
|
|
CMD ["run"]
|
|
|
|
FROM python-base AS native-dependencies
|
|
|
|
ADD --checksum=sha256:0144068502a1eddd2a0280ede10ef607d1ec592ce819940991203941564e8e76 https://www.postgresql.org/media/keys/ACCC4CF8.asc /usr/share/keyrings/postgresql.asc
|
|
|
|
RUN <<'SH'
|
|
set -eu
|
|
chmod 0644 /usr/share/keyrings/postgresql.asc
|
|
rm -f /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources
|
|
printf '%s\n' \
|
|
'Types: deb' \
|
|
'URIs: https://snapshot.debian.org/archive/debian/20260914T000000Z/' \
|
|
'Suites: bookworm bookworm-updates' \
|
|
'Components: main' \
|
|
'Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg' \
|
|
'Check-Valid-Until: no' \
|
|
'' \
|
|
'Types: deb' \
|
|
'URIs: https://snapshot.debian.org/archive/debian-security/20260914T000000Z/' \
|
|
'Suites: bookworm-security' \
|
|
'Components: main' \
|
|
'Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg' \
|
|
'Check-Valid-Until: no' \
|
|
> /etc/apt/sources.list.d/debian.sources
|
|
printf '%s\n' \
|
|
'deb [signed-by=/usr/share/keyrings/postgresql.asc] https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main' \
|
|
> /etc/apt/sources.list.d/postgresql.list
|
|
# Only these exact PGDG packages may supplement the immutable Debian snapshot.
|
|
printf '%s\n' \
|
|
'Package: postgresql-16 postgresql-client-16 libpq5' \
|
|
'Pin: version 16.15-1.pgdg12+2' \
|
|
'Pin-Priority: 1001' \
|
|
'' \
|
|
'Package: postgresql-common postgresql-client-common' \
|
|
'Pin: version 293.pgdg12+1' \
|
|
'Pin-Priority: 1001' \
|
|
'' \
|
|
'Package: *' \
|
|
'Pin: origin apt-archive.postgresql.org' \
|
|
'Pin-Priority: -1' \
|
|
> /etc/apt/preferences.d/postgresql
|
|
printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d
|
|
chmod 0755 /usr/sbin/policy-rc.d
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get -o Acquire::Retries=3 -o Acquire::https::Timeout=30 -o APT::Update::Error-Mode=any update
|
|
apt-get install -y --no-install-recommends \
|
|
postgresql-common=293.pgdg12+1 \
|
|
postgresql-client-common=293.pgdg12+1
|
|
# Set this after common is installed but before installing any server package.
|
|
printf '\ncreate_main_cluster = false\n' >> /etc/postgresql-common/createcluster.conf
|
|
apt-get install -y --no-install-recommends \
|
|
ca-certificates=20250419~deb12u1 \
|
|
git=1:2.39.5-0+deb12u3 \
|
|
tini=0.19.0-1+b3 \
|
|
postgresql-16=16.15-1.pgdg12+2 \
|
|
postgresql-client-16=16.15-1.pgdg12+2 \
|
|
libpq5=16.15-1.pgdg12+2
|
|
test ! -d /var/lib/postgresql/16/main
|
|
rm -f /etc/ssl/private/ssl-cert-snakeoil.key /etc/ssl/certs/ssl-cert-snakeoil.pem
|
|
rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /var/log/apt/*
|
|
/usr/sbin/groupadd --gid 10001 truf
|
|
/usr/sbin/useradd --uid 10001 --gid 10001 --no-create-home --home-dir /data/home --shell /usr/sbin/nologin truf
|
|
install -d -o 10001 -g 10001 -m 0700 /data /data/home
|
|
SH
|
|
|
|
RUN python3 -I -S -B - <<'PY'
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# Keep the package's complete Git helper tree; regular hard links preserve argv[0].
|
|
for directory in (Path("/usr/local/bin"), Path("/usr/lib/git-core")):
|
|
for path in directory.iterdir():
|
|
if path.is_symlink() and os.access(path, os.X_OK):
|
|
target = path.resolve(strict=True)
|
|
if not target.is_file() or target.stat().st_uid != 0:
|
|
raise SystemExit(f"Untrusted executable target: {path}")
|
|
path.unlink()
|
|
os.link(target, path)
|
|
# Prefer native PG16 clients over the distribution's symlinked version wrappers.
|
|
for target in Path("/usr/lib/postgresql/16/bin").iterdir():
|
|
path = Path("/usr/bin") / target.name
|
|
if path.is_symlink():
|
|
path.unlink()
|
|
os.link(target, path)
|
|
for name in ("/usr/local/bin/python3", "/usr/bin/git", "/usr/lib/git-core/git-remote-https", "/usr/bin/tini"):
|
|
path = Path(name)
|
|
details = path.lstat()
|
|
if path.is_symlink() or not path.is_file() or details.st_uid != 0 or details.st_mode & 0o022:
|
|
raise SystemExit(f"Untrusted native executable: {path}")
|
|
PY
|
|
|
|
FROM native-dependencies AS dependencies
|
|
|
|
COPY docker/requirements.lock /tmp/requirements.lock
|
|
RUN python3 -m pip --isolated install --index-url=https://pypi.org/simple \
|
|
--require-hashes --only-binary=:all: --no-compile --no-cache-dir \
|
|
-r /tmp/requirements.lock \
|
|
&& python3 -m pip --isolated check \
|
|
&& rm /tmp/requirements.lock
|
|
USER 10001:10001
|
|
|
|
# Both public targets inherit these exact runtime contents; the default stays runtime.
|
|
FROM dependencies AS runtime-base
|
|
|
|
USER 0:0
|
|
RUN install -d -o 10001 -g 10001 -m 0700 /opt/truf /opt/truf/app /opt/truf/tests
|
|
USER 10001:10001
|
|
COPY --chown=10001:10001 app/ /opt/truf/app/
|
|
RUN python3 -I -S -B - <<'PY'
|
|
import os
|
|
import stat
|
|
|
|
for directory, directories, files in os.walk("/opt/truf/app", followlinks=False):
|
|
for path in [directory, *(os.path.join(directory, name) for name in directories + files)]:
|
|
details = os.lstat(path)
|
|
if not (stat.S_ISDIR(details.st_mode) or stat.S_ISREG(details.st_mode)):
|
|
raise SystemExit(f"Application links/special files are forbidden: {path}")
|
|
if os.path.basename(path) == "__pycache__" or path.endswith((".pyc", ".pyo")):
|
|
raise SystemExit(f"Application bytecode is forbidden: {path}")
|
|
if (details.st_uid, details.st_gid) != (10001, 10001):
|
|
raise SystemExit(f"Application ownership mismatch: {path}")
|
|
os.chmod(path, 0o700 if stat.S_ISDIR(details.st_mode) else 0o600)
|
|
PY
|
|
WORKDIR /opt/truf/app
|
|
ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/python3", "-u", "-I", "-S", "-B", "/opt/truf/app/container_runtime.py"]
|
|
CMD ["run"]
|
|
|
|
FROM runtime-base AS test
|
|
|
|
USER 0:0
|
|
COPY --from=trufflehog-download --chown=0:0 --chmod=0755 /trufflehog /usr/local/bin/trufflehog
|
|
COPY docker/requirements-test.lock /tmp/requirements-test.lock
|
|
RUN python3 -m pip --isolated install --index-url=https://pypi.org/simple \
|
|
--require-hashes --only-binary=:all: --no-compile --no-cache-dir \
|
|
-r /tmp/requirements-test.lock \
|
|
&& python3 -m pip --isolated check \
|
|
&& rm /tmp/requirements-test.lock
|
|
USER 10001:10001
|
|
COPY --chown=10001:10001 tests/ /opt/truf/tests/
|
|
COPY --chown=10001:10001 --chmod=0600 .dockerignore /opt/truf/.dockerignore
|
|
COPY --chown=10001:10001 --chmod=0600 Dockerfile /opt/truf/Dockerfile
|
|
COPY --chown=10001:10001 --chmod=0600 start_runtime.ps1 start_core_runtime.ps1 stop_runtime.ps1 /opt/truf/
|
|
COPY --chown=10001:10001 --chmod=0600 compose.yaml compose.edge.yaml /opt/truf/
|
|
COPY --chown=10001:10001 deploy/ /opt/truf/deploy/
|
|
COPY --chown=10001:10001 docker/ /opt/truf/docker/
|
|
RUN python3 -I -S -B - <<'PY'
|
|
import os
|
|
from pathlib import Path
|
|
import stat
|
|
|
|
edge_e2e = {
|
|
path.name for path in Path('/opt/truf/tests').glob('edge_e2e_*.py')
|
|
}
|
|
if edge_e2e != {'edge_e2e_backend.py', 'edge_e2e_client.py'}:
|
|
raise SystemExit(f'Unexpected edge E2E test-stage inputs: {sorted(edge_e2e)}')
|
|
for directory, directories, files in os.walk("/opt/truf/tests", followlinks=False):
|
|
for path in [directory, *(os.path.join(directory, name) for name in directories + files)]:
|
|
details = os.lstat(path)
|
|
if not (stat.S_ISDIR(details.st_mode) or stat.S_ISREG(details.st_mode)):
|
|
raise SystemExit(f"Test links/special files are forbidden: {path}")
|
|
if os.path.basename(path) == "__pycache__" or path.endswith((".pyc", ".pyo")):
|
|
raise SystemExit(f"Test bytecode is forbidden: {path}")
|
|
if (details.st_uid, details.st_gid) != (10001, 10001):
|
|
raise SystemExit(f"Test ownership mismatch: {path}")
|
|
os.chmod(path, 0o700 if stat.S_ISDIR(details.st_mode) else 0o600)
|
|
PY
|
|
WORKDIR /opt/truf
|
|
ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/python3", "-u", "-I", "-S", "-B", "/opt/truf/tests/container_unit.py"]
|
|
CMD []
|
|
|
|
FROM runtime-base AS runtime
|