JMR-dev/android-assistant
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

Ensure python builds from source and fix linting/formatting errors

Browse Source
This commit is contained in:
Jason Ross
2026-03-20 20:29:13 -05:00
parent 3a5d4e7dbd
commit 298819fc91
8 changed files with 4017 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci/config.py
+28 -14
View File
@@ -4,6 +4,15 @@ from dataclasses import dataclass, field
from pathlib import Path
@dataclass(frozen=True)
class PythonBuildConfig:
"""Configuration for building Python from source."""
version: str = "3.14.3"
source_url: str = "https://www.python.org/ftp/python/3.14.3/Python-3.14.3.tgz"
sha256: str = "d7fe130d0501ae047ca318fa92aa642603ab6f217901015a1df6ce650d5470cd"
@dataclass(frozen=True)
class DistroConfig:
"""Configuration for a Linux distribution build."""
@@ -22,12 +31,15 @@ class PipelineConfig:
"""Top-level pipeline configuration."""
fpm_version: str = "1.16.0"
project_root: Path = field(default_factory=lambda: Path(__file__).parent.parent.resolve())
project_root: Path = field(
default_factory=lambda: Path(__file__).parent.parent.resolve()
)
python_build: PythonBuildConfig = field(default_factory=PythonBuildConfig)
# Container images (from GHCR)
debian_image: str = "ghcr.io/jmr-dev/android-file-handler-debian-builder:debian13-trixie"
arch_image: str = "ghcr.io/jmr-dev/android-file-handler-arch-builder:latest"
rhel_image: str = "ghcr.io/jmr-dev/android-file-handler-rhel-builder:fedora42"
# Base container images (Python and tools are installed by the pipeline)
debian_image: str = "debian:13"
arch_image: str = "archlinux:latest"
rhel_image: str = "fedora:42"
@property
def distros(self) -> list[DistroConfig]:
@@ -63,12 +75,14 @@ class PipelineConfig:
]
# Artifact patterns for each distro
artifact_patterns: dict[str, list[str]] = field(default_factory=lambda: {
"debian": ["dist/android-file-handler_*.deb"],
"arch": ["dist/android-file-handler-*.pkg.tar.zst"],
"rhel": ["dist/android-file-handler-*.rpm"],
"windows": [
"dist/android-file-handler-windows.exe",
"dist/android-file-handler-setup.exe",
],
})
artifact_patterns: dict[str, list[str]] = field(
default_factory=lambda: {
"debian": ["dist/android-file-handler_*.deb"],
"arch": ["dist/android-file-handler-*.pkg.tar.zst"],
"rhel": ["dist/android-file-handler-*.rpm"],
"windows": [
"dist/android-file-handler-windows.exe",
"dist/android-file-handler-setup.exe",
],
}
)
ci/dagger_pipeline.py
+218 -15
View File
@@ -1,14 +1,18 @@
"""Dagger pipeline for building Linux distribution packages.
Uses the Dagger Python SDK to run containerized builds for each
Linux distribution (Debian, Arch, RHEL) using pre-built builder images.
Linux distribution (Debian, Arch, RHEL) starting from base OS images,
downloading and compiling Python from source with SHA256 verification,
and building the application packages.
"""
# pyright: reportUnknownMemberType=false
# pyright: reportUnknownVariableType=false
# pyright: reportUnknownArgumentType=false
# pyright: reportUnknownParameterType=false
import asyncio
import os
import sys
from pathlib import Path
@@ -17,33 +21,226 @@ import dagger # type: ignore[import-not-found]
from ci.config import DistroConfig, PipelineConfig
def _get_registry_token() -> str | None:
"""Read GHCR token from GITHUB_TOKEN or GHCR_TOKEN environment variable."""
return os.environ.get("GITHUB_TOKEN") or os.environ.get("GHCR_TOKEN")
def _get_system_deps_cmd(distro_type: str) -> list[str]:
"""Get the shell command to install system build dependencies.
Args:
distro_type: One of 'debian', 'arch', 'rhel'.
Returns:
Shell command as list for with_exec.
"""
if distro_type == "debian":
return [
"sh",
"-c",
"apt-get update && apt-get install -y --no-install-recommends "
"curl git build-essential ruby ruby-dev gcc make "
"zlib1g-dev ca-certificates tcl-dev tk-dev "
"libx11-6 libxext6 libxrender1 libxcb1 "
"libbz2-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev "
"wget tar liblzma-dev patch && "
"apt-get clean && rm -rf /var/lib/apt/lists/*",
]
elif distro_type == "arch":
return [
"sh",
"-c",
"pacman -Syu --noconfirm "
"ruby ruby-bundler ruby-rake base-devel curl git tar "
"ca-certificates ca-certificates-utils "
"tk tcl libx11 libxext libxrender libxcb "
"gcc make zlib bzip2 readline sqlite openssl libffi "
"wget xz patch && "
"update-ca-trust && pacman -Scc --noconfirm",
]
elif distro_type == "rhel":
return [
"sh",
"-c",
"dnf -y update && dnf -y install "
"gcc make zlib-devel bzip2 bzip2-devel readline-devel "
"sqlite-devel openssl-devel libffi-devel wget tar git curl "
"ruby rubygems rpm-build redhat-rpm-config gcc-c++ patch which "
"xz-devel tk-devel tcl-devel libX11-devel libXext-devel "
"libXrender-devel && dnf clean all",
]
raise ValueError(f"Unknown distro type: {distro_type}")
def _get_python_configure_env(distro_type: str) -> str:
"""Get distro-specific LDFLAGS and CPPFLAGS for Python configure.
Args:
distro_type: One of 'debian', 'arch', 'rhel'.
Returns:
String with environment variable exports for the configure step.
"""
if distro_type == "debian":
return 'LDFLAGS="-L/usr/lib/x86_64-linux-gnu" CPPFLAGS="-I/usr/include/tcl8.6"'
elif distro_type == "arch":
return 'LDFLAGS="-L/usr/lib" CPPFLAGS="-I/usr/include"'
elif distro_type == "rhel":
return 'LDFLAGS="-L/usr/lib64" CPPFLAGS="-I/usr/include"'
return ""
def _install_fpm(
container: dagger.Container,
distro_type: str,
fpm_version: str,
) -> dagger.Container:
"""Install fpm (Effing Package Management) in the container.
Args:
container: Dagger container to install fpm in.
distro_type: One of 'debian', 'arch', 'rhel'.
fpm_version: Version of fpm to install.
Returns:
Container with fpm installed.
"""
if distro_type == "arch":
container = container.with_exec(
["gem", "install", "--no-document", "erb"]
).with_exec(
[
"sh",
"-c",
f'gem install --no-document -v "{fpm_version}" fpm && '
"GEM_BIN_DIR=$(ruby -e 'puts Gem.user_dir')/bin && "
'ln -sf "${GEM_BIN_DIR}/fpm" /usr/local/bin/fpm',
]
)
else:
container = container.with_exec(
["gem", "install", "--no-document", "-v", fpm_version, "fpm"]
)
return container
async def build_linux_distro(
client: dagger.Client,
config: PipelineConfig,
distro: DistroConfig,
registry_token: str | None = None,
) -> dict[str, Path]:
"""Build a single Linux distribution package inside a Dagger container.
Starts from a base OS image, compiles Python from source with SHA256
verification, installs build tools (Poetry, fpm), and builds the package.
Args:
client: Active Dagger client connection.
config: Pipeline configuration.
distro: Distribution-specific build configuration.
registry_token: Optional registry auth token.
Returns:
Dictionary mapping artifact names to their local output paths.
"""
print(f"[dagger] Starting {distro.name} build using {distro.container_image}")
# Mount the project source into the container, excluding local venv
source = client.host().directory(
str(config.project_root),
exclude=[".venv", "__pycache__", "dist", "dist_*", "pkg_dist_*", ".git"],
)
python = config.python_build
configure_env = _get_python_configure_env(distro.distro_type)
base = client.container()
if registry_token:
secret = client.set_secret("ghcr_token", registry_token)
base = base.with_registry_auth("ghcr.io", "_token", secret)
# Start from base image and install system dependencies
container = base.from_(distro.container_image).with_exec(
_get_system_deps_cmd(distro.distro_type)
)
# Download Python source and verify SHA256 against python.org
print(f"[dagger] Downloading Python {python.version} and verifying SHA256")
container = container.with_exec(
["wget", "-q", python.source_url, "-O", f"/tmp/Python-{python.version}.tgz"]
).with_exec(
[
"sh",
"-c",
f'echo "{python.sha256} /tmp/Python-{python.version}.tgz" '
f"| sha256sum -c -",
]
)
# Build and install Python from source
print(f"[dagger] Compiling Python {python.version} from source")
container = (
client.container()
.from_(distro.container_image)
.with_directory("/workspace", source)
container.with_exec(
["tar", "xzf", f"/tmp/Python-{python.version}.tgz", "-C", "/tmp"]
)
.with_exec(
[
"sh",
"-c",
f"cd /tmp/Python-{python.version} && "
f"{configure_env} ./configure --enable-shared "
f"--with-ensurepip=install --prefix=/usr/local && "
f"make -j$(nproc) && "
f"make install",
]
)
.with_exec(
[
"sh",
"-c",
'echo "/usr/local/lib" > /etc/ld.so.conf.d/python.conf && ldconfig',
]
)
.with_exec(["ln", "-sf", "/usr/local/bin/python3", "/usr/local/bin/python"])
.with_exec(
[
"sh",
"-c",
f"rm -rf /tmp/Python-{python.version} "
f"/tmp/Python-{python.version}.tgz",
]
)
.with_exec(
[
"python3",
"-c",
"import tkinter; import _tkinter; print('tkinter support verified')",
]
)
)
# Install Poetry
container = (
container.with_exec(
[
"sh",
"-c",
"curl -sSL https://install.python-poetry.org | python3 - --yes",
]
)
.with_env_variable(
"PATH", "/root/.local/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
)
.with_exec(["poetry", "--version"])
)
# Install fpm
container = _install_fpm(container, distro.distro_type, config.fpm_version)
# Mount workspace and run build
container = (
container.with_directory("/workspace", source)
.with_workdir("/workspace")
.with_env_variable("CI_CD", "true")
.with_env_variable("DISTRO_TYPE", distro.distro_type)
@@ -63,7 +260,9 @@ async def build_linux_distro(
str(pkg_dist_output)
)
print(f"[dagger] {distro.name} build complete — artifacts exported to {dist_output}")
print(
f"[dagger] {distro.name} build complete — artifacts exported to {dist_output}"
)
return {
"dist": dist_output,
@@ -71,7 +270,9 @@ async def build_linux_distro(
}
async def build_all_linux(config: PipelineConfig | None = None) -> dict[str, dict[str, Path]]:
async def build_all_linux(
config: PipelineConfig | None = None,
) -> dict[str, dict[str, Path]]:
"""Build all Linux distribution packages in parallel via Dagger.
Args:
@@ -85,16 +286,18 @@ async def build_all_linux(config: PipelineConfig | None = None) -> dict[str, dic
results: dict[str, dict[str, Path]] = {}
token = _get_registry_token()
async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
tasks = {
distro.distro_type: build_linux_distro(client, config, distro)
distro.distro_type: build_linux_distro(
client, config, distro, registry_token=token
)
for distro in config.distros
}
# Run all distro builds concurrently
completed = await asyncio.gather(
*tasks.values(), return_exceptions=True
)
completed = await asyncio.gather(*tasks.values(), return_exceptions=True)
for distro_type, result in zip(tasks.keys(), completed):
if isinstance(result, Exception):
@@ -120,16 +323,16 @@ async def build_single_linux(
if config is None:
config = PipelineConfig()
distro = next(
(d for d in config.distros if d.distro_type == distro_type), None
)
distro = next((d for d in config.distros if d.distro_type == distro_type), None)
if distro is None:
raise ValueError(
f"Unknown distro type '{distro_type}'. Valid: debian, arch, rhel"
)
token = _get_registry_token()
async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
return await build_linux_distro(client, config, distro)
return await build_linux_distro(client, config, distro, registry_token=token)
if __name__ == "__main__":
ci/prefect_flow.py
+10 -7
View File
@@ -22,7 +22,6 @@ import asyncio
import argparse
import os
import subprocess
import sys
from pathlib import Path
from prefect import flow, task
@@ -31,14 +30,13 @@ from ci.config import PipelineConfig
from ci.dagger_pipeline import build_all_linux
from ci.signing import sign_and_hash
# ---------------------------------------------------------------------------
# Tasks
# ---------------------------------------------------------------------------
@task(name="build-linux-distros", retries=1, retry_delay_seconds=30)
def task_build_linux(config: PipelineConfig) -> dict:
def task_build_linux(config: PipelineConfig) -> dict[str, dict[str, Path]]: # type: ignore[type-arg]
"""Build all Linux distribution packages via Dagger containers."""
print("=== Building Linux packages via Dagger ===")
results = asyncio.run(build_all_linux(config))
@@ -116,6 +114,7 @@ def task_prepare_release_files(dist_dir: Path, release_dir: Path) -> list[Path]:
dst = release_dir / src.name
if not dst.exists() or src.stat().st_mtime > dst.stat().st_mtime:
import shutil
shutil.copy2(src, dst)
copied.append(dst)
print(f" {src.name}")
@@ -146,8 +145,12 @@ def task_create_github_release(
env = {**os.environ, "GH_TOKEN": github_token}
cmd = [
"gh", "release", "create", tag,
"--title", f"Release {tag}",
"gh",
"release",
"create",
tag,
"--title",
f"Release {tag}",
"--latest",
] + [str(f) for f in files]
@@ -190,10 +193,10 @@ def task_upload_s3(
@flow(name="build-linux-flow", log_prints=True)
def flow_build_linux() -> dict:
def flow_build_linux() -> dict[str, dict[str, Path]]: # type: ignore[type-arg]
"""Build all Linux distribution packages."""
config = PipelineConfig()
return task_build_linux(config)
return task_build_linux(config) # type: ignore[return-value]
@flow(name="sign-flow", log_prints=True)
ci/signing.py
+7 -3
View File
@@ -31,9 +31,13 @@ def gpg_sign_file(file_path: Path, passphrase: str) -> Path:
subprocess.run(
[
"gpg", "--batch", "--yes",
"--passphrase-file", passfile.name,
"--detach-sign", "--armor",
"gpg",
"--batch",
"--yes",
"--passphrase-file",
passfile.name,
"--detach-sign",
"--armor",
str(file_path),
],
check=True,
poetry.lock
Generated
+3697 -15
View File
File diff suppressed because it is too large Load Diff
scripts/docker/Dockerfile.arch
+19 -15
View File
@@ -9,8 +9,10 @@
# For reproducibility, pin to a specific date tag like: archlinux:base-20251016
FROM archlinux:latest
# Set build argument for fpm version (can be overridden at build time)
# Set build arguments
ARG FPM_VERSION=1.16.0
ARG PYTHON_VERSION=3.14.3
ARG PYTHON_SHA256=d7fe130d0501ae047ca318fa92aa642603ab6f217901015a1df6ce650d5470cd
# Install system dependencies (Arch) including Python build dependencies
RUN pacman -Syu --noconfirm \
@@ -56,24 +58,26 @@ RUN gem install --no-document -v "${FPM_VERSION}" fpm && \
# Install pyenv
ENV PYENV_ROOT="/root/.pyenv"
ENV PATH="$PYENV_ROOT/bin:$PATH"
# Download Python source and verify SHA256 checksum against python.org
RUN wget -q "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" \
-O /tmp/Python-${PYTHON_VERSION}.tgz && \
echo "${PYTHON_SHA256} /tmp/Python-${PYTHON_VERSION}.tgz" | sha256sum -c -
RUN git clone https://github.com/pyenv/pyenv.git /root/.pyenv
# Install Python 3.12 via pyenv with tkinter support
# The tk and tcl packages must be installed before this step for _tkinter to be compiled
RUN eval "$(pyenv init -)" && \
# Build and install Python from source
RUN cd /tmp && tar xzf Python-${PYTHON_VERSION}.tgz && \
cd Python-${PYTHON_VERSION} && \
LDFLAGS="-L/usr/lib" \
CPPFLAGS="-I/usr/include" \
PYTHON_CONFIGURE_OPTS="--enable-shared" \
pyenv install 3.13 && \
pyenv global 3.13 && \
pyenv rehash
./configure --enable-shared --with-ensurepip=install --prefix=/usr/local && \
make -j$(nproc) && \
make install && \
echo "/usr/local/lib" > /etc/ld.so.conf.d/python.conf && \
ldconfig && \
ln -sf /usr/local/bin/python3 /usr/local/bin/python && \
rm -rf /tmp/Python-${PYTHON_VERSION} /tmp/Python-${PYTHON_VERSION}.tgz
# Update PATH to include pyenv shims
ENV PATH="/root/.pyenv/shims:$PATH"
# Update PATH to include Python installation
ENV PATH="/usr/local/bin:$PATH"
# Verify Python has tkinter support
RUN python3 -c "import tkinter; import _tkinter; print('tkinter support verified')" || \
scripts/docker/Dockerfile.debian
+19 -15
View File
@@ -8,8 +8,10 @@
# Use Debian 13 "Trixie" (latest stable release)
FROM debian:13
# Set build argument for fpm version (can be overridden at build time)
# Set build arguments
ARG FPM_VERSION=1.16.0
ARG PYTHON_VERSION=3.14.3
ARG PYTHON_SHA256=d7fe130d0501ae047ca318fa92aa642603ab6f217901015a1df6ce650d5470cd
# Install system dependencies including Python build dependencies
RUN apt-get update && \
@@ -41,24 +43,26 @@ RUN apt-get update && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Install pyenv
ENV PYENV_ROOT="/root/.pyenv"
ENV PATH="$PYENV_ROOT/bin:$PATH"
# Download Python source and verify SHA256 checksum against python.org
RUN wget -q "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" \
-O /tmp/Python-${PYTHON_VERSION}.tgz && \
echo "${PYTHON_SHA256} /tmp/Python-${PYTHON_VERSION}.tgz" | sha256sum -c -
RUN git clone https://github.com/pyenv/pyenv.git /root/.pyenv
# Install Python 3.12 via pyenv with tkinter support
# The tk8.6-dev package must be installed before this step for _tkinter to be compiled
RUN eval "$(pyenv init -)" && \
# Build and install Python from source
RUN cd /tmp && tar xzf Python-${PYTHON_VERSION}.tgz && \
cd Python-${PYTHON_VERSION} && \
LDFLAGS="-L/usr/lib/x86_64-linux-gnu" \
CPPFLAGS="-I/usr/include/tcl8.6" \
PYTHON_CONFIGURE_OPTS="--enable-shared" \
pyenv install 3.13 && \
pyenv global 3.13 && \
pyenv rehash
./configure --enable-shared --with-ensurepip=install --prefix=/usr/local && \
make -j$(nproc) && \
make install && \
echo "/usr/local/lib" > /etc/ld.so.conf.d/python.conf && \
ldconfig && \
ln -sf /usr/local/bin/python3 /usr/local/bin/python && \
rm -rf /tmp/Python-${PYTHON_VERSION} /tmp/Python-${PYTHON_VERSION}.tgz
# Update PATH to include pyenv shims
ENV PATH="/root/.pyenv/shims:$PATH"
# Update PATH to include Python installation
ENV PATH="/usr/local/bin:$PATH"
# Verify Python has tkinter support
RUN python3 -c "import tkinter; import _tkinter; print('tkinter support verified')" || \
scripts/docker/Dockerfile.rhel
+19 -15
View File
@@ -7,8 +7,10 @@
FROM fedora:42
# Set build argument for fpm version (can be overridden at build time)
# Set build arguments
ARG FPM_VERSION=1.16.0
ARG PYTHON_VERSION=3.14.3
ARG PYTHON_SHA256=d7fe130d0501ae047ca318fa92aa642603ab6f217901015a1df6ce650d5470cd
# Install system dependencies including tk8-devel for Python tkinter support
# Using tk8 (version 8.6) instead of tk (version 9.0) for Python 3.12 compatibility
@@ -42,24 +44,26 @@ RUN dnf -y update && \
libXrender-devel && \
dnf clean all
# Install pyenv
ENV PYENV_ROOT="/root/.pyenv"
ENV PATH="$PYENV_ROOT/bin:$PATH"
# Download Python source and verify SHA256 checksum against python.org
RUN wget -q "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" \
-O /tmp/Python-${PYTHON_VERSION}.tgz && \
echo "${PYTHON_SHA256} /tmp/Python-${PYTHON_VERSION}.tgz" | sha256sum -c -
RUN git clone https://github.com/pyenv/pyenv.git /root/.pyenv
# Install Python 3.13 via pyenv with tkinter support
# The tk8-devel package must be installed before this step for _tkinter to be compiled
RUN eval "$(pyenv init -)" && \
# Build and install Python from source
RUN cd /tmp && tar xzf Python-${PYTHON_VERSION}.tgz && \
cd Python-${PYTHON_VERSION} && \
LDFLAGS="-L/usr/lib64" \
CPPFLAGS="-I/usr/include" \
PYTHON_CONFIGURE_OPTS="--enable-shared" \
pyenv install 3.13 && \
pyenv global 3.13 && \
pyenv rehash
./configure --enable-shared --with-ensurepip=install --prefix=/usr/local && \
make -j$(nproc) && \
make install && \
echo "/usr/local/lib" > /etc/ld.so.conf.d/python.conf && \
ldconfig && \
ln -sf /usr/local/bin/python3 /usr/local/bin/python && \
rm -rf /tmp/Python-${PYTHON_VERSION} /tmp/Python-${PYTHON_VERSION}.tgz
# Update PATH to include pyenv shims
ENV PATH="/root/.pyenv/shims:$PATH"
# Update PATH to include Python installation
ENV PATH="/usr/local/bin:$PATH"
# Verify Python has tkinter support
RUN python3 -c "import tkinter; import _tkinter; print('tkinter support verified')" || \