diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c2c4007..44f37db 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -2,6 +2,7 @@ name: Integration Tests env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + BOOTSTRAP_CI: 'true' on: push: diff --git a/.github/workflows/macos-dispatch.yml b/.github/workflows/macos-dispatch.yml index 2e51ffe..ef46c9a 100644 --- a/.github/workflows/macos-dispatch.yml +++ b/.github/workflows/macos-dispatch.yml @@ -2,6 +2,7 @@ name: macOS Manual Integration env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' + BOOTSTRAP_CI: 'true' on: workflow_dispatch: diff --git a/ci/main.go b/ci/main.go index 9bf9961..13e74e0 100644 --- a/ci/main.go +++ b/ci/main.go @@ -47,10 +47,12 @@ func main() { targets = []string{"archlinux:latest"} case "fedora": targets = []string{"fedora:latest"} + case "ubuntu": + targets = []string{"ubuntu:latest"} case "all": - targets = []string{"debian:latest", "archlinux:latest", "fedora:latest"} + targets = []string{"debian:latest", "archlinux:latest", "fedora:latest", "ubuntu:latest"} default: - fmt.Fprintf(os.Stderr, "Unsupported OS: %s. Supported: debian, arch, fedora, all\n", *osFlag) + fmt.Fprintf(os.Stderr, "Unsupported OS: %s. Supported: debian, arch, fedora, ubuntu, all\n", *osFlag) os.Exit(1) } @@ -63,23 +65,28 @@ func main() { // 1. Prepare target container base and setup script based on OS distro var testContainer *dagger.Container - if strings.Contains(target, "debian") { + if strings.Contains(target, "debian") || strings.Contains(target, "ubuntu") { testContainer = client.Container(). From(target). WithExec([]string{"apt-get", "update"}). - WithExec([]string{"apt-get", "install", "-y", "sudo", "curl", "git", "wget", "tar", "unzip", "xz-utils", "make", "python3", "which"}) + WithExec([]string{"apt-get", "install", "-y", "sudo", "curl", "git", "wget", "tar", "unzip", "xz-utils", "make", "python3", "which", "zstd"}) } else if strings.Contains(target, "fedora") { testContainer = client.Container(). From(target). - WithExec([]string{"dnf", "install", "-y", "sudo", "curl", "git", "wget", "tar", "unzip", "xz", "make", "python3", "which"}) + WithExec([]string{"dnf", "install", "-y", "sudo", "curl", "git", "wget", "tar", "unzip", "xz", "make", "python3", "which", "zstd"}) } else if strings.Contains(target, "archlinux") { testContainer = client.Container(). From(target). - WithExec([]string{"pacman", "-Sy", "--noconfirm", "sudo", "curl", "git", "wget", "tar", "unzip", "xz", "make", "python", "which"}) + WithExec([]string{"pacman", "-Sy", "--noconfirm", "sudo", "curl", "git", "wget", "tar", "unzip", "xz", "make", "python", "which", "zstd"}) } else { return fmt.Errorf("unsupported target OS: %s", target) } + // Mock sync to prevent hangs during package updates/update-shells on overlayfs/FUSE + testContainer = testContainer. + WithExec([]string{"ln", "-sf", "/bin/true", "/bin/sync"}). + WithExec([]string{"ln", "-sf", "/bin/true", "/usr/bin/sync"}) + // 2. Pre-create the ~/.pyenv directory to skip python source compilation in integration test (saves ~10 minutes) testContainer = testContainer.WithExec([]string{"mkdir", "-p", "/root/.pyenv"}) @@ -88,6 +95,10 @@ func main() { WithFile("/usr/local/bin/bootstrap_environment", binaryFile). WithWorkdir("/tmp") + if val := os.Getenv("BOOTSTRAP_CI"); val != "" { + testContainer = testContainer.WithEnvVariable("BOOTSTRAP_CI", val) + } + // Forward GitHub token so API calls are authenticated (avoids 403 rate-limits) if tok := os.Getenv("GITHUB_TOKEN"); tok != "" { secret := client.SetSecret("github-token", tok) @@ -96,21 +107,47 @@ func main() { WithSecretVariable("GH_TOKEN", secret) } - // 4. Run bootstrap binary against the full package set (no scope flags). + // 4. Run bootstrap binary. + // We run it twice: + // Run 1: Step 1 (zsh, git, curl, oh-my-zsh) which configures the default shell and exits. + // Run 2: Step 2 (the rest of the packages) once the shell configuration is complete. // We pipe 'y' to satisfy the "Proceed? [y/N]" prompt. - fmt.Printf("[%s] Executing bootstrap_environment...\n", target) - testContainer = testContainer.WithExec([]string{"sh", "-c", "echo y | bootstrap_environment --gui"}) + var bootstrapCmd []string + if strings.Contains(target, "ubuntu") { + bootstrapCmd = []string{"sh", "-c", "echo y | bootstrap_environment --local-ai"} + } else { + bootstrapCmd = []string{"sh", "-c", "echo y | bootstrap_environment --gui"} + } + + fmt.Printf("[%s] Executing bootstrap_environment (Run 1: ZSH/Step 1)...\n", target) + testContainer = testContainer.WithExec(bootstrapCmd) + + fmt.Printf("[%s] Executing bootstrap_environment (Run 2: Step 2)...\n", target) + testContainer = testContainer.WithExec(bootstrapCmd) // 5. Verify all installed custom packages return a path and zero exit code from version command fmt.Printf("[%s] Verifying package installations on PATH and running version checks...\n", target) - verifyCmd := []string{ - "sh", "-c", - "set -e -x; " + - "export PATH=$PATH:/usr/local/go/bin:/usr/local/bin; " + - "which go && go version && " + - "which nvim && nvim --version && " + - "which zig && zig version && " + - "which firecracker && firecracker --version", + var verifyCmd []string + if strings.Contains(target, "ubuntu") { + verifyCmd = []string{ + "sh", "-c", + "set -e -x; " + + "export PATH=$PATH:/usr/local/bin:/root/.local/bin; " + + "which nvim && nvim --version && " + + "which firecracker && firecracker --version && " + + "which ollama && ollama --version && " + + "which hf && hf --help", + } + } else { + verifyCmd = []string{ + "sh", "-c", + "set -e -x; " + + "export PATH=$PATH:/usr/local/go/bin:/usr/local/bin; " + + "which go && go version && " + + "which nvim && nvim --version && " + + "which zig && zig version && " + + "which firecracker && firecracker --version", + } } verifyOutput, err := testContainer.WithExec(verifyCmd).Stdout(ctx) if err != nil { @@ -118,7 +155,6 @@ func main() { return fmt.Errorf("verification failed on %s: %v", target, err) } - fmt.Printf("[%s] Verification Output:\n%s\n", target, verifyOutput) fmt.Printf("--- PASS: Integration test on %s completed successfully ---\n", target) return nil diff --git a/main.go b/main.go index d14f44e..2489c9d 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,7 @@ func runMain(args []string) { gui := fs.Bool("gui", false, "Include GUI applications (headed environments).") noVM := fs.Bool("no-vm", false, "macOS only: skip provisioning the Fedora-on-QEMU VM that backs the firecracker() zsh wrapper.") noAI := fs.Bool("no-ai", false, "Skip installation of LLM/AI CLI tools (agy, claude, codex, copilot).") + localAI := fs.Bool("local-ai", false, "Ubuntu only: Install local ML inference environment (Nvidia drivers, CUDA, Nvidia Container Toolkit, Ollama, Hugging Face CLI).") _ = fs.Parse(args[1:]) switch *only { @@ -54,22 +55,61 @@ func runMain(args []string) { initPkgMgr() + if *localAI { + if isMacOS || pkgMgr != "apt-get" { + fmt.Fprintln(os.Stderr, "Error: --local-ai flag is only supported on Ubuntu/Debian (apt-get package manager)") + osExit(1) + return + } + } + systemPkgs := append([]string(nil), SystemPackages...) flatpakPkgs := append([]string(nil), FlatpakPackages...) + + if *localAI { + var filteredSys []string + excludeSys := map[string]bool{ + "ansible": true, "ansible-core": true, "buildah": true, + "dotnet-sdk-10.0": true, "helm": true, "kubectl": true, + "minikube": true, "pulumi": true, "sops": true, + "vagrant": true, "virt-manager": true, "wireshark": true, + "zoom": true, "obs-studio": true, "webcamoid": true, + "obsidian": true, + } + for _, p := range systemPkgs { + if !excludeSys[p] { + filteredSys = append(filteredSys, p) + } + } + filteredSys = append(filteredSys, "nvidia-drivers", "cuda-toolkit", "nvidia-container-toolkit", "ollama", "huggingface-cli") + systemPkgs = filteredSys + flatpakPkgs = nil + } + custom := customPackages() customPtrs := make([]*CustomPackage, 0, len(custom)) for i := range custom { + name := strings.ToLower(custom[i].Name) // Drop firecracker on macOS — it's provisioned inside the Fedora VM // (see setupFirecrackerVM), not on the host. - if isMacOS && strings.ToLower(custom[i].Name) == "firecracker" { + if isMacOS && name == "firecracker" { continue } if *noAI { - name := strings.ToLower(custom[i].Name) if name == "agy" || name == "claude" || name == "codex" || name == "copilot" { continue } } + if *localAI { + excludeCust := map[string]bool{ + "go": true, "zig": true, "rustup": true, "dagger": true, + "cosign": true, "agy": true, "claude": true, "copilot": true, + "codex": true, "playwright": true, + } + if excludeCust[name] { + continue + } + } customPtrs = append(customPtrs, &custom[i]) } diff --git a/main_test.go b/main_test.go index cdbdc95..41631f1 100644 --- a/main_test.go +++ b/main_test.go @@ -127,3 +127,43 @@ func TestRunMainMacos(t *testing.T) { t.Error("expected program to complete successfully on macOS mock run") } } + +func TestRunMainLocalAIValidation(t *testing.T) { + defer resetMocks() + + // 1. macOS validation + isMacOS = true + pkgMgr = "brew" + var exited bool + var exitCode int + osExit = func(code int) { + exited = true + exitCode = code + } + + runMain([]string{"bootstrap_environment", "--local-ai"}) + if !exited { + t.Error("expected runMain with --local-ai on macOS to exit") + } + if exitCode != 1 { + t.Errorf("expected exit code 1, got %d", exitCode) + } + + // 2. Linux non-apt validation + resetMocks() + isMacOS = false + pkgMgr = "dnf" + exited = false + osExit = func(code int) { + exited = true + exitCode = code + } + + runMain([]string{"bootstrap_environment", "--local-ai"}) + if !exited { + t.Error("expected runMain with --local-ai on Fedora/dnf to exit") + } + if exitCode != 1 { + t.Errorf("expected exit code 1, got %d", exitCode) + } +} diff --git a/repos.go b/repos.go index ad3fb9d..90ab94d 100644 --- a/repos.go +++ b/repos.go @@ -244,3 +244,42 @@ func repoGroups() []repoGroup { {mk("lazygit"), setupLazygitCoprRepo}, } } + +func setupCUDARepo() { + if pkgMgr != "apt-get" { + return + } + if repoFileExists("/etc/apt/sources.list.d/cuda.list") || repoFileExists("/etc/apt/sources.list.d/cuda-keyring.list") { + return + } + versionID := strings.Trim(osReleaseField("VERSION_ID"), `"`) + ubuntuVer := strings.ReplaceAll(versionID, ".", "") + if ubuntuVer == "" { + ubuntuVer = "2204" + } + debURL := fmt.Sprintf("https://developer.download.nvidia.com/compute/cuda/repos/ubuntu%s/x86_64/cuda-keyring_1.1-1_all.deb", ubuntuVer) + runShell( + fmt.Sprintf("curl -fsSL %s -o /tmp/cuda-keyring.deb && "+ + "sudo dpkg -i /tmp/cuda-keyring.deb && "+ + "sudo apt-get update", debURL), + CmdOpts{}, + ) +} + +func setupNvidiaContainerToolkitRepo() { + if pkgMgr != "apt-get" { + return + } + if repoFileExists("/etc/apt/sources.list.d/nvidia-container-toolkit.list") { + return + } + runShell( + "curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | "+ + "sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg && "+ + "curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | "+ + "sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | "+ + "sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list >/dev/null && "+ + "sudo apt-get update", + CmdOpts{}, + ) +} diff --git a/system.go b/system.go index d3af1d3..9fb633d 100644 --- a/system.go +++ b/system.go @@ -1,10 +1,13 @@ package main import ( + "flag" "fmt" "os" + "os/exec" "path/filepath" "strings" + "time" ) // Special packages: installed outside the regular package manager because @@ -18,6 +21,9 @@ func specialPkgs() map[string]bool { "github-desktop": true, "zoom": true, "obsidian": true, "minikube": true, "pipx": true, "poetry": true, "pulumi": true, "semgrep": true, + "nvidia-drivers": true, "cuda-toolkit": true, + "nvidia-container-toolkit": true, "ollama": true, + "huggingface-cli": true, } } @@ -51,6 +57,16 @@ func isSpecialPkgInstalled(pkg string) bool { return hasCmd("poetry") case "semgrep": return hasCmd("semgrep") + case "nvidia-drivers": + return hasCmd("nvidia-smi") + case "cuda-toolkit": + return hasCmd("nvcc") || exists("/usr/local/cuda/bin/nvcc") + case "nvidia-container-toolkit": + return hasCmd("nvidia-ctk") + case "ollama": + return hasCmd("ollama") + case "huggingface-cli": + return hasCmd("huggingface-cli") } return isSystemPkgInstalled(pkg) } @@ -357,6 +373,158 @@ func installSpecialPkg(pkg, tmp string) { installPoetry(tmp) case "semgrep": installSemgrep(tmp) + case "nvidia-drivers": + installNvidiaDrivers() + case "cuda-toolkit": + installCUDAToolkit() + case "nvidia-container-toolkit": + installNvidiaContainerToolkit() + case "ollama": + installOllama() + case "huggingface-cli": + installHuggingFaceCLI() + } +} + +func installNvidiaDrivers() { + if pkgMgr != "apt-get" { + errLog("nvidia-drivers can only be automatically installed via apt-get on Ubuntu/Debian") + return + } + fmt.Println(" Installing NVIDIA driver (nvidia-driver-550) ...") + res := pkgInstall("nvidia-driver-550") + if !res.OK() { + errLog(fmt.Sprintf("failed to install nvidia-driver-550: %v", res.Err)) + } +} + +func installCUDAToolkit() { + if pkgMgr != "apt-get" { + errLog("cuda-toolkit can only be automatically installed via apt-get on Ubuntu/Debian") + return + } + fmt.Println(" Setting up CUDA repository keyring ...") + setupCUDARepo() + fmt.Println(" Installing cuda-toolkit ...") + res := pkgInstall("cuda-toolkit") + if !res.OK() { + errLog(fmt.Sprintf("failed to install cuda-toolkit: %v", res.Err)) + return + } + fmt.Println(" Configuring system path for CUDA ...") + appendProfileLine("cuda", `export PATH="/usr/local/cuda/bin:$PATH"`) + appendProfileLine("cuda", `export LD_LIBRARY_PATH="/usr/local/cuda/lib64:$LD_LIBRARY_PATH"`) +} + +func installNvidiaContainerToolkit() { + if pkgMgr != "apt-get" { + errLog("nvidia-container-toolkit can only be automatically installed via apt-get on Ubuntu/Debian") + return + } + fmt.Println(" Setting up NVIDIA Container Toolkit repository ...") + setupNvidiaContainerToolkitRepo() + fmt.Println(" Installing nvidia-container-toolkit ...") + res := pkgInstall("nvidia-container-toolkit") + if !res.OK() { + errLog(fmt.Sprintf("failed to install nvidia-container-toolkit: %v", res.Err)) + return + } + fmt.Println(" Configuring Docker runtime for NVIDIA Container Toolkit ...") + configureRes := runCmd([]string{"nvidia-ctk", "runtime", "configure", "--runtime=docker"}, CmdOpts{AsSudo: true}) + if !configureRes.OK() { + warn(fmt.Sprintf("failed to configure docker runtime: %v", configureRes.Err)) + } + fmt.Println(" Restarting Docker service ...") + restartRes := runCmd([]string{"systemctl", "restart", "docker"}, CmdOpts{AsSudo: true}) + if !restartRes.OK() { + warn(fmt.Sprintf("failed to restart docker service: %v", restartRes.Err)) + } +} + +func installOllama() { + fmt.Println(" Installing Ollama via official install script ...") + res := runShell("curl -fsSL https://ollama.com/install.sh | sh", CmdOpts{}) + if !res.OK() { + errLog(fmt.Sprintf("Ollama installation failed: %v", res.Err)) + return + } + + isCI := os.Getenv("BOOTSTRAP_CI") == "true" + if isCI { + fmt.Println(" [CI] Skipping pulling large Ollama models in integration test container.") + return + } + + // Pull Gemma 4 E4B and Qwen2.5-Coder 7B + fmt.Println(" Pulling Gemma 4 E4B and Qwen2.5-Coder 7B models ...") + + isTesting := flag.Lookup("test.v") != nil || os.Getenv("GO_ENV") == "test" + serverRunning := false + + // Check if Ollama server is already responding + for i := 0; i < 5; i++ { + r := runCmd([]string{"ollama", "list"}, CmdOpts{Capture: true}) + if r.OK() { + serverRunning = true + break + } + if isTesting { + break + } + time.Sleep(1 * time.Second) + } + + var cmd *exec.Cmd + if !serverRunning && !isTesting { + fmt.Println(" Ollama server not running. Starting in background for model pulling ...") + cmd = exec.Command("ollama", "serve") + cmd.Stdout = nil + cmd.Stderr = nil + if err := cmd.Start(); err != nil { + warn(fmt.Sprintf("Failed to start Ollama server in background: %v", err)) + } else { + // Wait up to 30 seconds for server to start + for i := 0; i < 30; i++ { + r := runCmd([]string{"ollama", "list"}, CmdOpts{Capture: true}) + if r.OK() { + serverRunning = true + break + } + time.Sleep(1 * time.Second) + } + } + } + + if serverRunning || isTesting { + fmt.Println(" Pulling Gemma 4 E4B (gemma4:e4b) ...") + pull1 := runCmd([]string{"ollama", "pull", "gemma4:e4b"}, CmdOpts{}) + if !pull1.OK() { + errLog(fmt.Sprintf("Failed to pull gemma4:e4b: %v", pull1.Err)) + } + fmt.Println(" Pulling Qwen2.5-Coder 7B (qwen2.5-coder:7b) ...") + pull2 := runCmd([]string{"ollama", "pull", "qwen2.5-coder:7b"}, CmdOpts{}) + if !pull2.OK() { + errLog(fmt.Sprintf("Failed to pull qwen2.5-coder:7b: %v", pull2.Err)) + } + } else { + errLog("Ollama server failed to start — cannot pull models") + } + + if cmd != nil && cmd.Process != nil { + fmt.Println(" Stopping background Ollama server ...") + cmd.Process.Kill() + } +} + +func installHuggingFaceCLI() { + if !hasCmd("pipx") { + errLog("pipx is not installed — cannot install huggingface-cli") + return + } + fmt.Println(" Installing huggingface-cli via pipx ...") + res := runCmd([]string{"pipx", "install", "huggingface_hub[cli]"}, CmdOpts{}) + if !res.OK() { + errLog(fmt.Sprintf("failed to install huggingface-cli: %v", res.Err)) } } diff --git a/system_test.go b/system_test.go index 4b9a5ed..45fd946 100644 --- a/system_test.go +++ b/system_test.go @@ -528,3 +528,69 @@ func TestInstallSystemPackagesAria2First(t *testing.T) { } } +func TestInstallLocalAISpecialPkgs(t *testing.T) { + defer resetMocks() + + pkgMgr = "apt-get" + isMacOS = false + + var runCmdCalls [][]string + runCmd = func(argv []string, opts CmdOpts) CmdResult { + runCmdCalls = append(runCmdCalls, argv) + return CmdResult{ExitCode: 0} + } + + var runShellCalls []string + runShell = func(cmd string, opts CmdOpts) CmdResult { + runShellCalls = append(runShellCalls, cmd) + return CmdResult{ExitCode: 0} + } + + // 1. nvidia-drivers + installSpecialPkg("nvidia-drivers", "/tmp") + if len(runCmdCalls) != 1 || runCmdCalls[0][0] != "apt-get" || runCmdCalls[0][3] != "nvidia-driver-550" { + t.Errorf("expected apt-get install nvidia-driver-550, got calls: %v", runCmdCalls) + } + + // 2. cuda-toolkit + runCmdCalls = nil + runShellCalls = nil + installSpecialPkg("cuda-toolkit", "/tmp") + if len(runCmdCalls) < 1 || runCmdCalls[0][3] != "cuda-toolkit" { + t.Errorf("expected apt-get install cuda-toolkit, got calls: %v", runCmdCalls) + } + if len(runShellCalls) != 1 || !strings.Contains(runShellCalls[0], "cuda-keyring") { + t.Errorf("expected setupCUDARepo runShell call, got calls: %v", runShellCalls) + } + + // 3. nvidia-container-toolkit + runCmdCalls = nil + runShellCalls = nil + installSpecialPkg("nvidia-container-toolkit", "/tmp") + if len(runCmdCalls) < 3 || runCmdCalls[0][3] != "nvidia-container-toolkit" { + t.Errorf("expected apt-get install nvidia-container-toolkit, got calls: %v", runCmdCalls) + } + if len(runShellCalls) != 1 || !strings.Contains(runShellCalls[0], "nvidia-container-toolkit.list") { + t.Errorf("expected setupNvidiaContainerToolkitRepo runShell call, got calls: %v", runShellCalls) + } + + // 4. ollama + runCmdCalls = nil + runShellCalls = nil + installSpecialPkg("ollama", "/tmp") + if len(runShellCalls) != 1 || !strings.Contains(runShellCalls[0], "ollama.com/install.sh") { + t.Errorf("expected ollama installer runShell call, got calls: %v", runShellCalls) + } + + // 5. huggingface-cli + runCmdCalls = nil + runShellCalls = nil + hasCmd = func(name string) bool { + return name == "pipx" + } + installSpecialPkg("huggingface-cli", "/tmp") + if len(runCmdCalls) != 1 || runCmdCalls[0][0] != "pipx" || runCmdCalls[0][2] != "huggingface_hub[cli]" { + t.Errorf("expected pipx install huggingface_hub[cli], got calls: %v", runCmdCalls) + } +} +