deploy.go
package main
import (
"cmp"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/readysite/readysite/pkg/platform"
)
// deployToInstance deploys services to a specific instance.
// binPaths maps binary name to local path (e.g., {"example": "build/example"})
func deployToInstance(
p *platform.Platform,
binPaths interface{}, // string or map[string]string
cfg *InfraConfig,
serverType string,
inst *Instance,
) error {
spec := cfg.Servers[serverType]
log.Printf("Deploying to %s (%s)...", inst.Name, inst.IP)
// Use the instance info directly instead of looking up by name
// (there may be multiple droplets with same name from failed attempts)
server := &platform.Server{
ID: inst.ID,
Name: inst.Name,
IP: inst.IP,
Region: inst.Region,
}
// 1. Ensure volumes attached (if specified)
region := platform.Region(cfg.Platform.Region)
for _, vol := range spec.getVolumes() {
if err := ensureVolume(p, server, &vol, region); err != nil {
return fmt.Errorf("volume setup for %s: %w", vol.Name, err)
}
}
// 2. Run setup.sh for each binary if exists in its source directory
for _, binName := range spec.Binaries {
svcSpec := cfg.Binaries[binName]
if err := runSetupScript(server, svcSpec.Source); err != nil {
return err
}
}
// 3. Build/upload all binary images BEFORE stopping (minimizes downtime)
imageNames := make(map[string]string)
for _, binName := range spec.Binaries {
var localPath string
switch bp := binPaths.(type) {
case string:
localPath = bp
case map[string]string:
localPath = bp[binName]
}
svcSpec := cfg.Binaries[binName]
var imageName string
var err error
if localPath == "" {
// Build on server (local Docker unavailable)
imageName, err = buildOnServer(server, binName, svcSpec.Source)
} else {
// Upload pre-built image
imageName, err = uploadImage(server, localPath)
}
if err != nil {
return fmt.Errorf("prepare %s: %w", binName, err)
}
imageNames[binName] = imageName
}
// 4. Backup and stop existing services (after build, before start - minimal downtime)
// Backup binary containers for potential rollback
for _, svcName := range spec.Binaries {
backupContainer(server, svcName)
}
// Just remove infrastructure services (they use remote images, no rollback needed)
for _, svcName := range spec.Services {
server.SSH("docker", "rm", "-f", svcName)
}
// 5. Create Docker networks
networks := make(map[string]bool)
for _, svcName := range spec.Services {
if n := cfg.Services[svcName].Network; n != "" {
networks[n] = true
}
}
for _, binName := range spec.Binaries {
if n := cfg.Binaries[binName].Network; n != "" {
networks[n] = true
}
}
for network := range networks {
log.Printf(" Creating network %s...", network)
server.SSH("docker", "network", "create", network, "--driver", "bridge")
}
// 6. Start infrastructure services (no binary)
for _, svcName := range spec.Services {
svcSpec := cfg.Services[svcName]
if err := startService(server, svcName, &svcSpec, ""); err != nil {
return fmt.Errorf("start %s: %w", svcName, err)
}
}
// 7. Start binary services with pre-built images
for _, binName := range spec.Binaries {
svcSpec := cfg.Binaries[binName]
if err := startService(server, binName, &svcSpec, imageNames[binName]); err != nil {
return fmt.Errorf("start %s: %w", binName, err)
}
}
log.Printf(" Deployed to %s ✓", inst.Name)
return nil
}
// startService starts a Docker service on a server with health checking and rollback support.
// For binaries, imageName is the locally-built Docker image name.
// For services, imageName is empty and spec.Image is used.
func startService(server *platform.Server, name string, spec *ServiceSpec, imageName string) error {
log.Printf(" Starting %s...", name)
// Use the built image name if provided, otherwise use spec.Image
image := cmp.Or(imageName, spec.Image)
svc := &platform.Service{
Name: name,
Image: image,
Network: spec.Network,
Restart: "always",
Privileged: spec.Privileged,
}
if len(spec.Command) > 0 {
svc.Command = spec.Command
}
// Ports
for _, p := range spec.Ports {
svc.Ports = append(svc.Ports, platform.Port{
Host: p.Host,
Container: p.Container,
Bind: p.Bind,
})
}
// Volumes
for _, v := range spec.Volumes {
svc.Volumes = append(svc.Volumes, platform.Mount{Source: v.Source, Target: v.Target})
}
// Environment variables
svc.Env = make(map[string]string)
for k, v := range spec.Env {
svc.Env[k] = v
}
// Read env_files from server
for envName, filePath := range spec.EnvFiles {
value, err := server.SSH("cat", filePath)
if err != nil {
log.Printf(" Warning: could not read %s from %s", envName, filePath)
continue
}
svc.Env[envName] = strings.TrimSpace(value)
}
// Add health check if ports are exposed
if len(spec.Ports) > 0 {
healthCmd := fmt.Sprintf("curl -sf http://localhost:%d/health || exit 1", spec.Ports[0].Container)
if spec.Healthcheck != "" {
healthCmd = spec.Healthcheck
}
svc.Healthcheck = &platform.Healthcheck{
Cmd: healthCmd,
Interval: "10s",
Timeout: "5s",
Retries: 3,
StartPeriod: "10s",
}
}
if err := server.Start(svc); err != nil {
return err
}
// Wait for container to be healthy
if svc.Healthcheck != nil {
if err := waitForHealthy(server, name, 60); err != nil {
log.Printf(" Health check failed, attempting rollback...")
rollback(server, name)
return fmt.Errorf("health check failed: %w", err)
}
}
return nil
}
// waitForHealthy waits for a container to become healthy.
func waitForHealthy(server *platform.Server, name string, timeoutSecs int) error {
log.Printf(" Waiting for %s to become healthy...", name)
for i := 0; i < timeoutSecs; i += 5 {
status, err := server.SSH("docker", "inspect", "--format={{.State.Health.Status}}", name)
if err == nil {
status = strings.TrimSpace(status)
switch status {
case "healthy":
log.Printf(" %s is healthy", name)
return nil
case "unhealthy":
logs, _ := server.SSH("docker", "logs", "--tail=20", name)
return fmt.Errorf("container unhealthy, logs:\n%s", logs)
}
}
// Still starting or no health status yet
time.Sleep(5 * time.Second)
}
return fmt.Errorf("health check timeout after %ds", timeoutSecs)
}
// backupContainer stops and renames the current container for rollback.
func backupContainer(server *platform.Server, name string) {
backupName := name + "-backup"
// Remove old backup if exists
server.SSH("docker", "rm", "-f", backupName)
// Stop current container (releases ports)
server.SSH("docker", "stop", name)
// Rename current to backup (if it exists)
server.SSH("docker", "rename", name, backupName)
}
// rollback restores the previous container version.
func rollback(server *platform.Server, name string) error {
backupName := name + "-backup"
// Check if backup exists
if _, err := server.SSH("docker", "inspect", backupName); err != nil {
log.Printf(" No backup found for %s, cannot rollback", name)
return nil
}
log.Printf(" Rolling back %s...", name)
// Stop the failed container
server.SSH("docker", "rm", "-f", name)
// Restore the backup
if _, err := server.SSH("docker", "rename", backupName, name); err != nil {
return fmt.Errorf("rename backup: %w", err)
}
// Start the old container
if _, err := server.SSH("docker", "start", name); err != nil {
return fmt.Errorf("start backup: %w", err)
}
log.Printf(" Rolled back to previous version of %s", name)
return nil
}
// ensureVolume creates and mounts a volume if it doesn't exist.
func ensureVolume(p *platform.Platform, server *platform.Server, spec *VolumeSpec, region platform.Region) error {
log.Printf(" Ensuring volume %s...", spec.Name)
vol, err := p.GetVolume(spec.Name)
isNewVolume := err == platform.ErrNotFound
if isNewVolume {
log.Printf(" Creating volume %s (%dGB)...", spec.Name, spec.Size)
vol, err = p.CreateVolume(spec.Name, spec.Size, region)
if err != nil {
return fmt.Errorf("create volume: %w", err)
}
} else if err != nil {
return fmt.Errorf("get volume: %w", err)
}
// Attach volume to server
if err := p.AttachVolume(vol.ID, server.ID); err != nil {
log.Printf(" Volume attach: %v (may already be attached)", err)
}
// Determine filesystem type (default to ext4)
filesystem := cmp.Or(spec.Filesystem, "ext4")
// Format new volumes if needed
devicePath := fmt.Sprintf("/dev/disk/by-id/scsi-0DO_Volume_%s", spec.Name)
if isNewVolume {
log.Printf(" Formatting volume %s as %s...", spec.Name, filesystem)
formatScript := fmt.Sprintf(`
for i in $(seq 1 30); do [ -e %s ] && break; sleep 1; done
if ! blkid %s >/dev/null 2>&1; then mkfs.%s %s; fi
`, devicePath, devicePath, filesystem, devicePath)
if _, err := server.SSH("bash", "-c", formatScript); err != nil {
return fmt.Errorf("format volume: %w", err)
}
}
// Mount and persist in fstab
mountOpts := "defaults,nofail,discard"
if filesystem == "xfs" {
mountOpts = "defaults,nofail,discard,prjquota"
}
mountScript := fmt.Sprintf(`
mkdir -p %s
grep -qF "%s" /etc/fstab || echo "%s %s %s %s 0 2" >> /etc/fstab
mountpoint -q %s || mount %s || true
`, spec.Mount, spec.Name, devicePath, spec.Mount, filesystem, mountOpts, spec.Mount, spec.Mount)
if _, err := server.SSH("bash", "-c", mountScript); err != nil {
return fmt.Errorf("mount volume: %w", err)
}
log.Printf(" Volume %s mounted at %s", spec.Name, spec.Mount)
return nil
}
// runSetupScript runs setup.sh if it exists in the source directory.
func runSetupScript(server *platform.Server, sourceDir string) error {
setupPath := filepath.Join(sourceDir, "setup.sh")
if _, err := os.Stat(setupPath); os.IsNotExist(err) {
return nil
}
log.Printf(" Running %s...", setupPath)
if err := server.Copy(setupPath, "/tmp/setup.sh"); err != nil {
return fmt.Errorf("upload setup.sh: %w", err)
}
output, err := server.SSH("bash", "/tmp/setup.sh")
if err != nil {
return fmt.Errorf("setup.sh: %w\n%s", err, output)
}
for _, line := range strings.Split(strings.TrimSpace(output), "\n") {
if line != "" {
log.Printf(" %s", line)
}
}
return nil
}
// uploadImage uploads a Docker image tar.gz to the server and loads it.
// If imagePath is empty, returns empty string to signal remote build needed.
func uploadImage(server *platform.Server, imagePath string) (string, error) {
if imagePath == "" {
return "", nil // Signal to build on server
}
log.Printf(" Uploading image...")
// Get image name from path (e.g., "build/example.tar.gz" -> "example")
base := filepath.Base(imagePath)
imageName := strings.TrimSuffix(base, ".tar.gz")
remotePath := "/tmp/" + base
if err := server.Copy(imagePath, remotePath); err != nil {
return "", err
}
log.Printf(" Loading image...")
if _, err := server.SSH("bash", "-c", fmt.Sprintf("gunzip -c %s | docker load", remotePath)); err != nil {
return "", fmt.Errorf("docker load: %w", err)
}
// Clean up
server.SSH("rm", remotePath)
return imageName, nil
}
// buildOnServer uploads source and builds Docker image on the server.
func buildOnServer(server *platform.Server, name, source string) (string, error) {
log.Printf(" Building on server...")
// Clean source path (remove ./ prefix)
source = strings.TrimPrefix(source, "./")
remoteBuildDir := "/tmp/build-" + name
// Clean up any previous build
server.SSH("rm", "-rf", remoteBuildDir)
server.SSH("mkdir", "-p", remoteBuildDir)
// Upload go.mod and go.sum
if err := server.Copy("go.mod", remoteBuildDir+"/go.mod"); err != nil {
return "", fmt.Errorf("upload go.mod: %w", err)
}
if err := server.Copy("go.sum", remoteBuildDir+"/go.sum"); err != nil {
return "", fmt.Errorf("upload go.sum: %w", err)
}
// Upload pkg/ directory (tar and extract)
log.Printf(" Uploading pkg/...")
if err := uploadDir(server, "pkg", remoteBuildDir+"/pkg"); err != nil {
return "", fmt.Errorf("upload pkg: %w", err)
}
// Upload source directory
log.Printf(" Uploading %s/...", source)
if err := uploadDir(server, source, remoteBuildDir+"/"+source); err != nil {
return "", fmt.Errorf("upload %s: %w", source, err)
}
// Build on server with BuildKit for better caching
log.Printf(" Running docker build...")
dockerfile := source + "/Dockerfile"
buildCmd := fmt.Sprintf("cd %s && DOCKER_BUILDKIT=1 docker build -t %s -f %s .", remoteBuildDir, name, dockerfile)
if _, err := server.SSH("bash", "-c", buildCmd); err != nil {
return "", fmt.Errorf("remote docker build: %w", err)
}
// Clean up
server.SSH("rm", "-rf", remoteBuildDir)
return name, nil
}
// uploadDir uploads a directory to the server using tar.
func uploadDir(server *platform.Server, localDir, remoteDir string) error {
// Create tar locally
tarPath := "/tmp/" + filepath.Base(localDir) + ".tar.gz"
tarCmd := exec.Command("tar", "-czf", tarPath, "-C", filepath.Dir(localDir), filepath.Base(localDir))
if err := tarCmd.Run(); err != nil {
return fmt.Errorf("tar: %w", err)
}
defer os.Remove(tarPath)
// Upload tar
remoteTar := "/tmp/" + filepath.Base(localDir) + ".tar.gz"
if err := server.Copy(tarPath, remoteTar); err != nil {
return fmt.Errorf("upload: %w", err)
}
// Extract on server
destDir := filepath.Dir(remoteDir)
if _, err := server.SSH("mkdir", "-p", destDir); err != nil {
return fmt.Errorf("mkdir: %w", err)
}
if _, err := server.SSH("tar", "-xzf", remoteTar, "-C", destDir); err != nil {
return fmt.Errorf("extract: %w", err)
}
// Clean up remote tar
server.SSH("rm", remoteTar)
return nil
}