readysite / cmd / launch / deploy.go
13.4 KB
deploy.go
package main

import (
	"cmp"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/readysite/readysite/pkg/platform"
)

// deployToInstance deploys services to a specific instance.
// binPaths maps binary name to local path (e.g., {"example": "build/example"})
func deployToInstance(
	p *platform.Platform,
	binPaths interface{}, // string or map[string]string
	cfg *InfraConfig,
	serverType string,
	inst *Instance,
) error {
	spec := cfg.Servers[serverType]

	log.Printf("Deploying to %s (%s)...", inst.Name, inst.IP)

	// Use the instance info directly instead of looking up by name
	// (there may be multiple droplets with same name from failed attempts)
	server := &platform.Server{
		ID:     inst.ID,
		Name:   inst.Name,
		IP:     inst.IP,
		Region: inst.Region,
	}

	// 1. Ensure volumes attached (if specified)
	region := platform.Region(cfg.Platform.Region)
	for _, vol := range spec.getVolumes() {
		if err := ensureVolume(p, server, &vol, region); err != nil {
			return fmt.Errorf("volume setup for %s: %w", vol.Name, err)
		}
	}

	// 2. Run setup.sh for each binary if exists in its source directory
	for _, binName := range spec.Binaries {
		svcSpec := cfg.Binaries[binName]
		if err := runSetupScript(server, svcSpec.Source); err != nil {
			return err
		}
	}

	// 3. Build/upload all binary images BEFORE stopping (minimizes downtime)
	imageNames := make(map[string]string)
	for _, binName := range spec.Binaries {
		var localPath string
		switch bp := binPaths.(type) {
		case string:
			localPath = bp
		case map[string]string:
			localPath = bp[binName]
		}

		svcSpec := cfg.Binaries[binName]
		var imageName string
		var err error

		if localPath == "" {
			// Build on server (local Docker unavailable)
			imageName, err = buildOnServer(server, binName, svcSpec.Source)
		} else {
			// Upload pre-built image
			imageName, err = uploadImage(server, localPath)
		}

		if err != nil {
			return fmt.Errorf("prepare %s: %w", binName, err)
		}
		imageNames[binName] = imageName
	}

	// 4. Backup and stop existing services (after build, before start - minimal downtime)
	// Backup binary containers for potential rollback
	for _, svcName := range spec.Binaries {
		backupContainer(server, svcName)
	}
	// Just remove infrastructure services (they use remote images, no rollback needed)
	for _, svcName := range spec.Services {
		server.SSH("docker", "rm", "-f", svcName)
	}

	// 5. Create Docker networks
	networks := make(map[string]bool)
	for _, svcName := range spec.Services {
		if n := cfg.Services[svcName].Network; n != "" {
			networks[n] = true
		}
	}
	for _, binName := range spec.Binaries {
		if n := cfg.Binaries[binName].Network; n != "" {
			networks[n] = true
		}
	}
	for network := range networks {
		log.Printf("   Creating network %s...", network)
		server.SSH("docker", "network", "create", network, "--driver", "bridge")
	}

	// 6. Start infrastructure services (no binary)
	for _, svcName := range spec.Services {
		svcSpec := cfg.Services[svcName]
		if err := startService(server, svcName, &svcSpec, ""); err != nil {
			return fmt.Errorf("start %s: %w", svcName, err)
		}
	}

	// 7. Start binary services with pre-built images
	for _, binName := range spec.Binaries {
		svcSpec := cfg.Binaries[binName]
		if err := startService(server, binName, &svcSpec, imageNames[binName]); err != nil {
			return fmt.Errorf("start %s: %w", binName, err)
		}
	}

	log.Printf("   Deployed to %s ✓", inst.Name)
	return nil
}

// startService starts a Docker service on a server with health checking and rollback support.
// For binaries, imageName is the locally-built Docker image name.
// For services, imageName is empty and spec.Image is used.
func startService(server *platform.Server, name string, spec *ServiceSpec, imageName string) error {
	log.Printf("   Starting %s...", name)

	// Use the built image name if provided, otherwise use spec.Image
	image := cmp.Or(imageName, spec.Image)

	svc := &platform.Service{
		Name:       name,
		Image:      image,
		Network:    spec.Network,
		Restart:    "always",
		Privileged: spec.Privileged,
	}

	if len(spec.Command) > 0 {
		svc.Command = spec.Command
	}

	// Ports
	for _, p := range spec.Ports {
		svc.Ports = append(svc.Ports, platform.Port{
			Host:      p.Host,
			Container: p.Container,
			Bind:      p.Bind,
		})
	}

	// Volumes
	for _, v := range spec.Volumes {
		svc.Volumes = append(svc.Volumes, platform.Mount{Source: v.Source, Target: v.Target})
	}

	// Environment variables
	svc.Env = make(map[string]string)
	for k, v := range spec.Env {
		svc.Env[k] = v
	}

	// Read env_files from server
	for envName, filePath := range spec.EnvFiles {
		value, err := server.SSH("cat", filePath)
		if err != nil {
			log.Printf("   Warning: could not read %s from %s", envName, filePath)
			continue
		}
		svc.Env[envName] = strings.TrimSpace(value)
	}

	// Add health check if ports are exposed
	if len(spec.Ports) > 0 {
		healthCmd := fmt.Sprintf("curl -sf http://localhost:%d/health || exit 1", spec.Ports[0].Container)
		if spec.Healthcheck != "" {
			healthCmd = spec.Healthcheck
		}
		svc.Healthcheck = &platform.Healthcheck{
			Cmd:         healthCmd,
			Interval:    "10s",
			Timeout:     "5s",
			Retries:     3,
			StartPeriod: "10s",
		}
	}

	if err := server.Start(svc); err != nil {
		return err
	}

	// Wait for container to be healthy
	if svc.Healthcheck != nil {
		if err := waitForHealthy(server, name, 60); err != nil {
			log.Printf("   Health check failed, attempting rollback...")
			rollback(server, name)
			return fmt.Errorf("health check failed: %w", err)
		}
	}

	return nil
}

// waitForHealthy waits for a container to become healthy.
func waitForHealthy(server *platform.Server, name string, timeoutSecs int) error {
	log.Printf("   Waiting for %s to become healthy...", name)

	for i := 0; i < timeoutSecs; i += 5 {
		status, err := server.SSH("docker", "inspect", "--format={{.State.Health.Status}}", name)
		if err == nil {
			status = strings.TrimSpace(status)
			switch status {
			case "healthy":
				log.Printf("   %s is healthy", name)
				return nil
			case "unhealthy":
				logs, _ := server.SSH("docker", "logs", "--tail=20", name)
				return fmt.Errorf("container unhealthy, logs:\n%s", logs)
			}
		}
		// Still starting or no health status yet
		time.Sleep(5 * time.Second)
	}

	return fmt.Errorf("health check timeout after %ds", timeoutSecs)
}

// backupContainer stops and renames the current container for rollback.
func backupContainer(server *platform.Server, name string) {
	backupName := name + "-backup"
	// Remove old backup if exists
	server.SSH("docker", "rm", "-f", backupName)
	// Stop current container (releases ports)
	server.SSH("docker", "stop", name)
	// Rename current to backup (if it exists)
	server.SSH("docker", "rename", name, backupName)
}

// rollback restores the previous container version.
func rollback(server *platform.Server, name string) error {
	backupName := name + "-backup"

	// Check if backup exists
	if _, err := server.SSH("docker", "inspect", backupName); err != nil {
		log.Printf("   No backup found for %s, cannot rollback", name)
		return nil
	}

	log.Printf("   Rolling back %s...", name)

	// Stop the failed container
	server.SSH("docker", "rm", "-f", name)

	// Restore the backup
	if _, err := server.SSH("docker", "rename", backupName, name); err != nil {
		return fmt.Errorf("rename backup: %w", err)
	}

	// Start the old container
	if _, err := server.SSH("docker", "start", name); err != nil {
		return fmt.Errorf("start backup: %w", err)
	}

	log.Printf("   Rolled back to previous version of %s", name)
	return nil
}

// ensureVolume creates and mounts a volume if it doesn't exist.
func ensureVolume(p *platform.Platform, server *platform.Server, spec *VolumeSpec, region platform.Region) error {
	log.Printf("   Ensuring volume %s...", spec.Name)

	vol, err := p.GetVolume(spec.Name)
	isNewVolume := err == platform.ErrNotFound
	if isNewVolume {
		log.Printf("   Creating volume %s (%dGB)...", spec.Name, spec.Size)
		vol, err = p.CreateVolume(spec.Name, spec.Size, region)
		if err != nil {
			return fmt.Errorf("create volume: %w", err)
		}
	} else if err != nil {
		return fmt.Errorf("get volume: %w", err)
	}

	// Attach volume to server
	if err := p.AttachVolume(vol.ID, server.ID); err != nil {
		log.Printf("   Volume attach: %v (may already be attached)", err)
	}

	// Determine filesystem type (default to ext4)
	filesystem := cmp.Or(spec.Filesystem, "ext4")

	// Format new volumes if needed
	devicePath := fmt.Sprintf("/dev/disk/by-id/scsi-0DO_Volume_%s", spec.Name)
	if isNewVolume {
		log.Printf("   Formatting volume %s as %s...", spec.Name, filesystem)
		formatScript := fmt.Sprintf(`
			for i in $(seq 1 30); do [ -e %s ] && break; sleep 1; done
			if ! blkid %s >/dev/null 2>&1; then mkfs.%s %s; fi
		`, devicePath, devicePath, filesystem, devicePath)
		if _, err := server.SSH("bash", "-c", formatScript); err != nil {
			return fmt.Errorf("format volume: %w", err)
		}
	}

	// Mount and persist in fstab
	mountOpts := "defaults,nofail,discard"
	if filesystem == "xfs" {
		mountOpts = "defaults,nofail,discard,prjquota"
	}

	mountScript := fmt.Sprintf(`
		mkdir -p %s
		grep -qF "%s" /etc/fstab || echo "%s %s %s %s 0 2" >> /etc/fstab
		mountpoint -q %s || mount %s || true
	`, spec.Mount, spec.Name, devicePath, spec.Mount, filesystem, mountOpts, spec.Mount, spec.Mount)

	if _, err := server.SSH("bash", "-c", mountScript); err != nil {
		return fmt.Errorf("mount volume: %w", err)
	}

	log.Printf("   Volume %s mounted at %s", spec.Name, spec.Mount)
	return nil
}

// runSetupScript runs setup.sh if it exists in the source directory.
func runSetupScript(server *platform.Server, sourceDir string) error {
	setupPath := filepath.Join(sourceDir, "setup.sh")
	if _, err := os.Stat(setupPath); os.IsNotExist(err) {
		return nil
	}

	log.Printf("   Running %s...", setupPath)
	if err := server.Copy(setupPath, "/tmp/setup.sh"); err != nil {
		return fmt.Errorf("upload setup.sh: %w", err)
	}

	output, err := server.SSH("bash", "/tmp/setup.sh")
	if err != nil {
		return fmt.Errorf("setup.sh: %w\n%s", err, output)
	}

	for _, line := range strings.Split(strings.TrimSpace(output), "\n") {
		if line != "" {
			log.Printf("   %s", line)
		}
	}
	return nil
}

// uploadImage uploads a Docker image tar.gz to the server and loads it.
// If imagePath is empty, returns empty string to signal remote build needed.
func uploadImage(server *platform.Server, imagePath string) (string, error) {
	if imagePath == "" {
		return "", nil // Signal to build on server
	}

	log.Printf("   Uploading image...")

	// Get image name from path (e.g., "build/example.tar.gz" -> "example")
	base := filepath.Base(imagePath)
	imageName := strings.TrimSuffix(base, ".tar.gz")

	remotePath := "/tmp/" + base
	if err := server.Copy(imagePath, remotePath); err != nil {
		return "", err
	}

	log.Printf("   Loading image...")
	if _, err := server.SSH("bash", "-c", fmt.Sprintf("gunzip -c %s | docker load", remotePath)); err != nil {
		return "", fmt.Errorf("docker load: %w", err)
	}

	// Clean up
	server.SSH("rm", remotePath)

	return imageName, nil
}

// buildOnServer uploads source and builds Docker image on the server.
func buildOnServer(server *platform.Server, name, source string) (string, error) {
	log.Printf("   Building on server...")

	// Clean source path (remove ./ prefix)
	source = strings.TrimPrefix(source, "./")

	remoteBuildDir := "/tmp/build-" + name

	// Clean up any previous build
	server.SSH("rm", "-rf", remoteBuildDir)
	server.SSH("mkdir", "-p", remoteBuildDir)

	// Upload go.mod and go.sum
	if err := server.Copy("go.mod", remoteBuildDir+"/go.mod"); err != nil {
		return "", fmt.Errorf("upload go.mod: %w", err)
	}
	if err := server.Copy("go.sum", remoteBuildDir+"/go.sum"); err != nil {
		return "", fmt.Errorf("upload go.sum: %w", err)
	}

	// Upload pkg/ directory (tar and extract)
	log.Printf("   Uploading pkg/...")
	if err := uploadDir(server, "pkg", remoteBuildDir+"/pkg"); err != nil {
		return "", fmt.Errorf("upload pkg: %w", err)
	}

	// Upload source directory
	log.Printf("   Uploading %s/...", source)
	if err := uploadDir(server, source, remoteBuildDir+"/"+source); err != nil {
		return "", fmt.Errorf("upload %s: %w", source, err)
	}

	// Build on server with BuildKit for better caching
	log.Printf("   Running docker build...")
	dockerfile := source + "/Dockerfile"
	buildCmd := fmt.Sprintf("cd %s && DOCKER_BUILDKIT=1 docker build -t %s -f %s .", remoteBuildDir, name, dockerfile)
	if _, err := server.SSH("bash", "-c", buildCmd); err != nil {
		return "", fmt.Errorf("remote docker build: %w", err)
	}

	// Clean up
	server.SSH("rm", "-rf", remoteBuildDir)

	return name, nil
}

// uploadDir uploads a directory to the server using tar.
func uploadDir(server *platform.Server, localDir, remoteDir string) error {
	// Create tar locally
	tarPath := "/tmp/" + filepath.Base(localDir) + ".tar.gz"
	tarCmd := exec.Command("tar", "-czf", tarPath, "-C", filepath.Dir(localDir), filepath.Base(localDir))
	if err := tarCmd.Run(); err != nil {
		return fmt.Errorf("tar: %w", err)
	}
	defer os.Remove(tarPath)

	// Upload tar
	remoteTar := "/tmp/" + filepath.Base(localDir) + ".tar.gz"
	if err := server.Copy(tarPath, remoteTar); err != nil {
		return fmt.Errorf("upload: %w", err)
	}

	// Extract on server
	destDir := filepath.Dir(remoteDir)
	if _, err := server.SSH("mkdir", "-p", destDir); err != nil {
		return fmt.Errorf("mkdir: %w", err)
	}
	if _, err := server.SSH("tar", "-xzf", remoteTar, "-C", destDir); err != nil {
		return fmt.Errorf("extract: %w", err)
	}

	// Clean up remote tar
	server.SSH("rm", remoteTar)

	return nil
}
← Back