path: root/client/go/cmd/clone.go

// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// vespa clone command
// author: bratseth

package cmd

import (
	"archive/zip"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
)

const sampleAppsNamePrefix = "sample-apps-master"

func newCloneCmd(cli *CLI) *cobra.Command {
	var (
		listApps bool
		noCache  bool
	)
	cmd := &cobra.Command{
		Use:   "clone sample-application-path target-directory",
		Short: "Create files and directory structure for a new Vespa application from a sample application",
		Long: `Create files and directory structure for a new Vespa application
from a sample application.

Sample applications are downloaded from
https://github.com/vespa-engine/sample-apps.

By default sample applications are cached in the user's cache directory. This
directory can be overriden by setting the VESPA_CLI_CACHE_DIR environment
variable.`,
		Example:           "$ vespa clone album-recommendation my-app",
		DisableAutoGenTag: true,
		SilenceUsage:      true,
		RunE: func(cmd *cobra.Command, args []string) error {
			if listApps {
				apps, err := listSampleApps(cli.httpClient)
				if err != nil {
					return fmt.Errorf("could not list sample applications: %w", err)
				}
				for _, app := range apps {
					log.Print(app)
				}
				return nil
			}
			if len(args) != 2 {
				return fmt.Errorf("expected exactly 2 arguments, got %d", len(args))
			}
			cloner := &cloner{cli: cli, noCache: noCache}
			return cloner.Clone(args[0], args[1])
		},
	}
	cmd.Flags().BoolVarP(&listApps, "list", "l", false, "List available sample applications")
	cmd.Flags().BoolVarP(&noCache, "force", "f", false, "Ignore cache and force downloading the latest sample application from GitHub")
	return cmd
}

type cloner struct {
	cli     *CLI
	noCache bool
}

type zipFile struct {
	path    string
	etag    string
	modTime time.Time
}

// Clone copies the application identified by applicationName into given path. If the cached copy of sample applications
// has expired (as determined by its entity tag), a current copy will be downloaded from GitHub automatically.
func (c *cloner) Clone(applicationName, path string) error {
	zipPath, err := c.zipPath()
	if err != nil {
		return err
	}

	r, err := zip.OpenReader(zipPath)
	if err != nil {
		return fmt.Errorf("could not open sample apps zip '%s': %w", color.CyanString(zipPath), err)
	}
	defer r.Close()

	found := false
	for _, f := range r.File {
		dirPrefix := "sample-apps-master/" + applicationName + "/"
		if strings.HasPrefix(f.Name, dirPrefix) {
			if !found { // Create destination directory lazily when source is found
				createErr := os.Mkdir(path, 0755)
				if createErr != nil {
					return fmt.Errorf("could not create directory '%s': %w", color.CyanString(path), createErr)
				}
			}
			found = true

			if err := copyFromZip(f, path, dirPrefix); err != nil {
				return fmt.Errorf("could not copy zip entry '%s': %w", color.CyanString(f.Name), err)
			}
		}
	}

	if !found {
		return errHint(fmt.Errorf("could not find source application '%s'", color.CyanString(applicationName)), "Use -f to ignore the cache")
	} else {
		log.Print("Created ", color.CyanString(path))
	}
	return nil
}

// zipPath returns the path to the latest sample application ZIP file.
func (c *cloner) zipPath() (string, error) {
	zipFiles, err := c.listZipFiles()
	if err != nil {
		return "", nil
	}
	cacheCandidates := zipFiles
	if c.noCache {
		cacheCandidates = nil
	}
	zipPath, cacheHit, err := c.downloadZip(cacheCandidates)
	if err != nil {
		if cacheHit {
			c.cli.printWarning(err)
		} else {
			return "", err
		}
	}
	if cacheHit {
		log.Print(color.YellowString("Using cached sample apps ..."))
	}
	// Remove obsolete files
	for _, zf := range zipFiles {
		if zf.path != zipPath {
			os.Remove(zf.path)
		}
	}
	return zipPath, nil
}

// listZipFiles list all sample apps ZIP files found in cacheDir.
func (c *cloner) listZipFiles() ([]zipFile, error) {
	dirEntries, err := os.ReadDir(c.cli.config.cacheDir)
	if err != nil {
		return nil, err
	}
	var zipFiles []zipFile
	for _, entry := range dirEntries {
		ext := filepath.Ext(entry.Name())
		if ext != ".zip" {
			continue
		}
		if !strings.HasPrefix(entry.Name(), sampleAppsNamePrefix) {
			continue
		}
		fi, err := entry.Info()
		if err != nil {
			return nil, err
		}
		name := fi.Name()
		etag := ""
		parts := strings.Split(name, "_")
		if len(parts) == 2 {
			etag = strings.TrimSuffix(parts[1], ext)
		}
		zipFiles = append(zipFiles, zipFile{
			path:    filepath.Join(c.cli.config.cacheDir, name),
			etag:    etag,
			modTime: fi.ModTime(),
		})
	}
	return zipFiles, nil
}

// downloadZip conditionally downloads the latest sample apps ZIP file. If any of the ZIP files among cacheFiles are
// usable, downloading is skipped.
func (c *cloner) downloadZip(cachedFiles []zipFile) (string, bool, error) {
	zipPath := ""
	etag := ""
	sort.Slice(cachedFiles, func(i, j int) bool { return cachedFiles[i].modTime.Before(cachedFiles[j].modTime) })
	if len(cachedFiles) > 0 {
		latest := cachedFiles[len(cachedFiles)-1]
		zipPath = latest.path
		etag = latest.etag
	}
	// The latest cached file, if any, is considered a hit until we have downloaded a fresh one. This allows us to use
	// the cached copy if GitHub is unavailable.
	cacheHit := zipPath != ""
	err := c.cli.spinner(c.cli.Stderr, color.YellowString("Downloading sample apps ..."), func() error {
		request, err := http.NewRequest("GET", "https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip", nil)
		if err != nil {
			return fmt.Errorf("invalid url: %w", err)
		}
		if etag != "" {
			request.Header = make(http.Header)
			request.Header.Set("if-none-match", fmt.Sprintf(`W/"%s"`, etag))
		}
		response, err := c.cli.httpClient.Do(request, time.Minute*60)
		if err != nil {
			return fmt.Errorf("could not download sample apps: %w", err)
		}
		defer response.Body.Close()
		if response.StatusCode == http.StatusNotModified { // entity tag matched so our cached copy is current
			return nil
		}
		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("could not download sample apps: github returned status %d", response.StatusCode)
		}
		etag = trimEntityTagID(response.Header.Get("etag"))
		newPath, err := c.writeZip(response.Body, etag)
		if err != nil {
			return err
		}
		zipPath = newPath
		cacheHit = false
		return nil
	})
	return zipPath, cacheHit, err
}

// writeZip atomically writes the contents of reader zipReader to a file in the CLI cache directory.
func (c *cloner) writeZip(zipReader io.Reader, etag string) (string, error) {
	f, err := os.CreateTemp(c.cli.config.cacheDir, "sample-apps-tmp-")
	if err != nil {
		return "", fmt.Errorf("could not create temporary file: %w", err)
	}
	cleanTemp := true
	defer func() {
		f.Close()
		if cleanTemp {
			os.Remove(f.Name())
		}
	}()
	if _, err := io.Copy(f, zipReader); err != nil {
		return "", fmt.Errorf("could not write sample apps to file: %s: %w", f.Name(), err)
	}
	f.Close()
	path := filepath.Join(c.cli.config.cacheDir, sampleAppsNamePrefix)
	if etag != "" {
		path += "_" + etag
	}
	path += ".zip"
	if err := os.Rename(f.Name(), path); err != nil {
		return "", fmt.Errorf("could not move sample apps to %s", path)
	}
	cleanTemp = false
	return path, nil
}

func trimEntityTagID(s string) string {
	return strings.TrimSuffix(strings.TrimPrefix(s, `W/"`), `"`)
}

func copyFromZip(f *zip.File, destinationDir string, zipEntryPrefix string) error {
	destinationPath := filepath.Join(destinationDir, filepath.FromSlash(strings.TrimPrefix(f.Name, zipEntryPrefix)))
	if strings.HasSuffix(f.Name, "/") {
		if f.Name != zipEntryPrefix { // root is already created
			if err := os.Mkdir(destinationPath, 0755); err != nil {
				return err
			}
		}
	} else {
		r, err := f.Open()
		if err != nil {
			return err
		}
		defer r.Close()
		destination, err := os.Create(destinationPath)
		if err != nil {
			return err
		}
		if _, err := io.Copy(destination, r); err != nil {
			return err
		}
		if err := os.Chmod(destinationPath, f.Mode()); err != nil {
			return err
		}
	}
	return nil
}