Cache sample apps

4 files changed, 90 insertions, 42 deletions

diff --git a/client/go/cmd/clone.go b/client/go/cmd/clone.go
index 9503a81debf..508ad49438f 100644
--- a/client/go/cmd/clone.go
+++ b/client/go/cmd/clone.go
@@ -6,11 +6,10 @@ package cmd
 
 import (
 	"archive/zip"
+	"errors"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -20,22 +19,29 @@ import (
 	"github.com/vespa-engine/vespa/client/go/util"
 )
 
-// Set this to test without downloading this file from github
-var existingSampleAppsZip string
+const sampleAppsCacheTTL = time.Hour * 168 // 1 week
+
 var listApps bool
+var forceClone bool
 
 func init() {
 	rootCmd.AddCommand(cloneCmd)
 	cloneCmd.Flags().BoolVarP(&listApps, "list", "l", false, "List available sample applications")
+	cloneCmd.Flags().BoolVarP(&forceClone, "force", "f", false, "Ignore cache and force downloading the latest sample application from GitHub")
 }
 
 var cloneCmd = &cobra.Command{
-	// TODO: "application" and "list" subcommands?
 	Use:   "clone sample-application-path target-directory",
 	Short: "Create files and directory structure for a new Vespa application from a sample application",
-	Long: `Creates an application package file structure.
+	Long: `Create files and directory structure for a new Vespa application
+from a sample application.
+
+Sample applications are downloaded from
+https://github.com/vespa-engine/sample-apps.
 
-The application package is copied from a sample application in https://github.com/vespa-engine/sample-apps`,
+By default sample applications are cached in the user's cache directory. This
+directory can be overriden by setting the VESPA_CLI_CACHE_DIR environment
+variable.`,
 	Example:           "$ vespa clone vespa-cloud/album-recommendation my-app",
 	DisableAutoGenTag: true,
 	Run: func(cmd *cobra.Command, args []string) {
@@ -60,12 +66,7 @@ The application package is copied from a sample application in https://github.co
 
 func cloneApplication(source string, name string) {
 	zipFile := getSampleAppsZip()
-	if zipFile == nil {
-		return
-	}
-	if existingSampleAppsZip == "" { // Indicates we created a temp file now
-		defer os.Remove(zipFile.Name())
-	}
+	defer zipFile.Close()
 
 	zipReader, zipOpenError := zip.OpenReader(zipFile.Name())
 	if zipOpenError != nil {
@@ -101,45 +102,67 @@ func cloneApplication(source string, name string) {
 	}
 }
 
+func openOutputFile() (*os.File, error) {
+	cacheDir, err := vespaCliCacheDir()
+	if err != nil {
+		return nil, err
+	}
+	cacheFile := filepath.Join(cacheDir, "sample-apps-master.zip")
+	return os.OpenFile(cacheFile, os.O_RDWR|os.O_CREATE, 0755)
+}
+
+func useCache(cacheFile *os.File) (bool, error) {
+	if forceClone {
+		return false, nil
+	}
+	stat, err := cacheFile.Stat()
+	if errors.Is(err, os.ErrNotExist) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	expiry := stat.ModTime().Add(sampleAppsCacheTTL)
+	return stat.Size() > 0 && time.Now().Before(expiry), nil
+}
+
 func getSampleAppsZip() *os.File {
-	if existingSampleAppsZip != "" {
-		existing, openExistingError := os.Open(existingSampleAppsZip)
-		if openExistingError != nil {
-			printErr(openExistingError, "Could not open existing sample apps zip file '", color.Cyan(existingSampleAppsZip), "'")
-		}
-		return existing
+	f, err := openOutputFile()
+	if err != nil {
+		fatalErr(err, "Could not determine location of cache file")
+		return nil
+	}
+	useCache, err := useCache(f)
+	if err != nil {
+		fatalErr(err, "Could not determine cache status", "Try ignoring the cache with the -f flag")
+		return nil
+	}
+	if useCache {
+		log.Print(color.Yellow("Using cached sample apps ..."))
+		return f
 	}
 
-	// TODO: Cache it?
 	log.Print(color.Yellow("Downloading sample apps ...")) // TODO: Spawn thread to indicate progress
-	zipUrl, _ := url.Parse("https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip")
-	request := &http.Request{
-		URL:    zipUrl,
-		Method: "GET",
+	request, err := http.NewRequest("GET", "https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip", nil)
+	if err != nil {
+		fatalErr(err, "Invalid URL")
+		return nil
 	}
-	response, reqErr := util.HttpDo(request, time.Minute*60, "GitHub")
-	if reqErr != nil {
-		printErr(reqErr, "Could not download sample apps from GitHub")
+	response, err := util.HttpDo(request, time.Minute*60, "GitHub")
+	if err != nil {
+		fatalErr(err, "Could not download sample apps from GitHub")
 		return nil
 	}
 	defer response.Body.Close()
 	if response.StatusCode != 200 {
-		printErr(nil, "Could not download sample apps from GitHub: ", response.StatusCode)
+		fatalErr(nil, "Could not download sample apps from GitHub: ", response.StatusCode)
 		return nil
 	}
 
-	destination, tempFileError := ioutil.TempFile("", "prefix")
-	if tempFileError != nil {
-		printErr(tempFileError, "Could not create a temporary file to hold sample apps")
-	}
-	// destination, _ := os.Create("./" + name + "/sample-apps.zip")
-	// defer destination.Close()
-	_, err := io.Copy(destination, response.Body)
-	if err != nil {
-		printErr(err, "Could not download sample apps from GitHub")
+	if _, err := io.Copy(f, response.Body); err != nil {
+		fatalErr(err, "Could not write sample apps to file: ", f.Name())
 		return nil
 	}
-	return destination
+	return f
 }
 
 func copy(f *zip.File, destinationDir string, zipEntryPrefix string) error {
diff --git a/client/go/cmd/clone_test.go b/client/go/cmd/clone_test.go
index 5027c5bf972..6cf11dd4d40 100644
--- a/client/go/cmd/clone_test.go
+++ b/client/go/cmd/clone_test.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/vespa-engine/vespa/client/go/util"
@@ -18,10 +19,14 @@ func TestClone(t *testing.T) {
 }
 
 func assertCreated(sampleAppName string, app string, t *testing.T) {
-	existingSampleAppsZip = "testdata/sample-apps-master.zip"
-	standardOut := executeCommand(t, &mockHttpClient{}, []string{"clone", sampleAppName, app}, []string{})
+	testFile := filepath.Join("testdata", "sample-apps-master.zip")
+	now := time.Now()
+	if err := os.Chtimes(testFile, now, now); err != nil { // Ensure test file is considered new enough by cache mechanism
+		t.Fatal(err)
+	}
+	out, _ := execute(command{cacheDir: filepath.Dir(testFile), args: []string{"clone", sampleAppName, app}}, t, nil)
 	defer os.RemoveAll(app)
-	assert.Equal(t, "Created "+app+"\n", standardOut)
+	assert.Equal(t, "Using cached sample apps ...\nCreated "+app+"\n", out)
 	assert.True(t, util.PathExists(filepath.Join(app, "README.md")))
 	assert.True(t, util.PathExists(filepath.Join(app, "src", "main", "application")))
 	assert.True(t, util.IsDirectory(filepath.Join(app, "src", "main", "application")))
diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go
index a0c3183ca4c..71f821521df 100644
--- a/client/go/cmd/command_tester.go
+++ b/client/go/cmd/command_tester.go
@@ -22,6 +22,7 @@ import (
 
 type command struct {
 	homeDir  string
+	cacheDir string
 	args     []string
 	moreArgs []string
 }
@@ -31,12 +32,16 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string)
 		util.ActiveHttpClient = client
 	}
 
-	// Set config dir. Use a separate one per test if none is specified
+	// Set Vespa CLI directories. Use a separate one per test if none is specified
 	if cmd.homeDir == "" {
 		cmd.homeDir = filepath.Join(t.TempDir(), ".vespa")
 		viper.Reset()
 	}
+	if cmd.cacheDir == "" {
+		cmd.cacheDir = filepath.Join(t.TempDir(), ".cache", "vespa")
+	}
 	os.Setenv("VESPA_CLI_HOME", cmd.homeDir)
+	os.Setenv("VESPA_CLI_CACHE_DIR", cmd.cacheDir)
 
 	// Reset flags to their default value - persistent flags in Cobra persists over tests
 	rootCmd.Flags().VisitAll(func(f *pflag.Flag) {
diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go
index 9c3d9f3c0eb..2b0e1c35483 100644
--- a/client/go/cmd/helpers.go
+++ b/client/go/cmd/helpers.go
@@ -66,6 +66,21 @@ func vespaCliHome() (string, error) {
 	return home, nil
 }
 
+func vespaCliCacheDir() (string, error) {
+	cacheDir := os.Getenv("VESPA_CLI_CACHE_DIR")
+	if cacheDir == "" {
+		userCacheDir, err := os.UserCacheDir()
+		if err != nil {
+			return "", err
+		}
+		cacheDir = filepath.Join(userCacheDir, "vespa")
+	}
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		return "", err
+	}
+	return cacheDir, nil
+}
+
 func deploymentFromArgs() vespa.Deployment {
 	zone, err := vespa.ZoneFromString(zoneArg)
 	if err != nil {