use common curl utility

10 files changed, 172 insertions, 62 deletions

diff --git a/client/go/cmd/deploy/activate.go b/client/go/cmd/deploy/activate.go
index a6b7de81342..1f475ff0461 100644
--- a/client/go/cmd/deploy/activate.go
+++ b/client/go/cmd/deploy/activate.go
@@ -23,7 +23,7 @@ func RunActivate(opts *Options, args []string) error {
 	url = addUrlPropertyFromFlag(url, opts.Verbose, "verbose")
 	url = addUrlPropertyFromOption(url, strconv.Itoa(opts.Timeout), "timeout")
 	fmt.Printf("Activating session %s using %s\n", sessId, urlWithoutQuery(url))
-	output, err := curlPut(url, src)
+	output, err := curlPutNothing(url)
 	if err != nil {
 		return err
 	}
diff --git a/client/go/cmd/deploy/cmd.go b/client/go/cmd/deploy/cmd.go
index f5cf554834d..6c1d560fe79 100644
--- a/client/go/cmd/deploy/cmd.go
+++ b/client/go/cmd/deploy/cmd.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/vespa-engine/vespa/client/go/build"
+	"github.com/vespa-engine/vespa/client/go/vespa"
 )
 
 func reallySimpleHelp(cmd *cobra.Command, args []string) {
@@ -20,6 +21,9 @@ func NewDeployCmd() *cobra.Command {
 	var (
 		curOptions Options
 	)
+	if err := vespa.LoadDefaultEnv(); err != nil {
+		panic(err)
+	}
 	cobra.EnableCommandSorting = false
 	cmd := &cobra.Command{
 		Use:   "vespa-deploy [-h] [-v] [-f] [-t] [-c] [-p] [-z] [-V] [<command>] [args]",
@@ -69,8 +73,9 @@ func newUploadCmd(opts *Options) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			opts.Command = CmdUpload
 			if opts.Verbose {
-				fmt.Printf("upload %v [%v]\n", opts, args)
+				AdjustVerbosity(1)
 			}
+			PutTrace("upload with", opts, args)
 			err := RunUpload(opts, args)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%s\n", err.Error())
@@ -90,8 +95,9 @@ func newPrepareCmd(opts *Options) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			opts.Command = CmdPrepare
 			if opts.Verbose {
-				fmt.Printf("prepare %v [%v]\n", opts, args)
+				AdjustVerbosity(1)
 			}
+			PutTrace("prepare with", opts, args)
 			err := RunPrepare(opts, args)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%s\n", err.Error())
@@ -110,8 +116,9 @@ func newActivateCmd(opts *Options) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			opts.Command = CmdActivate
 			if opts.Verbose {
-				fmt.Printf("activate %v [%v]\n", opts, args)
+				AdjustVerbosity(1)
 			}
+			PutTrace("activate with", opts, args)
 			err := RunActivate(opts, args)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%s\n", err.Error())
@@ -130,8 +137,9 @@ func newFetchCmd(opts *Options) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			opts.Command = CmdFetch
 			if opts.Verbose {
-				fmt.Printf("fetch %v [%v]\n", opts, args)
+				AdjustVerbosity(1)
 			}
+			PutTrace("fetch with", opts, args)
 			err := RunFetch(opts, args)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%s\n", err.Error())
diff --git a/client/go/cmd/deploy/curl.go b/client/go/cmd/deploy/curl.go
index b5d392dad1e..6a0a722d502 100644
--- a/client/go/cmd/deploy/curl.go
+++ b/client/go/cmd/deploy/curl.go
@@ -12,31 +12,41 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/vespa-engine/vespa/client/go/curl"
 	"github.com/vespa-engine/vespa/client/go/vespa"
 )
 
-func curlPut(url string, cfgSrc string) (string, error) {
-	args := append(curlPutArgs(), url)
-	return runCurl(args, new(strings.Reader), cfgSrc)
+func curlPutNothing(url string) (string, error) {
+	cmd := newCurlCommand(url, curlPutArgs())
+	cmd.Method = "PUT"
+	var out bytes.Buffer
+	err := runCurl(cmd, &out)
+	return out.String(), err
 }
 
-func curlPost(url string, input io.Reader, cfgSrc string) (string, error) {
-	args := append(curlPostArgs(), url)
-	return runCurl(args, input, cfgSrc)
+func curlPost(url string, input io.Reader) (string, error) {
+	cmd := newCurlCommand(url, curlPostArgs())
+	cmd.Method = "POST"
+	cmd.Header("Content-Type", "application/x-gzip")
+	cmd.WithBodyInput(input)
+	var out bytes.Buffer
+	err := runCurl(cmd, &out)
+	return out.String(), err
 }
 
-func curlPostZip(url string, input io.Reader, cfgSrc string) (string, error) {
-	args := append(curlPostZipArgs(), url)
-	return runCurl(args, input, cfgSrc)
+func curlPostZip(url string, input io.Reader) (string, error) {
+	cmd := newCurlCommand(url, curlPostArgs())
+	cmd.Method = "POST"
+	cmd.Header("Content-Type", "application/zip")
+	cmd.WithBodyInput(input)
+	var out bytes.Buffer
+	err := runCurl(cmd, &out)
+	return out.String(), err
 }
 
 func curlGet(url string, output io.Writer) error {
-	args := append(curlGetArgs(), url)
-	cmd := exec.Command(curlWrapper(), args...)
-	cmd.Stdout = output
-	cmd.Stderr = os.Stderr
-	// fmt.Printf("running command: %v\n", cmd)
-	err := cmd.Run()
+	cmd := newCurlCommand(url, commonCurlArgs())
+	err := runCurl(cmd, output)
 	return err
 }
 
@@ -45,6 +55,32 @@ func urlWithoutQuery(url string) string {
 	return parts[0]
 }
 
+func newCurlCommand(url string, args []string) *curl.Command {
+	tls, err := vespa.LoadTlsConfig()
+	if err != nil {
+		panic(err)
+	}
+	if tls != nil && strings.HasPrefix(url, "http:") {
+		url = "https:" + url[5:]
+	}
+	cmd, err := curl.RawArgs(url, args...)
+	if err != nil {
+		panic(err)
+	}
+	if tls != nil {
+		if tls.DisableHostnameValidation {
+			cmd, err = curl.RawArgs(url, append(args, "--insecure")...)
+			if err != nil {
+				panic(err)
+			}
+		}
+		cmd.PrivateKey = tls.Files.PrivateKey
+		cmd.Certificate = tls.Files.Certificates
+		cmd.CaCertificate = tls.Files.CaCertificates
+	}
+	return cmd
+}
+
 func getOutputFromCmd(program string, args ...string) (string, error) {
 	cmd := exec.Command(program, args...)
 	var out bytes.Buffer
@@ -54,26 +90,18 @@ func getOutputFromCmd(program string, args ...string) (string, error) {
 	return out.String(), err
 }
 
-func runCurl(args []string, input io.Reader, cfgSrc string) (string, error) {
-	cmd := exec.Command(curlWrapper(), args...)
-	cmd.Stdin = input
-	var out bytes.Buffer
-	cmd.Stdout = &out
-	cmd.Stderr = os.Stderr
-	// fmt.Printf("running command: %v\n", cmd)
-	err := cmd.Run()
-	// fmt.Printf("output: %s\n", out.String())
+func runCurl(cmd *curl.Command, stdout io.Writer) error {
+	PutTrace("running curl:", cmd.String())
+	err := cmd.Run(stdout, os.Stderr)
 	if err != nil {
-		if cmd.ProcessState.ExitCode() == 7 {
-			return "", fmt.Errorf("HTTP request failed. Could not connect to %s", cfgSrc)
+		if ee, ok := err.(*exec.ExitError); ok {
+			if ee.ProcessState.ExitCode() == 7 {
+				return fmt.Errorf("HTTP request failed. Could not connect to %s", cmd.GetUrlPrefix())
+			}
 		}
-		return "", fmt.Errorf("HTTP request failed with curl %s", err.Error())
+		return fmt.Errorf("HTTP request failed with curl %s", err.Error())
 	}
-	return out.String(), err
-}
-
-func curlWrapper() string {
-	return vespa.FindHome() + "/libexec/vespa/vespa-curl-wrapper"
+	return err
 }
 
 func commonCurlArgs() []string {
@@ -88,27 +116,14 @@ func commonCurlArgs() []string {
 
 func curlPutArgs() []string {
 	return append(commonCurlArgs(),
-		"--write-out", "%{http_code}",
-		"--request", "PUT")
+		"--write-out", "\n%{http_code}")
 }
 
 func curlGetArgs() []string {
-	return append(commonCurlArgs(),
-		"--request", "GET")
+	return commonCurlArgs()
 }
 
 func curlPostArgs() []string {
 	return append(commonCurlArgs(),
-		"--write-out", "%{http_code}",
-		"--request", "POST",
-		"--header", "Content-Type: application/x-gzip",
-		"--data-binary", "@-")
-}
-
-func curlPostZipArgs() []string {
-	return append(commonCurlArgs(),
-		"--write-out", "%{http_code}",
-		"--request", "POST",
-		"--header", "Content-Type: application/zip",
-		"--data-binary", "@-")
+		"--write-out", "\n%{http_code}")
 }
diff --git a/client/go/cmd/deploy/options.go b/client/go/cmd/deploy/options.go
index 2f71f779044..21ee8c902ed 100644
--- a/client/go/cmd/deploy/options.go
+++ b/client/go/cmd/deploy/options.go
@@ -4,6 +4,11 @@
 
 package deploy
 
+import (
+	"strconv"
+	"strings"
+)
+
 type CmdType int
 
 const (
@@ -35,3 +40,31 @@ type Options struct {
 	Timeout    int
 	PortNumber int
 }
+
+func (opts *Options) String() string {
+	var buf strings.Builder
+	buf.WriteString("command-line options [")
+	if opts.DryRun {
+		buf.WriteString(" dry-run")
+	}
+	if opts.Force {
+		buf.WriteString(" force")
+	}
+	if opts.Hosted {
+		buf.WriteString(" hosted")
+	}
+	if opts.ServerHost != "" {
+		buf.WriteString(" server=")
+		buf.WriteString(opts.ServerHost)
+	}
+	if opts.PortNumber != 19071 {
+		buf.WriteString(" port=")
+		buf.WriteString(strconv.Itoa(opts.PortNumber))
+	}
+	if opts.From != "" {
+		buf.WriteString(" from=")
+		buf.WriteString(opts.From)
+	}
+	buf.WriteString(" ]")
+	return buf.String()
+}
diff --git a/client/go/cmd/deploy/persist.go b/client/go/cmd/deploy/persist.go
index ed719b0e935..31423050902 100644
--- a/client/go/cmd/deploy/persist.go
+++ b/client/go/cmd/deploy/persist.go
@@ -67,7 +67,7 @@ func writeSessionIdToFile(tenant, newSessionId string) {
 		dir := createTenantDir(tenant)
 		fn := filepath.Join(dir, sessionIdFileName)
 		os.WriteFile(fn, []byte(newSessionId), 0600)
-		// fmt.Printf("wrote %s to %s\n", newSessionId, fn)
+		PutTrace("wrote", newSessionId, "to", fn)
 	}
 }
 
@@ -76,7 +76,7 @@ func getSessionIdFromFile(tenant string) string {
 	fn := filepath.Join(dir, sessionIdFileName)
 	bytes, err := os.ReadFile(fn)
 	if err == nil {
-		// fmt.Printf("Session-id '%s' found from file %s\n", string(bytes), fn)
+		PutTrace("Session-id", string(bytes), "found from file", fn)
 		return string(bytes)
 	}
 	panic("Could not read session id from file, and no session id supplied as argument. Exiting.")
diff --git a/client/go/cmd/deploy/prepare.go b/client/go/cmd/deploy/prepare.go
index a9d1cf73d32..4e048883746 100644
--- a/client/go/cmd/deploy/prepare.go
+++ b/client/go/cmd/deploy/prepare.go
@@ -78,6 +78,6 @@ func doPrepare(opts *Options, sessionId string) (output string, err error) {
 	url = addUrlPropertyFromOption(url, opts.Rotations, "rotations")
 	url = addUrlPropertyFromOption(url, opts.VespaVersion, "vespaVersion")
 	fmt.Printf("Preparing session %s using %s\n", sessionId, urlWithoutQuery(url))
-	output, err = curlPut(url, src)
+	output, err = curlPutNothing(url)
 	return
 }
diff --git a/client/go/cmd/deploy/trace.go b/client/go/cmd/deploy/trace.go
new file mode 100644
index 00000000000..ece708d3174
--- /dev/null
+++ b/client/go/cmd/deploy/trace.go
@@ -0,0 +1,50 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Author: arnej
+
+package deploy
+
+import (
+	"fmt"
+	"os"
+)
+
+// handling of informational output
+
+type outputLevel int
+
+const (
+	levelNone outputLevel = iota
+	levelInfo
+	levelTrace
+	levelDebug
+	levelSpam
+)
+
+var currentOutputLevel outputLevel = levelInfo
+
+func AdjustVerbosity(howMuch int) {
+	currentOutputLevel = (outputLevel)(howMuch + int(currentOutputLevel))
+}
+
+func outputStderr(l outputLevel, v ...interface{}) {
+	if l > currentOutputLevel {
+		return
+	}
+	fmt.Fprintln(os.Stderr, v...)
+}
+
+func PutInfo(v ...interface{}) {
+	outputStderr(levelInfo, v...)
+}
+
+func PutTrace(v ...interface{}) {
+	outputStderr(levelTrace, v...)
+}
+
+func PutDebug(v ...interface{}) {
+	outputStderr(levelDebug, v...)
+}
+
+func PutWarning(v ...interface{}) {
+	fmt.Fprintln(os.Stderr, v...)
+}
diff --git a/client/go/cmd/deploy/upload.go b/client/go/cmd/deploy/upload.go
index 57d84e9923c..20272a4b2da 100644
--- a/client/go/cmd/deploy/upload.go
+++ b/client/go/cmd/deploy/upload.go
@@ -80,9 +80,8 @@ func uploadFrom(opts *Options, src string) (string, error) {
 	url := src + pathPrefix(opts)
 	url = addUrlPropertyFromOption(url, opts.From, "from")
 	url = addUrlPropertyFromFlag(url, opts.Verbose, "verbose")
-	// disallowed by system test:
-	// fmt.Printf("Upload from URL %s using %s\n", opts.From, urlWithoutQuery(url))
-	output, err := curlPost(url, nil, src)
+	PutTrace("Upload from URL", opts.From, "using", urlWithoutQuery(url))
+	output, err := curlPost(url, nil)
 	return output, err
 }
 
@@ -90,7 +89,7 @@ func uploadFile(opts *Options, src string, f *os.File, fileName string) (string,
 	url := src + pathPrefix(opts)
 	url = addUrlPropertyFromFlag(url, opts.Verbose, "verbose")
 	fmt.Printf("Uploading application '%s' using %s\n", fileName, urlWithoutQuery(url))
-	output, err := curlPostZip(url, f, src)
+	output, err := curlPostZip(url, f)
 	return output, err
 }
 
@@ -107,7 +106,7 @@ func uploadDirectory(opts *Options, src string, dirName string) (string, error)
 	if err != nil {
 		return "", err
 	}
-	output, err := curlPost(url, pipe, src)
+	output, err := curlPost(url, pipe)
 	tarCmd.Wait()
 	return output, err
 }
diff --git a/client/go/cmd/deploy/urls.go b/client/go/cmd/deploy/urls.go
index a865006df0d..9c56656ec6b 100644
--- a/client/go/cmd/deploy/urls.go
+++ b/client/go/cmd/deploy/urls.go
@@ -32,11 +32,12 @@ func makeConfigsourceUrls(opts *Options) []string {
 			if len(colonParts) > 1 {
 				// XXX overwrites port number from above - is this sensible?
 				src = fmt.Sprintf("%s:%s:%d", colonParts[0], colonParts[1], opts.PortNumber)
+				PutTrace("can use config server at", src)
 				results = append(results, src)
 			}
 		}
 		if len(results) == 0 {
-			fmt.Println("Could not get url to config server, make sure that VESPA_CONFIGSERVERS is set")
+			PutWarning("Could not get url to config server, make sure that VESPA_CONFIGSERVERS is set")
 			results = append(results, fmt.Sprintf("http://localhost:%d", opts.PortNumber))
 		}
 	} else {
diff --git a/client/go/curl/curl.go b/client/go/curl/curl.go
index 9da7fbe9cac..d3b805e16da 100644
--- a/client/go/curl/curl.go
+++ b/client/go/curl/curl.go
@@ -28,6 +28,10 @@ type Command struct {
 	rawArgs       []string
 }
 
+func (c *Command) GetUrlPrefix() string {
+	return c.url.Scheme + "://" + c.url.Host
+}
+
 func (c *Command) WithBodyFile(fn string) {
 	if c.bodyInput != nil {
 		panic("cannot use both WithBodyFile and WithBodyInput")