diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-05-25 16:27:21 +0200 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2023-05-25 18:30:54 +0200 |
commit | 2a3d0867074ee6abc4efbdc8d6400dc8c55a66c1 (patch) | |
tree | f3e52882a78b33fc873cb1424bc093aeba639a65 /client/go | |
parent | bb5e1faabbe68fa5aa536834e7669d1d92dc6776 (diff) |
Use document package in vespa document commands
Diffstat (limited to 'client/go')
-rw-r--r-- | client/go/internal/cli/cmd/document.go | 190 | ||||
-rw-r--r-- | client/go/internal/cli/cmd/document_test.go | 46 | ||||
-rw-r--r-- | client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Put-Id.json | 15 | ||||
-rw-r--r-- | client/go/internal/vespa/document.go | 197 | ||||
-rw-r--r-- | client/go/internal/vespa/document/http.go | 93 | ||||
-rw-r--r-- | client/go/internal/vespa/document/http_test.go | 41 | ||||
-rw-r--r-- | client/go/internal/vespa/document/stats.go | 1 |
7 files changed, 293 insertions, 290 deletions
diff --git a/client/go/internal/cli/cmd/document.go b/client/go/internal/cli/cmd/document.go index b5b63fd32df..07a98d2e626 100644 --- a/client/go/internal/cli/cmd/document.go +++ b/client/go/internal/cli/cmd/document.go @@ -5,15 +5,22 @@ package cmd import ( + "bytes" + "errors" "fmt" "io" + "net/http" + "os" + "strconv" "strings" "time" "github.com/fatih/color" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/curl" "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/vespa" + "github.com/vespa-engine/vespa/client/go/internal/vespa/document" ) func addDocumentFlags(cmd *cobra.Command, printCurl *bool, timeoutSecs *int) { @@ -21,6 +28,128 @@ func addDocumentFlags(cmd *cobra.Command, printCurl *bool, timeoutSecs *int) { cmd.PersistentFlags().IntVarP(timeoutSecs, "timeout", "T", 60, "Timeout for the document request in seconds") } +type serviceWithCurl struct { + curlCmdWriter io.Writer + bodyFile string + service *vespa.Service +} + +func (s *serviceWithCurl) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { + cmd, err := curl.RawArgs(request.URL.String()) + if err != nil { + return nil, err + } + cmd.Method = request.Method + for k, vs := range request.Header { + for _, v := range vs { + cmd.Header(k, v) + } + } + if s.bodyFile != "" { + cmd.WithBodyFile(s.bodyFile) + } + cmd.Certificate = s.service.TLSOptions.CertificateFile + cmd.PrivateKey = s.service.TLSOptions.PrivateKeyFile + out := cmd.String() + "\n" + if _, err := io.WriteString(s.curlCmdWriter, out); err != nil { + return nil, err + } + return s.service.Do(request, timeout) +} + +func documentClient(cli *CLI, timeoutSecs int, printCurl bool) (*document.Client, *serviceWithCurl, error) { + docService, err := documentService(cli) + if err != nil { + return nil, nil, err + } + service := &serviceWithCurl{curlCmdWriter: io.Discard, service: docService} + if printCurl { + service.curlCmdWriter = cli.Stderr + } + client, err := document.NewClient(document.ClientOptions{ + Compression: document.CompressionAuto, + Timeout: time.Duration(timeoutSecs) * time.Second, + BaseURL: docService.BaseURL, + NowFunc: time.Now, + }, []util.HTTPClient{service}) + if err != nil { + return nil, nil, err + } + return client, service, nil +} + +func sendOperation(op document.Operation, args []string, timeoutSecs int, printCurl bool, cli *CLI) error { + client, service, err := documentClient(cli, timeoutSecs, printCurl) + if err != nil { + return err + } + id := "" + filename := args[0] + if len(args) > 1 { + id = args[0] + filename = args[1] + } + f, err := os.Open(filename) + if err != nil { + return err + } + defer f.Close() + doc, err := document.NewDecoder(f).Decode() + if errors.Is(err, document.ErrMissingId) { + if id == "" { + return fmt.Errorf("no document id given neither as argument or as a 'put', 'update' or 'remove' key in the JSON file") + } + } else if err != nil { + return err + } + if id != "" { + docId, err := document.ParseId(id) + if err != nil { + return err + } + doc.Id = docId + } + if op > -1 { + if id == "" && op != doc.Operation { + return fmt.Errorf("wanted document operation is %s, but JSON file specifies %s", op, doc.Operation) + } + doc.Operation = op + } + if doc.Body != nil { + service.bodyFile = f.Name() + } + result := client.Send(doc) + return printResult(cli, operationResult(false, doc, service.service, result), false) +} + +func readDocument(id string, timeoutSecs int, printCurl bool, cli *CLI) error { + client, service, err := documentClient(cli, timeoutSecs, printCurl) + if err != nil { + return err + } + docId, err := document.ParseId(id) + if err != nil { + return err + } + result := client.Get(docId) + return printResult(cli, operationResult(true, document.Document{Id: docId}, service.service, result), true) +} + +func operationResult(read bool, doc document.Document, service *vespa.Service, result document.Result) util.OperationResult { + bodyReader := bytes.NewReader(result.Body) + if result.HTTPStatus == 200 { + if read { + return util.SuccessWithPayload("Read "+doc.Id.String(), util.ReaderToJSON(bodyReader)) + } else { + return util.Success(doc.Operation.String() + " " + doc.Id.String()) + } + } + if result.HTTPStatus/100 == 4 { + return util.FailureWithPayload("Invalid document operation: Status "+strconv.Itoa(result.HTTPStatus), util.ReaderToJSON(bodyReader)) + } + return util.FailureWithPayload(service.Description()+" at "+service.BaseURL+": Status "+strconv.Itoa(result.HTTPStatus), util.ReaderToJSON(bodyReader)) +} + func newDocumentCmd(cli *CLI) *cobra.Command { var ( printCurl bool @@ -44,11 +173,7 @@ should be used instead of this.`, SilenceUsage: true, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService(cli) - if err != nil { - return err - } - return printResult(cli, vespa.Send(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + return sendOperation(-1, args, timeoutSecs, printCurl, cli) }, } addDocumentFlags(cmd, &printCurl, &timeoutSecs) @@ -72,15 +197,7 @@ $ vespa document put id:mynamespace:music::a-head-full-of-dreams src/test/resour DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService(cli) - if err != nil { - return err - } - if len(args) == 1 { - return printResult(cli, vespa.Put("", args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) - } else { - return printResult(cli, vespa.Put(args[0], args[1], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) - } + return sendOperation(document.OperationPut, args, timeoutSecs, printCurl, cli) }, } addDocumentFlags(cmd, &printCurl, &timeoutSecs) @@ -103,15 +220,7 @@ $ vespa document update id:mynamespace:music::a-head-full-of-dreams src/test/res DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService(cli) - if err != nil { - return err - } - if len(args) == 1 { - return printResult(cli, vespa.Update("", args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) - } else { - return printResult(cli, vespa.Update(args[0], args[1], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) - } + return sendOperation(document.OperationUpdate, args, timeoutSecs, printCurl, cli) }, } addDocumentFlags(cmd, &printCurl, &timeoutSecs) @@ -134,14 +243,20 @@ $ vespa document remove id:mynamespace:music::a-head-full-of-dreams`, DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService(cli) - if err != nil { - return err - } if strings.HasPrefix(args[0], "id:") { - return printResult(cli, vespa.RemoveId(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + client, service, err := documentClient(cli, timeoutSecs, printCurl) + if err != nil { + return err + } + id, err := document.ParseId(args[0]) + if err != nil { + return err + } + doc := document.Document{Id: id, Operation: document.OperationRemove} + result := client.Send(doc) + return printResult(cli, operationResult(false, doc, service.service, result), false) } else { - return printResult(cli, vespa.RemoveOperation(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), false) + return sendOperation(document.OperationRemove, args, timeoutSecs, printCurl, cli) } }, } @@ -162,11 +277,7 @@ func newDocumentGetCmd(cli *CLI) *cobra.Command { SilenceUsage: true, Example: `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`, RunE: func(cmd *cobra.Command, args []string) error { - service, err := documentService(cli) - if err != nil { - return err - } - return printResult(cli, vespa.Get(args[0], service, operationOptions(cli.Stderr, printCurl, timeoutSecs)), true) + return readDocument(args[0], timeoutSecs, printCurl, cli) }, } addDocumentFlags(cmd, &printCurl, &timeoutSecs) @@ -181,17 +292,6 @@ func documentService(cli *CLI) (*vespa.Service, error) { return cli.service(target, vespa.DocumentService, 0, cli.config.cluster()) } -func operationOptions(stderr io.Writer, printCurl bool, timeoutSecs int) vespa.OperationOptions { - curlOutput := io.Discard - if printCurl { - curlOutput = stderr - } - return vespa.OperationOptions{ - CurlOutput: curlOutput, - Timeout: time.Second * time.Duration(timeoutSecs), - } -} - func printResult(cli *CLI, result util.OperationResult, payloadOnlyOnSuccess bool) error { out := cli.Stdout if !result.Success { diff --git a/client/go/internal/cli/cmd/document_test.go b/client/go/internal/cli/cmd/document_test.go index bf9cc0404dc..00f98ee1333 100644 --- a/client/go/internal/cli/cmd/document_test.go +++ b/client/go/internal/cli/cmd/document_test.go @@ -5,6 +5,7 @@ package cmd import ( + "encoding/json" "os" "strconv" "testing" @@ -20,6 +21,11 @@ func TestDocumentSendPut(t *testing.T) { "put", "POST", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Put.json", t) } +func TestDocumentSendPutWithIdInFile(t *testing.T) { + assertDocumentSend([]string{"document", "testdata/A-Head-Full-of-Dreams-Put-Id.json"}, + "put", "POST", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Put-Id.json", t) +} + func TestDocumentSendPutVerbose(t *testing.T) { assertDocumentSend([]string{"document", "-v", "testdata/A-Head-Full-of-Dreams-Put.json"}, "put", "POST", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Put.json", t) @@ -32,7 +38,7 @@ func TestDocumentSendUpdate(t *testing.T) { func TestDocumentSendRemove(t *testing.T) { assertDocumentSend([]string{"document", "testdata/A-Head-Full-of-Dreams-Remove.json"}, - "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Remove.json", t) + "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "", t) } func TestDocumentPutWithIdArg(t *testing.T) { @@ -57,19 +63,24 @@ func TestDocumentUpdateWithoutIdArg(t *testing.T) { func TestDocumentRemoveWithIdArg(t *testing.T) { assertDocumentSend([]string{"document", "remove", "id:mynamespace:music::a-head-full-of-dreams"}, - "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Remove.json", t) + "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "", t) } func TestDocumentRemoveWithoutIdArg(t *testing.T) { assertDocumentSend([]string{"document", "remove", "testdata/A-Head-Full-of-Dreams-Remove.json"}, - "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Remove.json", t) + "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "", t) +} + +func TestDocumentRemoveWithoutIdArgVerbose(t *testing.T) { + assertDocumentSend([]string{"document", "remove", "-v", "testdata/A-Head-Full-of-Dreams-Remove.json"}, + "remove", "DELETE", "id:mynamespace:music::a-head-full-of-dreams", "", t) } func TestDocumentSendMissingId(t *testing.T) { cli, _, stderr := newTestCLI(t) assert.NotNil(t, cli.Run("document", "put", "testdata/A-Head-Full-of-Dreams-Without-Operation.json")) assert.Equal(t, - "Error: No document id given neither as argument or as a 'put' key in the json file\n", + "Error: no document id given neither as argument or as a 'put', 'update' or 'remove' key in the JSON file\n", stderr.String()) } @@ -77,7 +88,7 @@ func TestDocumentSendWithDisagreeingOperations(t *testing.T) { cli, _, stderr := newTestCLI(t) assert.NotNil(t, cli.Run("document", "update", "testdata/A-Head-Full-of-Dreams-Put.json")) assert.Equal(t, - "Error: Wanted document operation is update but the JSON file specifies put\n", + "Error: wanted document operation is update, but JSON file specifies put\n", stderr.String()) } @@ -103,7 +114,7 @@ func assertDocumentSend(arguments []string, expectedOperation string, expectedMe t.Fatal(err) } expectedPath, _ := vespa.IdToURLPath(expectedDocumentId) - expectedURL := documentURL + "/document/v1/" + expectedPath + expectedURL := documentURL + "/document/v1/" + expectedPath + "?timeout=60000ms" assert.Nil(t, cli.Run(arguments...)) verbose := false @@ -113,16 +124,29 @@ func assertDocumentSend(arguments []string, expectedOperation string, expectedMe } } if verbose { - expectedCurl := "curl -X " + expectedMethod + " -H 'Content-Type: application/json' --data-binary @" + expectedPayloadFile + " " + expectedURL + "\n" + expectedCurl := "curl -X " + expectedMethod + " -H 'Content-Type: application/json; charset=utf-8' -H 'User-Agent: Vespa CLI/0.0.0-devel'" + if expectedPayloadFile != "" { + expectedCurl += " --data-binary @" + expectedPayloadFile + } + expectedCurl += " '" + expectedURL + "'\n" assert.Equal(t, expectedCurl, stderr.String()) } assert.Equal(t, "Success: "+expectedOperation+" "+expectedDocumentId+"\n", stdout.String()) assert.Equal(t, expectedURL, client.LastRequest.URL.String()) - assert.Equal(t, "application/json", client.LastRequest.Header.Get("Content-Type")) + assert.Equal(t, "application/json; charset=utf-8", client.LastRequest.Header.Get("Content-Type")) assert.Equal(t, expectedMethod, client.LastRequest.Method) - expectedPayload, _ := os.ReadFile(expectedPayloadFile) - assert.Equal(t, string(expectedPayload), util.ReaderToString(client.LastRequest.Body)) + if expectedPayloadFile != "" { + data, err := os.ReadFile(expectedPayloadFile) + assert.Nil(t, err) + var expectedPayload struct { + Fields json.RawMessage `json:"fields"` + } + assert.Nil(t, json.Unmarshal(data, &expectedPayload)) + assert.Equal(t, `{"fields":`+string(expectedPayload.Fields)+"}", util.ReaderToString(client.LastRequest.Body)) + } else { + assert.Nil(t, client.LastRequest.Body) + } } func assertDocumentGet(arguments []string, documentId string, t *testing.T) { @@ -170,7 +194,7 @@ func assertDocumentServerError(t *testing.T, status int, errorMessage string) { "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Put.json")) assert.Equal(t, - "Error: Container (document API) at 127.0.0.1:8080: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", + "Error: Container (document API) at http://127.0.0.1:8080: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", stderr.String()) } diff --git a/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Put-Id.json b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Put-Id.json new file mode 100644 index 00000000000..d39b22782ab --- /dev/null +++ b/client/go/internal/cli/cmd/testdata/A-Head-Full-of-Dreams-Put-Id.json @@ -0,0 +1,15 @@ +{ + "id": "id:mynamespace:music::a-head-full-of-dreams", + "fields": { + "album": "A Head Full of Dreams", + "artist": "Coldplay", + "year": 2015, + "category_scores": { + "cells": [ + { "address" : { "cat" : "pop" }, "value": 1 }, + { "address" : { "cat" : "rock" }, "value": 0.2 }, + { "address" : { "cat" : "jazz" }, "value": 0 } + ] + } + } +} diff --git a/client/go/internal/vespa/document.go b/client/go/internal/vespa/document.go deleted file mode 100644 index 9e4c8e7d136..00000000000 --- a/client/go/internal/vespa/document.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// vespa document API client -// Author: bratseth - -package vespa - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "time" - - "github.com/vespa-engine/vespa/client/go/internal/curl" - "github.com/vespa-engine/vespa/client/go/internal/util" -) - -// Sends the operation given in the file -func Send(jsonFile string, service *Service, options OperationOptions) util.OperationResult { - return sendOperation("", jsonFile, service, anyOperation, options) -} - -func Put(documentId string, jsonFile string, service *Service, options OperationOptions) util.OperationResult { - return sendOperation(documentId, jsonFile, service, putOperation, options) -} - -func Update(documentId string, jsonFile string, service *Service, options OperationOptions) util.OperationResult { - return sendOperation(documentId, jsonFile, service, updateOperation, options) -} - -func RemoveId(documentId string, service *Service, options OperationOptions) util.OperationResult { - return sendOperation(documentId, "", service, removeOperation, options) -} - -func RemoveOperation(jsonFile string, service *Service, options OperationOptions) util.OperationResult { - return sendOperation("", jsonFile, service, removeOperation, options) -} - -const ( - anyOperation string = "any" - putOperation string = "put" - updateOperation string = "update" - removeOperation string = "remove" -) - -type OperationOptions struct { - CurlOutput io.Writer - Timeout time.Duration -} - -func sendOperation(documentId string, jsonFile string, service *Service, operation string, options OperationOptions) util.OperationResult { - header := http.Header{} - header.Add("Content-Type", "application/json") - - var documentData []byte - if operation == "remove" && jsonFile == "" { - documentData = []byte("{\n \"remove\": \"" + documentId + "\"\n}\n") - } else { - fileReader, err := os.Open(jsonFile) - if err != nil { - return util.FailureWithDetail("Could not open file '"+jsonFile+"'", err.Error()) - } - defer fileReader.Close() - documentData, err = io.ReadAll(fileReader) - if err != nil { - return util.FailureWithDetail("Failed to read '"+jsonFile+"'", err.Error()) - } - } - - var doc map[string]interface{} - if err := json.Unmarshal(documentData, &doc); err != nil { - return util.Failure(fmt.Sprintf("Document is not valid JSON: %s", err)) - } - - operationInFile := operationIn(doc) - if operation == anyOperation { // Operation is decided by file content - operation = operationInFile - } else if operationInFile != "" && operationInFile != operation { // Otherwise operation must match - return util.Failure("Wanted document operation is " + operation + " but the JSON file specifies " + operationInFile) - } - - if documentId == "" { // Document id is decided by file content - if doc[operation] == nil { - return util.Failure("No document id given neither as argument or as a '" + operation + "' key in the json file") - } - documentId = doc[operation].(string) // document feeder format - } - - documentPath, documentPathError := IdToURLPath(documentId) - if documentPathError != nil { - return util.Failure("Invalid document id '" + documentId + "': " + documentPathError.Error()) - } - - url, urlParseError := url.Parse(service.BaseURL + "/document/v1/" + documentPath) - if urlParseError != nil { - return util.Failure("Invalid request path: '" + service.BaseURL + "/document/v1/" + documentPath + "': " + urlParseError.Error()) - } - - request := &http.Request{ - URL: url, - Method: operationToHTTPMethod(operation), - Header: header, - Body: io.NopCloser(bytes.NewReader(documentData)), - } - response, err := serviceDo(service, request, jsonFile, options) - if err != nil { - return util.Failure("Request failed: " + err.Error()) - } - - defer response.Body.Close() - if response.StatusCode == 200 { - return util.Success(operation + " " + documentId) - } else if response.StatusCode/100 == 4 { - return util.FailureWithPayload("Invalid document operation: "+response.Status, util.ReaderToJSON(response.Body)) - } else { - return util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) - } -} - -func operationIn(doc map[string]interface{}) string { - if doc["put"] != nil { - return "put" - } else if doc["update"] != nil { - return "update" - } else if doc["remove"] != nil { - return "remove" - } else { - return "" - } -} - -func operationToHTTPMethod(operation string) string { - switch operation { - case "put": - return "POST" - case "update": - return "PUT" - case "remove": - return "DELETE" - } - util.JustExitMsg("Unexpected document operation ''" + operation + "'") - panic("unreachable") -} - -func serviceDo(service *Service, request *http.Request, filename string, options OperationOptions) (*http.Response, error) { - cmd, err := curl.RawArgs(request.URL.String()) - if err != nil { - return nil, err - } - cmd.Method = request.Method - for k, vs := range request.Header { - for _, v := range vs { - cmd.Header(k, v) - } - } - cmd.WithBodyFile(filename) - cmd.Certificate = service.TLSOptions.CertificateFile - cmd.PrivateKey = service.TLSOptions.PrivateKeyFile - out := cmd.String() + "\n" - if _, err := io.WriteString(options.CurlOutput, out); err != nil { - return nil, err - } - return service.Do(request, options.Timeout) -} - -func Get(documentId string, service *Service, options OperationOptions) util.OperationResult { - documentPath, documentPathError := IdToURLPath(documentId) - if documentPathError != nil { - return util.Failure("Invalid document id '" + documentId + "': " + documentPathError.Error()) - } - - url, urlParseError := url.Parse(service.BaseURL + "/document/v1/" + documentPath) - if urlParseError != nil { - return util.Failure("Invalid request path: '" + service.BaseURL + "/document/v1/" + documentPath + "': " + urlParseError.Error()) - } - - request := &http.Request{ - URL: url, - Method: "GET", - } - response, err := serviceDo(service, request, "", options) - if err != nil { - return util.Failure("Request failed: " + err.Error()) - } - - defer response.Body.Close() - if response.StatusCode == 200 { - return util.SuccessWithPayload("Read "+documentId, util.ReaderToJSON(response.Body)) - } else if response.StatusCode/100 == 4 { - return util.FailureWithPayload("Invalid document operation: "+response.Status, util.ReaderToJSON(response.Body)) - } else { - return util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) - } -} diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 0c27b0d5440..277ef50ab33 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -127,37 +127,38 @@ func writeQueryParam(sb *bytes.Buffer, start int, escape bool, k, v string) { } } -func (c *Client) methodAndURL(d Document, sb *bytes.Buffer) (string, string) { - httpMethod := "" - switch d.Operation { - case OperationPut: - httpMethod = "POST" - case OperationUpdate: - httpMethod = "PUT" - case OperationRemove: - httpMethod = "DELETE" - } - // Base URL and path - sb.WriteString(c.options.BaseURL) - if !strings.HasSuffix(c.options.BaseURL, "/") { - sb.WriteString("/") - } - sb.WriteString("document/v1/") - sb.WriteString(url.PathEscape(d.Id.Namespace)) +func (c *Client) writeDocumentPath(id Id, sb *bytes.Buffer) { + sb.WriteString(strings.TrimSuffix(c.options.BaseURL, "/")) + sb.WriteString("/document/v1/") + sb.WriteString(url.PathEscape(id.Namespace)) sb.WriteString("/") - sb.WriteString(url.PathEscape(d.Id.Type)) - if d.Id.Number != nil { + sb.WriteString(url.PathEscape(id.Type)) + if id.Number != nil { sb.WriteString("/number/") - n := uint64(*d.Id.Number) + n := uint64(*id.Number) sb.WriteString(strconv.FormatUint(n, 10)) - } else if d.Id.Group != "" { + } else if id.Group != "" { sb.WriteString("/group/") - sb.WriteString(url.PathEscape(d.Id.Group)) + sb.WriteString(url.PathEscape(id.Group)) } else { sb.WriteString("/docid") } sb.WriteString("/") - sb.WriteString(url.PathEscape(d.Id.UserSpecific)) + sb.WriteString(url.PathEscape(id.UserSpecific)) +} + +func (c *Client) methodAndURL(d Document, sb *bytes.Buffer) (string, string) { + httpMethod := "" + switch d.Operation { + case OperationPut: + httpMethod = http.MethodPost + case OperationUpdate: + httpMethod = http.MethodPut + case OperationRemove: + httpMethod = http.MethodDelete + } + // Base URL and path + c.writeDocumentPath(d.Id, sb) // Query part queryStart := sb.Len() if c.options.Timeout > 0 { @@ -290,7 +291,28 @@ func (c *Client) Send(document Document) Result { } defer resp.Body.Close() elapsed := c.now().Sub(start) - return resultWithResponse(resp, bodySize, result, elapsed, buf) + return resultWithResponse(resp, bodySize, result, elapsed, buf, true) +} + +// Get retrieves document with given ID.nnnnnn +func (c *Client) Get(id Id) Result { + start := c.now() + buf := c.buffer() + defer c.buffers.Put(buf) + c.writeDocumentPath(id, buf) + url := buf.String() + result := Result{Id: id} + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return resultWithErr(result, err) + } + resp, err := c.leastBusyClient().Do(req, c.clientTimeout()) + if err != nil { + return resultWithErr(result, err) + } + defer resp.Body.Close() + elapsed := c.now().Sub(start) + return resultWithResponse(resp, 0, result, elapsed, buf, false) } func resultWithErr(result Result, err error) Result { @@ -299,7 +321,7 @@ func resultWithErr(result Result, err error) Result { return result } -func resultWithResponse(resp *http.Response, sentBytes int, result Result, elapsed time.Duration, buf *bytes.Buffer) Result { +func resultWithResponse(resp *http.Response, sentBytes int, result Result, elapsed time.Duration, buf *bytes.Buffer, parseBody bool) Result { result.HTTPStatus = resp.StatusCode switch resp.StatusCode { case 200: @@ -316,15 +338,20 @@ func resultWithResponse(resp *http.Response, sentBytes int, result Result, elaps if err != nil { result = resultWithErr(result, err) } else { - var body struct { - Message string `json:"message"` - Trace json.RawValue `json:"trace"` - } - if err := json.Unmarshal(buf.Bytes(), &body); err != nil { - result = resultWithErr(result, fmt.Errorf("failed to decode json response: %w", err)) + if resp.StatusCode == 200 && parseBody { + var body struct { + Message string `json:"message"` + Trace json.RawValue `json:"trace"` + } + if err := json.Unmarshal(buf.Bytes(), &body); err != nil { + result = resultWithErr(result, fmt.Errorf("failed to decode json response: %w", err)) + } else { + result.Message = body.Message + result.Trace = string(body.Trace) + } } else { - result.Message = body.Message - result.Trace = string(body.Trace) + result.Body = make([]byte, buf.Len()) + copy(result.Body, buf.Bytes()) } } result.Latency = elapsed diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index b34b322f6eb..2382e5f07cb 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -95,11 +95,12 @@ func TestClientSend(t *testing.T) { wantRes.Message = "All good!" wantRes.BytesRecv = 23 } else { - httpClient.NextResponseString(502, `{"message":"Good bye, cruel world!"}`) + errMsg := `something went wront` + httpClient.NextResponseString(502, errMsg) wantRes.Status = StatusVespaFailure wantRes.HTTPStatus = 502 - wantRes.Message = "Good bye, cruel world!" - wantRes.BytesRecv = 36 + wantRes.Body = []byte(errMsg) + wantRes.BytesRecv = 20 } res := client.Send(doc) wantRes.BytesSent = int64(len(httpClient.LastBody)) @@ -134,13 +135,45 @@ func TestClientSend(t *testing.T) { MinLatency: time.Second, MaxLatency: time.Second, BytesSent: 75, - BytesRecv: 105, + BytesRecv: 89, } if !reflect.DeepEqual(want, stats) { t.Errorf("got %+v, want %+v", stats, want) } } +func TestClientGet(t *testing.T) { + httpClient := mock.HTTPClient{ReadBody: true} + client, _ := NewClient(ClientOptions{ + BaseURL: "https://example.com:1337", + Timeout: time.Duration(5 * time.Second), + }, []util.HTTPClient{&httpClient}) + clock := manualClock{t: time.Now(), tick: time.Second} + client.now = clock.now + doc := `{ + "pathId": "/document/v1/mynamespace/music/docid/doc1", + "id": "id:mynamespace:music::doc1", + "fields": { + "artist": "Metallica", + "album": "Master of Puppets" + } +}` + id := Id{Namespace: "mynamespace", Type: "music", UserSpecific: "doc1"} + httpClient.NextResponseString(200, doc) + result := client.Get(id) + want := Result{ + Id: id, + Body: []byte(doc), + Status: StatusSuccess, + HTTPStatus: 200, + Latency: time.Second, + BytesRecv: 192, + } + if !reflect.DeepEqual(want, result) { + t.Errorf("got %+v, want %+v", result, want) + } +} + func TestClientSendCompressed(t *testing.T) { httpClient := &mock.HTTPClient{ReadBody: true} client, _ := NewClient(ClientOptions{ diff --git a/client/go/internal/vespa/document/stats.go b/client/go/internal/vespa/document/stats.go index 7696648f703..e1b5520a779 100644 --- a/client/go/internal/vespa/document/stats.go +++ b/client/go/internal/vespa/document/stats.go @@ -26,6 +26,7 @@ type Result struct { Id Id Message string Trace string + Body []byte Status Status HTTPStatus int Latency time.Duration |