1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
|
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.http.client.runner;
import com.yahoo.vespa.http.client.FeedClient;
import com.yahoo.vespa.http.client.FeedClientFactory;
import com.yahoo.vespa.http.client.SimpleLoggerResultCallback;
import com.yahoo.vespa.http.client.core.JsonReader;
import com.yahoo.vespa.http.client.core.XmlFeedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.time.Clock;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author Einar M R Rosenvinge
* @author dybis
*/
public class Runner {
/**
* Feed data from inputFile to session.
*
* @param feedClient where to send data to
* @param inputStream source of data
* @param isJson if input stream is of json formatted data
* @param numSent is updated while sending by this method
* @param verbose if true will print some information to stderr
* @return send time in ms, not including validating
*/
public static long send(FeedClient feedClient,
InputStream inputStream,
boolean isJson,
AtomicInteger numSent,
boolean verbose) {
Clock clock = Clock.systemUTC();
if (verbose)
System.err.println("Now sending data.");
long sendStartTime = clock.millis();
if (isJson) {
JsonReader.read(inputStream, feedClient, numSent);
} else {
try {
XmlFeedReader.read(inputStream, feedClient, numSent);
} catch (Exception e) {
System.err.println("Stopped reading feed, got problems with XML: " + e.getMessage());
}
}
long sendTotalTime = clock.millis() - sendStartTime;
if (verbose)
System.err.println("Waiting for all results, sent " + numSent.get() + " docs.");
feedClient.close();
if (verbose)
System.err.println("Session closed.");
return sendTotalTime;
}
public static void main(String[] args) throws IOException {
CommandLineArguments commandLineArgs = CommandLineArguments.build(args);
if (commandLineArgs == null)
System.exit(1);
FormatInputStream formatInputStream = new FormatInputStream(System.in,
Optional.ofNullable(commandLineArgs.getFile()),
commandLineArgs.getAddRootElementToXml());
int intervalOfLogging =
commandLineArgs.getVerbose()
? commandLineArgs.getWhenVerboseEnabledPrintMessageForEveryXDocuments()
: Integer.MAX_VALUE;
AtomicInteger numSent = new AtomicInteger(0);
SimpleLoggerResultCallback callback = new SimpleLoggerResultCallback(numSent, intervalOfLogging);
FeedClient feedClient = FeedClientFactory.create(commandLineArgs.createSessionParams(formatInputStream.getFormat()== FormatInputStream.Format.JSON),
callback);
long sendTotalTimeMs = send(feedClient,
formatInputStream.getInputStream(),
formatInputStream.getFormat() == FormatInputStream.Format.JSON,
numSent,
commandLineArgs.getVerbose());
if (commandLineArgs.getVerbose()) {
System.err.println(feedClient.getStatsAsJson());
double transferTimeSec = ((double) sendTotalTimeMs) / 1000.0;
if (transferTimeSec > 0)
System.err.printf("Docs/sec %.3f%n", numSent.get() / transferTimeSec);
if (commandLineArgs.getFile() != null) {
double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0;
System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds.");
System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " +
"(not taking compression into account)");
}
}
callback.printProgress();
}
}
|
