diff options
14 files changed, 52 insertions, 35 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java index 98e230cb8ba..dd9e057e2fa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java @@ -90,7 +90,9 @@ public class FileDistributorService extends AbstractService implements @Override public void getConfig(FiledistributorrpcConfig.Builder builder) { - builder.connectionspec("tcp/" + getHostName() + ":" + getRelativePort(0)); + // If disabled config proxy should act as file distributor, so use config proxy port + int port = (fileDistributionOptions.disabled()) ? 19090 : getRelativePort(0); + builder.connectionspec("tcp/" + getHostName() + ":" + port); } @Override diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 1b0d487a388..a88109663eb 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -84,7 +84,7 @@ LogServer = element logserver { FileDistribution = element filedistribution { element uploadbitrate { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? & element downloadbitrate { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? & - element disabled { xsd:boolean }? + element disabled { xsd:boolean }? # Nov. 2017: Temporary, should not be documented } Metrics = element metrics { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index 552758bcfa4..46b1ffc721e 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -138,11 +138,13 @@ public class FileDistributionRpcServer { String filename = req.parameters().get(1).asString(); byte[] content = req.parameters().get(2).asData(); long xxhash = req.parameters().get(3).asInt64(); - int errorCode = req.parameters().get(3).asInt32(); - String errorDescription = req.parameters().get(4).asString(); + int errorCode = req.parameters().get(4).asInt32(); + String errorDescription = req.parameters().get(5).asString(); if (errorCode == 0) { - //downloader.receive(fileReference, filename, content); + // TODO: Remove when system test works + log.log(LogLevel.INFO, "Receiving file reference '" + fileReference.value() + "'"); + downloader.receiveFile(fileReference, filename, content); req.returnValues().add(new Int32Value(0)); } else { log.log(LogLevel.WARNING, "Receiving file reference '" + fileReference.value() + "' failed: " + errorDescription); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java index 611ad67a5d8..4b32ffab2b7 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferenceDownloader.java @@ -118,7 +118,8 @@ class FileReferenceDownloader { } private synchronized void completedDownloading(FileReference fileReference, File file) { - downloads.get(fileReference).future().set(Optional.of(file)); + if (downloads.containsKey(fileReference)) + downloads.get(fileReference).future().set(Optional.of(file)); downloadStatus.put(fileReference, 100.0); } diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java index c44e19f9f03..cad3d2d0330 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileDownloaderTest.java @@ -55,8 +55,8 @@ public class FileDownloaderTest { String fileReferenceString = "foo"; String filename = "foo.jar"; - File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReferenceString); FileReference fileReference = new FileReference(fileReferenceString); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); writeFileReference(downloadDir, fileReferenceString, filename); // Check that we get correct path and content when asking for file reference @@ -76,7 +76,7 @@ public class FileDownloaderTest { connection.setResponseHandler(new MockConnection.UnknownFileReferenceResponseHandler()); FileReference fileReference = new FileReference("bar"); - File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference.value()); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); // Verify download status when unable to download @@ -87,7 +87,7 @@ public class FileDownloaderTest { // fileReference does not exist on disk, needs to be downloaded) FileReference fileReference = new FileReference("fileReference"); - File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference.value()); + File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); // Verify download status @@ -125,13 +125,24 @@ public class FileDownloaderTest { assertDownloadStatus(fileDownloader, bar, 0.0); } + @Test + public void receiveFile() throws IOException { + File downloadDir = Files.createTempDirectory("filedistribution").toFile(); + FileDownloader fileDownloader = new FileDownloader(null, downloadDir, Duration.ofMillis(200)); + FileReference foo = new FileReference("foo"); + String filename = "foo.jar"; + fileDownloader.receiveFile(foo, filename, Utf8.toBytes("content")); + File downloadedFile = new File(fileReferenceFullPath(downloadDir, foo), filename); + assertEquals("content", IOUtils.readFile(downloadedFile)); + } + private void writeFileReference(File dir, String fileReferenceString, String fileName) throws IOException { File file = new File(new File(dir, fileReferenceString), fileName); IOUtils.writeFile(file, "content", false); } - private File fileReferenceFullPath(File dir, String fileReferenceString) { - return new File(dir, fileReferenceString); + private File fileReferenceFullPath(File dir, FileReference fileReference) { + return new File(dir, fileReference.value()); } private void assertDownloadStatus(FileDownloader fileDownloader, FileReference fileReference, double expectedDownloadStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index a878a6afe61..ceb04d88026 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -102,7 +102,9 @@ public class JobStatus { public DeploymentJobs.JobType type() { return type; } /** Returns true unless this job last completed with a failure */ - public boolean isSuccess() { return ! jobError.isPresent(); } + public boolean isSuccess() { + return lastCompleted().isPresent() && ! jobError.isPresent(); + } /** Returns true if last triggered is newer than last completed and was started after timeoutLimit */ public boolean isRunning(Instant timeoutLimit) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index adb11f681d1..06b62b6bcee 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -305,6 +305,7 @@ public class DeploymentTrigger { private boolean isFailing(Change change, JobStatus status) { return status != null && ! status.isSuccess() + && status.lastCompleted().isPresent() && status.lastCompleted().get().lastCompletedWas(change); } diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index f2a4ab3a23d..ff6c1ab0b1d 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -45,7 +45,7 @@ public: void SyncPacket::Free() { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _done = true; if (_waiting) { _cond.notify_one(); @@ -560,7 +560,7 @@ FNET_Connection::OpenChannel() uint32_t chid; { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); chid = GetNextID(); AddRef_NoLock(); } @@ -652,7 +652,7 @@ FNET_Connection::PostPacket(FNET_Packet *packet, uint32_t chid) uint32_t FNET_Connection::GetQueueLen() { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); return _queue.GetPacketCnt_NoLock() + _myQueue.GetPacketCnt_NoLock(); } @@ -710,7 +710,7 @@ FNET_Connection::HandleReadEvent() bool FNET_Connection::writePendingAfterConnect() { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); _state = FNET_CONNECTED; // SetState(FNET_CONNECTED) LOG(debug, "Connection(%s): State transition: %s -> %s", GetSpec(), GetStateString(FNET_CONNECTING), GetStateString(FNET_CONNECTED)); diff --git a/fnet/src/vespa/fnet/frt/invoker.cpp b/fnet/src/vespa/fnet/frt/invoker.cpp index 5afc355c68f..ce4daa988de 100644 --- a/fnet/src/vespa/fnet/frt/invoker.cpp +++ b/fnet/src/vespa/fnet/frt/invoker.cpp @@ -32,7 +32,7 @@ void FRT_SingleReqWait::RequestDone(FRT_RPCRequest *req) { (void) req; - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _done = true; if (_waiting) { _cond.notify_one(); diff --git a/fnet/src/vespa/fnet/iocomponent.cpp b/fnet/src/vespa/fnet/iocomponent.cpp index ec51c1f080e..8276da57e2e 100644 --- a/fnet/src/vespa/fnet/iocomponent.cpp +++ b/fnet/src/vespa/fnet/iocomponent.cpp @@ -48,7 +48,7 @@ FNET_IOComponent::UpdateTimeOut() { void FNET_IOComponent::AddRef() { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); assert(_ioc_refcnt > 0); _ioc_refcnt++; } @@ -66,7 +66,7 @@ void FNET_IOComponent::SubRef() { { - std::unique_lock<std::mutex> guard(_ioc_lock); + std::lock_guard<std::mutex> guard(_ioc_lock); assert(_ioc_refcnt > 0); if (--_ioc_refcnt > 0) { return; diff --git a/fnet/src/vespa/fnet/packetqueue.cpp b/fnet/src/vespa/fnet/packetqueue.cpp index 4fdb5842a9d..4331819d3f5 100644 --- a/fnet/src/vespa/fnet/packetqueue.cpp +++ b/fnet/src/vespa/fnet/packetqueue.cpp @@ -192,7 +192,7 @@ void FNET_PacketQueue::QueuePacket(FNET_Packet *packet, FNET_Context context) { assert(packet != nullptr); - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); EnsureFree(); _buf[_in_pos]._packet = packet; // insert packet ref. _buf[_in_pos]._context = context; @@ -257,7 +257,7 @@ FNET_PacketQueue::DequeuePacket(uint32_t maxwait, FNET_Context *context) void FNET_PacketQueue::Print(uint32_t indent) { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); uint32_t i = _out_pos; uint32_t cnt = _bufused; diff --git a/fnet/src/vespa/fnet/scheduler.cpp b/fnet/src/vespa/fnet/scheduler.cpp index ef67407cb44..6c8340c1ff8 100644 --- a/fnet/src/vespa/fnet/scheduler.cpp +++ b/fnet/src/vespa/fnet/scheduler.cpp @@ -40,7 +40,7 @@ FNET_Scheduler::~FNET_Scheduler() bool empty = true; std::stringstream dump; { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); dump << "FNET_Scheduler {" << std::endl; dump << " [slot=" << _currSlot << "][iter=" << _currIter << "]" << std::endl; for (int i = 0; i <= NUM_SLOTS; i++) { @@ -70,7 +70,7 @@ FNET_Scheduler::Schedule(FNET_Task *task, double seconds) { uint32_t ticks = 1 + (uint32_t) (seconds * (1000 / SLOT_TICK) + 0.5); - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); if (!task->_killed) { if (IsActive(task)) LinkOut(task); @@ -84,7 +84,7 @@ FNET_Scheduler::Schedule(FNET_Task *task, double seconds) void FNET_Scheduler::ScheduleNow(FNET_Task *task) { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); if (!task->_killed) { if (IsActive(task)) LinkOut(task); @@ -119,7 +119,7 @@ FNET_Scheduler::Kill(FNET_Task *task) void FNET_Scheduler::Print(FILE *dst) { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); fprintf(dst, "FNET_Scheduler {\n"); fprintf(dst, " [slot=%d][iter=%d]\n", _currSlot, _currIter); for (int i = 0; i <= NUM_SLOTS; i++) { diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 61fcc26f1c6..443c90f1af4 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -173,7 +173,7 @@ FNET_TransportThread::UpdateStats() comp->FlushDirectWriteStats(); } { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _stats.Update(&_counters, ms / 1000.0); } _counters.Clear(); @@ -238,7 +238,7 @@ FNET_TransportThread::FNET_TransportThread(FNET_Transport &owner_in) FNET_TransportThread::~FNET_TransportThread() { { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _deleted = true; } if (_started && !_finished) { @@ -379,7 +379,7 @@ FNET_TransportThread::ShutDown(bool waitFinished) { bool wasEmpty = false; { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); if (!_shutdown) { _shutdown = true; wasEmpty = _queue.IsEmpty_NoLock(); @@ -413,7 +413,7 @@ FNET_TransportThread::InitEventLoop() bool wasStarted; bool wasDeleted; { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); wasStarted = _started; wasDeleted = _deleted; if (!_started && !_deleted) { @@ -440,7 +440,7 @@ void FNET_TransportThread::handle_wakeup() { { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); CountEvent(_queue.FlushPackets_NoLock(&_myQueue)); } @@ -590,7 +590,7 @@ FNET_TransportThread::EventLoopIteration() // flush event queue { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _queue.FlushPackets_NoLock(&_myQueue); } @@ -623,7 +623,7 @@ FNET_TransportThread::EventLoopIteration() _myQueue.IsEmpty_NoLock()); { - std::unique_lock<std::mutex> guard(_lock); + std::lock_guard<std::mutex> guard(_lock); _finished = true; if (_waitFinished) { _cond.notify_all(); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java index 787c976f6a0..b52e923662f 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java @@ -46,7 +46,7 @@ import static com.yahoo.jdisc.http.server.jetty.Exceptions.throwUnchecked; * @author Tony Vaagenes * @author bjorncs */ -class PemKeyStore extends KeyStoreSpi { +public class PemKeyStore extends KeyStoreSpi { private static String KEY_ALIAS = "KEY"; @@ -61,8 +61,6 @@ class PemKeyStore extends KeyStoreSpi { @GuardedBy("this") private final Map<String, Certificate> aliasToCertificate = new LinkedHashMap<>(); - PemKeyStore() {} - /** * The user is responsible for closing any readers given in the parameter. */ |