diff options
4 files changed, 35 insertions, 95 deletions
diff --git a/dist/post_install.sh b/dist/post_install.sh index 660aec141f5..f97cb0b1c0e 100755 --- a/dist/post_install.sh +++ b/dist/post_install.sh @@ -77,86 +77,3 @@ fallback VESPA_HOME $PREFIX override VESPA_USER vespa EOF -# Temporary when renaming programs in filedistribution -ln -s vespa-filedistributor $INSTALLPATH/sbin/filedistributor -ln -s vespa-filedistributor-bin $INSTALLPATH/sbin/filedistributor-bin - -# Temporary when renaming binaries in fnet -ln -s vespa-rpc-info $INSTALLPATH/bin/rpc_info -ln -s vespa-rpc-invoke $INSTALLPATH/bin/rpc_invoke -ln -s vespa-rpc-proxy $INSTALLPATH/bin/rpc_proxy - -# Temporary when renaming binaries in fsa -ln -s vespa-fsadump $INSTALLPATH/bin/fsadump -ln -s vespa-fsainfo $INSTALLPATH/bin/fsainfo -ln -s vespa-makefsa $INSTALLPATH/bin/makefsa - -# Temporary when renaming binaries in searchcore -ln -s vespa-proton-bin $INSTALLPATH/sbin/proton-bin -ln -s vespa-dispatch-bin $INSTALLPATH/sbin/fdispatch-bin -ln -s vespa-verify-ranksetup-bin $INSTALLPATH/bin/verify_ranksetup-bin -ln -s vespa-proton $INSTALLPATH/sbin/proton -ln -s vespa-dispatch $INSTALLPATH/sbin/fdispatch -ln -s vespa-verify-ranksetup $INSTALLPATH/bin/verify_ranksetup - -# Temporary when renaming binaries in storageserver -ln -s vespa-storaged-bin $INSTALLPATH/sbin/storaged-bin -ln -s vespa-distributord-bin $INSTALLPATH/sbin/distributord-bin -ln -s vespa-storaged $INSTALLPATH/sbin/storaged -ln -s vespa-distributord $INSTALLPATH/sbin/distributord - -# Temporary when renaming binaries in vespaclient -ln -s vespa-spoolmaster $INSTALLPATH/bin/spoolmaster - -# Temporary when renaming binaries in vespaclient-java -ln -s vespa-document-statistics $INSTALLPATH/bin/vds-document-statistics -ln -s vespa-stat $INSTALLPATH/bin/vdsstat -ln -s vespa-destination $INSTALLPATH/bin/vespadestination -ln -s vespa-feeder $INSTALLPATH/bin/vespafeeder -ln -s vespa-get $INSTALLPATH/bin/vespaget -ln -s vespa-visit $INSTALLPATH/bin/vespavisit -ln -s vespa-visit-target $INSTALLPATH/bin/vespavisittarget - -# Temporary when renaming binaries in vespalog -ln -s vespa-log-conv $INSTALLPATH/bin/vlogconv -ln -s vespa-logctl $INSTALLPATH/bin/logctl -ln -s vespa-logfmt $INSTALLPATH/bin/logfmt -ln -s vespa-log-replay $INSTALLPATH/bin/vlogreplay - -# Temporary when renaming binaries in logd -ln -s vespa-logd $INSTALLPATH/sbin/logd - -# Temporary when renaming binaries in memfilepersistence -ln -s vespa-vds-dump-slotfile $INSTALLPATH/bin/dumpslotfile -ln -s vespa-vds-disktool-bin $INSTALLPATH/bin/vdsdisktool-bin -ln -s vespa-vds-disktool $INSTALLPATH/bin/vdsdisktool - -# Temporary when renaming programs in config-model -ln -s vespa-deploy $INSTALLPATH/bin/deploy -ln -s vespa-deploy-application $INSTALLPATH/bin/deploy-application -ln -s vespa-expand-config.pl $INSTALLPATH/bin/expand-config.pl -ln -s vespa-validate-application $INSTALLPATH/bin/validate-application - -# Temporary when renaming programs in config-model -ln -s vespa-activate-application $INSTALLPATH/bin/activate-application -ln -s vespa-configproxy-cmd $INSTALLPATH/bin/configproxy-cmd -ln -s vespa-get-config $INSTALLPATH/bin/getvespaconfig -ln -s vespa-get-config-bin $INSTALLPATH/bin/getvespaconfig-bin -ln -s vespa-ping-configproxy $INSTALLPATH/bin/pingproxy - -# Temporary when renaming programs in fbench -ln -s vespa-fbench $INSTALLPATH/bin/fbench -ln -s vespa-fbench-filter-file $INSTALLPATH/bin/filterfile -ln -s vespa-fbench-geturl $INSTALLPATH/bin/geturl -ln -s vespa-fbench-split-file $INSTALLPATH/bin/splitfile - -# Temporary when renaming programs in slobrok -ln -s vespa-slobrok $INSTALLPATH/bin/slobrok -ln -s vespa-slobrok-cmd $INSTALLPATH/bin/sbcmd - -# Temporary when renaming programs in configd -ln -s vespa-run-as-vespa-user $INSTALLPATH/bin/run-as-yahoo -ln -s vespa-config-sentinel $INSTALLPATH/sbin/config-sentinel - -# Temporary when renaming programs in logserver -ln -s vespa-logserver-start $INSTALLPATH/bin/logserver-start diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java index 90e2049a868..588e5293066 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java @@ -53,7 +53,7 @@ class ActiveContainerDeactivationWatchdog implements ActiveContainerMetrics, Aut ActiveContainerDeactivationWatchdog() { this( Clock.systemUTC(), - new ScheduledThreadPoolExecutor(3, runnable -> { + new ScheduledThreadPoolExecutor(1, runnable -> { Thread thread = new Thread(runnable, "active-container-deactivation-watchdog"); thread.setDaemon(true); return thread; @@ -63,11 +63,12 @@ class ActiveContainerDeactivationWatchdog implements ActiveContainerMetrics, Aut ActiveContainerDeactivationWatchdog(Clock clock, ScheduledExecutorService scheduler) { this.clock = clock; this.scheduler = scheduler; + // NOTE: Make sure to update the unit test if the order commands are registered is changed. this.scheduler.scheduleAtFixedRate(this::warnOnStaleContainers, WATCHDOG_FREQUENCY.getSeconds(), WATCHDOG_FREQUENCY.getSeconds(), TimeUnit.SECONDS); - this.scheduler.scheduleAtFixedRate(System::gc, + this.scheduler.scheduleAtFixedRate(ActiveContainerDeactivationWatchdog::triggerGc, GC_TRIGGER_FREQUENCY.getSeconds(), GC_TRIGGER_FREQUENCY.getSeconds(), TimeUnit.SECONDS); @@ -112,6 +113,7 @@ class ActiveContainerDeactivationWatchdog implements ActiveContainerMetrics, Aut } private void warnOnStaleContainers() { + log.log(Level.FINE, "Checking for stale containers"); try { List<DeactivatedContainer> snapshot = getDeactivatedContainersSnapshot(); if (snapshot.isEmpty()) return; @@ -121,7 +123,14 @@ class ActiveContainerDeactivationWatchdog implements ActiveContainerMetrics, Aut } } + private static void triggerGc() { + log.log(Level.FINE, "Triggering GC"); + System.gc(); + System.runFinalization(); // this is required to trigger enqueuing of phantom references on some Linux systems + } + private void enforceDestructionOfGarbageCollectedContainers() { + log.log(Level.FINE, "Enforcing destruction of GCed containers"); ActiveContainerPhantomReference reference; while ((reference = (ActiveContainerPhantomReference) garbageCollectedContainers.poll()) != null) { try { diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java index 05207728c88..c83458796ed 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdogTest.java @@ -6,7 +6,6 @@ import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.statistics.ActiveContainerMetrics; import com.yahoo.jdisc.test.TestDriver; import com.yahoo.test.ManualClock; -import org.junit.Ignore; import org.junit.Test; import java.lang.ref.WeakReference; @@ -69,7 +68,6 @@ public class ActiveContainerDeactivationWatchdogTest { } @Test - @Ignore("JVM does not give any guarantee when phantom references will be enqueued to reference queues") public void deactivated_container_destructed_if_its_reference_counter_is_nonzero() { ExecutorMock executor = new ExecutorMock(); ActiveContainerDeactivationWatchdog watchdog = @@ -86,11 +84,14 @@ public class ActiveContainerDeactivationWatchdogTest { WeakReference<ActiveContainer> containerWeakReference = new WeakReference<>(container); container = null; // make container instance collectable by GC - System.gc(); + + executor.triggerGcCommand.run(); assertNull("Container is not GCed - probably because the watchdog has a concrete reference to it", containerWeakReference.get()); - executor.containerDestructorCommand.run(); + + executor.enforceDestructionOfGarbageCollectedContainersCommand.run(); + assertTrue("Destructor is not called on deactivated container", destructed.get()); } @@ -125,7 +126,9 @@ public class ActiveContainerDeactivationWatchdogTest { private static class ExecutorMock extends ScheduledThreadPoolExecutor { - public Runnable containerDestructorCommand; + public Runnable warnOnStaleContainersCommand; + public Runnable triggerGcCommand; + public Runnable enforceDestructionOfGarbageCollectedContainersCommand; private int registrationCounter = 0; public ExecutorMock() { @@ -134,14 +137,19 @@ public class ActiveContainerDeactivationWatchdogTest { @Override public ScheduledFuture<?> scheduleAtFixedRate(Runnable command, long initialDelay, long period, TimeUnit unit) { - if (registrationCounter == 2) { - containerDestructorCommand = command; - } else if (registrationCounter > 2){ + if (registrationCounter == 0) { + warnOnStaleContainersCommand = command; + } else if (registrationCounter == 1) { + triggerGcCommand = command; + } else if (registrationCounter == 2) { + enforceDestructionOfGarbageCollectedContainersCommand = command; + } else { throw new IllegalStateException("Unexpected registration"); } ++registrationCounter; return null; } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 63ba0946316..4e2d6332a46 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -63,10 +63,15 @@ public class AclMaintainer implements Runnable { } final Command flush = new FlushCommand(Chain.INPUT); final Command rollback = new PolicyCommand(Chain.INPUT, Action.ACCEPT); + log.info("Start modifying ACL rules"); try { + log.info("Running ACL command '" + flush.asString() + "'"); dockerOperations.executeCommandInNetworkNamespace(containerName, flush.asArray(IPTABLES_COMMAND)); - acl.toCommands().forEach(command -> dockerOperations.executeCommandInNetworkNamespace(containerName, - command.asArray(IPTABLES_COMMAND))); + acl.toCommands().forEach(command -> { + log.info("Running ACL command '" + command.asString() + "'"); + dockerOperations.executeCommandInNetworkNamespace(containerName, + command.asArray(IPTABLES_COMMAND)); + }); containerAcls.put(containerName, acl); } catch (Exception e) { log.error("Exception occurred while configuring ACLs, attempting rollback", e); @@ -76,6 +81,7 @@ public class AclMaintainer implements Runnable { log.error("Rollback failed, giving up", ne); } } + log.info("Finished modifying ACL rules"); } private synchronized void configureAcls() { |