Share embedded stack for all JobCriteriaQueryTest tests (#794)

+ do not run in the security manager which add 30sec delay on startup due to slowness in
java.io.UnixFileSystem.canonicalize0
(more detailed analysis available here: facebookarchive/nailgun#134)

build.gradle

@@ -159,8 +159,10 @@ subprojects {
         testCompile "org.assertj:assertj-core:${assertjVersion}"
     }
 
+    def processorTarget = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+
     test {
-        maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+        maxParallelForks = processorTarget
 
         useJUnit {
             excludeCategories 'com.netflix.titus.testkit.junit.category.IntegrationTest', 'com.netflix.titus.testkit.junit.category.IntegrationNotParallelizableTest',
@@ -182,7 +184,7 @@ subprojects {
     }
 
     task integrationTest(type: Test) {
-        maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+        maxParallelForks = processorTarget
         shouldRunAfter test
 
         useJUnit {
@@ -225,7 +227,7 @@ subprojects {
     }
 
     task remoteIntegrationTest(type: Test) {
-        maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+        maxParallelForks = processorTarget
         shouldRunAfter test
 
         useJUnit {

titus-api/src/main/java/com/netflix/titus/api/jobmanager/model/job/JobGroupInfo.java

@@ -84,7 +84,7 @@ public String toString() {
     }
 
     public Builder toBuilder() {
-        return newBuilder();
+        return newBuilder().withStack(stack).withDetail(detail).withSequence(sequence);
     }
 
     public static Builder newBuilder() {

titus-common/src/main/java/com/netflix/titus/common/runtime/TitusRuntime.java

@@ -70,6 +70,11 @@ public interface TitusRuntime {
      */
     LocalScheduler getLocalScheduler();
 
+    /**
+     * If true, fatal errors cause JVM termination via System.exit.
+     */
+    boolean isSystemExitOnFailure();
+
     /**
      * In a few places in TitusMaster a component may decide to break the bootstrap process or terminate the JVM process
      * abruptly. This method should be called before that happens, so the {@link SystemAbortListener} implementation can

titus-common/src/main/java/com/netflix/titus/common/runtime/TitusRuntimes.java

@@ -41,6 +41,7 @@ public static TitusRuntime internal(Duration localSchedulerLoopInterval) {
                 new LoggingCodePointTracker(),
                 LoggingCodeInvariants.getDefault(),
                 LoggingSystemLogService.getInstance(),
+                false,
                 LoggingSystemAbortListener.getDefault(),
                 localSchedulerLoopInterval,
                 new DefaultRegistry(),
@@ -58,6 +59,7 @@ public static TitusRuntime internal(boolean fitEnabled) {
                 new LoggingCodePointTracker(),
                 LoggingCodeInvariants.getDefault(),
                 LoggingSystemLogService.getInstance(),
+                false,
                 LoggingSystemAbortListener.getDefault(),
                 LOCAL_SCHEDULER_LOOP_INTERVAL_MS,
                 new DefaultRegistry(),
@@ -71,6 +73,7 @@ public static TitusRuntime test() {
                 new LoggingCodePointTracker(),
                 new RecordingCodeInvariants(),
                 LoggingSystemLogService.getInstance(),
+                false,
                 LoggingSystemAbortListener.getDefault(),
                 LOCAL_SCHEDULER_LOOP_INTERVAL_MS,
                 new DefaultRegistry(),
@@ -84,6 +87,7 @@ public static TitusRuntime test(TestClock clock) {
                 new LoggingCodePointTracker(),
                 new RecordingCodeInvariants(),
                 LoggingSystemLogService.getInstance(),
+                false,
                 LoggingSystemAbortListener.getDefault(),
                 LOCAL_SCHEDULER_LOOP_INTERVAL_MS,
                 new DefaultRegistry(),
@@ -97,6 +101,7 @@ public static TitusRuntime test(TestScheduler testScheduler) {
                 new LoggingCodePointTracker(),
                 new RecordingCodeInvariants(),
                 LoggingSystemLogService.getInstance(),
+                false,
                 LoggingSystemAbortListener.getDefault(),
                 LOCAL_SCHEDULER_LOOP_INTERVAL_MS,
                 new DefaultRegistry(),

titus-common/src/main/java/com/netflix/titus/common/runtime/internal/DefaultTitusRuntime.java

@@ -22,6 +22,7 @@
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
+import javax.inject.Named;
 import javax.inject.Singleton;
 
 import com.netflix.spectator.api.BasicTag;
@@ -50,6 +51,8 @@
 @Singleton
 public class DefaultTitusRuntime implements TitusRuntime {
 
+    public static final String SYSTEM_EXIT_ON_FAILURE = "systemExitOnFailure";
+
     private static final Logger logger = LoggerFactory.getLogger(DefaultTitusRuntime.class);
 
     public static final String FIT_ACTIVATION_PROPERTY = "titus.runtime.fit.enabled";
@@ -67,6 +70,7 @@ public class DefaultTitusRuntime implements TitusRuntime {
     private final CodePointTracker codePointTracker;
     private final CodeInvariants codeInvariants;
     private final SystemLogService systemLogService;
+    private final boolean systemExitOnFailure;
     private final SystemAbortListener systemAbortListener;
     private final Registry registry;
     private final Clock clock;
@@ -76,12 +80,14 @@ public class DefaultTitusRuntime implements TitusRuntime {
     @Inject
     public DefaultTitusRuntime(CodeInvariants codeInvariants,
                                SystemLogService systemLogService,
+                               @Named(SYSTEM_EXIT_ON_FAILURE) boolean systemExitOnFailure,
                                SystemAbortListener systemAbortListener,
                                Registry registry) {
         this(
                 new SpectatorCodePointTracker(registry),
                 codeInvariants,
                 systemLogService,
+                systemExitOnFailure,
                 systemAbortListener,
                 LOCAL_SCHEDULER_LOOP_INTERVAL,
                 registry,
@@ -93,6 +99,7 @@ public DefaultTitusRuntime(CodeInvariants codeInvariants,
     public DefaultTitusRuntime(CodePointTracker codePointTracker,
                                CodeInvariants codeInvariants,
                                SystemLogService systemLogService,
+                               boolean systemExitOnFailure,
                                SystemAbortListener systemAbortListener,
                                Duration localSchedulerLoopInterval,
                                Registry registry,
@@ -101,6 +108,7 @@ public DefaultTitusRuntime(CodePointTracker codePointTracker,
         this.codePointTracker = codePointTracker;
         this.codeInvariants = codeInvariants;
         this.systemLogService = systemLogService;
+        this.systemExitOnFailure = systemExitOnFailure;
         this.systemAbortListener = systemAbortListener;
         this.registry = registry;
         this.clock = clock;
@@ -175,6 +183,11 @@ public LocalScheduler getLocalScheduler() {
         return localScheduler;
     }
 
+    @Override
+    public boolean isSystemExitOnFailure() {
+        return systemExitOnFailure;
+    }
+
     @Override
     public void beforeAbort(SystemAbortEvent event) {
         logger.error("System abort requested: {}", event);

titus-server-federation/src/main/java/com/netflix/titus/federation/startup/TitusFederationModule.java

@@ -95,7 +95,7 @@ public TitusRuntime getTitusRuntime(SystemLogService systemLogService, SystemAbo
                 LoggingCodeInvariants.getDefault(),
                 new SpectatorCodeInvariants(registry.createId("titus.runtime.invariant.violations"), registry)
         );
-        return new DefaultTitusRuntime(codeInvariants, systemLogService, systemAbortListener, registry);
+        return new DefaultTitusRuntime(codeInvariants, systemLogService, false, systemAbortListener, registry);
     }
 
     @Provides

titus-server-gateway/src/main/java/com/netflix/titus/gateway/startup/TitusGatewayModule.java

@@ -140,7 +140,7 @@ public TitusRuntime getTitusRuntime(SystemLogService systemLogService, SystemAbo
                 LoggingCodeInvariants.getDefault(),
                 new SpectatorCodeInvariants(registry.createId("titus.runtime.invariant.violations"), registry)
         );
-        return new DefaultTitusRuntime(codeInvariants, systemLogService, systemAbortListener, registry);
+        return new DefaultTitusRuntime(codeInvariants, systemLogService, false, systemAbortListener, registry);
     }
 
     @Provides

titus-server-master/src/main/java/com/netflix/titus/master/TitusMaster.java

@@ -55,7 +55,7 @@ public static void main(String[] args) {
             String resourceDir = TitusMaster.class.getClassLoader().getResource("static").toExternalForm();
 
             LifecycleInjector injector = InjectorBuilder.fromModules(
-                    Modules.override(new TitusRuntimeModule()).with(
+                    Modules.override(new TitusRuntimeModule(true)).with(
                             new AbstractModule() {
                                 @Override
                                 protected void configure() {

titus-server-master/src/main/java/com/netflix/titus/master/TitusRuntimeModule.java

@@ -71,6 +71,12 @@ public class TitusRuntimeModule extends AbstractModule {
 
     public static final String FIT_CONFIGURATION_PREFIX = "titusMaster.runtime.fitActions.";
 
+    private final boolean systemExitOnFailure;
+
+    public TitusRuntimeModule(boolean systemExitOnFailure) {
+        this.systemExitOnFailure = systemExitOnFailure;
+    }
+
     @Override
     protected void configure() {
         // Framework services
@@ -95,7 +101,7 @@ public TitusRuntime getTitusRuntime(SystemLogService systemLogService, SystemAbo
                 LoggingCodeInvariants.getDefault(),
                 new SpectatorCodeInvariants(registry.createId("titus.runtime.invariant.violations"), registry)
         );
-        DefaultTitusRuntime titusRuntime = new DefaultTitusRuntime(codeInvariants, systemLogService, systemAbortListener, registry);
+        DefaultTitusRuntime titusRuntime = new DefaultTitusRuntime(codeInvariants, systemLogService, systemExitOnFailure, systemAbortListener, registry);
 
         // Setup FIT component hierarchy
         FitFramework fitFramework = titusRuntime.getFitFramework();

titus-server-master/src/main/java/com/netflix/titus/master/supervisor/service/leader/GuiceLeaderActivator.java

@@ -63,6 +63,7 @@ private enum State {
 
     private final Injector injector;
     private final Clock clock;
+    private final TitusRuntime titusRuntime;
     private final ActivationLifecycle activationLifecycle;
 
     private volatile boolean leader;
@@ -83,6 +84,7 @@ public GuiceLeaderActivator(Injector injector,
         this.injector = injector;
         this.activationLifecycle = activationLifecycle;
         this.clock = titusRuntime.getClock();
+        this.titusRuntime = titusRuntime;
 
         Registry registry = titusRuntime.getRegistry();
 
@@ -177,12 +179,16 @@ public void stopBeingLeader() {
         leader = false;
         activated = false;
 
-        // Various services may have built in-memory state that is currently not easy to revert to initialization state.
-        // Until we create such a lifecycle feature for each service and all of their references, best thing to do is to
-        //  exit the process and depend on a watcher process to restart us right away. Especially since restart isn't
-        // very expensive.
-        logger.error("Exiting due to losing leadership after running as leader");
-        SystemExt.forcedProcessExit(1);
+        if (titusRuntime.isSystemExitOnFailure()) {
+            // Various services may have built in-memory state that is currently not easy to revert to initialization state.
+            // Until we create such a lifecycle feature for each service and all of their references, best thing to do is to
+            //  exit the process and depend on a watcher process to restart us right away. Especially since restart isn't
+            // very expensive.
+            logger.error("Exiting due to losing leadership after running as leader");
+            SystemExt.forcedProcessExit(1);
+        } else {
+            deactivate();
+        }
     }
 
     @Override
@@ -213,15 +219,29 @@ private void activate() {
             } catch (Exception e) {
                 stopBeingLeader();
 
-                // As stopBeingLeader method not always terminates the process, lets make sure it does.
-                SystemExt.forcedProcessExit(-1);
+                if (titusRuntime.isSystemExitOnFailure()) {
+                    // As stopBeingLeader method not always terminates the process, lets make sure it does.
+                    SystemExt.forcedProcessExit(-1);
+                }
+                throw e;
             }
         } catch (Throwable e) {
-            SystemExt.forcedProcessExit(-1);
+            if (titusRuntime.isSystemExitOnFailure()) {
+                SystemExt.forcedProcessExit(-1);
+            }
+            throw e;
         }
 
         this.activated = true;
         this.activationEndTimestamp = clock.wallTime();
         this.activationTime = activationEndTimestamp - activationStartTimestamp;
     }
+
+    private void deactivate() {
+        try {
+            activationLifecycle.deactivate();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
 }

titus-server-master/src/test/java/com/netflix/titus/master/integration/BaseIntegrationTest.java

@@ -16,10 +16,7 @@
 
 package com.netflix.titus.master.integration;
 
-import java.security.Permission;
-
 import com.netflix.titus.testkit.junit.category.IntegrationTest;
-import org.junit.BeforeClass;
 import org.junit.experimental.categories.Category;
 
 @Category(IntegrationTest.class)
@@ -28,31 +25,4 @@ public class BaseIntegrationTest {
     protected static final long TEST_TIMEOUT_MS = 30_000;
 
     protected static final long LONG_TEST_TIMEOUT_MS = 60_000;
-
-    static class PreventSystemExitSecurityManager extends SecurityManager {
-        @Override
-        public void checkPermission(Permission perm) {
-        }
-
-        @Override
-        public void checkPermission(Permission perm, Object context) {
-        }
-
-        @Override
-        public void checkExit(int status) {
-            if (status != 0) {
-                String message = "System exit requested with error " + status;
-                throw new IllegalStateException(message);
-            }
-        }
-    }
-
-    private static final SecurityManager securityManager = new PreventSystemExitSecurityManager();
-
-    @BeforeClass
-    public static void setSecurityManager() {
-        if (System.getSecurityManager() != securityManager) {
-            System.setSecurityManager(securityManager);
-        }
-    }
 }