Set default session options for ONNX-RT stateless eval

1 files changed, 3 insertions, 0 deletions

diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java
index 51d54d719c2..b782a79f14b 100644
--- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java
+++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java
@@ -30,6 +30,9 @@ public class OnnxEvaluator {
             environment = OrtEnvironment.getEnvironment();
             OrtSession.SessionOptions options = new OrtSession.SessionOptions();
             options.setOptimizationLevel(OrtSession.SessionOptions.OptLevel.ALL_OPT);
+            options.setIntraOpNumThreads(Math.max(1, Runtime.getRuntime().availableProcessors() / 4));
+            options.setInterOpNumThreads(1);
+            options.setExecutionMode(OrtSession.SessionOptions.ExecutionMode.SEQUENTIAL);
             session = environment.createSession(modelPath, options);
         } catch (OrtException e) {
             throw new RuntimeException("ONNX Runtime exception", e);