Model and genai fixes (#14481)

* disable mem arena in options for cpu only

* add try/except around ollama initialization

* update docs
This commit is contained in:
Josh Hawkins
2024-10-21 09:00:45 -06:00
committed by GitHub
parent 4bb420d049
commit 8364e68667
3 changed files with 21 additions and 13 deletions
+2 -6
View File
@@ -20,7 +20,7 @@ def get_ort_providers(
["CPUExecutionProvider"],
[
{
"arena_extend_strategy": "kSameAsRequested",
"enable_cpu_mem_arena": False,
}
],
)
@@ -53,7 +53,7 @@ def get_ort_providers(
providers.append(provider)
options.append(
{
"arena_extend_strategy": "kSameAsRequested",
"enable_cpu_mem_arena": False,
}
)
else:
@@ -85,12 +85,8 @@ class ONNXModelRunner:
else:
# Use ONNXRuntime
self.type = "ort"
options = ort.SessionOptions()
if device == "CPU":
options.enable_cpu_mem_arena = False
self.ort = ort.InferenceSession(
model_path,
sess_options=options,
providers=providers,
provider_options=options,
)