mirror of
https://github.com/OPM/opm-simulators.git
synced 2026-10-24 22:57:42 -05:00
move hybrid newton test to an integration test directory
this is not a unit test for the python binaries
This commit is contained in:
Arne Morten Kvarving committed
1 parent
66b037d9a7
commit
3c365a428a
6 files changed
No files matched your search
@@ -1,13 +0,0 @@
|
||||
requirements (same as opm-common/python/opm/ml/ml_tools/requirements.txt)
|
||||
|
||||
# Python v.3.9.0 and above
|
||||
python>=3.9.0<=3.12.0
|
||||
|
||||
# Numpy 1.23.0 and above
|
||||
numpy>=1.23.0
|
||||
|
||||
# TensorFlow v.2.1 and above include both CPU and GPU versions.
|
||||
tensorflow>=2.1.0
|
||||
|
||||
# Keras v.2.12.0 and above
|
||||
keras>=2.12.0
|
||||
Whitespace-only changes.
@@ -1,104 +0,0 @@
|
||||
# Hybrid Newton Test Suite
|
||||
|
||||
This test suite validates the integration of **Hybrid Newton (HyNE)** with machine learning models inside the `flow` reservoir simulator.
|
||||
It ensures that adding ML-based approximations does not degrade solver performance and, in some cases, improves convergence.
|
||||
|
||||
---
|
||||
|
||||
## Dependencies
|
||||
|
||||
In a similar way as for opm-common ML module, these tests require **TensorFlow** to be installed.
|
||||
The CPU version is enough for this test and allows faster installation.
|
||||
```bash
|
||||
pip install tensorflow-cpu
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test Framework
|
||||
|
||||
* Implemented with Python’s built-in [`unittest`](https://docs.python.org/3/library/unittest.html).
|
||||
* Each test case defines a specific **Hybrid Newton configuration** via JSON.
|
||||
* Simulations are run using `flow`(therefore assuming it is installed):
|
||||
|
||||
* **Baseline** → Standard Newton solver (no ML).
|
||||
* **Hybrid Newton** → Newton solver with ML-based updates injected.
|
||||
|
||||
---
|
||||
|
||||
## Baseline Simulation
|
||||
|
||||
Before running hybrid cases:
|
||||
|
||||
* Runs a standard `flow` simulation with the given deck.
|
||||
* Extracts:
|
||||
|
||||
* **Baseline Newton iterations** (`baseline_iters`) from simulation logs.
|
||||
* **Reservoir state variables** (`PRESSURE`, `SGAS`, `SWAT`, etc.) from UNRST files.
|
||||
* Stores `n_cells` and `times` for consistency across tests.
|
||||
|
||||
---
|
||||
|
||||
## Hybrid Newton Simulation
|
||||
|
||||
For each test case:
|
||||
|
||||
1. A **minimal Keras model** is built that maps input → output features.
|
||||
|
||||
* Weights are initialized to copy output variables directly.
|
||||
* Models are exported using `kerasify.export_model`.
|
||||
2. A **JSON config** is generated with:
|
||||
|
||||
* Input/output features.
|
||||
* Scaling and feature engineering options.
|
||||
* Apply time and active cells.
|
||||
3. `flow` is run with:
|
||||
|
||||
```bash
|
||||
--use-hy-ne=true --hy-ne-config-file=config.json
|
||||
```
|
||||
4. Hybrid Newton iteration counts are extracted and compared to baseline.
|
||||
|
||||
---
|
||||
|
||||
## Assertions
|
||||
|
||||
Each test validates:
|
||||
|
||||
* **Convergence parity**
|
||||
Hybrid and baseline must produce the same number of timesteps.
|
||||
* **Iteration bounds**
|
||||
`hybrid_iters[t] <= baseline_iters[t]` for all timesteps.
|
||||
* **Zero Newton cases**
|
||||
Some cases require the first two timesteps to have `0` Newton iterations.
|
||||
* **Config integrity**
|
||||
Config JSONs must exist and be valid.
|
||||
|
||||
---
|
||||
|
||||
## Test Categories
|
||||
|
||||
| Category | Description |
|
||||
| --------------------------- | ----------------------------------------------------- |
|
||||
| `ABSOLUTE_CASES` | Tests absolute features. |
|
||||
| `RELATIVE_CASES` | Tests relative features. |
|
||||
| `FEATURE_ENGINEERING_CASES` | Tests features engineered inputs/outputs (e.g., log). |
|
||||
| `SCALING_CASES` | Applies scaling strategies (e.g., min-max, standard). |
|
||||
| `MULTI_MODEL_CASES` | Combines multiple models into one simulation. |
|
||||
| `ZERO_NEWTON_CASES` | Use all features, resulting in 0 Newton iterations. |
|
||||
| `ALL_CASES` | Runs all cases together as a stress test. |
|
||||
|
||||
---
|
||||
|
||||
## Utilities
|
||||
|
||||
The suite uses helper functions from `utils.py`:
|
||||
|
||||
* `collect_input_features` → Prepares input arrays from UNRST data.
|
||||
* `compute_output_vars` → Builds expected outputs from reservoir state.
|
||||
* `write_config` → Writes the HyNE JSON configuration.
|
||||
* `export_model` → Saves Keras models in a format usable by `flow`.
|
||||
* `extract_newtit_from_file` → Extracts iteration counts from solver logs.
|
||||
* `extract_unrst_variables` → Reads simulation outputs from UNRST files.
|
||||
|
||||
---
|
||||
Whitespace-only changes.
@@ -1,201 +0,0 @@
|
||||
import os
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import subprocess
|
||||
|
||||
from .utils import Dense, Sequential
|
||||
from .utils import collect_input_features, compute_output_vars, write_config, extract_newtit_from_file, extract_unrst_variables
|
||||
from .utils import (ABSOLUTE_CASES, RELATIVE_CASES, FEATURE_ENGINEERING_CASES, SCALING_CASES, MULTI_MODEL_CASES, ZERO_NEWTON_CASES, ALL_CASES)
|
||||
|
||||
from opm.ml.ml_tools.kerasify import export_model
|
||||
|
||||
class TestHybridNewton(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Run baseline simulation via subprocess and collect timesteps & fluid state."""
|
||||
test_dir = Path(__file__).parent
|
||||
cls.data_dir = test_dir.parent.parent.parent / "test_data/SPE1CASE1b"
|
||||
cls.deck_file = "SPE1CASE1.DATA"
|
||||
|
||||
cls.unrst = {}
|
||||
cls.rd_init = {"PERMX": []}
|
||||
cls.times = []
|
||||
|
||||
baseline_cmd = [
|
||||
os.environ.get("FLOW_BINARY", default="flow"),
|
||||
str(cls.data_dir / cls.deck_file),
|
||||
"--output-extra-convergence-info=steps,iterations",
|
||||
"--newton-min-iterations=0",
|
||||
"--output-dir=.",
|
||||
"--full-time-step-initially=true",
|
||||
]
|
||||
|
||||
print("Running baseline simulation... ")
|
||||
subprocess.run(baseline_cmd, stdout=subprocess.DEVNULL, check=True)
|
||||
|
||||
# Extract Newton iterations
|
||||
cls.baseline_iters = extract_newtit_from_file(Path('.') / Path(cls.deck_file).stem)
|
||||
print(f"Baseline Newton iterations: {cls.baseline_iters}")
|
||||
|
||||
# Extract fluid state from UNRST for all timesteps
|
||||
cls.unrst, cls.times = extract_unrst_variables(".", cls.deck_file)
|
||||
cls.rd_init["PERMX"] = [10**2] * len(cls.unrst["PRESSURE"][0])
|
||||
cls.n_cells = len(cls.unrst["PRESSURE"][0])
|
||||
|
||||
def _run_cases(self, cases):
|
||||
models_dir = Path('.') / "models"
|
||||
os.makedirs(models_dir, exist_ok=True)
|
||||
|
||||
for case in cases:
|
||||
with self.subTest(case=case["label"]):
|
||||
json_path = self._build_case_models(case, models_dir)
|
||||
self.assertTrue(json_path and os.path.isfile(json_path),
|
||||
f"Config JSON was not created for case {case['label']}")
|
||||
|
||||
# Run hybrid Newton test
|
||||
hybrid_iters = self._run_hybrid_newton_test(json_path)
|
||||
return hybrid_iters
|
||||
|
||||
def _build_case_models(self, case, models_dir):
|
||||
models_to_write = []
|
||||
|
||||
if case.get("mode", "single") == "single":
|
||||
model_entries = [case]
|
||||
multi_mode = False
|
||||
else:
|
||||
model_entries = case.get("models", [])
|
||||
multi_mode = True
|
||||
|
||||
for model_case in model_entries:
|
||||
start_idx = model_case.get("start_idx", case.get("start_idx"))
|
||||
end_idx = model_case.get("end_idx", case.get("end_idx"))
|
||||
input_features = model_case["input_features"]
|
||||
output_features = model_case["output_features"]
|
||||
fe_input = model_case.get("feature_engineering_input")
|
||||
fe_output = model_case.get("feature_engineering_output")
|
||||
scaling_input = model_case.get("scaling_input")
|
||||
scaling_output = model_case.get("scaling_output")
|
||||
|
||||
input_flat, input_scaling_params = collect_input_features(
|
||||
input_features, start_idx, self.unrst, self.rd_init, self.times,
|
||||
feat_eng=fe_input, scaling=scaling_input
|
||||
)
|
||||
output_flat, output_scaling_params = compute_output_vars(
|
||||
output_features, start_idx, end_idx, self.unrst, self.rd_init,
|
||||
feature_engineering=fe_output, scaling=scaling_output,
|
||||
)
|
||||
|
||||
input_flat = input_flat.reshape(-1)
|
||||
output_flat = output_flat.reshape(-1)
|
||||
|
||||
model = Sequential([
|
||||
Dense(input_dim=input_flat.shape[0], output_dim=output_flat.shape[0])
|
||||
])
|
||||
model.layers[0].set_weights([np.zeros((input_flat.shape[0], output_flat.shape[0])), output_flat])
|
||||
apply_time = self.times[start_idx]
|
||||
|
||||
active_cells = list(range(self.n_cells))
|
||||
|
||||
models_dir.mkdir(exist_ok=True)
|
||||
model_name = "_".join(input_features) + "__TO__" + "_".join(output_features) + f"_{start_idx}_{end_idx}.model"
|
||||
model_path = models_dir / model_name
|
||||
export_model(model, model_path)
|
||||
|
||||
models_to_write.append({
|
||||
"input_features": input_features,
|
||||
"output_features": output_features,
|
||||
"model_path": model_path,
|
||||
"active_cells": active_cells,
|
||||
"apply_time": apply_time,
|
||||
"feature_engineering_input": fe_input,
|
||||
"feature_engineering_output": fe_output,
|
||||
"scaling_input": scaling_input,
|
||||
"scaling_output": scaling_output,
|
||||
"input_scaling_params": input_scaling_params,
|
||||
"output_scaling_params": output_scaling_params
|
||||
})
|
||||
|
||||
if not models_to_write:
|
||||
print(f"No valid models for {case['label']}")
|
||||
return None
|
||||
|
||||
return write_config(models_to_write, models_dir, multi_mode=multi_mode, case_label=case['label'])
|
||||
|
||||
def _run_hybrid_newton_test(self, json_path):
|
||||
"""Run hybrid Newton via subprocess and extract Newton iterations."""
|
||||
cmd = [
|
||||
os.environ.get("FLOW_BINARY", default="flow"),
|
||||
str(self.data_dir / self.deck_file),
|
||||
f"--hy-ne-config-file={json_path}",
|
||||
"--use-hy-ne=true",
|
||||
"--output-extra-convergence-info=steps,iterations",
|
||||
"--newton-min-iterations=0",
|
||||
"--output-dir=.",
|
||||
"--full-time-step-initially=true"
|
||||
]
|
||||
|
||||
print(f"Running hybrid Newton simulation with config {json_path}...")
|
||||
subprocess.run(cmd,
|
||||
stdout=subprocess.DEVNULL,
|
||||
check=True)
|
||||
|
||||
hybrid_iters = extract_newtit_from_file(Path('.') / Path(self.deck_file).stem)
|
||||
return hybrid_iters
|
||||
|
||||
def test_absolute_cases(self):
|
||||
self._run_cases(ABSOLUTE_CASES)
|
||||
|
||||
def test_relative_cases(self):
|
||||
self._run_cases(RELATIVE_CASES)
|
||||
|
||||
def test_feature_engineering_cases(self):
|
||||
self._run_cases(FEATURE_ENGINEERING_CASES)
|
||||
|
||||
def test_scaling_cases(self):
|
||||
self._run_cases(SCALING_CASES)
|
||||
|
||||
def test_multi_model_cases(self):
|
||||
self._run_cases(MULTI_MODEL_CASES)
|
||||
|
||||
def test_zero_newton_cases(self):
|
||||
hybrid_iters = self._run_cases(ZERO_NEWTON_CASES)
|
||||
|
||||
# Length must match (failed newton convergence otherwise)
|
||||
assert len(hybrid_iters) == len(self.baseline_iters), (
|
||||
f"Hybrid iterations length {len(hybrid_iters)} "
|
||||
f"does not match baseline length {len(self.baseline_iters)}"
|
||||
)
|
||||
|
||||
# First two must be 0
|
||||
assert hybrid_iters[0] == 0, f"Expected hybrid_iters[0] == 0, got {hybrid_iters[0]}"
|
||||
assert hybrid_iters[1] == 0, f"Expected hybrid_iters[1] == 0, got {hybrid_iters[1]}"
|
||||
|
||||
# Hybrid must not exceed baseline
|
||||
for i, (h, b) in enumerate(zip(hybrid_iters, self.baseline_iters)):
|
||||
assert h <= b, (
|
||||
f"Hybrid iterations exceeded baseline at index {i}: "
|
||||
f"got {h}, expected <= {b}"
|
||||
)
|
||||
|
||||
def test_all_cases(self):
|
||||
hybrid_iters = self._run_cases(ALL_CASES)
|
||||
# Length must match (failed newton convergence otherwise)
|
||||
assert len(hybrid_iters) == len(self.baseline_iters), (
|
||||
f"Hybrid iterations length {len(hybrid_iters)} "
|
||||
f"does not match baseline length {len(self.baseline_iters)}"
|
||||
)
|
||||
|
||||
# First two must be 0
|
||||
assert hybrid_iters[0] == 0, f"Expected hybrid_iters[0] == 0, got {hybrid_iters[0]}"
|
||||
assert hybrid_iters[1] == 0, f"Expected hybrid_iters[1] == 0, got {hybrid_iters[1]}"
|
||||
|
||||
# Hybrid must not exceed baseline
|
||||
for i, (h, b) in enumerate(zip(hybrid_iters, self.baseline_iters)):
|
||||
assert h <= b, (
|
||||
f"Hybrid iterations exceeded baseline at index {i}: "
|
||||
f"got {h}, expected <= {b}"
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,435 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
|
||||
from opm.io.ecl import ERst, ESmry
|
||||
|
||||
# Avoid tensorflow dependency but maintain compability with kerasify export_model method
|
||||
class Dense:
|
||||
def __init__(self, input_dim, output_dim, activation='linear'):
|
||||
self.weights = np.zeros((input_dim, output_dim), dtype=np.float32)
|
||||
self.biases = np.zeros((output_dim,), dtype=np.float32)
|
||||
self.activation = activation
|
||||
|
||||
def set_weights(self, w_b):
|
||||
self.weights = w_b[0].astype(np.float32)
|
||||
self.biases = w_b[1].astype(np.float32)
|
||||
|
||||
def get_weights(self):
|
||||
return [self.weights, self.biases]
|
||||
|
||||
def get_config(self):
|
||||
return {'activation': self.activation}
|
||||
|
||||
class Sequential:
|
||||
def __init__(self, layers):
|
||||
self.layers = layers
|
||||
|
||||
|
||||
def extract_unrst_variables(data_dir, deck_file):
|
||||
"""
|
||||
Extracts fluid state variables and timesteps from the .UNRST file
|
||||
corresponding to a deck.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
data_dir : Path or str
|
||||
Directory containing the UNRST file.
|
||||
deck_file : str
|
||||
Name of the deck file (e.g., SPE1CASE1.DATA).
|
||||
|
||||
Returns
|
||||
-------
|
||||
unrst_vars : dict
|
||||
Dictionary with keys "PRESSURE", "SWAT", "SOIL", "SGAS", "RS"
|
||||
Each value is a list of np.ndarray for each report step.
|
||||
dts : list
|
||||
List of timestep durations (approximated from report steps if needed).
|
||||
"""
|
||||
deck_stem = Path(deck_file).stem
|
||||
unrst_file = str(Path(data_dir) / f"{deck_stem}.UNRST") # convert to string
|
||||
unsmry_file = str(Path(data_dir) / f"{deck_stem}.SMSPEC")
|
||||
rst = ERst(unrst_file)
|
||||
unsmry = ESmry(unsmry_file)
|
||||
# Load all report steps
|
||||
report_steps = sorted(rst.report_steps)
|
||||
for step in report_steps:
|
||||
rst.load_report_step(step)
|
||||
|
||||
variables_of_interest = ["PRESSURE", "SWAT", "SOIL", "SGAS", "RS"]
|
||||
unrst_vars = {var: [] for var in variables_of_interest}
|
||||
|
||||
for step in report_steps:
|
||||
for var in variables_of_interest:
|
||||
if (var, step) in rst:
|
||||
unrst_vars[var].append(np.array(rst[var, step], copy=True))
|
||||
else:
|
||||
# Fill with zeros if variable not in UNRST for this step
|
||||
n_cells = len(rst["PRESSURE", step]) # assume same size as PRESSURE
|
||||
unrst_vars[var].append(np.zeros(n_cells, dtype=np.float32))
|
||||
|
||||
# Compute approximate timesteps (differences between report steps)
|
||||
dates = unsmry.dates()
|
||||
times = [0.] + [(d - unsmry.start_date).total_seconds() for d in dates]
|
||||
|
||||
return unrst_vars, times
|
||||
|
||||
|
||||
def write_config(models, model_dir, multi_mode=False, case_label=None):
|
||||
"""
|
||||
Write one or multiple model configs to JSON in OPM-supported format.
|
||||
"""
|
||||
|
||||
model_dir = Path(model_dir)
|
||||
if multi_mode and not case_label:
|
||||
raise ValueError("case_label must be provided for multi_mode")
|
||||
|
||||
os.makedirs(model_dir, exist_ok=True)
|
||||
all_configs = {}
|
||||
|
||||
for idx, m in enumerate(models):
|
||||
# Ensure input/output features are lists
|
||||
input_features = m["input_features"] if isinstance(m["input_features"], list) else [m["input_features"]]
|
||||
output_features = m["output_features"] if isinstance(m["output_features"], list) else [m["output_features"]]
|
||||
|
||||
fe_input = m.get("feature_engineering_input") or [None]*len(input_features)
|
||||
fe_output = m.get("feature_engineering_output") or [None]*len(output_features)
|
||||
scaling_params_input = m.get("input_scaling_params") or [None]*len(input_features)
|
||||
scaling_params_output = m.get("output_scaling_params") or [None]*len(output_features)
|
||||
|
||||
# Build input/output blocks
|
||||
input_block = {}
|
||||
for i, f in enumerate(input_features):
|
||||
d = {}
|
||||
if fe_input[i] and fe_input[i].lower() != "none":
|
||||
d["feature_engineering"] = fe_input[i].lower()
|
||||
if scaling_params_input[i] is not None:
|
||||
# convert numpy float32 to native float if needed
|
||||
d["scaling_params"] = {k: float(v) for k, v in scaling_params_input[i].items()}
|
||||
input_block[f] = d
|
||||
|
||||
output_block = {}
|
||||
for i, f in enumerate(output_features):
|
||||
d = {}
|
||||
if fe_output[i] and fe_output[i].lower() != "none":
|
||||
d["feature_engineering"] = fe_output[i].lower()
|
||||
if scaling_params_output[i] is not None:
|
||||
d["scaling_params"] = {k: float(v) for k, v in scaling_params_output[i].items()}
|
||||
output_block[f] = d
|
||||
|
||||
# Active cells
|
||||
active_cells = m.get("active_cells")
|
||||
if active_cells is None:
|
||||
n_cells = getattr(m, "n_cells", None)
|
||||
if n_cells is None:
|
||||
raise ValueError("Cannot determine active_cells; provide n_cells or active_cells")
|
||||
active_cells = list(range(n_cells))
|
||||
|
||||
cells_file = model_dir / f"{os.path.splitext(os.path.basename(m['model_path']))[0]}_active_cells.txt"
|
||||
with open(cells_file, "w") as f:
|
||||
for cell in active_cells:
|
||||
f.write(f"{cell}\n")
|
||||
|
||||
# Apply time(s)
|
||||
apply_time = m["apply_time"]
|
||||
if isinstance(apply_time, float):
|
||||
apply_times_dict = {"0": float(apply_time)}
|
||||
elif isinstance(apply_time, list):
|
||||
apply_times_dict = {str(i): float(t) for i, t in enumerate(apply_time)}
|
||||
else:
|
||||
raise ValueError("apply_time must be float or list of floats")
|
||||
|
||||
model_name_rel = f"models/{os.path.basename(m['model_path'])}"
|
||||
cells_file_rel = f"models/{cells_file.name}"
|
||||
|
||||
all_configs[str(idx)] = {
|
||||
"model_path": model_name_rel,
|
||||
"cell_indices_file": cells_file_rel,
|
||||
"apply_times": apply_times_dict,
|
||||
"features": {"inputs": input_block, "outputs": output_block}
|
||||
}
|
||||
|
||||
# JSON filename
|
||||
if multi_mode:
|
||||
json_name = case_label + ".json"
|
||||
else:
|
||||
json_name = f"{os.path.splitext(os.path.basename(models[0]['model_path']))[0]}.json"
|
||||
|
||||
json_path = model_dir / json_name
|
||||
with open(json_path, "w") as f:
|
||||
json.dump(all_configs, f, indent=2)
|
||||
|
||||
return json_path
|
||||
|
||||
|
||||
def seconds_to_days(sec):
|
||||
return sec / 86400.0
|
||||
|
||||
|
||||
def apply_feature_engineering(data, method=None):
|
||||
if not method or method.lower() == "none":
|
||||
return data
|
||||
if method.lower() == "log":
|
||||
return np.log(data)
|
||||
if method.lower() == "log10":
|
||||
return np.log10(data)
|
||||
if method.lower() == "log1p":
|
||||
return np.log1p(data)
|
||||
raise ValueError(f"Unsupported feature engineering method: {method}")
|
||||
|
||||
|
||||
def apply_scaling_with_params(data, method):
|
||||
data = np.array(data)
|
||||
if method == "standard":
|
||||
mean, std = np.mean(data), np.std(data)
|
||||
return ((data - mean) / std if std != 0 else data - mean), {"mean": mean, "std": std}
|
||||
if method == "minmax":
|
||||
mn, mx = np.min(data), np.max(data)
|
||||
return ((data - mn) / (mx - mn) if mx != mn else data - mn), {"min": mn, "max": mx}
|
||||
return data, None
|
||||
|
||||
|
||||
def get_feature_data(name, timestep, unrst, rd_init, times=None, feat_eng=None):
|
||||
if name == "PERMX":
|
||||
val = np.array(rd_init["PERMX"])
|
||||
elif name == "SOIL":
|
||||
val = 1.0 - np.array(unrst["SGAS"][timestep]) - np.array(unrst["SWAT"][timestep])
|
||||
elif name == "TIMESTEP":
|
||||
if times is None:
|
||||
raise ValueError("times must be provided for TIMESTEP")
|
||||
val = times[timestep + 1] - times[timestep]
|
||||
else:
|
||||
val = np.array(unrst[name][timestep])
|
||||
return apply_feature_engineering(val, feat_eng)
|
||||
|
||||
|
||||
def collect_input_features(input_features, start_idx, unrst, rd_init, times=None, feat_eng=None, scaling=None):
|
||||
input_data = []
|
||||
scaling_params = []
|
||||
|
||||
for i, f in enumerate(input_features):
|
||||
fe = feat_eng[i] if feat_eng else None
|
||||
sc = scaling[i] if scaling else None
|
||||
val = get_feature_data(f, start_idx, unrst, rd_init, times=times, feat_eng=fe)
|
||||
val, params = apply_scaling_with_params(val, sc) if sc else (val, None)
|
||||
input_data.append(np.array(val).flatten())
|
||||
scaling_params.append(params)
|
||||
|
||||
return np.concatenate(input_data), scaling_params
|
||||
|
||||
|
||||
def compute_output_vars(output_features, start_idx, end_idx, unrst, rd_init, feature_engineering=None, scaling=None):
|
||||
output_data = []
|
||||
scaling_params = []
|
||||
|
||||
for i, f in enumerate(output_features):
|
||||
fe = feature_engineering[i] if feature_engineering else None
|
||||
sc = scaling[i] if scaling else None
|
||||
|
||||
if f.startswith("DELTA_"):
|
||||
base = f[6:]
|
||||
val = get_feature_data(base, end_idx, unrst, rd_init) - get_feature_data(base, start_idx, unrst, rd_init)
|
||||
else:
|
||||
val = get_feature_data(f, end_idx, unrst, rd_init)
|
||||
|
||||
val = apply_feature_engineering(val, fe)
|
||||
val, params = apply_scaling_with_params(val, sc) if sc else (val, None)
|
||||
output_data.append(np.array(val).flatten())
|
||||
scaling_params.append(params)
|
||||
|
||||
return np.stack(output_data), scaling_params
|
||||
|
||||
|
||||
def extract_newtit_from_file(filename):
|
||||
"""
|
||||
Parse an INFOSTEP file and return the values of the 'NewtIt' column.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str or Path
|
||||
Path to the file (with or without .INFOSTEP extension).
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[int]
|
||||
List of NewtIt values for each data row.
|
||||
"""
|
||||
path = Path(filename)
|
||||
if path.suffix.lower() != ".infostep":
|
||||
path = path.with_suffix(".INFOSTEP")
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"INFOSTEP file not found: {path}")
|
||||
|
||||
with path.open("r") as f:
|
||||
lines = [line.strip() for line in f if line.strip()]
|
||||
|
||||
# Locate header with "NewtIt"
|
||||
header_idx = next((i for i, line in enumerate(lines) if "NewtIt" in line), None)
|
||||
if header_idx is None:
|
||||
raise ValueError(f"'NewtIt' column not found in {path}")
|
||||
|
||||
header = lines[header_idx].split()
|
||||
idx = header.index("NewtIt")
|
||||
|
||||
newt_values = []
|
||||
for row in lines[header_idx + 1 :]:
|
||||
cols = row.split()
|
||||
if len(cols) > idx:
|
||||
try:
|
||||
newt_values.append(int(cols[idx]))
|
||||
except ValueError:
|
||||
continue # skip malformed rows
|
||||
|
||||
return newt_values
|
||||
|
||||
# -------------------------
|
||||
# Absolute (single timestep predictions)
|
||||
# -------------------------
|
||||
ABSOLUTE_CASES = [
|
||||
{
|
||||
"label": "single_RS",
|
||||
"mode": "single",
|
||||
"start_idx": 0, "end_idx": 1,
|
||||
"input_features": ["TIMESTEP"],
|
||||
"output_features": ["RS"],
|
||||
},
|
||||
{
|
||||
"label": "single_PRESSURE",
|
||||
"mode": "single",
|
||||
"start_idx": 0, "end_idx": 1,
|
||||
"input_features": ["PRESSURE"],
|
||||
"output_features": ["PRESSURE"],
|
||||
},
|
||||
{
|
||||
"label": "single_PERMX_to_SGAS",
|
||||
"mode": "single",
|
||||
"start_idx": 0, "end_idx": 1,
|
||||
"input_features": ["PERMX"],
|
||||
"output_features": ["SGAS"],
|
||||
},
|
||||
{
|
||||
"label": "single_SGAS_to_SWAT",
|
||||
"mode": "single",
|
||||
"start_idx": 0, "end_idx": 1,
|
||||
"input_features": ["SGAS"],
|
||||
"output_features": ["SWAT"],
|
||||
},
|
||||
]
|
||||
|
||||
# -------------------------
|
||||
# Relative / delta predictions
|
||||
# -------------------------
|
||||
RELATIVE_CASES = [
|
||||
{"label": "delta_RS", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["RS"], "output_features": ["DELTA_RS"]},
|
||||
{"label": "delta_SGAS", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["SGAS"], "output_features": ["DELTA_SGAS"]},
|
||||
{"label": "delta_SWAT", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["SWAT"], "output_features": ["DELTA_SWAT"]},
|
||||
{"label": "delta_PRESSURE", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["PRESSURE"], "output_features": ["DELTA_PRESSURE"]},
|
||||
{"label": "delta_PERMX_to_RS", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["PERMX"], "output_features": ["DELTA_RS"]},
|
||||
{"label": "delta_PERMX_to_PRESSURE", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["PERMX"], "output_features": ["DELTA_PRESSURE"]},
|
||||
]
|
||||
|
||||
# -------------------------
|
||||
# Feature engineering cases
|
||||
# -------------------------
|
||||
FEATURE_ENGINEERING_CASES = [
|
||||
{"label": "feat_eng_RS_to_PRESSURE", "mode": "single", "start_idx": 0, "end_idx": 1,
|
||||
"input_features": ["RS"], "feature_engineering_input": ["log"],
|
||||
"output_features": ["PRESSURE"], "feature_engineering_output": ["log1p"]},
|
||||
{"label": "feat_eng_PRESSURE", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["SGAS"], "feature_engineering_input": ["log1p"],
|
||||
"output_features": ["PRESSURE"], "feature_engineering_output": ["log10"]},
|
||||
{"label": "feat_eng_PERMX_to_PRESSURE", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["PERMX"], "feature_engineering_input": ["log"],
|
||||
"output_features": ["PRESSURE"], "feature_engineering_output": ["log"]},
|
||||
]
|
||||
|
||||
# -------------------------
|
||||
# Scaling cases
|
||||
# -------------------------
|
||||
SCALING_CASES = [
|
||||
{"label": "scaling_RS_minmax", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["RS"], "scaling_input": ["minmax"],
|
||||
"output_features": ["RS"], "scaling_output": ["minmax"]},
|
||||
{"label": "scaling_PRESSURE_standard", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["PRESSURE"], "scaling_input": ["standard"],
|
||||
"output_features": ["PRESSURE"], "scaling_output": ["standard"]},
|
||||
{"label": "scaling_SGAS_SWAT_standard", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["SGAS", "SWAT"], "scaling_input": ["standard", "none"],
|
||||
"output_features": ["SGAS", "SWAT"], "scaling_output": ["none", "standard"]},
|
||||
{"label": "scaling_PRESSURE_SWAT_minmax", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["PRESSURE", "SWAT"], "scaling_input": ["minmax", "none"],
|
||||
"output_features": ["SWAT", "PRESSURE"], "scaling_output": ["none", "minmax"]},
|
||||
{"label": "scaling_PERMX_TIMESTEP_standard", "mode": "single", "start_idx": 1, "end_idx": 2,
|
||||
"input_features": ["PRESSURE", "TIMESTEP"], "scaling_input": ["minmax", "none"],
|
||||
"output_features": ["SGAS", "SWAT"]},
|
||||
]
|
||||
|
||||
# -------------------------
|
||||
# Multi-model cases
|
||||
# -------------------------
|
||||
MULTI_MODEL_CASES = [
|
||||
{"label": "multi_same_index", "mode": "multi",
|
||||
"models": [
|
||||
{"input_features": ["PERMX"], "output_features": ["RS"], "start_idx": 1, "end_idx": 2},
|
||||
{"input_features": ["SGAS"], "output_features": ["SGAS"], "start_idx": 1, "end_idx": 2},
|
||||
]},
|
||||
{"label": "multi_diff_index", "mode": "multi",
|
||||
"models": [
|
||||
{"input_features": ["PERMX"], "output_features": ["RS"], "start_idx": 1, "end_idx": 2},
|
||||
{"input_features": ["PRESSURE", "TIMESTEP", "PERMX"], "output_features": ["PRESSURE"], "start_idx": 8, "end_idx": 9},
|
||||
{"input_features": ["SGAS"], "output_features": ["SGAS"], "start_idx": 10, "end_idx": 11},
|
||||
]},
|
||||
]
|
||||
|
||||
ZERO_NEWTON_CASES = [
|
||||
{
|
||||
"label": "all_features",
|
||||
"mode": "multi",
|
||||
"models": [
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 0, "end_idx": 1},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 1, "end_idx": 2},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 2, "end_idx": 3},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 3, "end_idx": 4},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 4, "end_idx": 5},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 5, "end_idx": 6},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 6, "end_idx": 7},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 7, "end_idx": 8},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 8, "end_idx": 9},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 9, "end_idx": 10},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 10, "end_idx": 11},
|
||||
{"input_features": ["PERMX"], "output_features": ["RS", "PRESSURE", "SGAS", "SOIL"], "start_idx": 11, "end_idx": 12},
|
||||
],
|
||||
|
||||
},
|
||||
]
|
||||
|
||||
ALL_CASES = [
|
||||
{
|
||||
"label": "all_cases",
|
||||
"mode": "multi",
|
||||
"models": [
|
||||
{"input_features": ["PERMX", "TIMESTEP", "RS", "SOIL", "SWAT", "PRESSURE"],
|
||||
"scaling_input": ["none", "none", "standard", "standard", "standard","minmax"],
|
||||
"feature_engineering_input": ["log", "log10", "log1p", "log1p", "log1p", "log10"],
|
||||
"output_features": ["RS", "PRESSURE", "DELTA_SGAS", "DELTA_SOIL"],
|
||||
"scaling_output": ["standard", "minmax", "standard", "standard"],
|
||||
"feature_engineering_output": ["log", "log10", "log1p", "log1p"],
|
||||
"start_idx": 0, "end_idx": 1},
|
||||
{"input_features": ["PERMX", "TIMESTEP", "RS", "SOIL", "SWAT", "PRESSURE"],
|
||||
"scaling_input": ["none", "none", "standard", "standard", "standard","minmax"],
|
||||
"feature_engineering_input": ["log", "log10", "log1p", "log1p", "log1p", "log10"],
|
||||
"output_features": ["DELTA_RS", "DELTA_PRESSURE", "DELTA_SGAS", "DELTA_SOIL"],
|
||||
"scaling_output": ["standard", "minmax", "standard", "standard"],
|
||||
"feature_engineering_output": ["none", "none", "log1p", "log1p"],
|
||||
"start_idx": 1, "end_idx": 2},
|
||||
],
|
||||
|
||||
},
|
||||
]
|
||||
Reference in new issue
Block a user