# debian-based distros
sudo apt-get install build-essential cmake ninja-build

find_package(Optimium-Runtime REQUIRED)
target_link_libraries(MyExecutable PRIVATE Optimium::Runtime)

# Use C++14
set(CMAKE_CXX_STANDARD 14)

pkg-config is supported for non-CMake users.
You can get compiler options via `pkg-config --libs --cflags optimium-runtime`.

set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH)
find_package(Optimium-Runtime REQUIRED)
target_link_libraries(MyExecutable PRIVATE Optimium::Runtime)

pkg-config is supported for non-CMake users.
You can get compiler options via `pkg-config --libs --cflags optimium-runtime`.

 <application ...
              android:extractNativeLibs="true"
              ...>

#include <Optimium/Runtime.h>
#include <Optimium/Runtime/Logging/LogSettings.h>
#include <Optimium/Runtime/Logging/AndroidLogWriter.h>
#include <Optimium/Runtime/Logging/ConsoleLogWriter.h>
#include <Optimium/Runtime/Logging/FileLogWriter.h>

int main(...) {
    // change verbosity to debug level
    rt::logging::setLogLevel(rt::LogLevel::Debug);

    // add console log writer.
    rt::logging::addLogWriter(std::make_unique<rt::logging::ConsoleWriter>());

    // add file log writer to "output.log" file.
    rt::logging::addLogWriter(std::make_unique<rt::logging::FileWriter>("output.log"));

    // add Android Logcat log writer. this logger is only available for Android.
    rt::logging::addLogWriter(std::make_unique<rt::logging::AndroidWriter>());

    // Explicitly initialize and finalize the runtime.
    rt::initialize();

    // ... (load model, run inference, etc.)

    rt::finalize();
}

int main(...) {
    rt::initialize();

    // ... load models, run inference ...

    rt::finalize();
}

// Recommended: static global — runtime lives for the entire process lifetime.
static rt::AutoInit Init;

int main(...) {
    // ... load models, run inference ...
    // rt::finalize() is called automatically when the process exits.
}

// Also OK: top of main() — runtime lives until main() returns.
int main(...) {
    rt::AutoInit Init;

    // ... load models, run inference ...
    // rt::finalize() is called when Init goes out of scope.
}

int main(...) {
    // ...

    // Iterate every devices to check the device exists.
    rt::HostInfo Local = rt::getLocalInfo();

    bool Found = false;
    for (rt::DeviceID ID : Local.Devices) {
        if (ID.getPlatform() == rt::PlatformKind::Native) {
            Found = true;
            break;
        }
    }

    if (!Found) {
        std::cout << "error: cannot find needed device."
                  << std::endl;
    }
}

int main(...) {
    // ...

    // load a model with default options (auto-detected devices).
    rt::Model Model = rt::loadModel("path/to/model");


    // load a model with manual configurations.
    rt::ModelLoadOptions Options;

    // set devices for running the model.
    Options.Devices = { ... };

    // if true, fail when the exact device is not available (no fallback to localhost).
    Options.Strict = false;

    // enable memory optimization (share buffers between non-overlapping tensors).
    Options.EnableMemoryOptimization = true;

    // enable runtime checks for debugging. (NOT YET IMPLEMENTED)
    Options.EnableRuntimeChecks = false;

    // treat denormal floats as zero.
    Options.DisableDenormals = true;

    // path to save intermediate tensors for debugging.
    Options.IntermediateSavePath = "";

    // passphrase for encrypted models.
    Options.Passphrase = "";

    // set the number of threads to be used for running the model.
    Options.ThreadCount = 4;

    // set which cores the threads used for inference will be assigned to.
    Options.Cores = {0, 1, 2, 3};

    // set the scheduler type.
    // Simple: sequential execution (default)
    // Exclusive: first-come first-served with resource queuing
    // Pipeline: pipelined multi-stage execution across devices
    Options.Scheduler = rt::SchedulerType::Simple;

    rt::Model Model = rt::loadModel("path/to/model", Options);
    // ..
}

int main(...) {
    // ...

    // print model name
    std::cout << "model name: " << Model.getName() << std::endl;

    // print list of input tensor info
    std::cout << "input tensors" << std::endl;
    for (rt::StringRef Name : Model.getInputNames())
        std::cout << Model.getTensorInfo(Name) << std::endl;

    // print list of output tensor info
    std::cout << "output tensors" << std::endl;
    for (rt::StringRef Name : Model.getOutputNames())
        std::cout << Model.getTensorInfo(Name) << std::endl;

    // access by index
    const rt::TensorInfo &FirstInput = Model.getInputTensorInfo(0);
    const rt::TensorInfo &FirstOutput = Model.getOutputTensorInfo(0);

    // list operations
    for (const auto &Op : Model.getOperations())
        std::cout << "op: " << Op.Name << " on device: " << Op.Device << std::endl;
}

int main(...) {
    // ...

    const rt::TensorInfo &Info = Model.getTensorInfo("input_0");

    std::cout << "name of tensor: "
              << Info.Name << std::endl;
    std::cout << "shape of tensor: "
              << Info.Shape << std::endl;
    std::cout << "alignment of tensor: "
              << Info.Alignment << std::endl;
    std::cout << "type of tensor: "
              << Info.Type << std::endl;
    std::cout << "padding of tensor: "
              << Info.Padding << std::endl;
    std::cout << "tensor size in bytes: "
              << Info.getTensorSize() << std::endl;

    if (Info.Scheme)
        std::cout << "scheme of tensor: "
                  << *(Info.Scheme) << std::endl;

    // ...
}

int main(...) {
    // ...

    // infer output shape from input shapes (by-name)
    std::map<std::string, rt::TensorShape> InputShapes;
    InputShapes["input_0"] = rt::TensorShape({1, 3, 224, 224});
    rt::TensorShape OutputShape = Model.inferShape("output_0", InputShapes);

    // infer output shape from input shapes (by-index)
    std::vector<rt::TensorShape> InputShapeList = { rt::TensorShape({1, 3, 224, 224}) };
    rt::TensorShape OutputShape2 = Model.inferShape("output_0", rt::make_array(InputShapeList));

    // ...
}

int main(...) {
    // ...

    rt::InferRequest Request = Model.createRequest();

    // ...
}

int main(...) {
    // ...

    // Create float32 tensor shaped 32x32.
    rt::TypedTensor<float> f32_tensor = rt::tensor<float>({32, 32});

    // Create generic tensor shaped 8.
    rt::Tensor i16_tensor = rt::tensor(rt::ElementType::I16, {8});

    // Create float16 tensor shaped 32x32 with user-provided buffer.
    rt::float16* Data = new rt::float16[32 * 32];
    rt::TypedTensor<rt::float16> f16_tensor = rt::tensor<rt::float16>({32, 32}, Data);
}

int main(...) {
    // ...

    // Example for load data from existing data.
    // Assume variable 'Data' contains data.
    std::vector<float> Data;

    rt::TypedTensor<float> Tensor = rt::tensor<float>({32, 32});
    std::copy(Data.begin(), Data.end(), Tensor.data());

    // Example for load data from file.
    // <fstream> is required.
    std::ifstream File("path/to/file", std::ios::in | std::ios::binary);
    File.read(reinterpret_cast<char *>(Tensor.data()), Tensor.getTensorSize());

    // Fill scalar value. 'Tensor' must be 'TypedTensor' type.
    Tensor.fill(1.0f);

    // Save tensor to file.
    Tensor.save("tensor.bin");

    // Load tensor from file.
    Tensor.load("tensor.bin");

    // Release tensor data explicitly.
    Tensor.release();

    // ...
}

int main(...) {
    // ...

    // running inference with input and output list
    std::vector<rt::Tensor> Inputs;
    std::vector<rt::Tensor> Outputs;

    // Create empty tensors.
    for (rt::StringRef Name : Model.getInputNames()) {
        const rt::TensorInfo& Info = Model.getTensorInfo(Name);
        Inputs.push_back(rt::tensor(Info.Type, Info.Shape));
    }

    for (rt::StringRef Name : Model.getOutputNames()) {
        const rt::TensorInfo& Info = Model.getTensorInfo(Name);
        Outputs.push_back(rt::tensor(Info.Type, Info.Shape));
    }

    // rt::make_array is helper function that creates rt::ArrayRef.
    Request.infer(rt::make_array(Inputs), rt::make_array(Outputs));
    Request.wait();

    // ...
}

int main(...) {
    // ...

    // running inference with input and output map
    std::map<std::string, rt::Tensor> Inputs;
    std::map<std::string, rt::Tensor> Outputs;

    // Create empty tensors
    for (rt::StringRef Name : Model.getInputNames()) {
        const rt::TensorInfo& Info = Model.getTensorInfo(Name);
        Inputs[Name] = rt::tensor(Info.Type, Info.Shape);
    }

    for (rt::StringRef Name : Model.getOutputNames()) {
        const rt::TensorInfo& Info = Model.getTensorInfo(Name);
        Outputs[Name] = rt::tensor(Info.Type, Info.Shape);
    }

    Request.infer(Inputs, Outputs);
    Request.wait();

    // ...
}

int main(...) {
    // ...

    // waits until inference is finished.
    Request.infer(...);
    Request.wait();

    // waits 500 milliseconds to finish inference
    using namespace std::chrono_literals;

    Request.infer(...);
    if (Request.wait(500ms))
        std::cout << "inference not finished after 500ms"
                  << std::endl;
    else
        std::cout << "inference was finished within 500ms"
                  << std::endl;

    // check the state of the request
    std::cout << "status of request: "
              << Request.getStatus()
              << std::endl;
}

int main(...) {
    // ...

    Request.addCallback([](rt::InferStatus Status, std::exception_ptr Err) {
        if (Status == rt::InferStatus::Fault) {
            try { std::rethrow_exception(Err); }
            catch (const std::exception &E) {
                std::cerr << "Inference error: " << E.what() << std::endl;
            }
        }
    });

    Request.infer(...);
    Request.wait();
}

Request.infer(...);
// ... later:
Request.cancel();

int main(...) {
    // ...

    rt::ProfileOptions Options;
    Options.Repeat = 100;          // number of repetitions
    Options.WarmUp = 10;           // warm-up count
    Options.WarmUpTime = std::chrono::microseconds(1000000); // warm-up duration
    Options.StopThreshold = std::chrono::microseconds(0);    // stop threshold
    Options.CheckPeriod = 0;       // check period
    Options.EventBufferSize = rt::kDefaultEventBufferSize;   // event buffer size (default: 4MB)

    // profile() is blocking (unlike infer() which is async)
    Request.profile(Inputs, Outputs, Options);

    // access profiling events
    for (const auto &Event : Request.getProfileEvents()) {
        std::cout << "Event: " << rt::toString(Event.Kind)
                  << " at " << Event.TimeStamp.time_since_epoch().count()
                  << std::endl;
    }

    // access model metadata from the request
    std::cout << "Model: " << Request.getModelName() << std::endl;
    for (const auto &Op : Request.getModelOperations())
        std::cout << "  Op: " << Op.Name << std::endl;
}

auto Recorder = std::make_shared<rt::ProfileEventRecorder>(1024 * 1024);
Request.setRecorder(Recorder);
Request.profile(Inputs, Outputs, Options);
// Recorder now contains events with lock-free access

rt::ModelLoadOptions Options;
Options.EnableMemoryOptimization = false;
rt::Model Model = rt::loadModel("path/to/model", Options);

auto Request = Model.createRequest();
Request.infer(Inputs, Outputs);
Request.wait();

// access intermediate tensor
rt::Tensor IntermediateTensor = Request.getTensor("intermediate_tensor_name");

int main(...) {
    // ...
    rt::loadExtension("path/to/extension.so");
    // ...
}

import optimium.runtime as rt

def main():
    # change verbosity to debug level
    rt.logging.set_loglevel(rt.LogLevel.DEBUG)

    # enable logger that writes logs on console
    rt.logging.enable_console_log()

    # enable logger that writes logs on file
    rt.logging.enable_file_log("output.log")

import os
os.environ["OPTIMIUM_RT_DEFER_INIT"] = "TRUE"

import optimium.runtime as rt
rt.initialize()  # must be called before use any runtime components.

version = rt.get_version()
print(f"Optimium Runtime v{version.major}.{version.minor}.{version.patch}")
print(f"Build: {version.build_info}")

def main():
    # ...

    # Iterate every devices to check the device exists.
    local = rt.get_local_info()

    found = False
    for dev in local.devices:
        if dev.platform == rt.PlatformKind.NATIVE:
            found = True

    if not found:
        print("cannot find needed device")

def main():
    # ...

    # load a model with auto-detected devices.
    model = rt.load_model("path/to/model")

    # load a model with manual configurations.
    model = rt.load_model("path/to/model",
                          devices=[...],
                          # if true, fail when exact device is unavailable (no fallback).
                          strict=False,
                          # enable memory optimization (share buffers between non-overlapping tensors).
                          memory_optimization=True,
                          # treat denormal floats as zero.
                          disable_denormals=True,
                          # path to save intermediate tensors for debugging.
                          intermediate_save_path=None,
                          # passphrase for encrypted models.
                          passphrase=None,
                          # number of threads for inference.
                          threads=4,
                          # which cores the threads will be assigned to.
                          cores=[0, 1, 2, 3],
                          # scheduler type: SIMPLE (default), EXCLUSIVE, or PIPELINE.
                          scheduler_type=rt.SchedulerType.SIMPLE)

def main():
    # ...

    # print model name
    print(f"model name: {model.name}")

    # print list of input tensor info
    print("input tensors")
    for name in model.input_names:
        print(model.get_tensor(name))

    # print list of output tensor info
    print("output tensors")
    for name in model.output_names:
        print(model.get_tensor(name))

    # access by index
    first_input = model.get_input_tensor(0)
    first_output = model.get_output_tensor(0)

    # list operations
    for op in model.operations:
        print(f"op: {op.name} on device: {op.device}")

def main():
    # ...

    info = model.get_tensor("input_0")

    print(f"name of tensor: {info.name}")
    print(f"shape of tensor: {info.shape}")
    print(f"alignment of tensor: {info.alignment}")
    print(f"type of tensor: {info.type}")
    print(f"padding of tensor: {info.padding}")
    print(f"tensor size in bytes: {info.size}")
    print(f"opt out: {info.opt_out}")
    print(f"constant: {info.constant}")

    if info.scheme:
        print(f"quantization scheme of tensor: {info.scheme}")
        if info.scheme.per_channel:
            print(f"per-channel on axis: {info.scheme.axis}")
            for i in range(len(info.scheme)):
                param = info.scheme[i]
                print(f"  channel {i}: scale={param.scale}, zero_point={param.zero_point}")
        else:
            param = info.scheme.get_param()
            print(f"per-tensor: scale={param.scale}, zero_point={param.zero_point}")

def main():
    # ...

    # infer output shape from input shapes (by-name dict)
    output_shape = model.infer_shape("output_0", {
        "input_0": rt.TensorShape(1, 3, 224, 224)
    })
    print(f"inferred output shape: {output_shape}")

    # infer output shape from input shapes (by-index list)
    output_shape = model.infer_shape("output_0", [
        rt.TensorShape(1, 3, 224, 224)
    ])

def main():
    # ...

    request = model.create_request()

# ...
import numpy as np

def main():
    # ...

    # create float32 tensor shaped 32x32 (uninitialized)
    f32_tensor = rt.tensor(shape=(32, 32), dtype=rt.ElementType.F32)

    # you can use numpy-style alias for dtype
    f32_tensor = rt.tensor(shape=(32, 32), dtype=rt.float32)

    # create int16 tensor filled with 123
    i16_tensor = rt.tensor(123, shape=(8,), dtype=rt.int16)

    # create tensor from list
    tensor_from_list = rt.tensor([[1,2,3], [4,5,6], [7,8,9]])  # default dtype is rt.float32

    # create tensor from numpy (with copy by default)
    arr = np.random.random((32, 32)).astype(np.float16)
    tensor_from_np = rt.tensor(arr)

    # create tensor from numpy without copy (zero-copy, shares memory)
    tensor_zero_copy = rt.tensor(arr, copy=False)

# ElementType -> numpy dtype
np_dtype = rt.ElementType.F32.to_dtype()

# numpy dtype -> ElementType
elem_type = rt.ElementType.from_dtype(np.float32)

def main():
    # ...

    tensor = rt.tensor(...)
    # not supported
    # val = tensor[...]

    arr = tensor.to_numpy()
    val = arr[...]

# set the maximum number of elements displayed per dimension (default: 8, -1 = unlimited)
rt.config.set_print_threshold(10)
# set the decimal precision for floating point output (default: 6, range 0-15)
rt.config.set_print_precision(4)

def main():
    # ...

    # running inference with input and output list
    inputs = []
    outputs = []

    # create empty tensors
    for name in model.input_names:
        info = model.get_tensor(name)
        inputs.append(rt.tensor(shape=info.shape, dtype=info.type))

    for name in model.output_names:
        info = model.get_tensor(name)
        outputs.append(rt.tensor(shape=info.shape, dtype=info.type))

    request.infer(inputs, outputs)
    request.wait()

def main():
    # ...

    # running inference with input and output dict
    inputs = {}
    outputs = {}

    # create empty tensors
    for name in model.input_names:
        info = model.get_tensor(name)
        inputs[name] = rt.tensor(shape=info.shape, dtype=info.type)

    for name in model.output_names:
        info = model.get_tensor(name)
        outputs[name] = rt.tensor(shape=info.shape, dtype=info.type)

    request.infer(inputs, outputs)
    request.wait()

def main():
    # ...

    # running inference with numpy arrays
    # assume the model accepts two float32 tensors and
    # returns one float32 tensor.
    inputs = [
        np.random.random((32, 32)).astype(np.float32),
        np.random.random((32, 32)).astype(np.float32)
    ]
    outputs = [
        np.zeros((32, 32), dtype=np.float32)
    ]

    request.infer(inputs, outputs)
    request.wait()

def main():
    # ...

    # waits until inference is finished.
    request.infer(...)
    request.wait()

    # waits 500000 microseconds (500ms) to finish inference
    request.infer(...)

    if request.wait(500000):
        print("inference not finished after 500ms")
    else:
        print("inference was finished within 500ms")

    # check status of the inference.
    print(f"current state of request: {request.status}")

def on_complete(status, error):
    if status == rt.InferStatus.FAULT:
        print(f"Inference error: {error}")

request.set_callback(on_complete)
request.infer(inputs, outputs)
request.wait()

request.infer(inputs, outputs)
# ... later:
request.cancel()

def main():
    # ...

    # profile() is blocking (unlike infer() which is async)
    request.profile(inputs, outputs,
                    repeat=100,
                    warmup=10,
                    warmup_time=1000000,    # microseconds
                    stop_threshold=0,       # microseconds
                    check_period=0,
                    event_buffer_size=0)    # 0 means use default (4MB)

    # access raw profiling events
    for event in request.get_profile_events():
        print(f"Event: {event.kind} at {event.timestamp}")

    # access computed durations (nanoseconds, grouped by model/layer/copy)
    durations = request.get_profile_durations()
    print(f"Model: {durations.model_name}")
    print(f"Model durations (ns): {durations.model_durations}")
    for name, layer_dur in durations.layer_durations.items():
        print(f"  Layer '{name}': {layer_dur}")
    for name, copy_dur in durations.copy_durations.items():
        print(f"  Copy '{name}': {copy_dur}")

from optimium.runtime.profile import BasicProfiler

profiler = BasicProfiler(
    "path/to/model",
    memory_optimization=True,
    disable_denormals=False,
    threads=1,
    cores=None,
    intermediate_save_path=""
)

profiler.profile(
    repeat=100,
    warmup=10,
    warmup_time=1000000,    # microseconds
)

# access statistics
print(f"Model: min={profiler.model_stat.min}, "
      f"max={profiler.model_stat.max}, "
      f"mean={profiler.model_stat.mean:.2f}")

for stat in profiler.layer_stats:
    print(f"  Layer {stat.name}: mean={stat.mean:.2f}")

# save results to JSON
profiler.dump_json("profile_results.json")

profiler = BasicProfiler(
    "path/to/model",
    remote_address="192.168.1.100",
    remote_port=32264
)

from optimium.runtime.profile import BatchProfiler

profiler = BatchProfiler("path/to/model")
profiler.profile(
    repeat=100,
    inputs=[batch1_inputs, batch2_inputs, ...],
    outputs=[batch1_outputs, batch2_outputs, ...]
)

# access per-batch and total statistics
for i, stat in enumerate(profiler.batch_stat):
    print(f"Batch {i}: mean={stat.model_stat.mean:.2f}")
print(f"Total: mean={profiler.total_stat.model_stat.mean:.2f}")

model = rt.load_model("path/to/model", memory_optimization=False)
request = model.create_request()
request.infer(inputs, outputs)
request.wait()

# access intermediate tensor
intermediate = request.get_tensor("intermediate_tensor_name")
print(intermediate.to_numpy())

import optimium.runtime as rt
from optimium.runtime import remote

# connect to remote server
session = remote.connect(
    "192.168.1.100",
    port=32264,
    enable_secure_connection=True,
    enable_compression=True
)

# check remote host info
print(f"Remote host: {session.host_info}")

# load model on remote
model = session.load_model(
    "path/to/model",
    threads=4,
    cores=[0, 1, 2, 3],
    scheduler_type=rt.SchedulerType.SIMPLE
)

# model info
print(f"Model: {model.name}")
for info in model.input_tensor_infos:
    print(f"  Input: {info.name} {info.shape} {info.type}")
for info in model.output_tensor_infos:
    print(f"  Output: {info.name} {info.shape} {info.type}")

# run profiling (asynchronous - call wait() after)
inputs = {"input_0": rt.tensor(shape=(1, 3, 224, 224), dtype=rt.float32)}
outputs = {"output_0": rt.tensor(shape=(1, 1000), dtype=rt.float32)}

model.profile(inputs, outputs, repeat=100, warmup=10)
model.wait()

# get profile events and durations
events = model.get_profile_events()
durations = model.get_profile_durations()

# get specific tensor result
result = model.get_tensor("output_0")

try:
    model = rt.load_model("nonexistent/path")
except rt.ContainerError as e:
    print(f"Container error: {e}")
except rt.ModelError as e:
    print(f"Model error: {e}")

rt.load_extension("path/to/extension.so")