Generation Generation — Rust examples backed by real provider calls. rust examples examples/generation src/examples/rust/generation example Generation

These Rust examples are real runnable files. Edit the source file first; this page is rebuilt from the checked-in example and its metadata header.

Rust Prompt-Cached Generation

Runs GPT-5.6 structured generation with stable OpenAI prompt-cache affinity.

Rust
use axllm::{ax, AxResult, OpenAICompatibleClient};
use serde_json::json;
use std::env;


fn openai_client() -> AxResult<OpenAICompatibleClient> {
    let api_key = env::var("OPENAI_API_KEY").or_else(|_| env::var("OPENAI_APIKEY")).map_err(|_| axllm::AxError::runtime("Set OPENAI_API_KEY or OPENAI_APIKEY to run this example."))?;
    let model = env::var("AX_OPENAI_MODEL").unwrap_or_else(|_| "gpt-5.6-luna".to_string());
    Ok(OpenAICompatibleClient::new(api_key, model).with_model_config(json!({"temperature": 0})))
}

fn main() -> AxResult<()> {
    let mut client = openai_client()?;
    let mut program = ax("question:string -> answer:string")?;
    let output = program.forward_with_options(
        &mut client,
        json!({"question": "In one sentence, explain Ax as a language-agnostic LLM programming library."}),
        json!({"promptCacheKey": "ax-openai-example", "contextCache": {}}),
    )?;
    println!("{}", serde_json::to_string_pretty(&output)?);
    Ok(())
}

Rust Astra Generation

Runs Astra through the standard generator with automatic Responses routing and prompt caching.

Rust
use axllm::{ax, AxResult, OpenAICompatibleClient};
use serde_json::json;
use std::env;


fn openai_client() -> AxResult<OpenAICompatibleClient> {
    let api_key = env::var("OPENAI_API_KEY").or_else(|_| env::var("OPENAI_APIKEY")).map_err(|_| axllm::AxError::runtime("Set OPENAI_API_KEY or OPENAI_APIKEY to run this example."))?;
    let model = env::var("AX_OPENAI_MODEL").unwrap_or_else(|_| "gpt-6-astra".to_string());
    axllm::ai("openai", json!({"api_key": api_key, "model": model, "model_config": {"thinkingTokenBudget": "low", "max_tokens": 2048}}))
}

fn main() -> AxResult<()> {
    let mut client = openai_client()?;
    let mut program = ax("question:string -> answer:string")?;
    let output = program.forward_with_options(
        &mut client,
        json!({"question": "In one sentence, explain Ax as a language-agnostic LLM programming library."}),
        json!({"serviceTier": "standard", "promptCacheKey": "ax-openai-example", "contextCache": {}}),
    )?;
    println!("{}", serde_json::to_string_pretty(&output)?);
    Ok(())
}

Rust Model Catalog

Lists static models and named OpenAI-compatible profiles with portable thinking levels and service tiers.

Rust
use axllm::{get_supported_ai_models, AxResult};
use serde_json::{json, Value};

fn provider<'a>(catalog: &'a [Value], name: &str) -> &'a Value {
    catalog
        .iter()
        .find(|entry| entry["name"] == name)
        .unwrap_or_else(|| panic!("missing provider {name}"))
}

fn main() -> AxResult<()> {
    let catalog = get_supported_ai_models()?;
    let azure = provider(&catalog, "azure-openai");
    let openrouter = provider(&catalog, "openrouter");

    assert_eq!(azure["isDynamic"], true);
    assert_eq!(azure["models"], json!([]));
    assert!(azure["capabilities"]["thinkingLevels"]
        .as_array()
        .is_some_and(|levels| levels.iter().any(|level| level == "high")));
    assert!(azure["capabilities"]["serviceTiers"]
        .as_array()
        .is_some_and(|tiers| tiers.iter().any(|tier| tier == "priority")));
    assert!(openrouter["capabilities"]["serviceTiers"]
        .as_array()
        .is_some_and(|tiers| tiers.iter().any(|tier| tier == "flex")));

    println!(
        "{} providers; Azure and OpenRouter named profiles are available",
        catalog.len()
    );
    Ok(())
}

Rust Jev Signature Decisions

Converts Jev probabilities into boolean and class outputs with a provider threshold.

Rust
use axllm::{ai, ax, AxResult};
use serde_json::json;
use std::env;

fn main() -> AxResult<()> {
    let mut model = ai(
        "typesafe",
        json!({"api_key":env::var("TYPESAFE_APIKEY").expect("Set TYPESAFE_APIKEY"),"trueThreshold":0.9}),
    )?;
    let decision = ax("ticket:string -> urgent:boolean(true \"Customers cannot complete a core task\", false \"Routine request\") \"Needs immediate attention?\", team:class \"support, billing, engineering\"")?.forward(&mut model,json!({"ticket":"Checkout is unavailable for all customers after the latest deployment."}))?;
    assert!(decision["urgent"].is_boolean());
    let output = decision;
    println!("{}", serde_json::to_string_pretty(&output)?);
    Ok(())
}

Rust Meta Muse Spark

Selects any of Meta’s three protocols through the existing chat API.

Rust
use axllm::{ai, AxAIClient, AxResult};
use serde_json::json;

fn main() -> AxResult<()> {
    let key = std::env::var("MODEL_API_KEY")
        .map_err(|_| axllm::AxError::runtime("Set MODEL_API_KEY to run this example."))?;
    for profile in ["meta", "meta-chat", "meta-messages"] {
        let mut client = ai(profile, json!({"api_key": key, "model": "muse-spark-1.3"}))?;
        let response = client.chat(json!({
            "chat_prompt": [{"role": "user", "content": "Name a solar-powered sailboat."}],
            "model_config": {"thinking_token_budget": "highest"}
        }))?;
        println!("{profile} {response}");
    }
    Ok(())
}

Rust Structured Extraction

Extracts structured fields and labels from support text with OpenAI.

Rust
use axllm::{ax, AxResult, OpenAICompatibleClient};
use serde_json::json;
use std::env;


fn openai_client() -> AxResult<OpenAICompatibleClient> {
    let api_key = env::var("OPENAI_API_KEY").or_else(|_| env::var("OPENAI_APIKEY")).map_err(|_| axllm::AxError::runtime("Set OPENAI_API_KEY or OPENAI_APIKEY to run this example."))?;
    let model = env::var("AX_OPENAI_MODEL").unwrap_or_else(|_| "gpt-5.4-mini".to_string());
    Ok(OpenAICompatibleClient::new(api_key, model).with_model_config(json!({"temperature": 0})))
}

fn main() -> AxResult<()> {
    let mut client = openai_client()?;
    let mut program = ax("ticket:string -> priority:class \"high, normal, low\", summary:string, labels:string[]")?;
    let output = program.forward(&mut client, json!({"ticket": "Checkout has failed for enterprise customers since 09:00. Support wants a concise summary and tags."}))?;
    println!("{}", serde_json::to_string_pretty(&output)?);
    Ok(())
}

Rust Vertex Gemini Routing

Calls Gemini through Vertex with project and multi-region routing.

  • Provider: google-gemini
  • Env: GOOGLE_VERTEX_ACCESS_TOKEN, GOOGLE_PROJECT_ID, GOOGLE_REGION
  • Level: intermediate
  • Run: npm run example -- rust src/examples/rust/generation/vertex_gemini.rs
  • Source: src/examples/rust/generation/vertex_gemini.rs
Rust
use axllm::{ai, AxAIClient, AxError, AxResult};
use serde_json::json;
use std::env;

fn required(name: &str) -> AxResult<String> {
    env::var(name).map_err(|_| AxError::runtime(format!("Set {name} to run this example.")))
}

fn main() -> AxResult<()> {
    let model = env::var("AX_VERTEX_MODEL").unwrap_or_else(|_| "gemini-3.5-flash".to_string());
    let mut client = ai("google-gemini", json!({
        "api_key": required("GOOGLE_VERTEX_ACCESS_TOKEN")?,
        "project_id": required("GOOGLE_PROJECT_ID")?,
        "region": required("GOOGLE_REGION")?,
        "model": model,
    }))?;
    let out = client.chat(json!({
        "chat_prompt": [{"role": "user", "content": "Reply with the word ready."}]
    }))?;
    println!("{}", serde_json::to_string_pretty(&out)?);
    Ok(())
}

Rust Jev Hybrid Reply

Passes Jev decisions to a second Ax program to generate a customer reply.

Rust
use axllm::{ai, ax, AxResult};
use serde_json::json;
use std::env;

fn main() -> AxResult<()> {
    let mut model = ai(
        "typesafe",
        json!({"api_key":env::var("TYPESAFE_APIKEY").expect("Set TYPESAFE_APIKEY"),"trueThreshold":0.9}),
    )?;
    let decision = ax("ticket:string -> urgent:boolean(true \"Customers cannot complete a core task\", false \"Routine request\") \"Needs immediate attention?\", team:class \"support, billing, engineering\"")?.forward(&mut model,json!({"ticket":"Checkout is unavailable for all customers after the latest deployment."}))?;
    assert!(decision["urgent"].is_boolean());
    let key = env::var("OPENAI_API_KEY")
        .or_else(|_| env::var("OPENAI_APIKEY"))
        .expect("Set OPENAI_APIKEY");
    let mut writer = ai(
        "openai",
        json!({"api_key":key,"model":"gpt-5.6-luna","model_config":{"temperature":1}}),
    )?;
    let reply = ax("ticket:string, urgent:boolean, team:string -> reply:string")?.forward(&mut writer,json!({"ticket":"Checkout is unavailable for all customers after the latest deployment.","urgent":decision["urgent"],"team":decision["team"]}))?;
    assert!(!reply["reply"].as_str().unwrap().trim().is_empty());
    let output = json!({"decision":decision,"reply":reply});
    println!("{}", serde_json::to_string_pretty(&output)?);
    Ok(())
}

Rust Signature Constraints

Builds native constrained fields and runs the signature with OpenAI.

Rust
use axllm::{ax, f, AxResult, FieldType, OpenAICompatibleClient};
use serde_json::json;
use std::env;

fn openai_client() -> AxResult<OpenAICompatibleClient> {
    let api_key = env::var("OPENAI_API_KEY")
        .or_else(|_| env::var("OPENAI_APIKEY"))
        .map_err(|_| {
            axllm::AxError::runtime("Set OPENAI_API_KEY or OPENAI_APIKEY to run this example.")
        })?;
    let model = env::var("AX_OPENAI_MODEL").unwrap_or_else(|_| "gpt-5.4-mini".to_string());
    Ok(OpenAICompatibleClient::new(api_key, model).with_model_config(json!({"temperature": 0})))
}

fn main() -> AxResult<()> {
    let mut request_type = FieldType::string();
    request_type.min_length = Some(10.0);
    request_type.max_length = Some(500.0);
    request_type.description = Some("Booking request".to_string());
    let mut email_type = FieldType::string();
    email_type.format = Some("email".to_string());
    email_type.description = Some("Contact email".to_string());
    let mut party_type = FieldType::number();
    party_type.minimum = Some(1.0);
    party_type.maximum = Some(12.0);
    party_type.description = Some("Guests".to_string());
    let mut code_type = FieldType::string();
    code_type.pattern = Some(r"^[A-Z]{3}-\d{4}$".to_string());
    code_type.pattern_description = Some("Must look like ABC-1234".to_string());

    let signature = f()
        .input("requestText", request_type)
        .input("contactEmail", email_type)
        .output("partySize", party_type)
        .output("bookingCode", code_type)
        .build();
    let mut program = ax("requestText:string -> partySize:number, bookingCode:string")?;
    program.signature = signature;
    let mut client = openai_client()?;
    let output = program.forward(
        &mut client,
        json!({
            "requestText": "Book dinner for four people under the name Ada Lovelace.",
            "contactEmail": "ada@example.com"
        }),
    )?;
    println!("{}", serde_json::to_string_pretty(&output)?);
    Ok(())
}

Centralized Usage Observer

Attributes every completed model call to a tenant, user, and request from one global observer.

Rust
use axllm::{
    set_usage_observer, AxAIClient, AxError, AxResult, AxUsageEvent, OpenAICompatibleClient,
};
use serde_json::json;
use std::env;
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};

fn main() -> AxResult<()> {
    let api_key = env::var("OPENAI_API_KEY")
        .or_else(|_| env::var("OPENAI_APIKEY"))
        .map_err(|_| AxError::runtime("Set OPENAI_API_KEY or OPENAI_APIKEY to run this example."))?;
    let model = env::var("AX_OPENAI_MODEL").unwrap_or_else(|_| "gpt-5.4-mini".to_string());
    let events = Arc::new(Mutex::new(Vec::<AxUsageEvent>::new()));
    let captured = Arc::clone(&events);
    set_usage_observer(Some(Arc::new(move |event| {
        captured.lock().unwrap().push(event);
    })));

    let mut client = OpenAICompatibleClient::new(api_key, model).with_options(json!({
        "usageContext": {
            "tenantId": "tenant-42",
            "feature": "support-chat",
            "attributes": {"environment": "example"}
        }
    }));
    let request_id = format!(
        "request-{}",
        SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .map_err(|error| AxError::runtime(error.to_string()))?
            .as_nanos()
    );
    client.chat_with_options(
        json!({
            "chat_prompt": [
                {"role": "user", "content": "Reply with one short greeting."}
            ]
        }),
        json!({
            "usageContext": {"userId": "user-7", "requestId": request_id}
        }),
    )?;
    set_usage_observer(None);
    println!("{}", serde_json::to_string_pretty(&*events.lock().unwrap())?);
    Ok(())
}

Rust Incremental Provider Stream

Iterates OpenAI SSE events incrementally; dropping the iterator closes the response.

Rust
use axllm::{ai, AxAIClient, AxResult};
use serde_json::json;
use std::{env, time::Instant};

fn main() -> AxResult<()> {
    let api_key = env::var("OPENAI_API_KEY")
        .or_else(|_| env::var("OPENAI_APIKEY"))
        .map_err(|_| {
            axllm::AxError::runtime("Set OPENAI_API_KEY or OPENAI_APIKEY to run this example.")
        })?;
    let model = env::var("AX_OPENAI_MODEL").unwrap_or_else(|_| "gpt-5.6-luna".to_string());
    let mut client = ai(
        "openai",
        json!({"api_key": api_key, "model": model}),
    )?;
    let started = Instant::now();
    for event in client.stream_iter(json!({
        "chat_prompt": [{"role": "user", "content": "Reply with exactly: streaming works"}],
        "model_config": {"temperature": 1}
    }))? {
        let event = event?;
        if let Some(content) = event["results"][0]["content"]
            .as_str()
            .filter(|value| !value.is_empty())
        {
            print!("[{} ms] {content}", started.elapsed().as_millis());
        }
    }
    println!();
    Ok(())
}

Rust Portable Cancellation

Cancels a provider request before transport and preserves the first cancellation reason.

Rust
use axllm::{AxAIClient, AxCancellationToken, AxResult, AxTransport, OpenAICompatibleClient};
use serde_json::{json, Value};
use std::sync::{
    atomic::{AtomicUsize, Ordering},
    Arc,
};

struct CountingTransport(Arc<AtomicUsize>);

impl AxTransport for CountingTransport {
    fn send(&mut self, _request: Value) -> AxResult<Value> {
        self.0.fetch_add(1, Ordering::SeqCst);
        Ok(json!({"status": 200, "json": {}}))
    }
}

fn main() -> AxResult<()> {
    let calls = Arc::new(AtomicUsize::new(0));
    let mut client = OpenAICompatibleClient::new("test-key", "gpt-5.6-luna")
        .with_transport(CountingTransport(calls.clone()));
    let token = AxCancellationToken::default();
    assert!(token.cancel("user stopped") && !token.cancel("later reason"));

    let error = client
        .chat_with_cancellation(
            json!({"chat_prompt": [{"role": "user", "content": "This must not be sent."}]}),
            json!({}), &token,
        )
        .expect_err("pre-cancelled request unexpectedly completed");
    assert_eq!(error.error_type.as_deref(), Some("AxAIServiceAbortedError"));
    assert!(!error.retryable && error.to_string().contains("user stopped"));
    assert_eq!(calls.load(Ordering::SeqCst), 0);
    println!("cancelled before transport: user stopped");
    Ok(())
}

Rust Gemini Flex Inference

Sends latency-tolerant work through Gemini Flex and reports the applied tier.

Rust
use axllm::{ai, AxAIClient, AxError, AxResult};
use serde_json::json;
use std::env;

fn api_key() -> AxResult<String> {
    env::var("GOOGLE_API_KEY")
        .or_else(|_| env::var("GOOGLE_APIKEY"))
        .map_err(|_| AxError::runtime("Set GOOGLE_API_KEY or GOOGLE_APIKEY to run this example."))
}

fn main() -> AxResult<()> {
    let model = env::var("AX_GEMINI_MODEL").unwrap_or_else(|_| "gemini-3.8-flash".to_string());
    let mut client = ai(
        "google-gemini",
        json!({
            "api_key": api_key()?,
            "model": model,
        }),
    )?;
    let out = client.chat_with_options(
        json!({
            "chat_prompt": [{
                "role": "user",
                "content": "Explain in one sentence why batch evaluations save time."
            }]
        }),
        json!({"service_tier": "flex"}),
    )?;
    println!("{}", serde_json::to_string_pretty(&out)?);
    Ok(())
}

Rust Native File Routing

Summarizes a PDF through a provider router without replacing the native file with extracted text.

Rust
use axllm::{ai, ax, AxResult, ProviderRouter};
use serde_json::json;
use std::env;
fn main() -> AxResult<()> {
 let client=ai("openai",json!({"api_key":env::var("OPENAI_API_KEY").or_else(|_|env::var("OPENAI_APIKEY")).expect("Set OPENAI_API_KEY"),"model":"gpt-6-astra","model_config":{"thinkingTokenBudget":"low"}}))?;
 let mut router=ProviderRouter::new().with_provider("openai",client);
 let mut program=ax("document:file -> summary:string")?;
 let result=program.forward_with_options(&mut router,json!({"document":{"filename":"report.pdf","mimeType":"application/pdf","data":env::var("AX_PDF_BASE64").expect("Set AX_PDF_BASE64")}}),json!({"serviceTier":"standard"}))?;
 println!("{}",serde_json::to_string_pretty(&result)?);
 Ok(())
}

Rust Automatic Background Tools

Uses ordinary generation with background tools, steering, and a reasoning update.

Rust
use axllm::{ai, ax, run_control, tool, AxForwardOptions, AxResult};
use serde_json::json;
use std::{
    env,
    sync::{
        atomic::{AtomicBool, AtomicUsize, Ordering},
        Arc,
    },
    time::Duration,
};

fn main() -> AxResult<()> {
    let key = env::var("OPENAI_API_KEY")
        .or_else(|_| env::var("OPENAI_APIKEY"))
        .expect("Set OPENAI_API_KEY or OPENAI_APIKEY.");
    let mut client = ai(
        "openai",
        json!({"api_key":key,"model":"gpt-6-astra","model_config":{"thinkingTokenBudget":"low","max_tokens":4096}}),
    )?;
    let pending = Arc::new(AtomicBool::new(false));
    let finished = Arc::new(AtomicBool::new(false));
    let overlap = Arc::new(AtomicBool::new(false));
    let applied = Arc::new(AtomicUsize::new(0));
    let control = run_control();
    let steering = control.clone();
    let updates = applied.clone();
    control.on_event(move |event| {
        if event["type"] == "applied" {
            updates.fetch_add(1, Ordering::SeqCst);
        }
    });
    let started = pending.clone();
    let done = finished.clone();
    let steered = AtomicBool::new(false);
    let slow = tool("slow_reference")
        .description("Look up a reference; takes a few seconds.")
        .execution("background")
        .handler(move |_| {
            started.store(true, Ordering::SeqCst);
            if !steered.swap(true, Ordering::SeqCst) {
                steering.steer("Include the word VERIFIED in the final answer.")?;
                steering.set_thinking_token_budget("medium")?;
            }
            std::thread::sleep(Duration::from_secs(6));
            done.store(true, Ordering::SeqCst);
            Ok(json!("REF-42"))
        });
    let observed = overlap.clone();
    let label = tool("local_label")
        .description("Read an independent local label immediately.")
        .handler(move |_| {
            for _ in 0..300 {
                if pending.load(Ordering::SeqCst) {
                    break;
                }
                std::thread::sleep(Duration::from_millis(10));
            }
            observed.fetch_or(
                pending.load(Ordering::SeqCst) && !finished.load(Ordering::SeqCst),
                Ordering::SeqCst,
            );
            Ok(json!("LAUNCH"))
        });
    let mut program = ax("question -> answer")?.with_tool(slow).with_tool(label);
    let result=program.forward_with_options(&mut client,json!({"question":"First call slow_reference. While it is pending, call local_label. Call each tool only once; do not call a tool again while its result is pending. If a required tool result is still pending, end this response with a brief progress message. The application will continue with the result when it arrives; do not spend reasoning tokens waiting for it. Once both results arrive, return them in one sentence."}),AxForwardOptions::from(json!({"serviceTier":"standard","maxSteps":6})).with_control(control))?;
    let answer = result.to_string();
    for word in ["REF-42", "LAUNCH", "VERIFIED"] {
        assert!(answer.contains(word), "Missing final result: {answer}");
    }
    assert!(
        overlap.load(Ordering::SeqCst),
        "No independent work while background tool was pending"
    );
    assert_eq!(applied.load(Ordering::SeqCst), 2);
    println!("{answer}\nBackground overlap verified; steering and reasoning applied at the next response.");
    Ok(())
}

Rust Cancel Background Work

Cancels a live Astra run through the high-level controller and observes cooperative tool cancellation.

Rust
use axllm::{ai,ax,tool,run_control,AxForwardOptions,AxResult};
use serde_json::json;
use std::{env,sync::{Arc,Mutex,atomic::{AtomicBool,Ordering}},time::{Instant,Duration}};
fn main()->AxResult<()> {
 let key=env::var("OPENAI_API_KEY").or_else(|_|env::var("OPENAI_APIKEY")).expect("Set OPENAI_API_KEY or OPENAI_APIKEY.");
 let control=run_control();let abort=control.clone();let settled=Arc::new(AtomicBool::new(false));let done=settled.clone();let started=Arc::new(Mutex::new(None));let clock=started.clone();
 let lookup=tool("lookup").description("Look up the reference.").execution("background").context_handler(move |_,context|{*clock.lock().unwrap()=Some((Instant::now(),context.call_id.clone()));abort.abort();let now=Instant::now();while !context.is_cancelled()&&now.elapsed()<Duration::from_secs(2){std::thread::sleep(Duration::from_millis(1));}assert!(context.is_cancelled(),"Tool missed cancellation");done.store(true,Ordering::SeqCst);Ok(json!("LATE: discard this result"))});
 let mut program=ax("question -> answer")?.with_tool(lookup);
 let mut client=ai("openai",json!({"api_key":key,"model":"gpt-6-astra","model_config":{"thinkingTokenBudget":"low","max_tokens":2048}}))?;
 let error=program.forward_with_options(&mut client,json!({"question":"Call lookup once and return its result."}),AxForwardOptions::from(json!({"serviceTier":"standard"})).with_control(control)).expect_err("Cancelled run returned success");
 let (start,id)=started.lock().unwrap().clone().expect("Tool did not start");let elapsed=start.elapsed();assert!(elapsed<Duration::from_secs(2));assert!(error.to_string().contains(&id.expect("Missing call ID")),"{error}");let deadline=Instant::now();while !settled.load(Ordering::SeqCst)&&deadline.elapsed()<Duration::from_secs(2){std::thread::sleep(Duration::from_millis(1));}assert!(settled.load(Ordering::SeqCst));println!("Cancelled in {elapsed:?}; {error}");Ok(())
}

Rust Native Astra Steering

Steers a running generation through the high-level controller using the optional WebSocket transport.

Rust
use axllm::{ai, ax, run_control, AxForwardOptions, AxResult};
use serde_json::json;
use std::{
    env,
    sync::{
        atomic::{AtomicBool, Ordering},
        Arc,
    },
};

fn main() -> AxResult<()> {
    let key = env::var("OPENAI_API_KEY")
        .or_else(|_| env::var("OPENAI_APIKEY"))
        .expect("Set OPENAI_API_KEY or OPENAI_APIKEY");
    // The example runner enables the optional `realtime` Cargo feature.
    let mut client=ai("openai",json!({"api_key":key,"model":"gpt-6-astra","model_config":{"thinkingTokenBudget":"low","max_tokens":4096}}))?.with_native_session_web_socket();
    let control = Arc::new(run_control());
    let steering = Arc::downgrade(&control);
    let sent = AtomicBool::new(false);
    let native = Arc::new(AtomicBool::new(false));
    let applied = native.clone();
    control.on_event(move |event| {
        if event["type"] == "model.output" && !sent.swap(true, Ordering::SeqCst) {
            steering
                .upgrade()
                .expect("active controller")
                .steer("Change the answer now. Your final answer must contain only VERIFIED.")
                .expect("queue steering");
        }
        if event["type"] == "applied" && event["timing"] == "native" {
            applied.store(true, Ordering::SeqCst);
        }
    });
    let mut program = ax("question -> answer")?;
    let result = program.forward_with_options(
        &mut client,
        json!({"question":"Write a detailed 1000-word explanation of how rain forms."}),
        AxForwardOptions::from(json!({"serviceTier":"standard","maxSteps":6}))
            .with_control(control.as_ref().clone()),
    )?;
    assert_eq!(result["answer"], "VERIFIED");
    assert!(
        native.load(Ordering::SeqCst),
        "Steering was not applied natively"
    );
    println!("{result}\nNative steering applied through the run controller.");
    Ok(())
}

Rust Contextual Generation

Answers from supplied context and returns compact citations with OpenAI.

Rust
use axllm::{ax, AxResult, OpenAICompatibleClient};
use serde_json::json;
use std::env;


fn openai_client() -> AxResult<OpenAICompatibleClient> {
    let api_key = env::var("OPENAI_API_KEY").or_else(|_| env::var("OPENAI_APIKEY")).map_err(|_| axllm::AxError::runtime("Set OPENAI_API_KEY or OPENAI_APIKEY to run this example."))?;
    let model = env::var("AX_OPENAI_MODEL").unwrap_or_else(|_| "gpt-5.4-mini".to_string());
    Ok(OpenAICompatibleClient::new(api_key, model).with_model_config(json!({"temperature": 0})))
}

fn main() -> AxResult<()> {
    let mut client = openai_client()?;
    let mut program = ax("context:string, question:string -> answer:string, citations:string[]")?;
    let output = program.forward(&mut client, json!({"context": "Ax uses signatures, ai(), ax(), agent(), flow(), and optimize().", "question": "How should a new developer think about Ax?"}))?;
    println!("{}", serde_json::to_string_pretty(&output)?);
    Ok(())
}

Rust Jev Native Questions

Uses structured criteria, native scoring, model discovery, and probability-based decisions.

Rust
use axllm::{typesafe, AxResult, TypesafeAnswer, TypesafeQuestion, TypesafeRequest};
use serde_json::json;
use std::{collections::BTreeMap, env};

fn main() -> AxResult<()> {
    let mut client =
        typesafe(json!({"api_key":env::var("TYPESAFE_APIKEY").expect("Set TYPESAFE_APIKEY")}))?;
    assert!(!client.list_models()?.is_empty());
    let request = TypesafeRequest {
        state: json!({
            "ticket": "Checkout is unavailable for all customers after the latest deployment.",
            "account": {"tier": "enterprise", "notes": null}
        }),
        model: None,
        questions: BTreeMap::from([
            (
                "urgent".into(),
                TypesafeQuestion::Noul {
                    instructions: Some(json!({"question": "Does this need immediate attention?"})),
                    criteria: Some(serde_json::Map::from_iter([
                        (
                            "true".into(),
                            json!("Customers cannot complete a core task"),
                        ),
                        ("false".into(), json!("Routine request")),
                    ])),
                },
            ),
            (
                "team".into(),
                TypesafeQuestion::Choice {
                    instructions: Some(json!("Who should handle the ticket?")),
                    criteria: serde_json::Map::from_iter([
                        ("support".into(), json!("Usage guidance")),
                        ("billing".into(), json!({"scope": "Invoices and payments"})),
                        ("engineering".into(), json!("Product failures")),
                    ]),
                },
            ),
            (
                "severity".into(),
                TypesafeQuestion::Score {
                    instructions: Some(json!("Rate customer impact")),
                    criteria: vec![
                        json!("Minor inconvenience"),
                        json!("One task blocked"),
                        json!("Core task unavailable"),
                        json!("Widespread outage"),
                    ],
                },
            ),
        ]),
    };
    let response = client.system_one(request)?;
    let probability = match response.answers["urgent"] {
        TypesafeAnswer::Noul { noul } => noul,
        _ => panic!("Expected Noul"),
    };
    let score = match response.answers["severity"] {
        TypesafeAnswer::Score { score, .. } => score,
        _ => panic!("Expected Score"),
    };
    assert!((0.0..=1.0).contains(&probability) && (0.0..=3.0).contains(&score));
    // Apply thresholds and custom score scales in application code.
    let output = json!({"page_on_call":probability>=0.9,"severity_1_to_5":1.0+4.0*score/3.0,"response":response});
    println!("{}", serde_json::to_string_pretty(&output)?);
    Ok(())
}

Rust Adaptive Provider Balancing

Routes equivalent chat traffic using shared reliability, latency, and cost statistics.

Rust
use std::{env, sync::{Arc, Mutex}};

use axllm::{
    AxAIClient, AxBalancer, AxBalancerAdaptiveStrategy, AxBalancerOptions,
    AxInMemoryBalancerStatsStore, AxResult, OpenAICompatibleClient,
};
use serde_json::json;

fn main() -> AxResult<()> {
    let key = env::var("OPENAI_API_KEY").or_else(|_| env::var("OPENAI_APIKEY"))
        .map_err(|_| axllm::AxError::runtime("Set OPENAI_API_KEY or OPENAI_APIKEY to run this example."))?;
    let model = env::var("AX_OPENAI_MODEL").unwrap_or_else(|_| "gpt-5.4-mini".into());
    let clients: Vec<Box<dyn AxAIClient>> = vec![
        Box::new(OpenAICompatibleClient::new(&key, &model)),
        Box::new(OpenAICompatibleClient::new(&key, &model)),
    ];

    let store = Arc::new(AxInMemoryBalancerStatsStore::new());
    let route_keys = ["openai-primary".to_string(), "openai-backup".to_string()];
    let events = Arc::new(Mutex::new(Vec::new()));
    let event_sink = events.clone();
    let strategy = AxBalancerAdaptiveStrategy::new(6_000.0, 0.02)
        .with_expected_tokens(1_200, 300)
        .with_namespace("support-summary-v1")
        .with_store(store)
        .with_route_key(Arc::new(move |_service, index| route_keys[index].clone()))
        .with_slice(Arc::new(|context| if context["options"]["stream"] == true { "streaming".into() } else { "interactive".into() }))
        .on_routing_event(Arc::new(move |event| event_sink.lock().unwrap().push(event["type"].clone())));
    let mut balancer = AxBalancer::from_clients(clients, AxBalancerOptions { strategy: Some(strategy), ..AxBalancerOptions::default() })?;
    let response = balancer.chat(json!({"model": model, "chat_prompt": [{"role": "user", "content": "Summarize why shared routing state matters."}]}))?;
    println!("{}", serde_json::to_string_pretty(&response)?);
    println!("{:?}", events.lock().unwrap());
    Ok(())
}

Portable Runtime Hooks

Applies global and forward-scoped rate limiting, tracing, and metrics to AxGen, AxAgent, and AxFlow.

Rust
use axllm::runtime::quickjs::QuickJsCodeRuntime;
use axllm::{
    agent_with_options, ax, flow, set_meter, set_rate_limiter, set_tracer, AxCounter,
    AxError, AxGauge, AxHistogram, AxMeter, AxMetricInstrumentOptions, AxRateLimitInfo,
    AxRateLimiter, AxResult, AxRuntimeHooks, AxSpan, AxSpanStart, AxTracer,
    OpenAICompatibleClient,
};
use serde_json::{json, Value};
use std::collections::BTreeMap;
use std::env;
use std::fmt;
use std::sync::Arc;

struct LogSpan(String);
impl fmt::Debug for LogSpan { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("LogSpan").field(&self.0).finish() } }
impl AxSpan for LogSpan {
    fn add_event(&self, name: &str, _: &BTreeMap<String, Value>) { println!("[span:event] {} {}", self.0, name); }
    fn record_exception(&self, error: &AxError) { println!("[span:error] {} {}", self.0, error); }
    fn end(&self) { println!("[span:end] {}", self.0); }
}

struct LogTracer;
impl AxTracer for LogTracer {
    fn start_span(&self, start: AxSpanStart) -> Option<Arc<dyn AxSpan>> {
        println!("[span:start] {}", start.name);
        Some(Arc::new(LogSpan(start.name)))
    }
}

struct LogInstrument(String);
impl AxCounter for LogInstrument { fn add(&self, value: f64, _: &BTreeMap<String, Value>) { println!("[metric] {} += {}", self.0, value); } }
impl AxHistogram for LogInstrument { fn record(&self, value: f64, _: &BTreeMap<String, Value>) { println!("[metric] {} = {}", self.0, value); } }
impl AxGauge for LogInstrument { fn record(&self, value: f64, _: &BTreeMap<String, Value>) { println!("[metric] {} = {}", self.0, value); } }

struct LogMeter;
impl AxMeter for LogMeter {
    fn create_counter(&self, name: &str, _: &AxMetricInstrumentOptions) -> Option<Arc<dyn AxCounter>> { Some(Arc::new(LogInstrument(name.into()))) }
    fn create_histogram(&self, name: &str, _: &AxMetricInstrumentOptions) -> Option<Arc<dyn AxHistogram>> { Some(Arc::new(LogInstrument(name.into()))) }
    fn create_gauge(&self, name: &str, _: &AxMetricInstrumentOptions) -> Option<Arc<dyn AxGauge>> { Some(Arc::new(LogInstrument(name.into()))) }
}

fn limiter(label: &'static str) -> Arc<dyn AxRateLimiter> {
    Arc::new(move |next: &mut dyn FnMut() -> AxResult<Value>, info: &AxRateLimitInfo| {
        println!("[limit:{label}] {} {}/{} stream={}", info.operation, info.provider, info.model, info.streaming);
        next()
    })
}

fn main() -> AxResult<()> {
    let api_key = env::var("OPENAI_API_KEY").or_else(|_| env::var("OPENAI_APIKEY"))
        .map_err(|_| AxError::runtime("Set OPENAI_API_KEY or OPENAI_APIKEY to run this example."))?;
    let model = env::var("AX_OPENAI_MODEL").unwrap_or_else(|_| "gpt-5.4-mini".into());
    let mut client = OpenAICompatibleClient::new(api_key, model).with_model_config(json!({"temperature": 0}));
    let tracer: Arc<dyn AxTracer> = Arc::new(LogTracer);
    let meter: Arc<dyn AxMeter> = Arc::new(LogMeter);
    let override_hooks = AxRuntimeHooks { rate_limiter: Some(limiter("forward")), tracer: Some(Arc::clone(&tracer)), meter: Some(Arc::clone(&meter)) };

    set_rate_limiter(Some(limiter("global")));
    set_tracer(Some(Arc::clone(&tracer)));
    set_meter(Some(Arc::clone(&meter)));
    let result = (|| -> AxResult<()> {
        println!("{}", ax("topic:string -> summary:string")?.forward(&mut client, json!({"topic": "portable Ax runtime hooks"}))?);

        let mut helper = agent_with_options("question:string -> answer:string", json!({}))?
            .with_runtime(Box::new(QuickJsCodeRuntime::new()))?;
        println!("{}", helper.forward_with_hooks(&mut client, json!({"question": "What does a rate limiter wrap?"}), json!({"max_actor_steps": 12}), override_hooks.clone())?);

        let mut workflow = flow("examples.runtimeHooks")
            .execute("outline", ax("topic:string -> outline:string")?)
            .execute("polish", ax("outline:string -> answer:string")?)
            .returns(json!({"answer": "polish"}));
        println!("{}", workflow.forward_with_hooks(&mut client, json!({"topic": "Ax runtime hooks"}), Value::Null, override_hooks)?);
        Ok(())
    })();
    set_rate_limiter(None);
    set_tracer(None);
    set_meter(None);
    result
}
Docs