Check both comparison backends before building either (R2.9) (#1009)

* Check both comparison backends before building either (R2.9)

* Keep the API refusal test independent of provider keys (R2.9)

---------

Co-authored-by: Tanish Manku <[email protected]>
This commit is contained in:
Alex Joaquim Pereira
2026-11-19 16:06:46 -07:00
committed by GitHub
co-authored by Tanish Manku
parent 04ef228d1f
commit 936bfff9b6
5 changed files with 198 additions and 13 deletions
@@ -0,0 +1,46 @@
//! A named chat must not run an AI the person turned off (`ai off` says nothing uses it).
//!
//! The test gives the binary its own config, home and `PATH`, and a stand-in `claude` that writes
//! a marker if it is started at all, so neither the person's settings nor an installed agent can
//! decide the result.
#![cfg(unix)]
use craft_anything_platform::process::{Output, Process};
use std::path::Path;
use std::time::Duration;
fn cli(args: &[&str], config: &Path, home: &Path, bin: &Path) -> Output {
let path = format!("{}:/usr/bin:/bin", bin.display());
Process::new(env!("CARGO_BIN_EXE_craft-anything-cli"))
.args(args)
.env_clear()
.env("CRAFT_ANYTHING_CONFIG_DIR", config)
.env("HOME", home)
.env("PATH", path)
.timeout(Duration::from_secs(60))
.run()
.unwrap()
}
#[test]
fn a_named_chat_does_not_start_a_turned_off_agent() {
use std::os::unix::fs::PermissionsExt;
let config = tempfile::tempdir().unwrap();
let home = tempfile::tempdir().unwrap();
let bin = tempfile::tempdir().unwrap();
let marker = config.path().join("ran.txt");
let program = bin.path().join("claude");
std::fs::write(&program, format!("#!/bin/sh\nprintf '%s\\n' \"$*\" >> {}\nexit 0\n", marker.display())).unwrap();
std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap();
std::fs::write(
config.path().join("config.toml"),
format!("[[ai.backends]]\nname = \"Fake\"\nkind = \"agent_cli\"\nmodel = \"claude\"\nenabled = false\ncommand = \"{}\"\n", program.display()),
)
.unwrap();
let out = cli(&["chat", "hi", "--backend", "Fake"], config.path(), home.path(), bin.path());
let log = format!("{}{}", out.stdout_text(), out.stderr_text());
assert!(!out.success, "the chat must be refused before it starts the agent:\n{log}");
assert!(log.contains("turned off"), "the refusal must say why:\n{log}");
assert!(!marker.exists(), "the turned-off agent ran:\n{log}");
}
+13 -4
View File
@@ -95,18 +95,26 @@ enum KeychainGet {
pub struct KeyStore {
file: PathBuf,
use_keychain: bool,
use_env: bool,
}
impl KeyStore {
/// The default store: keychain + `<config dir>/ai-keys.toml`.
pub fn system() -> Self {
let no_chain = std::env::var(NO_KEYCHAIN_ENV).is_ok_and(|v| v == "1");
KeyStore { file: craft_anything_engine::settings::config_dir().join("ai-keys.toml"), use_keychain: !no_chain }
KeyStore { file: craft_anything_engine::settings::config_dir().join("ai-keys.toml"), use_keychain: !no_chain, use_env: true }
}
/// A file-only store (tests).
pub fn file_only(file: PathBuf) -> Self {
KeyStore { file, use_keychain: false }
KeyStore { file, use_keychain: false, use_env: true }
}
/// A file-only store that also ignores the environment, so a test that needs a backend to
/// have no key doesn't depend on the machine's variables.
#[cfg(test)]
pub(crate) fn file_only_without_env(file: PathBuf) -> Self {
KeyStore { file, use_keychain: false, use_env: false }
}
fn read_file(&self) -> toml::Table {
@@ -145,9 +153,10 @@ impl KeyStore {
self.read_file().get(backend).and_then(|v| v.as_str()).map(str::to_string).filter(|k| !k.is_empty())
}
/// The key to use: stored, else the kind's environment variable.
/// The key to use: stored, else the kind's environment variable (when this store reads it).
pub fn resolve(&self, backend: &str, kind: BackendKind) -> Option<String> {
self.get(backend).or_else(|| env_var(kind).and_then(|v| std::env::var(v).ok()).filter(|k| !k.trim().is_empty()))
let from_env = self.use_env.then(|| env_var(kind).and_then(|v| std::env::var(v).ok()).filter(|k| !k.trim().is_empty())).flatten();
self.get(backend).or(from_env)
}
/// Save a key. Returns where it went: `"keychain"` or `"file"`.
+32 -9
View File
@@ -1087,9 +1087,25 @@ impl AiHub {
}
fn entry(&self, name: &str) -> Result<BackendEntry, ToolError> {
self.config().get(name).cloned().ok_or_else(|| {
ToolError::invalid(format!("No AI backend named {name:?}")).with_remediation("See `craft-anything-cli ai list`, or add one in Settings → AI.")
})
let cfg = self.config();
cfg.get(name).cloned().ok_or_else(|| Self::no_backend(name))
}
fn no_backend(name: &str) -> ToolError {
ToolError::invalid(format!("No AI backend named {name:?}")).with_remediation("See `craft-anything-cli ai list`, or add one in Settings → AI.")
}
/// A backend that is named for a conversation must be on: `ai off` says nothing uses it, and
/// Settings treats an off AI as unusable. Listing models and testing a saved backend still
/// reach it through `entry`, so it can be configured before it is turned on. `cfg` is the
/// same snapshot the conversation is started from.
fn entry_on(cfg: &AiConfig, name: &str) -> Result<BackendEntry, ToolError> {
let e = cfg.get(name).cloned().ok_or_else(|| Self::no_backend(name))?;
if !e.config.enabled {
return Err(ToolError::new(ErrorCode::BadSettings, format!("{} is turned off.", e.config.name))
.with_remediation("Turn it on in Settings → AI, or run `craft-anything-cli ai on` with its name, then try again."));
}
Ok(e)
}
fn key_for(&self, e: &BackendEntry) -> Result<String, ToolError> {
@@ -1181,16 +1197,18 @@ impl AiHub {
pub fn start(&self, backend: Option<&str>, beginner: bool) -> Result<Box<dyn Conversation>, ToolError> {
let cfg = self.config();
let first = match backend {
Some(n) => self.entry(n)?,
Some(n) => Self::entry_on(&cfg, n)?,
None => cfg
.backend_for(Role::Lead)
.cloned()
.ok_or_else(|| ToolError::unavailable("No AI is set up yet.").with_remediation("Add one in Settings → AI."))?,
};
if !first.config.enabled {
return Err(ToolError::new(ErrorCode::BadSettings, format!("{} is turned off.", first.config.name))
.with_remediation("Turn it on in Settings → AI, or choose another backend for this conversation."));
}
self.start_selected(first, &cfg, beginner)
}
/// Start on an entry already chosen (and, when named, checked to be on), using the same
/// configuration snapshot for its fallback chain.
fn start_selected(&self, first: BackendEntry, cfg: &AiConfig, beginner: bool) -> Result<Box<dyn Conversation>, ToolError> {
if first.config.kind == BackendKind::AgentCli {
return Ok(Box::new(cli_agents::SharedConversation::new(self.engine.clone(), self.keys.clone(), first, beginner)?));
}
@@ -1261,7 +1279,12 @@ impl AiHub {
if a == b {
return Err(ToolError::invalid("Pick two different backends to compare."));
}
Ok(vec![(a.to_string(), self.start(Some(a), beginner)?), (b.to_string(), self.start(Some(b), beginner)?)])
// Check both before the first conversation starts, and start from those checked entries,
// so an off or unknown second name cannot leave a comparison half-started.
let cfg = self.config();
let a_entry = Self::entry_on(&cfg, a)?;
let b_entry = Self::entry_on(&cfg, b)?;
Ok(vec![(a.to_string(), self.start_selected(a_entry, &cfg, beginner)?), (b.to_string(), self.start_selected(b_entry, &cfg, beginner)?)])
}
/// Live model list for a backend. Blocking (bounded); run it off the UI thread.
+101
View File
@@ -329,6 +329,15 @@ fn hub() -> (AiHub, tempfile::TempDir) {
(h, dir)
}
/// A hub whose key store never reads the environment, so a test that expects no key is
/// deterministic even when the machine has the provider's variable set.
fn hub_without_env() -> (AiHub, tempfile::TempDir) {
let dir = tempfile::tempdir().unwrap();
let engine = Arc::new(Engine::new(craft_anything_engine::testing::session()));
let h = AiHub::with_stores(engine, dir.path().join("config.toml"), KeyStore::file_only_without_env(dir.path().join("ai-keys.toml")));
(h, dir)
}
/// A named backend stays off for chat, including before API or agent setup can run.
#[test]
fn disabled_backends_are_refused_when_chat_starts() {
@@ -587,6 +596,98 @@ fn an_app_bundle_is_refused_as_an_agents_program() {
assert_eq!(program(&h).as_deref(), Some("agy"));
}
/// Two stand-in agents (one on, one off) that write a marker if run, plus an enabled API backend
/// with no key: constructing that one would fail with "no API key" without doing any I/O.
#[cfg(unix)]
fn off_on_and_no_key_backends() -> (AiHub, tempfile::TempDir, std::path::PathBuf) {
use std::os::unix::fs::PermissionsExt;
let (h, dir) = hub_without_env();
let marker = dir.path().join("ran.txt");
let program = |name: &str| {
let p = dir.path().join(name);
std::fs::write(&p, format!("#!/bin/sh\nprintf '%s\\n' \"$*\" >> {}\nexit 0\n", marker.display())).unwrap();
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o700)).unwrap();
p
};
let agent = |name: &str, enabled: bool| BackendEntry {
config: BackendConfig { name: name.into(), kind: BackendKind::AgentCli, model: "claude".into(), base_url: None, ssh_host: None, enabled },
options: BackendOptions { command: Some(program(name).display().to_string()), ..Default::default() },
};
h.add_backend_with(agent("Off", false), None).unwrap();
h.add_backend_with(agent("On", true), None).unwrap();
let no_key = BackendConfig {
name: "NoKey".into(),
kind: BackendKind::Openrouter,
model: "test-model".into(),
base_url: Some("http://127.0.0.1:9".into()),
ssh_host: None,
enabled: true,
};
h.add_backend(no_key, None).unwrap();
(h, dir, marker)
}
/// R2.9: `ai off` says nothing uses an AI, so a named chat must not start one that is turned off.
#[cfg(unix)]
#[test]
fn a_turned_off_ai_is_not_used_by_a_named_chat() {
let (h, _d, marker) = off_on_and_no_key_backends();
let err = h.start(Some("Off"), true).err().expect("the turned-off agent must not start");
assert_eq!(err.code, ErrorCode::BadSettings, "{err:?}");
assert!(err.message.contains("turned off"), "{}", err.message);
assert!(err.remediation.as_deref().is_some_and(|r| r.contains("Settings") && r.contains("ai on")), "{err:?}");
assert!(!marker.exists(), "the turned-off agent ran");
}
/// The off check covers both names before either comparison conversation runs.
#[cfg(unix)]
#[test]
fn a_comparison_checks_both_names_before_it_starts_either() {
let (h, _d, marker) = off_on_and_no_key_backends();
for (a, b) in [("Off", "On"), ("On", "Off")] {
let err = h.compare(a, b, true).err().expect("a comparison with a turned-off name must not start");
assert_eq!(err.code, ErrorCode::BadSettings, "{err:?}");
assert!(err.message.contains("turned off"), "{}", err.message);
assert!(!marker.exists(), "compare started {a} before checking the other name");
}
}
/// `NoKey` is on but has no key, so constructing it would fail with a key error. The off name
/// must be reported first, in either order, without building the other conversation.
#[cfg(unix)]
#[test]
fn a_comparison_does_not_build_the_other_backend_before_rejecting_the_off_one() {
let (h, _d, _marker) = off_on_and_no_key_backends();
// The fixture's store ignores OPENROUTER_API_KEY, so this canary is deterministic.
let err = h.start(Some("NoKey"), true).err().expect("NoKey is meant to have no key");
assert!(err.message.contains("no API key"), "the test store must ignore the environment: {err:?}");
for (a, b) in [("NoKey", "Off"), ("Off", "NoKey")] {
let err = h.compare(a, b, true).err().expect("a comparison with a turned-off name must not start");
assert_eq!(err.code, ErrorCode::BadSettings, "{err:?}");
assert!(err.message.contains("turned off"), "compare built {a} before checking the off name: {err:?}");
assert!(!err.message.contains("no API key"), "construction was attempted: {err:?}");
}
}
/// A named backend that is turned off is refused before its key is looked up: the message is
/// about the choice, not about a key the person may not have.
#[test]
fn a_turned_off_api_backend_is_refused_before_its_key() {
let (h, _d) = hub_without_env();
let off = BackendConfig {
name: "Off".into(),
kind: BackendKind::Openrouter,
model: "test-model".into(),
base_url: Some("http://127.0.0.1:9".into()),
ssh_host: None,
enabled: false,
};
h.add_backend(off, None).unwrap();
let err = h.start(Some("Off"), true).err().expect("the turned-off API backend must not start");
assert_eq!(err.code, ErrorCode::BadSettings, "{err:?}");
assert!(err.message.contains("turned off"), "the key was looked up first: {err:?}");
}
/// #423: a saved AI that is switched off isn't set up, so a copy found on the computer is offered
/// again (to be turned on); one that is on is left out.
#[test]
+6
View File
@@ -1642,6 +1642,12 @@ Gaps found while using the app for real projects.
agent setup: `AiHub::start` checks `enabled` and says how to turn it on or choose another
— PR #1008; claimed: Codex, branch bugfix/r2.9-disabled-chat-backend
(the API regression uses invalid tunnel settings to detect setup before the guard)
- [x] `compare` checks both names before it builds either conversation, so an off or unknown
second side cannot leave a half-started comparison; a CLI regression covers `chat --backend`
with a turned-off agent
— claimed: AlexJoaquimPereira, branch `bugfix/r5.5-chat-turned-off-backend`
- [ ] The app's chat keeps a turned-off AI as its saved choice and refuses every send; offer to
turn it on or choose another enabled AI
- [x] Ctrl+V (Shift+Insert, Cmd+V) in a prompt box that takes pictures pastes a copied image (saved
as a PNG) or copied picture files; attachments show as square picture tiles, the whole
picture on hover, decoded on a worker