mirror of
https://github.com/storytold/craft-anything.git
synced 2026-11-21 20:08:24 +00:00
Check both comparison backends before building either (R2.9) (#1009)
* Check both comparison backends before building either (R2.9) * Keep the API refusal test independent of provider keys (R2.9) --------- Co-authored-by: Tanish Manku <[email protected]>
This commit is contained in:
co-authored by
Tanish Manku
parent
04ef228d1f
commit
936bfff9b6
@@ -0,0 +1,46 @@
|
||||
//! A named chat must not run an AI the person turned off (`ai off` says nothing uses it).
|
||||
//!
|
||||
//! The test gives the binary its own config, home and `PATH`, and a stand-in `claude` that writes
|
||||
//! a marker if it is started at all, so neither the person's settings nor an installed agent can
|
||||
//! decide the result.
|
||||
#![cfg(unix)]
|
||||
|
||||
use craft_anything_platform::process::{Output, Process};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
fn cli(args: &[&str], config: &Path, home: &Path, bin: &Path) -> Output {
|
||||
let path = format!("{}:/usr/bin:/bin", bin.display());
|
||||
Process::new(env!("CARGO_BIN_EXE_craft-anything-cli"))
|
||||
.args(args)
|
||||
.env_clear()
|
||||
.env("CRAFT_ANYTHING_CONFIG_DIR", config)
|
||||
.env("HOME", home)
|
||||
.env("PATH", path)
|
||||
.timeout(Duration::from_secs(60))
|
||||
.run()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_named_chat_does_not_start_a_turned_off_agent() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let config = tempfile::tempdir().unwrap();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
let bin = tempfile::tempdir().unwrap();
|
||||
let marker = config.path().join("ran.txt");
|
||||
let program = bin.path().join("claude");
|
||||
std::fs::write(&program, format!("#!/bin/sh\nprintf '%s\\n' \"$*\" >> {}\nexit 0\n", marker.display())).unwrap();
|
||||
std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
std::fs::write(
|
||||
config.path().join("config.toml"),
|
||||
format!("[[ai.backends]]\nname = \"Fake\"\nkind = \"agent_cli\"\nmodel = \"claude\"\nenabled = false\ncommand = \"{}\"\n", program.display()),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let out = cli(&["chat", "hi", "--backend", "Fake"], config.path(), home.path(), bin.path());
|
||||
let log = format!("{}{}", out.stdout_text(), out.stderr_text());
|
||||
assert!(!out.success, "the chat must be refused before it starts the agent:\n{log}");
|
||||
assert!(log.contains("turned off"), "the refusal must say why:\n{log}");
|
||||
assert!(!marker.exists(), "the turned-off agent ran:\n{log}");
|
||||
}
|
||||
+13
-4
@@ -95,18 +95,26 @@ enum KeychainGet {
|
||||
pub struct KeyStore {
|
||||
file: PathBuf,
|
||||
use_keychain: bool,
|
||||
use_env: bool,
|
||||
}
|
||||
|
||||
impl KeyStore {
|
||||
/// The default store: keychain + `<config dir>/ai-keys.toml`.
|
||||
pub fn system() -> Self {
|
||||
let no_chain = std::env::var(NO_KEYCHAIN_ENV).is_ok_and(|v| v == "1");
|
||||
KeyStore { file: craft_anything_engine::settings::config_dir().join("ai-keys.toml"), use_keychain: !no_chain }
|
||||
KeyStore { file: craft_anything_engine::settings::config_dir().join("ai-keys.toml"), use_keychain: !no_chain, use_env: true }
|
||||
}
|
||||
|
||||
/// A file-only store (tests).
|
||||
pub fn file_only(file: PathBuf) -> Self {
|
||||
KeyStore { file, use_keychain: false }
|
||||
KeyStore { file, use_keychain: false, use_env: true }
|
||||
}
|
||||
|
||||
/// A file-only store that also ignores the environment, so a test that needs a backend to
|
||||
/// have no key doesn't depend on the machine's variables.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn file_only_without_env(file: PathBuf) -> Self {
|
||||
KeyStore { file, use_keychain: false, use_env: false }
|
||||
}
|
||||
|
||||
fn read_file(&self) -> toml::Table {
|
||||
@@ -145,9 +153,10 @@ impl KeyStore {
|
||||
self.read_file().get(backend).and_then(|v| v.as_str()).map(str::to_string).filter(|k| !k.is_empty())
|
||||
}
|
||||
|
||||
/// The key to use: stored, else the kind's environment variable.
|
||||
/// The key to use: stored, else the kind's environment variable (when this store reads it).
|
||||
pub fn resolve(&self, backend: &str, kind: BackendKind) -> Option<String> {
|
||||
self.get(backend).or_else(|| env_var(kind).and_then(|v| std::env::var(v).ok()).filter(|k| !k.trim().is_empty()))
|
||||
let from_env = self.use_env.then(|| env_var(kind).and_then(|v| std::env::var(v).ok()).filter(|k| !k.trim().is_empty())).flatten();
|
||||
self.get(backend).or(from_env)
|
||||
}
|
||||
|
||||
/// Save a key. Returns where it went: `"keychain"` or `"file"`.
|
||||
|
||||
+32
-9
@@ -1087,9 +1087,25 @@ impl AiHub {
|
||||
}
|
||||
|
||||
fn entry(&self, name: &str) -> Result<BackendEntry, ToolError> {
|
||||
self.config().get(name).cloned().ok_or_else(|| {
|
||||
ToolError::invalid(format!("No AI backend named {name:?}")).with_remediation("See `craft-anything-cli ai list`, or add one in Settings → AI.")
|
||||
})
|
||||
let cfg = self.config();
|
||||
cfg.get(name).cloned().ok_or_else(|| Self::no_backend(name))
|
||||
}
|
||||
|
||||
fn no_backend(name: &str) -> ToolError {
|
||||
ToolError::invalid(format!("No AI backend named {name:?}")).with_remediation("See `craft-anything-cli ai list`, or add one in Settings → AI.")
|
||||
}
|
||||
|
||||
/// A backend that is named for a conversation must be on: `ai off` says nothing uses it, and
|
||||
/// Settings treats an off AI as unusable. Listing models and testing a saved backend still
|
||||
/// reach it through `entry`, so it can be configured before it is turned on. `cfg` is the
|
||||
/// same snapshot the conversation is started from.
|
||||
fn entry_on(cfg: &AiConfig, name: &str) -> Result<BackendEntry, ToolError> {
|
||||
let e = cfg.get(name).cloned().ok_or_else(|| Self::no_backend(name))?;
|
||||
if !e.config.enabled {
|
||||
return Err(ToolError::new(ErrorCode::BadSettings, format!("{} is turned off.", e.config.name))
|
||||
.with_remediation("Turn it on in Settings → AI, or run `craft-anything-cli ai on` with its name, then try again."));
|
||||
}
|
||||
Ok(e)
|
||||
}
|
||||
|
||||
fn key_for(&self, e: &BackendEntry) -> Result<String, ToolError> {
|
||||
@@ -1181,16 +1197,18 @@ impl AiHub {
|
||||
pub fn start(&self, backend: Option<&str>, beginner: bool) -> Result<Box<dyn Conversation>, ToolError> {
|
||||
let cfg = self.config();
|
||||
let first = match backend {
|
||||
Some(n) => self.entry(n)?,
|
||||
Some(n) => Self::entry_on(&cfg, n)?,
|
||||
None => cfg
|
||||
.backend_for(Role::Lead)
|
||||
.cloned()
|
||||
.ok_or_else(|| ToolError::unavailable("No AI is set up yet.").with_remediation("Add one in Settings → AI."))?,
|
||||
};
|
||||
if !first.config.enabled {
|
||||
return Err(ToolError::new(ErrorCode::BadSettings, format!("{} is turned off.", first.config.name))
|
||||
.with_remediation("Turn it on in Settings → AI, or choose another backend for this conversation."));
|
||||
}
|
||||
self.start_selected(first, &cfg, beginner)
|
||||
}
|
||||
|
||||
/// Start on an entry already chosen (and, when named, checked to be on), using the same
|
||||
/// configuration snapshot for its fallback chain.
|
||||
fn start_selected(&self, first: BackendEntry, cfg: &AiConfig, beginner: bool) -> Result<Box<dyn Conversation>, ToolError> {
|
||||
if first.config.kind == BackendKind::AgentCli {
|
||||
return Ok(Box::new(cli_agents::SharedConversation::new(self.engine.clone(), self.keys.clone(), first, beginner)?));
|
||||
}
|
||||
@@ -1261,7 +1279,12 @@ impl AiHub {
|
||||
if a == b {
|
||||
return Err(ToolError::invalid("Pick two different backends to compare."));
|
||||
}
|
||||
Ok(vec![(a.to_string(), self.start(Some(a), beginner)?), (b.to_string(), self.start(Some(b), beginner)?)])
|
||||
// Check both before the first conversation starts, and start from those checked entries,
|
||||
// so an off or unknown second name cannot leave a comparison half-started.
|
||||
let cfg = self.config();
|
||||
let a_entry = Self::entry_on(&cfg, a)?;
|
||||
let b_entry = Self::entry_on(&cfg, b)?;
|
||||
Ok(vec![(a.to_string(), self.start_selected(a_entry, &cfg, beginner)?), (b.to_string(), self.start_selected(b_entry, &cfg, beginner)?)])
|
||||
}
|
||||
|
||||
/// Live model list for a backend. Blocking (bounded); run it off the UI thread.
|
||||
|
||||
@@ -329,6 +329,15 @@ fn hub() -> (AiHub, tempfile::TempDir) {
|
||||
(h, dir)
|
||||
}
|
||||
|
||||
/// A hub whose key store never reads the environment, so a test that expects no key is
|
||||
/// deterministic even when the machine has the provider's variable set.
|
||||
fn hub_without_env() -> (AiHub, tempfile::TempDir) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let engine = Arc::new(Engine::new(craft_anything_engine::testing::session()));
|
||||
let h = AiHub::with_stores(engine, dir.path().join("config.toml"), KeyStore::file_only_without_env(dir.path().join("ai-keys.toml")));
|
||||
(h, dir)
|
||||
}
|
||||
|
||||
/// A named backend stays off for chat, including before API or agent setup can run.
|
||||
#[test]
|
||||
fn disabled_backends_are_refused_when_chat_starts() {
|
||||
@@ -587,6 +596,98 @@ fn an_app_bundle_is_refused_as_an_agents_program() {
|
||||
assert_eq!(program(&h).as_deref(), Some("agy"));
|
||||
}
|
||||
|
||||
/// Two stand-in agents (one on, one off) that write a marker if run, plus an enabled API backend
|
||||
/// with no key: constructing that one would fail with "no API key" without doing any I/O.
|
||||
#[cfg(unix)]
|
||||
fn off_on_and_no_key_backends() -> (AiHub, tempfile::TempDir, std::path::PathBuf) {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let (h, dir) = hub_without_env();
|
||||
let marker = dir.path().join("ran.txt");
|
||||
let program = |name: &str| {
|
||||
let p = dir.path().join(name);
|
||||
std::fs::write(&p, format!("#!/bin/sh\nprintf '%s\\n' \"$*\" >> {}\nexit 0\n", marker.display())).unwrap();
|
||||
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
p
|
||||
};
|
||||
let agent = |name: &str, enabled: bool| BackendEntry {
|
||||
config: BackendConfig { name: name.into(), kind: BackendKind::AgentCli, model: "claude".into(), base_url: None, ssh_host: None, enabled },
|
||||
options: BackendOptions { command: Some(program(name).display().to_string()), ..Default::default() },
|
||||
};
|
||||
h.add_backend_with(agent("Off", false), None).unwrap();
|
||||
h.add_backend_with(agent("On", true), None).unwrap();
|
||||
let no_key = BackendConfig {
|
||||
name: "NoKey".into(),
|
||||
kind: BackendKind::Openrouter,
|
||||
model: "test-model".into(),
|
||||
base_url: Some("http://127.0.0.1:9".into()),
|
||||
ssh_host: None,
|
||||
enabled: true,
|
||||
};
|
||||
h.add_backend(no_key, None).unwrap();
|
||||
(h, dir, marker)
|
||||
}
|
||||
|
||||
/// R2.9: `ai off` says nothing uses an AI, so a named chat must not start one that is turned off.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_turned_off_ai_is_not_used_by_a_named_chat() {
|
||||
let (h, _d, marker) = off_on_and_no_key_backends();
|
||||
let err = h.start(Some("Off"), true).err().expect("the turned-off agent must not start");
|
||||
assert_eq!(err.code, ErrorCode::BadSettings, "{err:?}");
|
||||
assert!(err.message.contains("turned off"), "{}", err.message);
|
||||
assert!(err.remediation.as_deref().is_some_and(|r| r.contains("Settings") && r.contains("ai on")), "{err:?}");
|
||||
assert!(!marker.exists(), "the turned-off agent ran");
|
||||
}
|
||||
|
||||
/// The off check covers both names before either comparison conversation runs.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_comparison_checks_both_names_before_it_starts_either() {
|
||||
let (h, _d, marker) = off_on_and_no_key_backends();
|
||||
for (a, b) in [("Off", "On"), ("On", "Off")] {
|
||||
let err = h.compare(a, b, true).err().expect("a comparison with a turned-off name must not start");
|
||||
assert_eq!(err.code, ErrorCode::BadSettings, "{err:?}");
|
||||
assert!(err.message.contains("turned off"), "{}", err.message);
|
||||
assert!(!marker.exists(), "compare started {a} before checking the other name");
|
||||
}
|
||||
}
|
||||
|
||||
/// `NoKey` is on but has no key, so constructing it would fail with a key error. The off name
|
||||
/// must be reported first, in either order, without building the other conversation.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_comparison_does_not_build_the_other_backend_before_rejecting_the_off_one() {
|
||||
let (h, _d, _marker) = off_on_and_no_key_backends();
|
||||
// The fixture's store ignores OPENROUTER_API_KEY, so this canary is deterministic.
|
||||
let err = h.start(Some("NoKey"), true).err().expect("NoKey is meant to have no key");
|
||||
assert!(err.message.contains("no API key"), "the test store must ignore the environment: {err:?}");
|
||||
for (a, b) in [("NoKey", "Off"), ("Off", "NoKey")] {
|
||||
let err = h.compare(a, b, true).err().expect("a comparison with a turned-off name must not start");
|
||||
assert_eq!(err.code, ErrorCode::BadSettings, "{err:?}");
|
||||
assert!(err.message.contains("turned off"), "compare built {a} before checking the off name: {err:?}");
|
||||
assert!(!err.message.contains("no API key"), "construction was attempted: {err:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A named backend that is turned off is refused before its key is looked up: the message is
|
||||
/// about the choice, not about a key the person may not have.
|
||||
#[test]
|
||||
fn a_turned_off_api_backend_is_refused_before_its_key() {
|
||||
let (h, _d) = hub_without_env();
|
||||
let off = BackendConfig {
|
||||
name: "Off".into(),
|
||||
kind: BackendKind::Openrouter,
|
||||
model: "test-model".into(),
|
||||
base_url: Some("http://127.0.0.1:9".into()),
|
||||
ssh_host: None,
|
||||
enabled: false,
|
||||
};
|
||||
h.add_backend(off, None).unwrap();
|
||||
let err = h.start(Some("Off"), true).err().expect("the turned-off API backend must not start");
|
||||
assert_eq!(err.code, ErrorCode::BadSettings, "{err:?}");
|
||||
assert!(err.message.contains("turned off"), "the key was looked up first: {err:?}");
|
||||
}
|
||||
|
||||
/// #423: a saved AI that is switched off isn't set up, so a copy found on the computer is offered
|
||||
/// again (to be turned on); one that is on is left out.
|
||||
#[test]
|
||||
|
||||
@@ -1642,6 +1642,12 @@ Gaps found while using the app for real projects.
|
||||
agent setup: `AiHub::start` checks `enabled` and says how to turn it on or choose another
|
||||
— PR #1008; claimed: Codex, branch bugfix/r2.9-disabled-chat-backend
|
||||
(the API regression uses invalid tunnel settings to detect setup before the guard)
|
||||
- [x] `compare` checks both names before it builds either conversation, so an off or unknown
|
||||
second side cannot leave a half-started comparison; a CLI regression covers `chat --backend`
|
||||
with a turned-off agent
|
||||
— claimed: AlexJoaquimPereira, branch `bugfix/r5.5-chat-turned-off-backend`
|
||||
- [ ] The app's chat keeps a turned-off AI as its saved choice and refuses every send; offer to
|
||||
turn it on or choose another enabled AI
|
||||
- [x] Ctrl+V (Shift+Insert, Cmd+V) in a prompt box that takes pictures pastes a copied image (saved
|
||||
as a PNG) or copied picture files; attachments show as square picture tiles, the whole
|
||||
picture on hover, decoded on a worker
|
||||
|
||||
Reference in New Issue
Block a user