Establish OA/SA prototype baseline and first-release work
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "oa-server"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "arcadia-server"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
oa-core = { path = "../oa-core" }
|
||||
axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] }
|
||||
tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal", "sync", "time", "io-util"] }
|
||||
tower-http = { version = "0.6", features = ["limit"] }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
uuid.workspace = true
|
||||
thiserror.workspace = true
|
||||
tempfile = "3"
|
||||
subtle = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
@@ -0,0 +1,111 @@
|
||||
use axum::{
|
||||
http::{StatusCode, header},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use serde::Serialize;
|
||||
|
||||
pub type ApiResult<T> = Result<T, ApiError>;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ApiError {
|
||||
#[serde(rename = "type")]
|
||||
problem_type: Box<str>,
|
||||
pub title: &'static str,
|
||||
pub status: u16,
|
||||
pub code: &'static str,
|
||||
pub detail: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub current_revision: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub issues: Vec<IssueDto>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct IssueDto {
|
||||
pub severity: &'static str,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub field: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub object_id: Option<oa_core::Id>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub line: Option<usize>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub column: Option<usize>,
|
||||
}
|
||||
|
||||
impl ApiError {
|
||||
pub fn new(status: StatusCode, code: &'static str, detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
problem_type: format!("urn:arcadia:problem:{code}").into_boxed_str(),
|
||||
title: status.canonical_reason().unwrap_or("Request failed"),
|
||||
status: status.as_u16(),
|
||||
code,
|
||||
detail: detail.into(),
|
||||
current_revision: None,
|
||||
issues: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bad_request(detail: impl Into<String>) -> Self {
|
||||
Self::new(StatusCode::BAD_REQUEST, "invalid_request", detail)
|
||||
}
|
||||
|
||||
pub fn missing() -> Self {
|
||||
Self::new(
|
||||
StatusCode::NOT_FOUND,
|
||||
"project_not_found",
|
||||
"Project does not exist.",
|
||||
)
|
||||
}
|
||||
|
||||
pub fn storage(error: std::io::Error) -> Self {
|
||||
eprintln!("arcadia-server storage error: {error}");
|
||||
Self::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"storage_unavailable",
|
||||
"The storage operation failed. Reload the project before retrying a write.",
|
||||
)
|
||||
}
|
||||
|
||||
pub fn internal(detail: &str) -> Self {
|
||||
Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal_error", detail)
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for ApiError {
|
||||
fn into_response(self) -> Response {
|
||||
let status = StatusCode::from_u16(self.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
let mut response = (status, axum::Json(self)).into_response();
|
||||
response.headers_mut().insert(
|
||||
header::CONTENT_TYPE,
|
||||
header::HeaderValue::from_static("application/problem+json"),
|
||||
);
|
||||
if status == StatusCode::UNAUTHORIZED {
|
||||
response.headers_mut().insert(
|
||||
header::WWW_AUTHENTICATE,
|
||||
header::HeaderValue::from_static("Bearer realm=\"arcadia\""),
|
||||
);
|
||||
}
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
pub fn issues(model: &oa_core::Project) -> Vec<IssueDto> {
|
||||
oa_core::validation::validate(model)
|
||||
.into_iter()
|
||||
.map(|issue| IssueDto {
|
||||
severity: match issue.severity {
|
||||
oa_core::validation::Severity::Error => "error",
|
||||
oa_core::validation::Severity::Warning => "warning",
|
||||
},
|
||||
message: issue.message,
|
||||
code: issue.code,
|
||||
field: issue.field,
|
||||
object_id: issue.element,
|
||||
line: None,
|
||||
column: None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
//! Headless REST service over the shared Arcadia model and command boundary.
|
||||
pub mod error;
|
||||
mod routes;
|
||||
pub mod store;
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
extract::{DefaultBodyLimit, Request, State},
|
||||
http::{StatusCode, header},
|
||||
middleware::{self, Next},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use error::{ApiError, ApiResult};
|
||||
use std::{net::SocketAddr, path::PathBuf, sync::Arc, time::Duration};
|
||||
use store::Store;
|
||||
use subtle::ConstantTimeEq;
|
||||
use tokio::sync::Semaphore;
|
||||
use tower_http::limit::RequestBodyLimitLayer;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct AppState {
|
||||
store: Arc<Store>,
|
||||
token: Option<Arc<str>>,
|
||||
requests: Arc<Semaphore>,
|
||||
workers: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
/// Authentication is optional on loopback; it is mandatory on any other bind.
|
||||
pub struct ServerConfig {
|
||||
pub bind: SocketAddr,
|
||||
pub data_dir: PathBuf,
|
||||
pub bearer_token: Option<String>,
|
||||
}
|
||||
|
||||
impl ServerConfig {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if let Some(token) = &self.bearer_token
|
||||
&& (token.len() < 32 || !token.bytes().all(|byte| byte.is_ascii_graphic()))
|
||||
{
|
||||
return Err("Bearer token must contain at least 32 printable ASCII characters without whitespace.".into());
|
||||
}
|
||||
if !self.bind.ip().is_loopback() && self.bearer_token.is_none() {
|
||||
return Err(
|
||||
"A non-loopback bind requires --token-file (or ARCADIA_SERVER_TOKEN).".into(),
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a router for an already opened, exclusively owned store.
|
||||
/// Call `ServerConfig::validate` before binding a network listener.
|
||||
pub fn app(store: Arc<Store>, bearer_token: Option<String>) -> Router {
|
||||
let state = AppState {
|
||||
store,
|
||||
token: bearer_token.map(Arc::from),
|
||||
requests: Arc::new(Semaphore::new(8)),
|
||||
workers: Arc::new(Semaphore::new(8)),
|
||||
};
|
||||
routes::router()
|
||||
.with_state(state.clone())
|
||||
.layer(DefaultBodyLimit::max(store::MAX_MODEL_BYTES))
|
||||
.layer(RequestBodyLimitLayer::new(store::MAX_MODEL_BYTES))
|
||||
.layer(middleware::from_fn_with_state(state, guard))
|
||||
}
|
||||
|
||||
async fn guard(State(state): State<AppState>, request: Request, next: Next) -> Response {
|
||||
let request_id = oa_core::Id::new_v4().to_string();
|
||||
let public = matches!(
|
||||
request.uri().path(),
|
||||
"/health" | "/health/live" | "/health/ready" | "/openapi.json" | "/api/v1/openapi.json"
|
||||
);
|
||||
let mut response = if public {
|
||||
next.run(request).await
|
||||
} else if request.headers().contains_key(header::ORIGIN) {
|
||||
ApiError::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"browser_origin_denied",
|
||||
"Browser origins are disabled; use a native client.",
|
||||
)
|
||||
.into_response()
|
||||
} else if !authorized(&state, request.headers()) {
|
||||
ApiError::new(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"unauthorized",
|
||||
"A valid bearer token is required.",
|
||||
)
|
||||
.into_response()
|
||||
} else if state.token.is_none() && !loopback_host(request.headers()) {
|
||||
ApiError::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"host_not_allowed",
|
||||
"Unauthenticated loopback access requires a localhost or loopback-IP Host header.",
|
||||
)
|
||||
.into_response()
|
||||
} else if request.uri().query().is_some() && !allows_query(&request) {
|
||||
ApiError::bad_request(
|
||||
"This route does not accept query parameters; historical revisions are not retained.",
|
||||
)
|
||||
.into_response()
|
||||
} else if request.headers().contains_key(header::CONTENT_ENCODING) {
|
||||
ApiError::new(
|
||||
StatusCode::UNSUPPORTED_MEDIA_TYPE,
|
||||
"content_encoding_unsupported",
|
||||
"Compressed or encoded request bodies are not supported.",
|
||||
)
|
||||
.into_response()
|
||||
} else {
|
||||
match state.requests.clone().try_acquire_owned() {
|
||||
Err(_) => ApiError::new(StatusCode::SERVICE_UNAVAILABLE, "overloaded", "All request slots are busy. Retry later.").into_response(),
|
||||
Ok(_permit) => match tokio::time::timeout(Duration::from_secs(30), next.run(request)).await {
|
||||
Ok(response) => response,
|
||||
Err(_) => ApiError::new(StatusCode::GATEWAY_TIMEOUT, "outcome_unknown", "Request deadline exceeded. An admitted write may still complete; read the project before retrying.").into_response(),
|
||||
},
|
||||
}
|
||||
};
|
||||
// Normalize middleware/framework rejections too, including body limits.
|
||||
if response.status().is_client_error() || response.status().is_server_error() {
|
||||
let is_problem = response
|
||||
.headers()
|
||||
.get(header::CONTENT_TYPE)
|
||||
.is_some_and(|value| value == "application/problem+json");
|
||||
if !is_problem {
|
||||
response = ApiError::new(
|
||||
response.status(),
|
||||
"request_rejected",
|
||||
"The HTTP request was rejected.",
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
}
|
||||
if response.status() == StatusCode::SERVICE_UNAVAILABLE {
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::RETRY_AFTER, header::HeaderValue::from_static("1"));
|
||||
}
|
||||
response.headers_mut().insert(
|
||||
"x-request-id",
|
||||
request_id.parse().expect("UUID is a header value"),
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
header::CACHE_CONTROL,
|
||||
header::HeaderValue::from_static("no-cache"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn loopback_host(headers: &header::HeaderMap) -> bool {
|
||||
let mut hosts = headers.get_all(header::HOST).iter();
|
||||
let authority = hosts
|
||||
.next()
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<axum::http::uri::Authority>().ok());
|
||||
if hosts.next().is_some() {
|
||||
return false;
|
||||
}
|
||||
authority.is_some_and(|authority| {
|
||||
let host = authority.host().trim_matches(['[', ']']);
|
||||
host.eq_ignore_ascii_case("localhost")
|
||||
|| host
|
||||
.parse::<std::net::IpAddr>()
|
||||
.is_ok_and(|ip| ip.is_loopback())
|
||||
})
|
||||
}
|
||||
|
||||
fn allows_query(request: &Request) -> bool {
|
||||
request.method() == axum::http::Method::GET
|
||||
&& (request.uri().path() == "/api/v1/projects"
|
||||
|| ["/elements", "/relations", "/artifact-links", "/traces"]
|
||||
.iter()
|
||||
.any(|suffix| request.uri().path().ends_with(suffix)))
|
||||
}
|
||||
|
||||
fn authorized(state: &AppState, headers: &header::HeaderMap) -> bool {
|
||||
let Some(expected) = &state.token else {
|
||||
return true;
|
||||
};
|
||||
let mut values = headers.get_all(header::AUTHORIZATION).iter();
|
||||
let supplied = values
|
||||
.next()
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.strip_prefix("Bearer "));
|
||||
if values.next().is_some() {
|
||||
return false;
|
||||
}
|
||||
supplied.is_some_and(|supplied| bool::from(supplied.as_bytes().ct_eq(expected.as_bytes())))
|
||||
}
|
||||
|
||||
pub(crate) async fn blocking<R: Send + 'static>(
|
||||
state: AppState,
|
||||
work: impl FnOnce(&Store) -> ApiResult<R> + Send + 'static,
|
||||
) -> ApiResult<R> {
|
||||
let permit = state.workers.clone().try_acquire_owned().map_err(|_| {
|
||||
ApiError::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"overloaded",
|
||||
"All model workers are busy. Retry later.",
|
||||
)
|
||||
})?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// Retain the work permit even if the HTTP future is cancelled.
|
||||
let _permit = permit;
|
||||
// Parsing, queries and validation are worker work, not catalog-lock
|
||||
// work. Store accessors capture snapshots and recheck writes at commit.
|
||||
work(&state.store)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ApiError::internal("Model worker failed."))?
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
use oa_server::{ServerConfig, app, store::Store};
|
||||
use std::{net::SocketAddr, path::PathBuf, sync::Arc};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
if let Err(error) = run().await {
|
||||
eprintln!("arcadia-server: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async fn run() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let Some(config) = tokio::task::spawn_blocking(configuration).await?? else {
|
||||
return Ok(());
|
||||
};
|
||||
config.validate()?;
|
||||
let data_dir = config.data_dir.clone();
|
||||
let store = Arc::new(tokio::task::spawn_blocking(move || Store::open(data_dir)).await??);
|
||||
let listener = tokio::net::TcpListener::bind(config.bind).await?;
|
||||
eprintln!(
|
||||
"arcadia-server {} listening on {}; data {}",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
listener.local_addr()?,
|
||||
config.data_dir.display()
|
||||
);
|
||||
axum::serve(listener, app(store, config.bearer_token))
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn configuration() -> Result<Option<ServerConfig>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut bind: SocketAddr = "127.0.0.1:8787".parse()?;
|
||||
let mut data_dir = default_data_dir();
|
||||
let mut token_file = None;
|
||||
let mut args = std::env::args().skip(1);
|
||||
while let Some(arg) = args.next() {
|
||||
match arg.as_str() {
|
||||
"--help" | "-h" => {
|
||||
println!(
|
||||
"Arcadia model REST service\n\nUsage: arcadia-server [--bind IP:PORT] [--data-dir PATH] [--token-file PATH]\n\nDefaults: 127.0.0.1:8787; platform-local ArcadiaStudio/server directory.\nToken required for non-loopback binds. ARCADIA_SERVER_TOKEN is an alternative.\nKeep the data directory on local storage. See docs/server.md for deployment and API limits."
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
"--version" | "-V" => {
|
||||
println!("arcadia-server {}", env!("CARGO_PKG_VERSION"));
|
||||
return Ok(None);
|
||||
}
|
||||
"--bind" => bind = args.next().ok_or("--bind requires IP:PORT")?.parse()?,
|
||||
"--data-dir" => {
|
||||
data_dir = PathBuf::from(args.next().ok_or("--data-dir requires a path")?)
|
||||
}
|
||||
"--token-file" => {
|
||||
token_file = Some(PathBuf::from(
|
||||
args.next().ok_or("--token-file requires a path")?,
|
||||
))
|
||||
}
|
||||
_ => return Err(format!("Unknown option {arg}; use --help.").into()),
|
||||
}
|
||||
}
|
||||
let bearer_token = if let Some(path) = token_file {
|
||||
let token = std::fs::read_to_string(path)?;
|
||||
Some(token.trim_end_matches(['\r', '\n']).to_owned())
|
||||
} else {
|
||||
std::env::var("ARCADIA_SERVER_TOKEN").ok()
|
||||
};
|
||||
Ok(Some(ServerConfig {
|
||||
bind,
|
||||
data_dir,
|
||||
bearer_token,
|
||||
}))
|
||||
}
|
||||
|
||||
fn default_data_dir() -> PathBuf {
|
||||
#[cfg(windows)]
|
||||
if let Some(path) = std::env::var_os("LOCALAPPDATA") {
|
||||
return PathBuf::from(path).join("ArcadiaStudio").join("server");
|
||||
}
|
||||
if let Some(path) = std::env::var_os("XDG_DATA_HOME") {
|
||||
return PathBuf::from(path).join("arcadia-studio").join("server");
|
||||
}
|
||||
if let Some(path) = std::env::var_os("HOME") {
|
||||
return PathBuf::from(path).join(".local/share/arcadia-studio/server");
|
||||
}
|
||||
std::env::temp_dir().join("arcadia-studio-server")
|
||||
}
|
||||
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
if let Err(error) = tokio::signal::ctrl_c().await {
|
||||
eprintln!("Could not listen for Ctrl+C: {error}");
|
||||
}
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) {
|
||||
Ok(mut signal) => {
|
||||
signal.recv().await;
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!("Could not listen for SIGTERM: {error}");
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let terminate = std::future::pending::<()>();
|
||||
tokio::select! { _ = ctrl_c => {}, _ = terminate => {} }
|
||||
eprintln!("arcadia-server stopping admission and draining requests");
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,434 @@
|
||||
//! Single-owner on-disk envelopes. All methods run on a blocking worker.
|
||||
use crate::error::{ApiError, ApiResult, issues};
|
||||
use axum::http::StatusCode;
|
||||
use oa_core::{Id, Project};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs::{self, File, OpenOptions},
|
||||
io::{Read, Write},
|
||||
path::{Path, PathBuf},
|
||||
sync::{Mutex, MutexGuard},
|
||||
};
|
||||
|
||||
pub const MAX_MODEL_BYTES: usize = 8 * 1024 * 1024;
|
||||
pub const MAX_OBJECTS: usize = 5_000;
|
||||
pub const MAX_PROJECTS: usize = 128;
|
||||
pub const MAX_STORE_BYTES: usize = 256 * 1024 * 1024;
|
||||
pub(crate) const MODEL_LIMITS: oa_core::language::ParseLimits = oa_core::language::ParseLimits {
|
||||
max_source_bytes: MAX_MODEL_BYTES,
|
||||
max_graph_objects: MAX_OBJECTS,
|
||||
max_records: 20_000,
|
||||
max_step_references: 50_000,
|
||||
};
|
||||
const MAX_ENVELOPE_BYTES: usize = MAX_MODEL_BYTES + 512;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StoredProject {
|
||||
pub resource_id: Id,
|
||||
pub revision: u64,
|
||||
pub model: Project,
|
||||
}
|
||||
|
||||
impl StoredProject {
|
||||
pub fn etag(&self) -> String {
|
||||
format!("\"p-{}-r{}\"", self.resource_id, self.revision)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct ProjectSummary {
|
||||
pub resource_id: Id,
|
||||
pub revision: u64,
|
||||
pub model_id: Id,
|
||||
pub schema_version: u32,
|
||||
pub name: String,
|
||||
pub elements: usize,
|
||||
pub relations: usize,
|
||||
}
|
||||
impl From<&StoredProject> for ProjectSummary {
|
||||
fn from(stored: &StoredProject) -> Self {
|
||||
Self {
|
||||
resource_id: stored.resource_id,
|
||||
revision: stored.revision,
|
||||
model_id: stored.model.id,
|
||||
schema_version: stored.model.schema_version,
|
||||
name: stored.model.name.clone(),
|
||||
elements: stored.model.elements.len(),
|
||||
relations: stored.model.relations.len(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum StoreError {
|
||||
#[error("Cannot open data directory: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("Data directory is already owned or cannot be locked: {0}")]
|
||||
Locked(String),
|
||||
#[error("Invalid stored project {path}: {detail}")]
|
||||
Corrupt { path: PathBuf, detail: String },
|
||||
#[error("Store capacity exceeded: {0}")]
|
||||
Capacity(String),
|
||||
}
|
||||
|
||||
pub struct Store {
|
||||
inner: Mutex<StoreInner>,
|
||||
// Keep the OS lock alive for the store's entire lifetime, including requests.
|
||||
_directory_lock: File,
|
||||
}
|
||||
|
||||
struct StoreInner {
|
||||
directory: PathBuf,
|
||||
pub projects: BTreeMap<Id, StoredProject>,
|
||||
byte_sizes: BTreeMap<Id, usize>,
|
||||
total_bytes: usize,
|
||||
}
|
||||
|
||||
/// Constructed only after complete candidate validation, outside the catalog
|
||||
/// mutex. Commit methods cannot accidentally accept an unchecked candidate.
|
||||
struct PreparedModel(Project);
|
||||
impl PreparedModel {
|
||||
fn new(model: Project) -> ApiResult<Self> {
|
||||
validate_model(&model)?;
|
||||
Ok(Self(model))
|
||||
}
|
||||
}
|
||||
|
||||
impl Store {
|
||||
fn lock(&self) -> ApiResult<MutexGuard<'_, StoreInner>> {
|
||||
self.inner.lock().map_err(|_| {
|
||||
ApiError::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"storage_unavailable",
|
||||
"Storage worker needs a restart.",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn get(&self, id: Id) -> ApiResult<StoredProject> {
|
||||
Ok(self.lock()?.get(id)?.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn checked(&self, id: Id, expected: &str) -> ApiResult<StoredProject> {
|
||||
Ok(self.lock()?.checked(id, expected)?.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn catalog(&self) -> ApiResult<Vec<ProjectSummary>> {
|
||||
Ok(self
|
||||
.lock()?
|
||||
.projects
|
||||
.values()
|
||||
.map(ProjectSummary::from)
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub(crate) fn create(&self, model: Project) -> ApiResult<StoredProject> {
|
||||
let prepared = PreparedModel::new(model)?;
|
||||
self.lock()?.create(prepared)
|
||||
}
|
||||
|
||||
pub(crate) fn replace(
|
||||
&self,
|
||||
id: Id,
|
||||
expected: &str,
|
||||
model: Project,
|
||||
) -> ApiResult<(StoredProject, bool)> {
|
||||
let prepared = PreparedModel::new(model)?;
|
||||
self.lock()?.replace(id, expected, prepared)
|
||||
}
|
||||
|
||||
pub(crate) fn delete(&self, id: Id, expected: &str) -> ApiResult<()> {
|
||||
self.lock()?.delete(id, expected)
|
||||
}
|
||||
|
||||
/// The caller must keep this synchronous disk work off async runtime threads.
|
||||
pub fn open(directory: impl AsRef<Path>) -> Result<Self, StoreError> {
|
||||
fs::create_dir_all(directory.as_ref())?;
|
||||
let directory = fs::canonicalize(directory.as_ref())?;
|
||||
let directory_lock = OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(directory.join(".arcadia.lock"))?;
|
||||
directory_lock
|
||||
.try_lock()
|
||||
.map_err(|error| StoreError::Locked(error.to_string()))?;
|
||||
let mut projects = BTreeMap::new();
|
||||
let mut byte_sizes = BTreeMap::new();
|
||||
let mut total_bytes: usize = 0;
|
||||
for entry in fs::read_dir(&directory)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
// Aborted sibling temporary files are not committed envelopes.
|
||||
if path.extension().is_none_or(|ext| ext != "json") {
|
||||
continue;
|
||||
}
|
||||
let corrupt = |detail: String| StoreError::Corrupt {
|
||||
path: path.clone(),
|
||||
detail,
|
||||
};
|
||||
if !entry.file_type()?.is_file() {
|
||||
return Err(corrupt(
|
||||
"Envelope must be a regular file, not a link/directory.".into(),
|
||||
));
|
||||
}
|
||||
let filename_id: Id = path
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.ok_or_else(|| corrupt("Filename is not a UUID.".into()))?
|
||||
.parse()
|
||||
.map_err(|_| corrupt("Filename is not a UUID.".into()))?;
|
||||
if path.file_stem().and_then(|stem| stem.to_str())
|
||||
!= Some(filename_id.to_string().as_str())
|
||||
{
|
||||
return Err(corrupt(
|
||||
"Filename must use the canonical lowercase hyphenated resource UUID.".into(),
|
||||
));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
File::open(&path)?
|
||||
.take((MAX_ENVELOPE_BYTES + 1) as u64)
|
||||
.read_to_end(&mut bytes)?;
|
||||
if bytes.len() > MAX_ENVELOPE_BYTES {
|
||||
return Err(corrupt("Envelope exceeds the size limit.".into()));
|
||||
}
|
||||
total_bytes = total_bytes.saturating_add(bytes.len());
|
||||
if projects.len() >= MAX_PROJECTS || total_bytes > MAX_STORE_BYTES {
|
||||
return Err(StoreError::Capacity(
|
||||
"At most 128 models and 256 MiB of serialized envelopes are supported.".into(),
|
||||
));
|
||||
}
|
||||
let stored: StoredProject =
|
||||
serde_json::from_slice(&bytes).map_err(|error| corrupt(error.to_string()))?;
|
||||
if stored.resource_id != filename_id || stored.revision == 0 {
|
||||
return Err(corrupt(
|
||||
"Resource ID/filename mismatch or zero revision.".into(),
|
||||
));
|
||||
}
|
||||
validate_model(&stored.model).map_err(|error| corrupt(error.detail))?;
|
||||
byte_sizes.insert(stored.resource_id, bytes.len());
|
||||
if projects.insert(stored.resource_id, stored).is_some() {
|
||||
return Err(corrupt("Duplicate hosted resource ID.".into()));
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
inner: Mutex::new(StoreInner {
|
||||
directory,
|
||||
projects,
|
||||
byte_sizes,
|
||||
total_bytes,
|
||||
}),
|
||||
_directory_lock: directory_lock,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl StoreInner {
|
||||
pub fn get(&self, id: Id) -> ApiResult<&StoredProject> {
|
||||
self.projects.get(&id).ok_or_else(ApiError::missing)
|
||||
}
|
||||
|
||||
pub fn checked(&self, id: Id, expected: &str) -> ApiResult<&StoredProject> {
|
||||
let current = self.get(id)?;
|
||||
if current.etag() != expected {
|
||||
let mut error = ApiError::new(
|
||||
StatusCode::PRECONDITION_FAILED,
|
||||
"revision_conflict",
|
||||
"Project changed. Reload and review the current model before retrying.",
|
||||
);
|
||||
error.current_revision = Some(current.revision);
|
||||
return Err(error);
|
||||
}
|
||||
Ok(current)
|
||||
}
|
||||
|
||||
fn create(&mut self, prepared: PreparedModel) -> ApiResult<StoredProject> {
|
||||
if self.projects.len() >= MAX_PROJECTS {
|
||||
return Err(capacity_error());
|
||||
}
|
||||
let model = prepared.0;
|
||||
let stored = StoredProject {
|
||||
resource_id: Id::new_v4(),
|
||||
revision: 1,
|
||||
model,
|
||||
};
|
||||
let size = self.write(&stored, true)?;
|
||||
self.adopt(stored.clone(), size);
|
||||
self.sync_directory()?;
|
||||
Ok(stored)
|
||||
}
|
||||
|
||||
pub fn replace(
|
||||
&mut self,
|
||||
id: Id,
|
||||
expected: &str,
|
||||
prepared: PreparedModel,
|
||||
) -> ApiResult<(StoredProject, bool)> {
|
||||
let current = self.checked(id, expected)?;
|
||||
let model = prepared.0;
|
||||
if current.model == model {
|
||||
return Ok((current.clone(), false));
|
||||
}
|
||||
let revision = current.revision.checked_add(1).ok_or_else(|| {
|
||||
ApiError::new(
|
||||
StatusCode::CONFLICT,
|
||||
"revision_exhausted",
|
||||
"Project revision counter is exhausted.",
|
||||
)
|
||||
})?;
|
||||
let stored = StoredProject {
|
||||
resource_id: id,
|
||||
revision,
|
||||
model,
|
||||
};
|
||||
let size = self.write(&stored, false)?;
|
||||
self.adopt(stored.clone(), size);
|
||||
self.sync_directory()?;
|
||||
Ok((stored, true))
|
||||
}
|
||||
|
||||
pub fn delete(&mut self, id: Id, expected: &str) -> ApiResult<()> {
|
||||
self.checked(id, expected)?;
|
||||
fs::remove_file(self.path(id)).map_err(ApiError::storage)?;
|
||||
self.projects.remove(&id);
|
||||
self.total_bytes -= self.byte_sizes.remove(&id).unwrap_or(0);
|
||||
self.sync_directory()
|
||||
}
|
||||
|
||||
fn path(&self, id: Id) -> PathBuf {
|
||||
self.directory.join(format!("{id}.json"))
|
||||
}
|
||||
|
||||
fn adopt(&mut self, stored: StoredProject, size: usize) {
|
||||
let previous = self
|
||||
.byte_sizes
|
||||
.insert(stored.resource_id, size)
|
||||
.unwrap_or(0);
|
||||
self.total_bytes = self.total_bytes - previous + size;
|
||||
self.projects.insert(stored.resource_id, stored);
|
||||
}
|
||||
|
||||
fn write(&self, stored: &StoredProject, create: bool) -> ApiResult<usize> {
|
||||
let bytes = serde_json::to_vec(stored)
|
||||
.map_err(|_| ApiError::internal("Could not serialize model."))?;
|
||||
if bytes.len() > MAX_ENVELOPE_BYTES {
|
||||
return Err(ApiError::new(
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"model_too_large",
|
||||
"Stored model exceeds 8 MiB.",
|
||||
));
|
||||
}
|
||||
let previous = self
|
||||
.byte_sizes
|
||||
.get(&stored.resource_id)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
if self.total_bytes - previous + bytes.len() > MAX_STORE_BYTES {
|
||||
return Err(capacity_error());
|
||||
}
|
||||
let mut temporary = tempfile::Builder::new()
|
||||
.prefix(".arcadia-write-")
|
||||
.tempfile_in(&self.directory)
|
||||
.map_err(ApiError::storage)?;
|
||||
temporary.write_all(&bytes).map_err(ApiError::storage)?;
|
||||
temporary.as_file().sync_all().map_err(ApiError::storage)?;
|
||||
if create {
|
||||
temporary.persist_noclobber(self.path(stored.resource_id))
|
||||
} else {
|
||||
temporary.persist(self.path(stored.resource_id))
|
||||
}
|
||||
.map_err(|error| ApiError::storage(error.error))?;
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn sync_directory(&self) -> ApiResult<()> {
|
||||
// The installed file and in-memory head already agree if this sync fails.
|
||||
// Clients receive an uncertain write outcome and must reload.
|
||||
#[cfg(unix)]
|
||||
File::open(&self.directory)
|
||||
.and_then(|file| file.sync_all())
|
||||
.map_err(ApiError::storage)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn capacity_error() -> ApiError {
|
||||
ApiError::new(
|
||||
StatusCode::INSUFFICIENT_STORAGE,
|
||||
"store_capacity",
|
||||
"Store supports at most 128 models and 256 MiB of serialized envelopes. Export/delete unneeded models before adding more data.",
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn check_model_size(model: &Project) -> ApiResult<()> {
|
||||
MODEL_LIMITS
|
||||
.check_model(model)
|
||||
.map_err(|detail| ApiError::new(StatusCode::UNPROCESSABLE_ENTITY, "model_limit", detail))?;
|
||||
let bytes = serde_json::to_vec(model)
|
||||
.map_err(|_| ApiError::bad_request("Model cannot be serialized."))?;
|
||||
if bytes.len() > MAX_MODEL_BYTES {
|
||||
return Err(ApiError::new(
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"model_too_large",
|
||||
"Serialized model exceeds 8 MiB.",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn validate_model(model: &Project) -> ApiResult<()> {
|
||||
check_model_size(model)?;
|
||||
let issues = issues(model);
|
||||
if issues.iter().any(|issue| issue.severity == "error") {
|
||||
let mut error = ApiError::new(
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
"invalid_model",
|
||||
"Model violates structural constraints.",
|
||||
);
|
||||
error.issues = issues;
|
||||
return Err(error);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn prepared_candidates_recheck_revision_at_commit_and_failed_loser_is_not_installed() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let store = Store::open(directory.path()).unwrap();
|
||||
let first = store.create(Project::empty()).unwrap();
|
||||
let expected = first.etag();
|
||||
let mut winner = store.checked(first.resource_id, &expected).unwrap().model;
|
||||
let mut loser = winner.clone();
|
||||
winner.name = "Winning review".into();
|
||||
loser.name = "Losing stale review".into();
|
||||
// Both requests completed preparation against the same original head.
|
||||
let winner = PreparedModel::new(winner).unwrap();
|
||||
let loser = PreparedModel::new(loser).unwrap();
|
||||
let committed = store
|
||||
.lock()
|
||||
.unwrap()
|
||||
.replace(first.resource_id, &expected, winner)
|
||||
.unwrap()
|
||||
.0;
|
||||
let error = store
|
||||
.lock()
|
||||
.unwrap()
|
||||
.replace(first.resource_id, &expected, loser)
|
||||
.unwrap_err();
|
||||
assert_eq!(error.status, StatusCode::PRECONDITION_FAILED.as_u16());
|
||||
assert_eq!(error.current_revision, Some(2));
|
||||
let installed: StoredProject = serde_json::from_slice(
|
||||
&fs::read(directory.path().join(format!("{}.json", first.resource_id))).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(installed.model, committed.model);
|
||||
assert_eq!(installed.revision, 2);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user