336 lines
12 KiB
Rust
336 lines
12 KiB
Rust
//! 后台文件/网络 provider。线程池只携带纯数据、取消控制与弱 Task 身份，不持有 VM。
|
|
//! Lease 由托管任务拥有，释放会撤销交付资格并取消可取消的后台工作；已经完成的结果还要由 VM worker 检查归属后才能交付。
|
|
//! 并发、队列、读取字节数、超时和 URL 协议均有边界，避免 provider 绕过运行时资源限制。
|
|
//!
|
|
//! Background providers carry plain data and weak task identities, never a VM.
|
|
use endor_core::{VmError, WeakObjectRef};
|
|
use std::{
|
|
collections::HashMap,
|
|
fs::File,
|
|
io::{Read, Write},
|
|
process::{Child, Command, Stdio},
|
|
sync::{
|
|
Arc, Mutex, OnceLock,
|
|
atomic::{AtomicBool, AtomicUsize, Ordering},
|
|
mpsc::{SyncSender, sync_channel},
|
|
},
|
|
thread,
|
|
time::Duration,
|
|
};
|
|
|
|
const MAX_TEXT: usize = 1024 * 1024;
|
|
const MAX_ACTIVE: usize = 16;
|
|
const WORKERS: usize = 4;
|
|
const QUEUE: usize = 64;
|
|
type Completion = Result<String, String>;
|
|
|
|
#[derive(Debug, Default)]
|
|
/// provider 共享的就绪结果与并发计数；内容只有纯数据和弱任务身份。
|
|
struct Shared {
|
|
ready: Mutex<HashMap<WeakObjectRef, Completion>>,
|
|
active: AtomicUsize,
|
|
}
|
|
#[derive(Clone, Debug, Default)]
|
|
/// 某个运行时拥有的后台 provider 能力，负责提交和取走完成结果。
|
|
pub struct Provider {
|
|
shared: Arc<Shared>,
|
|
}
|
|
#[derive(Debug, Default)]
|
|
/// 后台工作取消标记与可取消子进程，不拥有托管 VM。
|
|
struct Control {
|
|
cancelled: AtomicBool,
|
|
process: Mutex<Option<Child>>,
|
|
}
|
|
impl Control {
|
|
/// 撤销当前操作的后续交付资格，并执行对应取消清理。
|
|
fn cancel(&self) {
|
|
self.cancelled.store(true, Ordering::Release);
|
|
if let Ok(mut process) = self.process.lock()
|
|
&& let Some(child) = process.as_mut()
|
|
{
|
|
// Reaping belongs to the provider worker, never the VM thread.
|
|
let _ = child.kill();
|
|
}
|
|
}
|
|
}
|
|
#[derive(Debug)]
|
|
/// 后台工作种类及其纯文本参数，源调用的类型检查在提交前完成。
|
|
pub enum Request {
|
|
ReadText(String),
|
|
FetchText(String),
|
|
}
|
|
/// Only Task payloads own this lease. Provider threads hold plain control data.
|
|
#[derive(Debug)]
|
|
/// 仅托管 Task 拥有的交付租约；释放它会取消后台资格并清理晚到结果。
|
|
pub struct Lease {
|
|
shared: Arc<Shared>,
|
|
task: WeakObjectRef,
|
|
control: Arc<Control>,
|
|
}
|
|
impl Drop for Lease {
|
|
/// 在 Rust 所有者离开作用域时执行本类型的配套清理；不能用它代替需要报告错误的显式关闭。
|
|
fn drop(&mut self) {
|
|
// 租约释放立即取消交付资格；后台 OS 读取即使稍后结束，也不能再次唤醒旧 Task。
|
|
self.control.cancel();
|
|
if let Ok(mut ready) = self.shared.ready.lock() {
|
|
ready.remove(&self.task);
|
|
}
|
|
self.shared.active.fetch_sub(1, Ordering::AcqRel);
|
|
}
|
|
}
|
|
/// 线程池队列中的纯数据工作，关联弱任务身份、控制和共享完成表。
|
|
struct Job {
|
|
request: Request,
|
|
task: WeakObjectRef,
|
|
control: Arc<Control>,
|
|
shared: Arc<Shared>,
|
|
}
|
|
impl Provider {
|
|
/// 检查并发额度并取得任务 Lease，再把纯数据工作放入有界线程池队列。
|
|
pub fn submit(
|
|
&self,
|
|
task: WeakObjectRef,
|
|
request: Request,
|
|
replacing: bool,
|
|
) -> Result<Arc<Lease>, VmError> {
|
|
self.shared
|
|
.active
|
|
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| {
|
|
(active < MAX_ACTIVE + usize::from(replacing)).then_some(active + 1)
|
|
})
|
|
.map_err(|_| VmError::ExecutionLimit("16 I/O tasks per runtime"))?;
|
|
let lease = Arc::new(Lease {
|
|
shared: self.shared.clone(),
|
|
task,
|
|
control: Arc::new(Control::default()),
|
|
});
|
|
let sender = pool().map_err(|_| VmError::InvalidArgument("I/O provider unavailable"))?;
|
|
sender
|
|
.try_send(Job {
|
|
request,
|
|
task,
|
|
control: lease.control.clone(),
|
|
shared: self.shared.clone(),
|
|
})
|
|
.map_err(|_| VmError::ExecutionLimit("I/O provider queue full or unavailable"))?;
|
|
Ok(lease)
|
|
}
|
|
/// 取走指定弱任务身份的 provider 完成结果；实际托管写入仍由 VM worker 执行。
|
|
pub fn take_ready(&self) -> Result<Vec<(WeakObjectRef, Completion)>, VmError> {
|
|
Ok(self
|
|
.shared
|
|
.ready
|
|
.lock()
|
|
.map_err(|_| VmError::InvalidArgument("I/O mailbox poisoned"))?
|
|
.drain()
|
|
.collect())
|
|
}
|
|
}
|
|
/// 懒初始化共享工作线程和有界任务队列；初始化失败反馈给提交方。
|
|
fn pool() -> Result<&'static SyncSender<Job>, &'static str> {
|
|
static POOL: OnceLock<Result<SyncSender<Job>, String>> = OnceLock::new();
|
|
POOL.get_or_init(|| {
|
|
let (sender, receiver) = sync_channel::<Job>(QUEUE);
|
|
let receiver = Arc::new(Mutex::new(receiver));
|
|
for index in 0..WORKERS {
|
|
let receiver = receiver.clone();
|
|
thread::Builder::new()
|
|
.name(format!("tessera-io-{index}"))
|
|
.spawn(move || {
|
|
loop {
|
|
let job = match receiver.lock() {
|
|
Ok(receiver) => receiver.recv(),
|
|
Err(_) => break,
|
|
};
|
|
let Ok(job) = job else {
|
|
break;
|
|
};
|
|
if job.control.cancelled.load(Ordering::Acquire) {
|
|
continue;
|
|
}
|
|
let result = match &job.request {
|
|
Request::ReadText(path) => read_text(path),
|
|
Request::FetchText(url) => fetch_text(url, &job.control),
|
|
};
|
|
if let Ok(mut ready) = job.shared.ready.lock()
|
|
&& !job.control.cancelled.load(Ordering::Acquire)
|
|
{
|
|
ready.insert(job.task, result);
|
|
}
|
|
}
|
|
})
|
|
.map_err(|error| error.to_string())?;
|
|
}
|
|
Ok(sender)
|
|
})
|
|
.as_ref()
|
|
.map_err(|_| "I/O provider unavailable")
|
|
}
|
|
/// 读取受字节上限限制的 UTF-8 文件内容，错误作为任务结果交付。
|
|
fn read_text(path: &str) -> Completion {
|
|
let metadata = std::fs::metadata(path).map_err(|error| error.to_string())?;
|
|
if !metadata.is_file() {
|
|
return Err("text source must be a regular file".into());
|
|
}
|
|
if metadata.len() > MAX_TEXT as u64 {
|
|
return Err("text source exceeds 1 MiB".into());
|
|
}
|
|
let file = File::open(path).map_err(|error| error.to_string())?;
|
|
if !file
|
|
.metadata()
|
|
.map_err(|error| error.to_string())?
|
|
.is_file()
|
|
{
|
|
return Err("text source must be a regular file".into());
|
|
}
|
|
let mut bytes = Vec::new();
|
|
file.take(MAX_TEXT as u64 + 1)
|
|
.read_to_end(&mut bytes)
|
|
.map_err(|error| error.to_string())?;
|
|
if bytes.len() > MAX_TEXT {
|
|
return Err("text source exceeds 1 MiB".into());
|
|
}
|
|
String::from_utf8(bytes).map_err(|_| "text source must be UTF-8".into())
|
|
}
|
|
|
|
/// No shell interpretation, curlrc, URL globbing, or non-HTTP redirects.
|
|
/// Authentication/custom headers are deliberately outside this text GET API.
|
|
/// 检查网络请求的 URL 协议、长度与凭据等约束，拒绝不受支持的输入。
|
|
pub fn valid_url(url: &str) -> bool {
|
|
if url.len() > 8192 || url.chars().any(char::is_control) {
|
|
return false;
|
|
}
|
|
let Some((scheme, rest)) = url.split_once("://") else {
|
|
return false;
|
|
};
|
|
if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") {
|
|
return false;
|
|
}
|
|
let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
|
|
!authority.is_empty() && !authority.contains('@') && !authority.chars().any(char::is_whitespace)
|
|
}
|
|
|
|
/// Created only on an I/O worker. All early exits kill and reap the child.
|
|
/// 子进程作用域收尾保护，避免取消/失败路径遗留未回收进程。
|
|
struct Reaper<'a>(&'a Control);
|
|
impl Drop for Reaper<'_> {
|
|
/// 在 Rust 所有者离开作用域时执行本类型的配套清理；不能用它代替需要报告错误的显式关闭。
|
|
fn drop(&mut self) {
|
|
let child = self.0.process.lock().ok().and_then(|mut slot| slot.take());
|
|
if let Some(mut child) = child {
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
}
|
|
}
|
|
}
|
|
/// 在取消、超时、协议和大小限制下取得 HTTP 文本；不把网络执行放在 VM 线程。
|
|
fn fetch_text(url: &str, control: &Control) -> Completion {
|
|
if !valid_url(url) {
|
|
return Err("invalid HTTP(S) URL".into());
|
|
}
|
|
#[cfg(unix)]
|
|
let executable = "/usr/bin/curl";
|
|
#[cfg(not(unix))]
|
|
let executable = "curl.exe";
|
|
let mut child = Command::new(executable)
|
|
.args([
|
|
"--disable",
|
|
"--silent",
|
|
"--show-error",
|
|
"--globoff",
|
|
"--location",
|
|
"--max-redirs",
|
|
"5",
|
|
"--proto",
|
|
"=http,https",
|
|
"--proto-redir",
|
|
"=http,https",
|
|
"--max-time",
|
|
"30",
|
|
"--connect-timeout",
|
|
"10",
|
|
"--max-filesize",
|
|
"1048576",
|
|
"--compressed",
|
|
"--fail",
|
|
"--write-out",
|
|
"\n%{http_code}",
|
|
"--config",
|
|
"-",
|
|
])
|
|
.stdin(Stdio::piped())
|
|
.stdout(Stdio::piped())
|
|
.stderr(Stdio::null())
|
|
.spawn()
|
|
.map_err(|_| "HTTP transport unavailable: system curl is required".to_owned())?;
|
|
let mut input = child.stdin.take().expect("piped stdin");
|
|
let output = child.stdout.take().expect("piped stdout");
|
|
match control.process.lock() {
|
|
Ok(mut slot) => *slot = Some(child),
|
|
Err(_) => {
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
return Err("HTTP process lock poisoned".into());
|
|
}
|
|
}
|
|
let _reaper = Reaper(control);
|
|
if control.cancelled.load(Ordering::Acquire) {
|
|
return Err("request cancelled".into());
|
|
}
|
|
// URL stays out of process arguments. Reject controls before config escaping.
|
|
let escaped = url.replace('\\', "\\\\").replace('"', "\\\"");
|
|
writeln!(input, "url = \"{escaped}\"").map_err(|_| "HTTP transport input failed".to_owned())?;
|
|
drop(input);
|
|
let mut bytes = Vec::new();
|
|
output
|
|
.take(MAX_TEXT as u64 + 5)
|
|
.read_to_end(&mut bytes)
|
|
.map_err(|_| "HTTP response read failed".to_owned())?;
|
|
if bytes.len() > MAX_TEXT + 4 {
|
|
return Err("HTTP text exceeds 1 MiB".into());
|
|
}
|
|
let status = loop {
|
|
let mut slot = control
|
|
.process
|
|
.lock()
|
|
.map_err(|_| "HTTP process lock poisoned".to_owned())?;
|
|
let child = slot.as_mut().ok_or("HTTP process missing")?;
|
|
if let Some(status) = child
|
|
.try_wait()
|
|
.map_err(|_| "HTTP transport wait failed".to_owned())?
|
|
{
|
|
slot.take();
|
|
break status;
|
|
}
|
|
drop(slot);
|
|
thread::sleep(Duration::from_millis(2));
|
|
};
|
|
if control.cancelled.load(Ordering::Acquire) {
|
|
return Err("request cancelled".into());
|
|
}
|
|
let code = bytes
|
|
.len()
|
|
.checked_sub(4)
|
|
.and_then(|offset| (bytes[offset] == b'\n').then_some(&bytes[offset + 1..]))
|
|
.and_then(|digits| std::str::from_utf8(digits).ok())
|
|
.and_then(|digits| digits.parse::<u16>().ok());
|
|
if let Some(code) = code
|
|
&& code >= 300
|
|
{
|
|
return Err(format!("HTTP {code}"));
|
|
}
|
|
if !status.success() {
|
|
return Err(match status.code() {
|
|
Some(28) => "HTTP request timed out".into(),
|
|
Some(63) => "HTTP text exceeds 1 MiB".into(),
|
|
Some(code) => format!("HTTP transport failed (curl {code})"),
|
|
None => "HTTP transport terminated".into(),
|
|
});
|
|
}
|
|
if !code.is_some_and(|code| (200..300).contains(&code)) {
|
|
return Err("HTTP response status unavailable".into());
|
|
}
|
|
bytes.truncate(bytes.len() - 4);
|
|
String::from_utf8(bytes).map_err(|_| "HTTP text must be UTF-8".into())
|
|
}
|