feat: deliver cancellable HTTP text requests through owner-scoped tasks
This commit is contained in:
+182
-19
@@ -3,13 +3,15 @@ use endor_core::{VmError, WeakObjectRef};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::File,
|
||||
io::Read,
|
||||
io::{Read, Write},
|
||||
process::{Child, Command, Stdio},
|
||||
sync::{
|
||||
Arc, Mutex, OnceLock,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
mpsc::{SyncSender, sync_channel},
|
||||
},
|
||||
thread,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
const MAX_TEXT: usize = 1024 * 1024;
|
||||
@@ -27,16 +29,37 @@ struct Shared {
|
||||
pub struct Provider {
|
||||
shared: Arc<Shared>,
|
||||
}
|
||||
/// Only Task payloads own this lease. Provider threads hold just its flag.
|
||||
#[derive(Debug, Default)]
|
||||
struct Control {
|
||||
cancelled: AtomicBool,
|
||||
process: Mutex<Option<Child>>,
|
||||
}
|
||||
impl Control {
|
||||
fn cancel(&self) {
|
||||
self.cancelled.store(true, Ordering::Release);
|
||||
if let Ok(mut process) = self.process.lock()
|
||||
&& let Some(child) = process.as_mut()
|
||||
{
|
||||
// Reaping belongs to the provider worker, never the VM thread.
|
||||
let _ = child.kill();
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub enum Request {
|
||||
ReadText(String),
|
||||
FetchText(String),
|
||||
}
|
||||
/// Only Task payloads own this lease. Provider threads hold plain control data.
|
||||
#[derive(Debug)]
|
||||
pub struct Lease {
|
||||
shared: Arc<Shared>,
|
||||
task: WeakObjectRef,
|
||||
cancelled: Arc<AtomicBool>,
|
||||
control: Arc<Control>,
|
||||
}
|
||||
impl Drop for Lease {
|
||||
fn drop(&mut self) {
|
||||
self.cancelled.store(true, Ordering::Release);
|
||||
self.control.cancel();
|
||||
if let Ok(mut ready) = self.shared.ready.lock() {
|
||||
ready.remove(&self.task);
|
||||
}
|
||||
@@ -44,16 +67,16 @@ impl Drop for Lease {
|
||||
}
|
||||
}
|
||||
struct Job {
|
||||
path: String,
|
||||
request: Request,
|
||||
task: WeakObjectRef,
|
||||
cancelled: Arc<AtomicBool>,
|
||||
control: Arc<Control>,
|
||||
shared: Arc<Shared>,
|
||||
}
|
||||
impl Provider {
|
||||
pub fn submit(
|
||||
&self,
|
||||
task: WeakObjectRef,
|
||||
path: String,
|
||||
request: Request,
|
||||
replacing: bool,
|
||||
) -> Result<Arc<Lease>, VmError> {
|
||||
self.shared
|
||||
@@ -61,21 +84,21 @@ impl Provider {
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| {
|
||||
(active < MAX_ACTIVE + usize::from(replacing)).then_some(active + 1)
|
||||
})
|
||||
.map_err(|_| VmError::ExecutionLimit("16 file tasks per runtime"))?;
|
||||
.map_err(|_| VmError::ExecutionLimit("16 I/O tasks per runtime"))?;
|
||||
let lease = Arc::new(Lease {
|
||||
shared: self.shared.clone(),
|
||||
task,
|
||||
cancelled: Arc::new(AtomicBool::new(false)),
|
||||
control: Arc::new(Control::default()),
|
||||
});
|
||||
let sender = pool().map_err(|_| VmError::InvalidArgument("file provider unavailable"))?;
|
||||
let sender = pool().map_err(|_| VmError::InvalidArgument("I/O provider unavailable"))?;
|
||||
sender
|
||||
.try_send(Job {
|
||||
path,
|
||||
request,
|
||||
task,
|
||||
cancelled: lease.cancelled.clone(),
|
||||
control: lease.control.clone(),
|
||||
shared: self.shared.clone(),
|
||||
})
|
||||
.map_err(|_| VmError::ExecutionLimit("file provider queue full or unavailable"))?;
|
||||
.map_err(|_| VmError::ExecutionLimit("I/O provider queue full or unavailable"))?;
|
||||
Ok(lease)
|
||||
}
|
||||
pub fn take_ready(&self) -> Result<Vec<(WeakObjectRef, Completion)>, VmError> {
|
||||
@@ -83,7 +106,7 @@ impl Provider {
|
||||
.shared
|
||||
.ready
|
||||
.lock()
|
||||
.map_err(|_| VmError::InvalidArgument("file mailbox poisoned"))?
|
||||
.map_err(|_| VmError::InvalidArgument("I/O mailbox poisoned"))?
|
||||
.drain()
|
||||
.collect())
|
||||
}
|
||||
@@ -96,7 +119,7 @@ fn pool() -> Result<&'static SyncSender<Job>, &'static str> {
|
||||
for index in 0..WORKERS {
|
||||
let receiver = receiver.clone();
|
||||
thread::Builder::new()
|
||||
.name(format!("tessera-file-{index}"))
|
||||
.name(format!("tessera-io-{index}"))
|
||||
.spawn(move || {
|
||||
loop {
|
||||
let job = match receiver.lock() {
|
||||
@@ -106,12 +129,15 @@ fn pool() -> Result<&'static SyncSender<Job>, &'static str> {
|
||||
let Ok(job) = job else {
|
||||
break;
|
||||
};
|
||||
if job.cancelled.load(Ordering::Acquire) {
|
||||
if job.control.cancelled.load(Ordering::Acquire) {
|
||||
continue;
|
||||
}
|
||||
let result = read_text(&job.path);
|
||||
let result = match &job.request {
|
||||
Request::ReadText(path) => read_text(path),
|
||||
Request::FetchText(url) => fetch_text(url, &job.control),
|
||||
};
|
||||
if let Ok(mut ready) = job.shared.ready.lock()
|
||||
&& !job.cancelled.load(Ordering::Acquire)
|
||||
&& !job.control.cancelled.load(Ordering::Acquire)
|
||||
{
|
||||
ready.insert(job.task, result);
|
||||
}
|
||||
@@ -122,7 +148,7 @@ fn pool() -> Result<&'static SyncSender<Job>, &'static str> {
|
||||
Ok(sender)
|
||||
})
|
||||
.as_ref()
|
||||
.map_err(|_| "file provider unavailable")
|
||||
.map_err(|_| "I/O provider unavailable")
|
||||
}
|
||||
fn read_text(path: &str) -> Completion {
|
||||
let metadata = std::fs::metadata(path).map_err(|error| error.to_string())?;
|
||||
@@ -149,3 +175,140 @@ fn read_text(path: &str) -> Completion {
|
||||
}
|
||||
String::from_utf8(bytes).map_err(|_| "text source must be UTF-8".into())
|
||||
}
|
||||
|
||||
/// No shell interpretation, curlrc, URL globbing, or non-HTTP redirects.
|
||||
/// Authentication/custom headers are deliberately outside this text GET API.
|
||||
pub fn valid_url(url: &str) -> bool {
|
||||
if url.len() > 8192 || url.chars().any(char::is_control) {
|
||||
return false;
|
||||
}
|
||||
let Some((scheme, rest)) = url.split_once("://") else {
|
||||
return false;
|
||||
};
|
||||
if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") {
|
||||
return false;
|
||||
}
|
||||
let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
|
||||
!authority.is_empty() && !authority.contains('@') && !authority.chars().any(char::is_whitespace)
|
||||
}
|
||||
|
||||
/// Created only on an I/O worker. All early exits kill and reap the child.
|
||||
struct Reaper<'a>(&'a Control);
|
||||
impl Drop for Reaper<'_> {
|
||||
fn drop(&mut self) {
|
||||
let child = self.0.process.lock().ok().and_then(|mut slot| slot.take());
|
||||
if let Some(mut child) = child {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
fn fetch_text(url: &str, control: &Control) -> Completion {
|
||||
if !valid_url(url) {
|
||||
return Err("invalid HTTP(S) URL".into());
|
||||
}
|
||||
#[cfg(unix)]
|
||||
let executable = "/usr/bin/curl";
|
||||
#[cfg(not(unix))]
|
||||
let executable = "curl.exe";
|
||||
let mut child = Command::new(executable)
|
||||
.args([
|
||||
"--disable",
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--globoff",
|
||||
"--location",
|
||||
"--max-redirs",
|
||||
"5",
|
||||
"--proto",
|
||||
"=http,https",
|
||||
"--proto-redir",
|
||||
"=http,https",
|
||||
"--max-time",
|
||||
"30",
|
||||
"--connect-timeout",
|
||||
"10",
|
||||
"--max-filesize",
|
||||
"1048576",
|
||||
"--compressed",
|
||||
"--fail",
|
||||
"--write-out",
|
||||
"\n%{http_code}",
|
||||
"--config",
|
||||
"-",
|
||||
])
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.map_err(|_| "HTTP transport unavailable: system curl is required".to_owned())?;
|
||||
let mut input = child.stdin.take().expect("piped stdin");
|
||||
let output = child.stdout.take().expect("piped stdout");
|
||||
match control.process.lock() {
|
||||
Ok(mut slot) => *slot = Some(child),
|
||||
Err(_) => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
return Err("HTTP process lock poisoned".into());
|
||||
}
|
||||
}
|
||||
let _reaper = Reaper(control);
|
||||
if control.cancelled.load(Ordering::Acquire) {
|
||||
return Err("request cancelled".into());
|
||||
}
|
||||
// URL stays out of process arguments. Reject controls before config escaping.
|
||||
let escaped = url.replace('\\', "\\\\").replace('"', "\\\"");
|
||||
writeln!(input, "url = \"{escaped}\"").map_err(|_| "HTTP transport input failed".to_owned())?;
|
||||
drop(input);
|
||||
let mut bytes = Vec::new();
|
||||
output
|
||||
.take(MAX_TEXT as u64 + 5)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|_| "HTTP response read failed".to_owned())?;
|
||||
if bytes.len() > MAX_TEXT + 4 {
|
||||
return Err("HTTP text exceeds 1 MiB".into());
|
||||
}
|
||||
let status = loop {
|
||||
let mut slot = control
|
||||
.process
|
||||
.lock()
|
||||
.map_err(|_| "HTTP process lock poisoned".to_owned())?;
|
||||
let child = slot.as_mut().ok_or("HTTP process missing")?;
|
||||
if let Some(status) = child
|
||||
.try_wait()
|
||||
.map_err(|_| "HTTP transport wait failed".to_owned())?
|
||||
{
|
||||
slot.take();
|
||||
break status;
|
||||
}
|
||||
drop(slot);
|
||||
thread::sleep(Duration::from_millis(2));
|
||||
};
|
||||
if control.cancelled.load(Ordering::Acquire) {
|
||||
return Err("request cancelled".into());
|
||||
}
|
||||
let code = bytes
|
||||
.len()
|
||||
.checked_sub(4)
|
||||
.and_then(|offset| (bytes[offset] == b'\n').then_some(&bytes[offset + 1..]))
|
||||
.and_then(|digits| std::str::from_utf8(digits).ok())
|
||||
.and_then(|digits| digits.parse::<u16>().ok());
|
||||
if let Some(code) = code
|
||||
&& code >= 300
|
||||
{
|
||||
return Err(format!("HTTP {code}"));
|
||||
}
|
||||
if !status.success() {
|
||||
return Err(match status.code() {
|
||||
Some(28) => "HTTP request timed out".into(),
|
||||
Some(63) => "HTTP text exceeds 1 MiB".into(),
|
||||
Some(code) => format!("HTTP transport failed (curl {code})"),
|
||||
None => "HTTP transport terminated".into(),
|
||||
});
|
||||
}
|
||||
if !code.is_some_and(|code| (200..300).contains(&code)) {
|
||||
return Err("HTTP response status unavailable".into());
|
||||
}
|
||||
bytes.truncate(bytes.len() - 4);
|
||||
String::from_utf8(bytes).map_err(|_| "HTTP text must be UTF-8".into())
|
||||
}
|
||||
|
||||
+41
-16
@@ -196,9 +196,15 @@ pub fn cancel_owner(cx: &mut NativeContext<'_>, owner: ObjectRef) -> Result<(),
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Operation {
|
||||
Timer,
|
||||
ReadText,
|
||||
FetchText,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
struct After {
|
||||
file: bool,
|
||||
operation: Operation,
|
||||
provider: crate::io_tasks::Provider,
|
||||
signature: endor_core::CallSignature,
|
||||
index: Index,
|
||||
@@ -220,10 +226,10 @@ impl NativeCallable for After {
|
||||
}
|
||||
let (owner, key, delay, callback) = match args {
|
||||
[owner, key, delay, callback] => (owner, key, delay, Some(callback)),
|
||||
[owner, key, path] if self.file => (owner, key, path, None),
|
||||
[owner, key, path] if self.operation != Operation::Timer => (owner, key, path, None),
|
||||
_ => {
|
||||
return Err(invalid(
|
||||
"asynchronous request requires owner, key, delay/path and optional file callback",
|
||||
"asynchronous request requires owner, key, delay/path and optional I/O callback",
|
||||
));
|
||||
}
|
||||
};
|
||||
@@ -248,14 +254,28 @@ impl NativeCallable for After {
|
||||
if key.is_empty() || key.len() > 128 || key.chars().any(char::is_control) {
|
||||
return Err(invalid("invalid task key"));
|
||||
}
|
||||
let (milliseconds, path) = if self.file {
|
||||
let (milliseconds, request) = if self.operation != Operation::Timer {
|
||||
let Value::Text(path) = delay else {
|
||||
return Err(invalid("text path must be a String"));
|
||||
};
|
||||
if path.is_empty() || path.len() > 4096 || path.contains('\0') {
|
||||
return Err(invalid("invalid text path"));
|
||||
}
|
||||
(0, Some(path.clone()))
|
||||
let request = match self.operation {
|
||||
Operation::ReadText => {
|
||||
if path.is_empty() || path.len() > 4096 || path.contains('\0') {
|
||||
return Err(invalid("invalid text path"));
|
||||
}
|
||||
crate::io_tasks::Request::ReadText(path.clone())
|
||||
}
|
||||
Operation::FetchText => {
|
||||
if !crate::io_tasks::valid_url(path) {
|
||||
return Err(invalid(
|
||||
"expected HTTP(S) URL without credentials or control characters",
|
||||
));
|
||||
}
|
||||
crate::io_tasks::Request::FetchText(path.clone())
|
||||
}
|
||||
Operation::Timer => unreachable!(),
|
||||
};
|
||||
(0, Some(request))
|
||||
} else {
|
||||
let Value::Number(delay) = delay else {
|
||||
return Err(invalid(
|
||||
@@ -309,13 +329,13 @@ impl NativeCallable for After {
|
||||
.ok_or(invalid("task deadline overflow"))?,
|
||||
callback,
|
||||
state: State::Pending,
|
||||
ready: !self.file,
|
||||
ready: self.operation == Operation::Timer,
|
||||
text: None,
|
||||
error: None,
|
||||
lease: None,
|
||||
})?;
|
||||
cx.attach_type(task, desktop_type(cx.store(), "Task", self.desktop)?)?;
|
||||
if let Some(path) = path {
|
||||
if let Some(request) = request {
|
||||
let replacing = previous
|
||||
.map(|previous| {
|
||||
cx.store()
|
||||
@@ -326,7 +346,7 @@ impl NativeCallable for After {
|
||||
.unwrap_or(false);
|
||||
let lease = self
|
||||
.provider
|
||||
.submit(cx.store().downgrade(task)?, path, replacing)?;
|
||||
.submit(cx.store().downgrade(task)?, request, replacing)?;
|
||||
let mut value = cx.store().payload::<Task>(task, "Task")?.clone();
|
||||
value.lease = Some(lease);
|
||||
cx.replace_payload(task, value)?;
|
||||
@@ -683,18 +703,23 @@ pub fn install(
|
||||
let index = Arc::new(Mutex::new(Vec::new()));
|
||||
let definition = desktop_type(vm.object_store(), "Task", desktop)?;
|
||||
let provider = crate::io_tasks::Provider::default();
|
||||
for (name, file) in [("after", false), ("readText", true)] {
|
||||
for (name, operation) in [
|
||||
("after", Operation::Timer),
|
||||
("readText", Operation::ReadText),
|
||||
("fetchText", Operation::FetchText),
|
||||
] {
|
||||
let external = operation != Operation::Timer;
|
||||
let function = vm.allocate(ContextNativeFunction::new(
|
||||
name,
|
||||
After {
|
||||
file,
|
||||
operation,
|
||||
provider: provider.clone(),
|
||||
signature: {
|
||||
let mut signature = endor_core::CallSignature::exact(
|
||||
vec![
|
||||
endor_core::AnnotationType::Any,
|
||||
endor_core::AnnotationType::String,
|
||||
if file {
|
||||
if external {
|
||||
endor_core::AnnotationType::String
|
||||
} else {
|
||||
endor_core::AnnotationType::Number
|
||||
@@ -702,13 +727,13 @@ pub fn install(
|
||||
endor_core::AnnotationType::Function,
|
||||
],
|
||||
endor_core::AnnotationType::Nominal(definition),
|
||||
if file {
|
||||
if external {
|
||||
endor_core::CallEffect::External
|
||||
} else {
|
||||
endor_core::CallEffect::Mutating
|
||||
},
|
||||
);
|
||||
if file {
|
||||
if external {
|
||||
signature.arity = endor_core::CallArity::range(3, 4);
|
||||
}
|
||||
signature
|
||||
|
||||
Reference in New Issue
Block a user