From 6fa95b64b038bebb15438bab8cc50c561577b330 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sun, 18 Oct 2026 20:41:56 +0800 Subject: [PATCH] Implement validated wordcode execution with shared code objects and frames --- README.md | 12 +- crates/core/src/builtins.rs | 88 +--- crates/core/src/code.rs | 375 ++++++++++++++++ crates/core/src/eval.rs | 535 +++++++++++++++++++++++ crates/core/src/frame.rs | 79 ++++ crates/core/src/instruction.rs | 203 +++++++++ crates/core/src/lib.rs | 35 +- crates/core/src/object.rs | 11 +- crates/core/src/vm.rs | 51 ++- crates/core/src/vm_builtins.rs | 50 ++- crates/core/tests/builtins.rs | 91 ++-- crates/core/tests/execution.rs | 761 +++++++++++++++++++++++++++++++++ docs/architecture.md | 14 +- docs/bytecode-vm.md | 134 ++++++ docs/development.md | 10 +- docs/endor-builtins.md | 16 +- docs/endor-object.md | 6 +- docs/validation.md | 8 + 18 files changed, 2327 insertions(+), 152 deletions(-) create mode 100644 crates/core/src/code.rs create mode 100644 crates/core/src/eval.rs create mode 100644 crates/core/src/frame.rs create mode 100644 crates/core/src/instruction.rs create mode 100644 crates/core/tests/execution.rs create mode 100644 docs/bytecode-vm.md diff --git a/README.md b/README.md index 9c12bf2..47c65d2 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ Rust 编写的可嵌入运行时。当前实现 Endor 的 VM 基础设施：`EndorObject`、稳定对象分配、代际句柄、显式引用计数、循环回收和 Rust 原生操作分发。VM 从空堆启动，暂不实现 Python 的 type／元类体系，也不定义语言根类和继承规则。 -尚无词法／语法前端、字节码执行器或渲染器，不承诺 Python 兼容。数据库只保存宿主元数据，不保存或恢复 VM 堆。对象基础层的逐段说明见 [EndorObject](docs/endor-object.md)。 +已具备小型 wordcode 执行器，尚无词法／语法前端或渲染器，不承诺 Python 兼容。数据库只保存宿主元数据，不保存或恢复 VM 堆。对象基础层的逐段说明见 [EndorObject](docs/endor-object.md)。 -现有基础对象包括 `EndorFunctionObject`、`EndorNativeFunctionObject`、`EndorDictObject`、`EndorCellObject`、`EndorBaseExceptionObject`、`EndorStringObject` 和 `EndorNumberObject`。统一 `EndorNumber` 优先无损表示任意长度整数、精确十进制数、分数、IEEE 浮点和复数；不先实现重计算。语言函数及异常目前是数据基础，尚无语言函数执行和 raise／try／except。接口、示例与边界见 [基本对象和 Number](docs/endor-builtins.md)。 +现有基础对象包括 `EndorFunctionObject`、`EndorNativeFunctionObject`、`EndorDictObject`、`EndorCellObject`、`EndorBaseExceptionObject`、`EndorStringObject` 和 `EndorNumberObject`。统一 `EndorNumber` 优先无损表示任意长度整数、精确十进制数、分数、IEEE 浮点和复数；不先实现重计算。语言函数现可执行验证后的 wordcode，异常目前仍是数据基础，尚无 raise／try／except。接口、示例与边界见 [基本对象和 Number](docs/endor-builtins.md) 和 [字节码 VM](docs/bytecode-vm.md)。 ## 结构 @@ -164,11 +164,11 @@ docker stop aster-pg-test - `EndorObject` 是 Rust 运行时公共对象结构，包含身份、引用计数、原生载荷和托管属性。没有语言类或类型指针；`ObjectPayload` 是 Rust 操作接口，诊断名称不参与语言类型判断。 - `ObjectRef` 包含私有 VM 身份、槽位和代数。`Box` 保持存活对象地址稳定；槽位复用更新代数，旧句柄不会指向新对象。外部不构造堆索引，也不操作裸指针。 -- 分配返回一个外部强引用；复制句柄或含句柄的 `Value` 不增加计数。额外持有使用 `retain`，结束持有使用 `release`。对象属性和载荷声明的边各自持有内部强引用；读属性与原生调用返回借用值，需长期持有时显式 retain。 +- 分配返回一个外部强引用；复制句柄或含句柄的 `Value` 不增加计数。额外持有使用 `retain`，结束持有使用 `release`。对象属性和载荷声明的边各自持有内部强引用；读属性返回借用值，需长期持有时显式 retain；call／execute 的对象结果持有一份强引用，结束使用调用 release_value。 - 最后一个强引用释放时，迭代清理依赖对象并调用 Rust Drop。`collect_cycles()` 从外部强引用追踪，显式回收不可达环；不会自动定时执行。销毁 VM 时统一释放剩余对象。 -- 支持 repr／str、身份与内容比较、哈希、真值、属性读写删除、原生函数调用。操作由 Rust 载荷实现；不支持的操作返回错误。原生调用可返回标量或已有对象的借用句柄，当前不能在回调中修改 VM；没有语言函数执行器。 -- 标量与字符串可直接保存在 `Value`，数字统一为 `Value::Number`；需要对象身份时可分配 String／Number 对象。字典和 Cell 支持 VM 管理的可变引用；函数和异常保存数据并参与 GC。数值比较／哈希、语言函数执行、异常继承与传播、语言根类、元类、描述符及 VM 堆持久化留待以后实现。 +- 支持 repr／str、身份与内容比较、哈希、真值、属性读写删除、原生函数调用。操作由 Rust 载荷实现；不支持的操作返回错误。原生回调可返回标量或已有对象的借用句柄，由 VM 保活后作为持有式结果返回；当前不能在回调中修改 VM。语言函数通过显式执行帧运行 wordcode。 +- 标量与字符串可直接保存在 `Value`，数字统一为 `Value::Number`；需要对象身份时可分配 String／Number 对象。字典和 Cell 支持 VM 管理的可变引用；代码对象和函数共享不可变代码并参与 GC，执行栈管理对象根。数值比较／哈希、异常继承与传播、语言根类、元类、描述符及 VM 堆持久化留待以后实现。 -下一阶段可围绕执行栈的强引用根管理和可变 VM 调用上下文推进执行器；GC 已有手动全堆版本，尚无分代、增量、并行收集或语言终结器／复活语义。详见 [架构](docs/architecture.md)、[对象基础层](docs/endor-object.md) 与 [开发交接](docs/development.md)。 +下一阶段可扩充指令和源码编译入口，并按需求增加可变 VM 宿主调用上下文；GC 已有手动全堆版本，尚无分代、增量、并行收集或语言终结器／复活语义。详见 [架构](docs/architecture.md)、[对象基础层](docs/endor-object.md) 与 [开发交接](docs/development.md)。 依赖 API 依据 [SQLx 0.8.6 官方文档](https://docs.rs/sqlx/0.8.6/sqlx/) 和 [Axum 官方文档](https://docs.rs/axum/0.8.9/axum/) 实现。SQLx 0.9 要求更高 Rust 版本，本阶段使用与本机工具链兼容的 0.8.6；确切解析版本以锁文件为准。 diff --git a/crates/core/src/builtins.rs b/crates/core/src/builtins.rs index 879090e..3e7f07b 100644 --- a/crates/core/src/builtins.rs +++ b/crates/core/src/builtins.rs @@ -150,89 +150,42 @@ impl ObjectPayload for EndorCellObject { } } -/// Code data only: bytecode is opaque until an instruction format/executor -/// is defined. Constants may own managed objects through the enclosing function. -#[derive(Debug, Clone)] -pub struct EndorCode { - bytecode: Vec, - constants: Vec, - parameters: Vec, - free_variables: Vec, -} - -impl EndorCode { - pub fn new( - bytecode: Vec, - constants: Vec, - parameters: Vec, - free_variables: Vec, - ) -> Result { - for names in [¶meters, &free_variables] { - let mut seen = std::collections::BTreeSet::new(); - for name in names { - if name.is_empty() || !seen.insert(name) { - return Err(VmError::InvalidArgument( - "empty or duplicate code variable name", - )); - } - } - } - Ok(Self { - bytecode, - constants, - parameters, - free_variables, - }) - } - pub fn bytecode(&self) -> &[u8] { - &self.bytecode - } - pub fn constants(&self) -> &[Value] { - &self.constants - } - pub fn parameters(&self) -> &[String] { - &self.parameters - } - pub fn free_variables(&self) -> &[String] { - &self.free_variables - } -} - #[derive(Debug, Clone)] pub struct EndorFunctionObject { name: String, - code: EndorCode, + code: ObjectRef, + builtins: Option, globals: ObjectRef, defaults: Vec, closure: Vec, } impl EndorFunctionObject { - pub fn new( - name: impl Into, - code: EndorCode, + pub(crate) fn new( + name: String, + code: ObjectRef, globals: ObjectRef, + builtins: Option, defaults: Vec, closure: Vec, - ) -> Result { - if defaults.len() > code.parameters.len() || closure.len() != code.free_variables.len() { - return Err(VmError::InvalidArgument( - "function defaults or closure do not match code", - )); - } - Ok(Self { - name: name.into(), + ) -> Self { + Self { + name, code, globals, + builtins, defaults, closure, - }) + } } pub fn name(&self) -> &str { &self.name } - pub fn code(&self) -> &EndorCode { - &self.code + pub fn code(&self) -> ObjectRef { + self.code + } + pub fn builtins(&self) -> Option { + self.builtins } pub fn globals(&self) -> ObjectRef { self.globals @@ -256,20 +209,21 @@ impl ObjectPayload for EndorFunctionObject { true } fn trace(&self, visit: &mut dyn FnMut(ObjectRef)) { + visit(self.code); visit(self.globals); + if let Some(builtins) = self.builtins { + visit(builtins); + } for reference in &self.closure { visit(*reference); } - for value in self.defaults.iter().chain(&self.code.constants) { + for value in &self.defaults { trace_value(value, visit); } } fn repr(&self, _identity: ObjectRef) -> Result { Ok(format!("", self.name)) } - fn call(&self, _arguments: &[Value]) -> Result { - Err(VmError::UnsupportedOperation("language function execution")) - } } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/core/src/code.rs b/crates/core/src/code.rs new file mode 100644 index 0000000..884db8c --- /dev/null +++ b/crates/core/src/code.rs @@ -0,0 +1,375 @@ +use std::any::Any; +use std::collections::{BTreeSet, VecDeque}; +use std::sync::Arc; + +use crate::builtins::trace_value; +use crate::instruction::{compare_operation, decode, invalid}; +use crate::{DecodedInstruction, Instruction, ObjectPayload, ObjectRef, Opcode, Value, VmError}; + +/// Local slots start with positional parameters. Captured locals occupy their +/// original slots as cells; free-variable cells follow all local slots. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CodeLayout { + pub parameters: Vec, + pub locals: Vec, + pub cell_variables: Vec, + pub free_variables: Vec, + pub names: Vec, +} + +#[derive(Debug, Clone)] +pub struct EndorCode { + bytecode: Vec, + instructions: Vec, + constants: Vec, + layout: CodeLayout, + stack_size: usize, + name: String, + filename: String, + first_line: usize, +} + +impl EndorCode { + pub fn new( + bytecode: Vec, + constants: Vec, + parameters: Vec, + free_variables: Vec, + ) -> Result { + Self::with_layout( + bytecode, + constants, + CodeLayout { + locals: parameters.clone(), + parameters, + free_variables, + ..CodeLayout::default() + }, + ) + } + + pub fn from_instructions( + instructions: &[Instruction], + constants: Vec, + layout: CodeLayout, + ) -> Result { + Self::with_layout(Instruction::encode_all(instructions)?, constants, layout) + } + + pub fn with_layout( + bytecode: Vec, + constants: Vec, + layout: CodeLayout, + ) -> Result { + validate_layout(&layout)?; + let instructions = decode(&bytecode)?; + let stack_size = verify(&instructions, constants.len(), &layout)?; + Ok(Self { + bytecode, + instructions, + constants, + layout, + stack_size, + name: "".into(), + filename: "".into(), + first_line: 1, + }) + } + + pub fn with_location( + mut self, + name: impl Into, + filename: impl Into, + first_line: usize, + ) -> Self { + self.name = name.into(); + self.filename = filename.into(); + self.first_line = first_line; + self + } + + pub fn bytecode(&self) -> &[u8] { + &self.bytecode + } + pub fn instructions(&self) -> &[DecodedInstruction] { + &self.instructions + } + pub fn constants(&self) -> &[Value] { + &self.constants + } + pub fn parameters(&self) -> &[String] { + &self.layout.parameters + } + pub fn free_variables(&self) -> &[String] { + &self.layout.free_variables + } + pub fn layout(&self) -> &CodeLayout { + &self.layout + } + pub fn stack_size(&self) -> usize { + self.stack_size + } + pub fn name(&self) -> &str { + &self.name + } + pub fn filename(&self) -> &str { + &self.filename + } + pub fn first_line(&self) -> usize { + self.first_line + } + pub fn slot_count(&self) -> usize { + self.layout.locals.len() + self.layout.free_variables.len() + } + pub(crate) fn is_cell_slot(&self, slot: usize) -> bool { + is_cell_slot(&self.layout, slot) + } + pub(crate) fn slot_name(&self, slot: usize) -> &str { + if slot < self.layout.locals.len() { + &self.layout.locals[slot] + } else { + &self.layout.free_variables[slot - self.layout.locals.len()] + } + } + pub(crate) fn instruction_at(&self, offset: usize) -> Result { + instruction_index(&self.instructions, offset) + .map(|index| self.instructions[index]) + .ok_or_else(|| { + invalid( + offset, + "instruction pointer is not on an instruction boundary", + ) + }) + } +} + +/// Heap-allocated, immutable and shareable executable. Function objects hold a +/// managed reference to this object rather than embedding a copy of its code. +#[derive(Debug, Clone)] +pub struct EndorCodeObject { + code: Arc, +} + +impl EndorCodeObject { + pub fn new(code: EndorCode) -> Self { + Self { + code: Arc::new(code), + } + } + pub fn code(&self) -> &EndorCode { + &self.code + } + pub(crate) fn shared_code(&self) -> Arc { + Arc::clone(&self.code) + } +} + +impl ObjectPayload for EndorCodeObject { + fn as_any(&self) -> &dyn Any { + self + } + fn kind_name(&self) -> &'static str { + "code" + } + fn trace(&self, visit: &mut dyn FnMut(ObjectRef)) { + for value in self.code.constants() { + trace_value(value, visit); + } + } + fn repr(&self, _identity: ObjectRef) -> Result { + Ok(format!( + "", + self.code.name(), + self.code.filename(), + self.code.first_line() + )) + } +} + +fn validate_layout(layout: &CodeLayout) -> Result<(), VmError> { + for names in [ + &layout.parameters, + &layout.locals, + &layout.cell_variables, + &layout.free_variables, + &layout.names, + ] { + let mut seen = BTreeSet::new(); + for name in names { + if name.is_empty() || !seen.insert(name) { + return Err(VmError::InvalidArgument( + "empty or duplicate code variable name", + )); + } + } + } + if !layout.locals.starts_with(&layout.parameters) { + return Err(VmError::InvalidArgument( + "parameter names must lead local slots", + )); + } + if layout + .cell_variables + .iter() + .any(|name| !layout.locals.contains(name)) + || layout + .free_variables + .iter() + .any(|name| layout.locals.contains(name)) + { + return Err(VmError::InvalidArgument( + "invalid local/cell/free-variable layout", + )); + } + Ok(()) +} + +fn is_cell_slot(layout: &CodeLayout, slot: usize) -> bool { + if slot >= layout.locals.len() { + slot < layout.locals.len() + layout.free_variables.len() + } else { + layout.cell_variables.contains(&layout.locals[slot]) + } +} + +fn instruction_index(instructions: &[DecodedInstruction], offset: usize) -> Option { + instructions + .binary_search_by_key(&offset, |instruction| instruction.offset) + .ok() +} + +pub(crate) fn jump_target(instruction: DecodedInstruction) -> Result { + let argument = instruction.instruction.argument as usize; + let target = if instruction.instruction.opcode == Opcode::JumpBackward { + instruction.next_offset.checked_sub(argument) + } else { + instruction.next_offset.checked_add(argument) + }; + target.ok_or_else(|| invalid(instruction.offset, "jump offset overflow")) +} + +/// Validate every operand, then propagate stack heights through reachable +/// control flow. Bad jumps, underflow, mismatched joins and fallthrough fail +/// before execution; loops must return to the same stack height. +fn verify( + instructions: &[DecodedInstruction], + constants: usize, + layout: &CodeLayout, +) -> Result { + for instruction in instructions { + let Instruction { opcode, argument } = instruction.instruction; + let index = argument as usize; + let valid = match opcode { + Opcode::LoadConst => index < constants, + Opcode::LoadFast | Opcode::StoreFast | Opcode::DeleteFast => { + index < layout.locals.len() && !is_cell_slot(layout, index) + } + Opcode::LoadDeref | Opcode::StoreDeref | Opcode::DeleteDeref | Opcode::LoadClosure => { + is_cell_slot(layout, index) + } + Opcode::LoadGlobal + | Opcode::StoreGlobal + | Opcode::DeleteGlobal + | Opcode::LoadAttr + | Opcode::StoreAttr => index < layout.names.len(), + Opcode::CompareOp => compare_operation(argument).is_some(), + Opcode::Copy => argument >= 1, + Opcode::Swap => argument >= 2, + Opcode::JumpForward + | Opcode::JumpBackward + | Opcode::PopJumpIfFalse + | Opcode::PopJumpIfTrue => { + instruction_index(instructions, jump_target(*instruction)?).is_some() + } + Opcode::ExtendedArg => false, + _ => true, + }; + if !valid { + return Err(invalid( + instruction.offset, + "invalid operand or jump target", + )); + } + } + let mut heights = vec![None; instructions.len()]; + heights[0] = Some(0_usize); + let mut pending = VecDeque::from([0]); + let mut maximum = 0; + while let Some(index) = pending.pop_front() { + let instruction = instructions[index]; + let height = heights[index].unwrap(); + let Instruction { opcode, argument } = instruction.instruction; + let argument = argument as usize; + let (required, popped, pushed) = match opcode { + Opcode::LoadConst + | Opcode::LoadFast + | Opcode::LoadGlobal + | Opcode::LoadDeref + | Opcode::LoadClosure + | Opcode::PushNull => (0, 0, 1), + Opcode::StoreFast + | Opcode::StoreGlobal + | Opcode::PopTop + | Opcode::StoreDeref + | Opcode::PopJumpIfFalse + | Opcode::PopJumpIfTrue => (1, 1, 0), + Opcode::ToBool | Opcode::UnaryNot | Opcode::LoadAttr => (1, 1, 1), + Opcode::CompareOp => (2, 2, 1), + Opcode::StoreAttr => (2, 2, 0), + Opcode::Copy => (argument, 0, 1), + Opcode::Swap => (argument, 0, 0), + Opcode::Call => { + let consumed = argument + .checked_add(2) + .ok_or_else(|| invalid(instruction.offset, "call operand overflow"))?; + (consumed, consumed, 1) + } + Opcode::BuildMap => { + let consumed = argument + .checked_mul(2) + .ok_or_else(|| invalid(instruction.offset, "map operand overflow"))?; + (consumed, consumed, 1) + } + Opcode::ReturnValue => (1, 1, 0), + _ => (0, 0, 0), + }; + if height < required { + return Err(invalid(instruction.offset, "evaluation stack underflow")); + } + if opcode == Opcode::ReturnValue && height != 1 { + return Err(invalid( + instruction.offset, + "RETURN_VALUE must leave an empty frame stack", + )); + } + let next_height = height - popped + pushed; + maximum = maximum.max(height).max(next_height); + let successors: Vec = match opcode { + Opcode::ReturnValue => Vec::new(), + Opcode::JumpForward | Opcode::JumpBackward => vec![jump_target(instruction)?], + Opcode::PopJumpIfFalse | Opcode::PopJumpIfTrue => { + vec![jump_target(instruction)?, instruction.next_offset] + } + _ => vec![instruction.next_offset], + }; + for successor in successors { + let target = instruction_index(instructions, successor).ok_or_else(|| { + invalid(instruction.offset, "execution falls off the code object") + })?; + match heights[target] { + Some(existing) if existing != next_height => { + return Err(invalid( + successor, + "control-flow paths have different stack heights", + )); + } + Some(_) => {} + None => { + heights[target] = Some(next_height); + pending.push_back(target); + } + } + } + } + Ok(maximum) +} diff --git a/crates/core/src/eval.rs b/crates/core/src/eval.rs new file mode 100644 index 0000000..5e9dc2f --- /dev/null +++ b/crates/core/src/eval.rs @@ -0,0 +1,535 @@ +use crate::code::jump_target; +use crate::frame::StackEntry; +use crate::instruction::{compare_operation, invalid}; +use crate::{ + CompareOp, EndorCellObject, EndorCodeObject, EndorFrame, EndorFunctionObject, + EndorStringObject, ExecutionLimits, ExecutionLocation, ObjectRef, Opcode, Value, Vm, VmError, +}; + +impl Vm { + /// Execute a code object in a globals dictionary. Like CPython's EvalCode + /// entry path, this constructs a temporary function and enters its frame. + /// Object results own one external reference: release_value() when done. + pub fn execute(&mut self, code: ObjectRef, globals: ObjectRef) -> Result { + self.execute_with_limits(code, globals, ExecutionLimits::default()) + } + + pub fn execute_with_limits( + &mut self, + code: ObjectRef, + globals: ObjectRef, + limits: ExecutionLimits, + ) -> Result { + self.last_traceback.clear(); + let name = self + .payload::(code, "code")? + .code() + .name() + .to_owned(); + let function = self.new_function(name, code, globals, Vec::new(), Vec::new())?; + let result = self.call_with_limits(function, &[], limits); + self.release(function)?; + result + } + + /// Dispatch both language and host functions. Arguments are borrowed; + /// every Object result owns one external reference, including native + /// callbacks that return one of their borrowed arguments. + pub fn call(&mut self, function: ObjectRef, arguments: &[Value]) -> Result { + self.call_with_limits(function, arguments, ExecutionLimits::default()) + } + + pub fn call_with_limits( + &mut self, + function: ObjectRef, + arguments: &[Value], + limits: ExecutionLimits, + ) -> Result { + self.last_traceback.clear(); + if !self + .object(function)? + .payload() + .as_any() + .is::() + { + return self.invoke_native(function, arguments); + } + if limits.max_frames == 0 { + return Err(VmError::ExecutionLimit("frames")); + } + let frame = self.prepare_frame(function, arguments)?; + let mut frames = vec![frame]; + let result = self.eval_frames(&mut frames, limits); + if result.is_err() { + self.last_traceback = frames + .iter() + .map(|frame| ExecutionLocation { + function: frame.name.clone(), + filename: frame.executable.filename().into(), + first_line: frame.executable.first_line(), + instruction_offset: frame.current_offset, + }) + .collect(); + } + while let Some(frame) = frames.pop() { + self.clear_frame(frame); + } + result + } + + /// Outer-to-inner execution locations from the last failed call/execute. + /// These are copied metadata, not heap references or language exceptions. + pub fn last_traceback(&self) -> &[ExecutionLocation] { + &self.last_traceback + } + + pub fn release_value(&mut self, value: Value) -> Result<(), VmError> { + if let Value::Object(reference) = value { + self.release(reference)?; + } + Ok(()) + } + + fn retain_value(&mut self, value: &Value) -> Result<(), VmError> { + self.validate_value(value)?; + if let Value::Object(reference) = value { + self.retain(*reference)?; + } + Ok(()) + } + + fn drop_root(&mut self, value: Value) { + self.release_value(value) + .expect("execution owns every object root it releases"); + } + + fn clear_frame(&mut self, frame: EndorFrame) { + for entry in frame.stack { + if let StackEntry::Value(value) = entry { + self.drop_root(value); + } + } + for value in frame.localsplus.into_iter().flatten() { + self.drop_root(value); + } + self.release(frame.function) + .expect("frame owns its function root"); + } + + fn prepare_frame( + &mut self, + function: ObjectRef, + arguments: &[Value], + ) -> Result { + let data = self + .payload::(function, "function")? + .clone(); + let executable = self + .payload::(data.code(), "code")? + .shared_code(); + let maximum = executable.parameters().len(); + let required = maximum - data.defaults().len(); + if arguments.len() < required || arguments.len() > maximum { + return Err(VmError::ArgumentCount { + name: data.name().into(), + required, + maximum, + provided: arguments.len(), + }); + } + for value in arguments { + self.validate_value(value)?; + } + self.retain(function)?; + let mut frame = EndorFrame { + function, + previous: None, + code: data.code(), + globals: data.globals(), + builtins: data.builtins(), + name: data.name().into(), + instruction_offset: 0, + current_offset: 0, + localsplus: vec![None; executable.slot_count()], + stack: Vec::with_capacity(executable.stack_size()), + executable, + }; + let initialized = (|| -> Result<(), VmError> { + let defaults = data.defaults().iter().skip(arguments.len() - required); + for (slot, value) in frame + .localsplus + .iter_mut() + .zip(arguments.iter().chain(defaults)) + { + self.retain_value(value)?; + *slot = Some(value.clone()); + } + // Captured local cells occupy the original local slots, as in + // CPython's localsplus layout. Free cells follow the local slots. + let local_count = frame.executable.layout().locals.len(); + for (index, slot) in frame.localsplus[..local_count].iter_mut().enumerate() { + if frame.executable.is_cell_slot(index) { + let cell = self.new_cell(slot.clone())?; + if let Some(old) = slot.replace(Value::Object(cell)) { + self.drop_root(old); + } + } + } + let first_free = frame.executable.layout().locals.len(); + for (index, cell) in data.closure().iter().enumerate() { + self.retain(*cell)?; + frame.localsplus[first_free + index] = Some(Value::Object(*cell)); + } + Ok(()) + })(); + if let Err(error) = initialized { + self.clear_frame(frame); + return Err(error); + } + Ok(frame) + } + + fn invoke_native( + &mut self, + function: ObjectRef, + arguments: &[Value], + ) -> Result { + self.object(function)?; + for argument in arguments { + self.validate_value(argument)?; + } + let result = self.object(function)?.payload().call(arguments)?; + self.retain_value(&result)?; + Ok(result) + } + + fn stack_value(frame: &EndorFrame, index: usize) -> Result { + match frame.stack.get(index) { + Some(StackEntry::Value(value)) => Ok(value.clone()), + _ => Err(invalid( + frame.current_offset, + "NULL call marker used as a value", + )), + } + } + + fn top_value(frame: &EndorFrame) -> Result { + Self::stack_value(frame, frame.stack.len() - 1) + } + + fn push_borrowed(&mut self, frame: &mut EndorFrame, value: Value) -> Result<(), VmError> { + self.retain_value(&value)?; + frame.stack.push(StackEntry::Value(value)); + Ok(()) + } + + fn discard(&mut self, frame: &mut EndorFrame, count: usize) { + for _ in 0..count { + if let StackEntry::Value(value) = frame.stack.pop().expect("verified stack height") { + self.drop_root(value); + } + } + } + + fn cell_slot(&self, frame: &EndorFrame, index: usize) -> Result { + match &frame.localsplus[index] { + Some(Value::Object(reference)) => { + self.payload::(*reference, "cell")?; + Ok(*reference) + } + _ => Err(invalid( + frame.current_offset, + "cell slot is not initialized", + )), + } + } + + fn value_is_true(&self, value: &Value) -> Result { + match value { + Value::Nil => Ok(false), + Value::Bool(value) => Ok(*value), + Value::Text(value) => Ok(!value.is_empty()), + Value::Number(value) => Ok(!value.is_zero()), + Value::Object(reference) => self.is_true(*reference), + } + } + + fn text_value(&self, value: &Value) -> Result, VmError> { + Ok(match value { + Value::Text(text) => Some(text.clone()), + Value::Object(reference) => self + .object(*reference)? + .payload() + .as_any() + .downcast_ref::() + .map(|text| text.as_str().to_owned()), + _ => None, + }) + } + + fn compare_values( + &self, + left: &Value, + right: &Value, + operation: CompareOp, + ) -> Result { + if matches!(left, Value::Number(_)) || matches!(right, Value::Number(_)) { + return Err(VmError::UnsupportedOperation("numeric comparison")); + } + if let (Some(left), Some(right)) = (self.text_value(left)?, self.text_value(right)?) { + return Ok(match operation { + CompareOp::Eq => left == right, + CompareOp::Ne => left != right, + CompareOp::Lt => left < right, + CompareOp::Le => left <= right, + CompareOp::Gt => left > right, + CompareOp::Ge => left >= right, + }); + } + if let (Value::Object(left), Value::Object(right)) = (left, right) { + return self.compare(*left, *right, operation); + } + match operation { + CompareOp::Eq => Ok(left == right), + CompareOp::Ne => Ok(left != right), + _ => Err(VmError::UnsupportedOperation("ordering")), + } + } + + fn build_map(&mut self, frame: &EndorFrame, count: usize) -> Result { + let dictionary = self.new_dict()?; + let result = (|| -> Result<(), VmError> { + let start = frame.stack.len() - count * 2; + for index in 0..count { + let key = Self::stack_value(frame, start + index * 2)?; + let value = Self::stack_value(frame, start + index * 2 + 1)?; + self.dict_set(dictionary, key, value)?; + } + Ok(()) + })(); + if let Err(error) = result { + self.release(dictionary)?; + return Err(error); + } + Ok(dictionary) + } + + fn eval_frames( + &mut self, + frames: &mut Vec, + limits: ExecutionLimits, + ) -> Result { + let mut executed = 0; + loop { + if executed >= limits.max_instructions { + return Err(VmError::ExecutionLimit("instructions")); + } + executed += 1; + let frame = frames.last_mut().expect("an evaluation frame is active"); + let decoded = frame.executable.instruction_at(frame.instruction_offset)?; + frame.current_offset = decoded.offset; + frame.instruction_offset = decoded.next_offset; + let opcode = decoded.instruction.opcode; + let argument = decoded.instruction.argument as usize; + match opcode { + Opcode::Nop => {} + Opcode::LoadConst => { + let value = frame.executable.constants()[argument].clone(); + self.push_borrowed(frame, value)?; + } + Opcode::LoadFast => { + let value = frame.localsplus[argument].clone().ok_or_else(|| { + VmError::UnboundLocal(frame.executable.slot_name(argument).into()) + })?; + self.push_borrowed(frame, value)?; + } + Opcode::StoreFast => { + let value = Self::top_value(frame)?; + // The stack root is moved into the local slot. + frame.stack.pop(); + if let Some(old) = frame.localsplus[argument].replace(value) { + self.drop_root(old); + } + } + Opcode::DeleteFast => { + let value = frame.localsplus[argument].take().ok_or_else(|| { + VmError::UnboundLocal(frame.executable.slot_name(argument).into()) + })?; + self.drop_root(value); + } + Opcode::LoadGlobal => { + let name = &frame.executable.layout().names[argument]; + let key = Value::Text(name.clone()); + let mut value = self.dict_get(frame.globals, &key)?; + if value.is_none() + && let Some(builtins) = frame.builtins + { + value = self.dict_get(builtins, &key)?; + } + let value = value.ok_or_else(|| VmError::NameNotFound(name.clone()))?; + self.push_borrowed(frame, value)?; + } + Opcode::StoreGlobal => { + let value = Self::top_value(frame)?; + let key = Value::Text(frame.executable.layout().names[argument].clone()); + self.dict_set(frame.globals, key, value)?; + self.discard(frame, 1); + } + Opcode::DeleteGlobal => { + let name = &frame.executable.layout().names[argument]; + if !self.dict_delete(frame.globals, &Value::Text(name.clone()))? { + return Err(VmError::NameNotFound(name.clone())); + } + } + Opcode::LoadDeref | Opcode::LoadClosure => { + let cell = self.cell_slot(frame, argument)?; + let value = if opcode == Opcode::LoadClosure { + Value::Object(cell) + } else { + self.cell_get(cell).map_err(|error| { + if error == VmError::EmptyCell { + VmError::UnboundLocal(frame.executable.slot_name(argument).into()) + } else { + error + } + })? + }; + self.push_borrowed(frame, value)?; + } + Opcode::StoreDeref => { + let cell = self.cell_slot(frame, argument)?; + self.cell_set(cell, Self::top_value(frame)?)?; + self.discard(frame, 1); + } + Opcode::DeleteDeref => { + let cell = self.cell_slot(frame, argument)?; + if self.cell_get(cell).is_err() { + return Err(VmError::UnboundLocal( + frame.executable.slot_name(argument).into(), + )); + } + self.cell_clear(cell)?; + } + Opcode::PopTop => self.discard(frame, 1), + Opcode::Copy => { + let entry = frame.stack[frame.stack.len() - argument].clone(); + if let StackEntry::Value(value) = &entry { + self.retain_value(value)?; + } + frame.stack.push(entry); + } + Opcode::Swap => { + let length = frame.stack.len(); + frame.stack.swap(length - 1, length - argument); + } + Opcode::PushNull => frame.stack.push(StackEntry::Null), + Opcode::Call => { + let start = frame.stack.len() - argument - 2; + let Value::Object(function) = Self::stack_value(frame, start)? else { + return Err(VmError::UnsupportedOperation("call")); + }; + let mut arguments = Vec::with_capacity(argument + 1); + if let StackEntry::Value(value) = &frame.stack[start + 1] { + arguments.push(value.clone()); + } + for index in start + 2..frame.stack.len() { + arguments.push(Self::stack_value(frame, index)?); + } + let language_function = self + .object(function)? + .payload() + .as_any() + .is::(); + if language_function { + if frames.len() >= limits.max_frames { + return Err(VmError::ExecutionLimit("frames")); + } + let mut callee = self.prepare_frame(function, &arguments)?; + callee.previous = Some(frames.len() - 1); + self.discard(frames.last_mut().unwrap(), argument + 2); + frames.push(callee); + } else { + let value = self.invoke_native(function, &arguments)?; + self.discard(frames.last_mut().unwrap(), argument + 2); + frames + .last_mut() + .unwrap() + .stack + .push(StackEntry::Value(value)); + } + } + Opcode::ReturnValue => { + let result = Self::top_value(frame)?; + frame.stack.pop(); + self.clear_frame(frames.pop().unwrap()); + if let Some(caller) = frames.last_mut() { + caller.stack.push(StackEntry::Value(result)); + } else { + return Ok(result); + } + } + Opcode::JumpForward | Opcode::JumpBackward => { + frame.instruction_offset = jump_target(decoded)? + } + Opcode::PopJumpIfFalse | Opcode::PopJumpIfTrue => { + let truth = self.value_is_true(&Self::top_value(frame)?)?; + self.discard(frame, 1); + if truth == (opcode == Opcode::PopJumpIfTrue) { + frame.instruction_offset = jump_target(decoded)?; + } + } + Opcode::ToBool | Opcode::UnaryNot => { + let truth = self.value_is_true(&Self::top_value(frame)?)?; + self.discard(frame, 1); + frame.stack.push(StackEntry::Value(Value::Bool( + if opcode == Opcode::UnaryNot { + !truth + } else { + truth + }, + ))); + } + Opcode::CompareOp => { + let left = Self::stack_value(frame, frame.stack.len() - 2)?; + let right = Self::top_value(frame)?; + let result = self.compare_values( + &left, + &right, + compare_operation(decoded.instruction.argument).unwrap(), + )?; + self.discard(frame, 2); + frame.stack.push(StackEntry::Value(Value::Bool(result))); + } + Opcode::LoadAttr => { + let Value::Object(owner) = Self::top_value(frame)? else { + return Err(VmError::ObjectKindMismatch("object")); + }; + let value = self.get_attr(owner, &frame.executable.layout().names[argument])?; + self.retain_value(&value)?; + self.discard(frame, 1); + frame.stack.push(StackEntry::Value(value)); + } + Opcode::StoreAttr => { + let Value::Object(owner) = Self::top_value(frame)? else { + return Err(VmError::ObjectKindMismatch("object")); + }; + let value = Self::stack_value(frame, frame.stack.len() - 2)?; + self.set_attr( + owner, + frame.executable.layout().names[argument].clone(), + value, + )?; + self.discard(frame, 2); + } + Opcode::BuildMap => { + let dictionary = self.build_map(frame, argument)?; + self.discard(frame, argument * 2); + frame + .stack + .push(StackEntry::Value(Value::Object(dictionary))); + } + Opcode::ExtendedArg => unreachable!("EXTENDED_ARG was decoded before evaluation"), + } + } + } +} diff --git a/crates/core/src/frame.rs b/crates/core/src/frame.rs new file mode 100644 index 0000000..5f62100 --- /dev/null +++ b/crates/core/src/frame.rs @@ -0,0 +1,79 @@ +use std::sync::Arc; + +use crate::{EndorCode, ObjectRef, Value}; + +/// NULL is a call-protocol marker and can never escape as a language value. +/// Each Value::Object in a live frame owns one execution root. +#[derive(Debug, Clone)] +pub(crate) enum StackEntry { + Null, + Value(Value), +} + +/// Internal activation record, analogous to _PyInterpreterFrame. It has no +/// EndorObject header; introspection objects can be added independently later. +#[derive(Debug)] +pub struct EndorFrame { + pub(crate) function: ObjectRef, + /// Index of the caller in the execution's frame vector, never a pointer. + pub(crate) previous: Option, + pub(crate) code: ObjectRef, + pub(crate) executable: Arc, + pub(crate) globals: ObjectRef, + pub(crate) builtins: Option, + pub(crate) name: String, + pub(crate) instruction_offset: usize, + pub(crate) current_offset: usize, + pub(crate) localsplus: Vec>, + pub(crate) stack: Vec, +} + +impl EndorFrame { + pub fn previous(&self) -> Option { + self.previous + } + pub fn function(&self) -> ObjectRef { + self.function + } + pub fn code(&self) -> ObjectRef { + self.code + } + pub fn globals(&self) -> ObjectRef { + self.globals + } + pub fn builtins(&self) -> Option { + self.builtins + } + pub fn instruction_offset(&self) -> usize { + self.instruction_offset + } + pub fn localsplus(&self) -> &[Option] { + &self.localsplus + } + pub fn stack_depth(&self) -> usize { + self.stack.len() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExecutionLocation { + pub function: String, + pub filename: String, + pub first_line: usize, + pub instruction_offset: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExecutionLimits { + pub max_frames: usize, + pub max_instructions: usize, +} + +impl Default for ExecutionLimits { + fn default() -> Self { + Self { + max_frames: 1000, + max_instructions: usize::MAX, + } + } +} diff --git a/crates/core/src/instruction.rs b/crates/core/src/instruction.rs new file mode 100644 index 0000000..1e730e3 --- /dev/null +++ b/crates/core/src/instruction.rs @@ -0,0 +1,203 @@ +use crate::{CompareOp, VmError}; + +/// Endor wordcode, following CPython's opcode/oparg code-unit design. +/// Numeric opcode IDs are Endor's own, not a CPython bytecode ABI. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum Opcode { + Nop = 0, + LoadConst = 1, + LoadFast = 2, + StoreFast = 3, + DeleteFast = 4, + LoadGlobal = 5, + StoreGlobal = 6, + DeleteGlobal = 7, + PopTop = 8, + Copy = 9, + Swap = 10, + PushNull = 11, + Call = 12, + ReturnValue = 13, + JumpForward = 14, + JumpBackward = 15, + PopJumpIfFalse = 16, + PopJumpIfTrue = 17, + ToBool = 18, + UnaryNot = 19, + CompareOp = 20, + LoadDeref = 21, + StoreDeref = 22, + DeleteDeref = 23, + LoadClosure = 24, + LoadAttr = 25, + StoreAttr = 26, + BuildMap = 27, + ExtendedArg = 255, +} + +impl Opcode { + fn decode(byte: u8) -> Option { + Some(match byte { + 0 => Self::Nop, + 1 => Self::LoadConst, + 2 => Self::LoadFast, + 3 => Self::StoreFast, + 4 => Self::DeleteFast, + 5 => Self::LoadGlobal, + 6 => Self::StoreGlobal, + 7 => Self::DeleteGlobal, + 8 => Self::PopTop, + 9 => Self::Copy, + 10 => Self::Swap, + 11 => Self::PushNull, + 12 => Self::Call, + 13 => Self::ReturnValue, + 14 => Self::JumpForward, + 15 => Self::JumpBackward, + 16 => Self::PopJumpIfFalse, + 17 => Self::PopJumpIfTrue, + 18 => Self::ToBool, + 19 => Self::UnaryNot, + 20 => Self::CompareOp, + 21 => Self::LoadDeref, + 22 => Self::StoreDeref, + 23 => Self::DeleteDeref, + 24 => Self::LoadClosure, + 25 => Self::LoadAttr, + 26 => Self::StoreAttr, + 27 => Self::BuildMap, + 255 => Self::ExtendedArg, + _ => return None, + }) + } + + pub(crate) fn takes_argument(self) -> bool { + !matches!( + self, + Self::Nop + | Self::PopTop + | Self::PushNull + | Self::ReturnValue + | Self::ToBool + | Self::UnaryNot + ) + } +} + +/// A decoded logical instruction; EXTENDED_ARG is handled by the codec. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Instruction { + pub opcode: Opcode, + pub argument: u32, +} + +impl Instruction { + pub const fn new(opcode: Opcode, argument: u32) -> Self { + Self { opcode, argument } + } + pub const fn simple(opcode: Opcode) -> Self { + Self::new(opcode, 0) + } + + pub const fn compare(operation: CompareOp) -> Self { + Self::new( + Opcode::CompareOp, + match operation { + CompareOp::Lt => 0, + CompareOp::Le => 1, + CompareOp::Eq => 2, + CompareOp::Ne => 3, + CompareOp::Gt => 4, + CompareOp::Ge => 5, + }, + ) + } + + /// Encodes 8-bit operands directly and prefixes up to three EXTENDED_ARG + /// code units for larger operands. Each code unit is [opcode, operand]. + pub fn encode_all(instructions: &[Self]) -> Result, VmError> { + let mut bytes = Vec::new(); + for instruction in instructions { + if instruction.opcode == Opcode::ExtendedArg { + return Err(invalid(0, "EXTENDED_ARG is emitted by the encoder")); + } + if !instruction.opcode.takes_argument() && instruction.argument != 0 { + return Err(invalid(bytes.len() / 2, "unexpected instruction operand")); + } + let parts = instruction.argument.to_be_bytes(); + let first = parts.iter().position(|part| *part != 0).unwrap_or(3); + for part in &parts[first..3] { + bytes.extend([Opcode::ExtendedArg as u8, *part]); + } + bytes.extend([instruction.opcode as u8, parts[3]]); + } + Ok(bytes) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DecodedInstruction { + /// Offsets count two-byte code units, including EXTENDED_ARG prefixes. + pub offset: usize, + pub next_offset: usize, + pub instruction: Instruction, +} + +pub(crate) fn decode(bytes: &[u8]) -> Result, VmError> { + if bytes.is_empty() || !bytes.len().is_multiple_of(2) { + return Err(invalid( + 0, + "wordcode must contain complete two-byte code units", + )); + } + let mut instructions = Vec::new(); + let mut cursor = 0; + while cursor < bytes.len() { + let start = cursor / 2; + let mut argument = 0_u32; + let mut prefixes = 0; + loop { + if cursor >= bytes.len() { + return Err(invalid(start, "dangling EXTENDED_ARG")); + } + let opcode = Opcode::decode(bytes[cursor]) + .ok_or_else(|| invalid(cursor / 2, "unknown opcode"))?; + argument = (argument << 8) | u32::from(bytes[cursor + 1]); + cursor += 2; + if opcode == Opcode::ExtendedArg { + prefixes += 1; + if prefixes > 3 { + return Err(invalid(start, "too many EXTENDED_ARG prefixes")); + } + continue; + } + if !opcode.takes_argument() && (argument != 0 || prefixes != 0) { + return Err(invalid(start, "unexpected instruction operand")); + } + instructions.push(DecodedInstruction { + offset: start, + next_offset: cursor / 2, + instruction: Instruction::new(opcode, argument), + }); + break; + } + } + Ok(instructions) +} + +pub(crate) fn compare_operation(argument: u32) -> Option { + Some(match argument { + 0 => CompareOp::Lt, + 1 => CompareOp::Le, + 2 => CompareOp::Eq, + 3 => CompareOp::Ne, + 4 => CompareOp::Gt, + 5 => CompareOp::Ge, + _ => return None, + }) +} + +pub(crate) fn invalid(offset: usize, reason: &'static str) -> VmError { + VmError::InvalidBytecode { offset, reason } +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index da516ae..7873bd0 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,4 +1,5 @@ -//! Embeddable object runtime. No language type system, parser or executor. +//! Embeddable object runtime and a small wordcode interpreter. No parser or +//! Python language type system; wordcode is not CPython bytecode compatible. //! //! ``` //! use aster_core::{Value, Vm}; @@ -13,19 +14,49 @@ //! assert_eq!(vm.object_count(), 0); //! # Ok::<(), aster_core::VmError>(()) //! ``` +//! +//! Execute validated wordcode without a source parser: +//! ``` +//! use aster_core::{CodeLayout, EndorCode, Instruction, Opcode, Value, Vm}; +//! let mut vm = Vm::new()?; +//! let globals = vm.new_dict()?; +//! let code = EndorCode::from_instructions( +//! &[ +//! Instruction::new(Opcode::LoadConst, 0), +//! Instruction::simple(Opcode::ReturnValue), +//! ], +//! vec![Value::Text("Hello Endor".into())], +//! CodeLayout::default(), +//! )?; +//! let code = vm.new_code(code)?; +//! let result = vm.execute(code, globals)?; +//! assert_eq!(result, Value::Text("Hello Endor".into())); +//! vm.release_value(result)?; // Object results own one external reference. +//! vm.release(code)?; +//! vm.release(globals)?; +//! assert_eq!(vm.object_count(), 0); +//! # Ok::<(), aster_core::VmError>(()) +//! ``` mod builtins; +mod code; +mod eval; +mod frame; mod heap; +mod instruction; mod number; mod object; mod vm; mod vm_builtins; pub use builtins::{ - EndorBaseExceptionObject, EndorCellObject, EndorCode, EndorDictObject, EndorFunctionObject, + EndorBaseExceptionObject, EndorCellObject, EndorDictObject, EndorFunctionObject, EndorStringObject, EndorTracebackEntry, }; +pub use code::{CodeLayout, EndorCode, EndorCodeObject}; +pub use frame::{EndorFrame, ExecutionLimits, ExecutionLocation}; pub use heap::ObjectRef; +pub use instruction::{DecodedInstruction, Instruction, Opcode}; pub use number::{ EndorDecimal, EndorInteger, EndorNumber, EndorNumberObject, EndorRational, EndorReal, }; diff --git a/crates/core/src/object.rs b/crates/core/src/object.rs index 9187bbb..25b078f 100644 --- a/crates/core/src/object.rs +++ b/crates/core/src/object.rs @@ -6,8 +6,9 @@ use std::hash::{Hash, Hasher}; use crate::{EndorNumber, ObjectRef, VmError}; -/// Immediate values stay outside the object heap. Object values are borrowed -/// handles: copying or cloning one does not retain an external strong reference. +/// Immediate values stay outside the object heap. Copying an ObjectRef or +/// cloning a Value never retains a reference; ownership follows the API that +/// produced it. In particular, call/execute results own a root until released. #[derive(Debug, Clone, PartialEq)] pub enum Value { Nil, @@ -89,8 +90,9 @@ pub trait ObjectPayload: Any + fmt::Debug + Send + Sync { Ok(None) } - /// Object arguments and object results are borrowed handles. Native calls - /// compute values without mutating the VM; this is not a language executor. + /// Host-call entry: arguments and raw results are borrowed handles. The VM + /// validates/retains the result before exposing it as an owned call result. + /// Language functions are dispatched by Vm's frame evaluator instead. fn call(&self, _arguments: &[Value]) -> Result { Err(VmError::UnsupportedOperation("call")) } @@ -173,6 +175,7 @@ impl ObjectHeader { self.strong_refs } + /// Host-owned references and live execution roots, excluding heap edges. pub fn external_ref_count(&self) -> usize { self.external_refs } diff --git a/crates/core/src/vm.rs b/crates/core/src/vm.rs index c1fe400..0ea2e61 100644 --- a/crates/core/src/vm.rs +++ b/crates/core/src/vm.rs @@ -1,7 +1,7 @@ use std::fmt; use crate::heap::Heap; -use crate::{CompareOp, EndorObject, ObjectPayload, ObjectRef, Record, Value}; +use crate::{CompareOp, EndorObject, ExecutionLocation, ObjectPayload, ObjectRef, Record, Value}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum VmError { @@ -15,6 +15,19 @@ pub enum VmError { InvalidArgument(&'static str), ObjectKindMismatch(&'static str), EmptyCell, + InvalidBytecode { + offset: usize, + reason: &'static str, + }, + UnboundLocal(String), + NameNotFound(String), + ArgumentCount { + name: String, + required: usize, + maximum: usize, + provided: usize, + }, + ExecutionLimit(&'static str), } impl fmt::Display for VmError { @@ -32,6 +45,21 @@ impl fmt::Display for VmError { Self::InvalidArgument(message) => f.write_str(message), Self::ObjectKindMismatch(expected) => write!(f, "expected {expected} object"), Self::EmptyCell => f.write_str("cell is empty"), + Self::InvalidBytecode { offset, reason } => { + write!(f, "invalid bytecode at code unit {offset}: {reason}") + } + Self::UnboundLocal(name) => write!(f, "local variable is unbound: {name}"), + Self::NameNotFound(name) => write!(f, "name not found: {name}"), + Self::ArgumentCount { + name, + required, + maximum, + provided, + } => write!( + f, + "{name} expects {required}..={maximum} positional arguments, received {provided}" + ), + Self::ExecutionLimit(limit) => write!(f, "execution limit reached: {limit}"), } } } @@ -43,11 +71,15 @@ impl std::error::Error for VmError {} #[derive(Debug)] pub struct Vm { pub(crate) heap: Heap, + pub(crate) last_traceback: Vec, } impl Vm { pub fn new() -> Result { - Ok(Self { heap: Heap::new()? }) + Ok(Self { + heap: Heap::new()?, + last_traceback: Vec::new(), + }) } pub fn object_count(&self) -> usize { @@ -145,17 +177,6 @@ impl Vm { self.heap.del_attr(reference, name) } - /// Results are borrowed values; native operations cannot mutate this VM. - pub fn call(&self, reference: ObjectRef, arguments: &[Value]) -> Result { - let payload = self.heap.get(reference)?.payload(); - for argument in arguments { - self.validate_value(argument)?; - } - let result = payload.call(arguments)?; - self.validate_value(&result)?; - Ok(result) - } - pub(crate) fn validate_value(&self, value: &Value) -> Result<(), VmError> { if let Value::Object(reference) = value { self.heap.get(*reference)?; @@ -163,8 +184,8 @@ impl Vm { Ok(()) } - /// Explicit full collection of unreachable cycles, using external strong - /// references as roots. A tracing execution stack can be added later. + /// Explicit full collection. Execution slots use owned roots, so values + /// on live execution stacks participate in the same root accounting. pub fn collect_cycles(&mut self) -> usize { self.heap.collect_cycles() } diff --git a/crates/core/src/vm_builtins.rs b/crates/core/src/vm_builtins.rs index 83b4ee9..7fe663e 100644 --- a/crates/core/src/vm_builtins.rs +++ b/crates/core/src/vm_builtins.rs @@ -2,15 +2,28 @@ use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use crate::{ - CompareOp, EndorBaseExceptionObject, EndorCellObject, EndorDictObject, EndorFunctionObject, - EndorNumber, EndorNumberObject, EndorStringObject, ObjectPayload, ObjectRef, Value, Vm, - VmError, + CompareOp, EndorBaseExceptionObject, EndorCellObject, EndorCode, EndorCodeObject, + EndorDictObject, EndorFunctionObject, EndorNumber, EndorNumberObject, EndorStringObject, + ObjectPayload, ObjectRef, Value, Vm, VmError, }; impl Vm { pub(crate) fn validate_builtin(&self, payload: &dyn ObjectPayload) -> Result<(), VmError> { if let Some(function) = payload.as_any().downcast_ref::() { + let code = self + .payload::(function.code(), "code")? + .code(); + if function.defaults().len() > code.parameters().len() + || function.closure().len() != code.free_variables().len() + { + return Err(VmError::InvalidArgument( + "function defaults or closure do not match code", + )); + } self.payload::(function.globals(), "dict")?; + if let Some(builtins) = function.builtins() { + self.payload::(builtins, "dict")?; + } for cell in function.closure() { self.payload::(*cell, "cell")?; } @@ -42,6 +55,37 @@ impl Vm { self.allocate(EndorStringObject::new(text)) } + pub fn new_code(&mut self, code: EndorCode) -> Result { + self.allocate(EndorCodeObject::new(code)) + } + + pub fn new_function( + &mut self, + name: impl Into, + code: ObjectRef, + globals: ObjectRef, + defaults: Vec, + closure: Vec, + ) -> Result { + self.payload::(globals, "dict")?; + let builtins = match self.dict_get(globals, &Value::Text("__builtins__".into()))? { + Some(Value::Object(reference)) => { + self.payload::(reference, "dict")?; + Some(reference) + } + None => None, + _ => return Err(VmError::ObjectKindMismatch("dict")), + }; + self.allocate(EndorFunctionObject::new( + name.into(), + code, + globals, + builtins, + defaults, + closure, + )) + } + pub fn new_number(&mut self, number: EndorNumber) -> Result { self.allocate(EndorNumberObject::new(number)) } diff --git a/crates/core/tests/builtins.rs b/crates/core/tests/builtins.rs index 99d0b22..f12e85c 100644 --- a/crates/core/tests/builtins.rs +++ b/crates/core/tests/builtins.rs @@ -1,7 +1,7 @@ use aster_core::{ - CompareOp, EndorBaseExceptionObject, EndorCode, EndorDictObject, EndorFunctionObject, - EndorInteger, EndorNativeFunctionObject, EndorNumber, EndorNumberObject, EndorReal, - EndorStringObject, EndorTracebackEntry, Value, Vm, VmError, + CodeLayout, CompareOp, EndorBaseExceptionObject, EndorCode, EndorDictObject, + EndorFunctionObject, EndorInteger, EndorNativeFunctionObject, EndorNumber, EndorNumberObject, + EndorReal, EndorStringObject, EndorTracebackEntry, Instruction, Opcode, Value, Vm, VmError, }; fn text(value: &str) -> Value { @@ -208,31 +208,35 @@ fn functions_retain_globals_constants_defaults_and_shared_closure_cells() { let globals = vm.new_dict().unwrap(); let constant = vm.new_string("constant").unwrap(); let cell = vm.new_cell(Some(text("first"))).unwrap(); - let code = EndorCode::new( - vec![0], + let code = EndorCode::from_instructions( + &[ + Instruction::new(Opcode::LoadDeref, 1), + Instruction::simple(Opcode::ReturnValue), + ], vec![Value::Object(constant)], - vec!["argument".into()], - vec!["captured".into()], + CodeLayout { + parameters: vec!["argument".into()], + locals: vec!["argument".into()], + free_variables: vec!["captured".into()], + ..CodeLayout::default() + }, ) .unwrap(); + let code = vm.new_code(code).unwrap(); let function = vm - .allocate( - EndorFunctionObject::new( - "read", - code.clone(), - globals, - vec![Value::Object(constant)], - vec![cell], - ) - .unwrap(), + .new_function( + "read", + code, + globals, + vec![Value::Object(constant)], + vec![cell], ) .unwrap(); let second = vm - .allocate( - EndorFunctionObject::new("read_again", code, globals, vec![], vec![cell]).unwrap(), - ) + .new_function("read_again", code, globals, vec![], vec![cell]) .unwrap(); - assert_eq!(vm.ref_count(constant).unwrap(), 4); + assert_eq!(vm.ref_count(constant).unwrap(), 3); + vm.release(code).unwrap(); vm.release(constant).unwrap(); vm.release(cell).unwrap(); vm.cell_set(cell, text("updated")).unwrap(); @@ -242,32 +246,49 @@ fn functions_retain_globals_constants_defaults_and_shared_closure_cells() { .unwrap(); assert_eq!(vm.cell_get(f.closure()[0]).unwrap(), text("updated")); } - assert_eq!( - vm.call(function, &[]), - Err(VmError::UnsupportedOperation("language function execution")) - ); + assert_eq!(vm.call(function, &[]).unwrap(), text("updated")); vm.dict_set(globals, text("read"), Value::Object(function)) .unwrap(); vm.release(globals).unwrap(); vm.release(function).unwrap(); vm.release(second).unwrap(); - assert_eq!(vm.collect_cycles(), 4); + assert_eq!(vm.collect_cycles(), 5); } #[test] fn malformed_function_environments_are_rejected_without_retaining_any_edges() { let mut vm = Vm::new().unwrap(); let globals = vm.new_object().unwrap(); - let code = EndorCode::new(vec![], vec![], vec![], vec![]).unwrap(); - let function = EndorFunctionObject::new("bad", code, globals, vec![], vec![]).unwrap(); + let instructions = [ + Instruction::new(Opcode::LoadConst, 0), + Instruction::simple(Opcode::ReturnValue), + ]; + let code = EndorCode::from_instructions(&instructions, vec![Value::Nil], CodeLayout::default()) + .unwrap(); + let code = vm.new_code(code).unwrap(); assert_eq!( - vm.allocate(function), + vm.new_function("bad", code, globals, vec![], vec![]), Err(VmError::ObjectKindMismatch("dict")) ); assert_eq!(vm.ref_count(globals).unwrap(), 1); + assert_eq!(vm.ref_count(code).unwrap(), 1); assert!(EndorCode::new(vec![], vec![], vec!["x".into(), "x".into()], vec![]).is_err()); - let code = EndorCode::new(vec![], vec![], vec![], vec!["x".into()]).unwrap(); - assert!(EndorFunctionObject::new("bad", code, globals, vec![], vec![]).is_err()); + let code = EndorCode::from_instructions( + &instructions, + vec![Value::Nil], + CodeLayout { + free_variables: vec!["x".into()], + ..CodeLayout::default() + }, + ) + .unwrap(); + let code = vm.new_code(code).unwrap(); + let globals = vm.new_dict().unwrap(); + assert!( + vm.new_function("bad", code, globals, vec![], vec![]) + .is_err() + ); + assert_eq!(vm.ref_count(code).unwrap(), 1); } #[test] @@ -320,7 +341,7 @@ fn echo(arguments: &[Value]) -> Result { } #[test] -fn native_functions_pass_lossless_numbers_and_borrowed_objects() { +fn native_functions_pass_lossless_numbers_and_return_owned_objects() { let mut vm = Vm::new().unwrap(); let function = vm .allocate(EndorNativeFunctionObject::new("echo", echo)) @@ -331,10 +352,10 @@ fn native_functions_pass_lossless_numbers_and_borrowed_objects() { number ); let dict = vm.new_dict().unwrap(); - assert_eq!( - vm.call(function, &[Value::Object(dict)]).unwrap(), - Value::Object(dict) - ); + let returned = vm.call(function, &[Value::Object(dict)]).unwrap(); + assert_eq!(returned, Value::Object(dict)); + assert_eq!(vm.ref_count(dict).unwrap(), 2); + vm.release_value(returned).unwrap(); assert_eq!(vm.ref_count(dict).unwrap(), 1); assert_eq!( vm.payload::(dict, "dict").unwrap().len(), diff --git a/crates/core/tests/execution.rs b/crates/core/tests/execution.rs new file mode 100644 index 0000000..82f75af --- /dev/null +++ b/crates/core/tests/execution.rs @@ -0,0 +1,761 @@ +use std::any::Any; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use aster_core::{ + CodeLayout, CompareOp, EndorCode, EndorCodeObject, EndorNativeFunctionObject, ExecutionLimits, + Instruction, ObjectPayload, ObjectRef, Opcode, Value, Vm, VmError, +}; + +fn op(opcode: Opcode, argument: u32) -> Instruction { + Instruction::new(opcode, argument) +} +fn simple(opcode: Opcode) -> Instruction { + Instruction::simple(opcode) +} +fn text(value: &str) -> Value { + Value::Text(value.into()) +} +fn make_code( + vm: &mut Vm, + instructions: &[Instruction], + constants: Vec, + layout: CodeLayout, +) -> ObjectRef { + vm.new_code(EndorCode::from_instructions(instructions, constants, layout).unwrap()) + .unwrap() +} +fn return_constant(vm: &mut Vm, value: Value) -> ObjectRef { + make_code( + vm, + &[op(Opcode::LoadConst, 0), simple(Opcode::ReturnValue)], + vec![value], + CodeLayout::default(), + ) +} + +#[test] +fn wordcode_extended_operands_decode_and_execute_without_truncation() { + let encoded = Instruction::encode_all(&[op(Opcode::LoadConst, 0x01020304)]).unwrap(); + assert_eq!(encoded, vec![255, 1, 255, 2, 255, 3, 1, 4]); + let mut vm = Vm::new().unwrap(); + let mut constants = vec![Value::Nil; 257]; + constants[256] = text("extended"); + let code = make_code( + &mut vm, + &[op(Opcode::LoadConst, 256), simple(Opcode::ReturnValue)], + constants, + CodeLayout::default(), + ); + let executable = vm.payload::(code, "code").unwrap().code(); + assert_eq!(executable.instructions()[0].offset, 0); + assert_eq!(executable.instructions()[0].next_offset, 2); + assert_eq!(executable.stack_size(), 1); + let globals = vm.new_dict().unwrap(); + assert_eq!(vm.execute(code, globals).unwrap(), text("extended")); +} + +#[test] +fn malformed_wordcode_and_unaligned_jumps_are_rejected_at_construction() { + for bytes in [ + vec![], + vec![0], + vec![254, 0], + vec![255, 1], + vec![255, 1, 255, 1, 255, 1, 255, 1, 1, 0], + vec![13, 1], + ] { + assert!(matches!( + EndorCode::new(bytes, vec![Value::Nil], vec![], vec![]), + Err(VmError::InvalidBytecode { .. }) + )); + } + // Jump target 2 is inside a prefixed LOAD_CONST starting at code unit 1. + let bytes = vec![14, 1, 255, 1, 1, 0, 13, 0]; + assert!(EndorCode::new(bytes, vec![Value::Nil; 257], vec![], vec![]).is_err()); + assert!( + EndorCode::from_instructions( + &[op(Opcode::LoadConst, 1), simple(Opcode::ReturnValue)], + vec![Value::Nil], + CodeLayout::default() + ) + .is_err() + ); +} + +#[test] +fn verifier_rejects_underflow_fallthrough_bad_cells_and_inconsistent_stack_joins() { + let cases = [ + vec![simple(Opcode::PopTop), simple(Opcode::ReturnValue)], + vec![op(Opcode::LoadConst, 0)], + vec![op(Opcode::LoadConst, 0), op(Opcode::JumpBackward, 2)], + vec![op(Opcode::LoadDeref, 0), simple(Opcode::ReturnValue)], + vec![ + op(Opcode::LoadConst, 0), + op(Opcode::PopJumpIfFalse, 2), + op(Opcode::LoadConst, 0), + op(Opcode::JumpForward, 1), + simple(Opcode::Nop), + op(Opcode::LoadConst, 0), + simple(Opcode::ReturnValue), + ], + ]; + for instructions in cases { + assert!( + EndorCode::from_instructions(&instructions, vec![Value::Nil], CodeLayout::default()) + .is_err() + ); + } + let layout = CodeLayout { + locals: vec!["captured".into()], + cell_variables: vec!["captured".into()], + ..CodeLayout::default() + }; + assert!( + EndorCode::from_instructions( + &[op(Opcode::LoadFast, 0), simple(Opcode::ReturnValue)], + vec![], + layout + ) + .is_err() + ); +} + +#[test] +fn execute_moves_values_between_constants_local_slots_and_globals() { + let mut vm = Vm::new().unwrap(); + assert_eq!(vm.object_count(), 0); + let globals = vm.new_dict().unwrap(); + let code = make_code( + &mut vm, + &[ + op(Opcode::LoadConst, 0), + op(Opcode::StoreFast, 0), + op(Opcode::LoadFast, 0), + op(Opcode::StoreGlobal, 0), + op(Opcode::LoadGlobal, 0), + simple(Opcode::ReturnValue), + ], + vec![text("hello")], + CodeLayout { + locals: vec!["local".into()], + names: vec!["saved".into()], + ..CodeLayout::default() + }, + ); + assert_eq!(vm.execute(code, globals).unwrap(), text("hello")); + assert_eq!( + vm.dict_get(globals, &text("saved")).unwrap(), + Some(text("hello")) + ); + assert!(vm.last_traceback().is_empty()); + vm.release(code).unwrap(); + vm.release(globals).unwrap(); + assert_eq!(vm.object_count(), 0); +} + +#[derive(Debug)] +struct Tick(Arc); +impl ObjectPayload for Tick { + fn as_any(&self) -> &dyn Any { + self + } + fn kind_name(&self) -> &'static str { + "tick" + } + fn call(&self, arguments: &[Value]) -> Result { + assert!(arguments.is_empty()); + Ok(Value::Bool(self.0.fetch_add(1, Ordering::SeqCst) < 3)) + } +} + +#[test] +fn relative_branches_and_back_edges_execute_a_loop() { + let mut vm = Vm::new().unwrap(); + let count = Arc::new(AtomicUsize::new(0)); + let tick = vm.allocate(Tick(count.clone())).unwrap(); + let globals = vm.new_dict().unwrap(); + vm.dict_set(globals, text("tick"), Value::Object(tick)) + .unwrap(); + let code = make_code( + &mut vm, + &[ + op(Opcode::LoadGlobal, 0), + simple(Opcode::PushNull), + op(Opcode::Call, 0), + op(Opcode::PopJumpIfFalse, 1), + op(Opcode::JumpBackward, 5), + op(Opcode::LoadConst, 0), + simple(Opcode::ReturnValue), + ], + vec![text("done")], + CodeLayout { + names: vec!["tick".into()], + ..CodeLayout::default() + }, + ); + assert_eq!(vm.execute(code, globals).unwrap(), text("done")); + assert_eq!(count.load(Ordering::SeqCst), 4); +} + +#[test] +fn nested_language_calls_bind_defaults_and_return_owned_argument_objects() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let identity = make_code( + &mut vm, + &[op(Opcode::LoadFast, 0), simple(Opcode::ReturnValue)], + vec![], + CodeLayout { + parameters: vec!["value".into()], + locals: vec!["value".into()], + ..CodeLayout::default() + }, + ); + let default = vm.new_string("default").unwrap(); + let function = vm + .new_function( + "identity", + identity, + globals, + vec![Value::Object(default)], + vec![], + ) + .unwrap(); + let before = vm.ref_count(default).unwrap(); + let returned = vm.call(function, &[]).unwrap(); + assert_eq!(returned, Value::Object(default)); + assert_eq!(vm.ref_count(default).unwrap(), before + 1); + vm.release_value(returned).unwrap(); + assert_eq!(vm.ref_count(default).unwrap(), before); + vm.dict_set(globals, text("identity"), Value::Object(function)) + .unwrap(); + let argument = vm.new_string("argument").unwrap(); + let wrapper = make_code( + &mut vm, + &[ + op(Opcode::LoadGlobal, 0), + simple(Opcode::PushNull), + op(Opcode::LoadConst, 0), + op(Opcode::Call, 1), + simple(Opcode::ReturnValue), + ], + vec![Value::Object(argument)], + CodeLayout { + names: vec!["identity".into()], + ..CodeLayout::default() + }, + ); + let before = vm.ref_count(argument).unwrap(); + let returned = vm.execute(wrapper, globals).unwrap(); + assert_eq!(returned, Value::Object(argument)); + assert_eq!(vm.ref_count(argument).unwrap(), before + 1); + vm.release_value(returned).unwrap(); + assert_eq!(vm.ref_count(argument).unwrap(), before); +} + +#[test] +fn argument_count_and_foreign_argument_failures_do_not_retain_inputs() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let code = make_code( + &mut vm, + &[op(Opcode::LoadFast, 0), simple(Opcode::ReturnValue)], + vec![], + CodeLayout { + parameters: vec!["value".into()], + locals: vec!["value".into()], + ..CodeLayout::default() + }, + ); + let function = vm + .new_function("identity", code, globals, vec![], vec![]) + .unwrap(); + assert!(matches!( + vm.call(function, &[]), + Err(VmError::ArgumentCount { + required: 1, + provided: 0, + .. + }) + )); + assert!(matches!( + vm.call(function, &[Value::Nil, Value::Nil]), + Err(VmError::ArgumentCount { + maximum: 1, + provided: 2, + .. + }) + )); + let mut other = Vm::new().unwrap(); + let foreign = other.new_object().unwrap(); + let before = vm.ref_count(function).unwrap(); + assert_eq!( + vm.call(function, &[Value::Object(foreign)]), + Err(VmError::ForeignHandle) + ); + assert_eq!(vm.ref_count(function).unwrap(), before); +} + +#[test] +fn freshly_built_maps_and_their_children_survive_frame_cleanup() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let child = vm.new_string("owned by result").unwrap(); + let code = make_code( + &mut vm, + &[ + op(Opcode::LoadConst, 0), + op(Opcode::LoadConst, 1), + op(Opcode::BuildMap, 1), + simple(Opcode::ReturnValue), + ], + vec![text("child"), Value::Object(child)], + CodeLayout::default(), + ); + vm.release(child).unwrap(); + let result = vm.execute(code, globals).unwrap(); + vm.release(code).unwrap(); + vm.release(globals).unwrap(); + let Value::Object(dictionary) = result else { + panic!("expected a map"); + }; + assert_eq!(vm.ref_count(dictionary).unwrap(), 1); + assert_eq!( + vm.dict_get(dictionary, &text("child")).unwrap(), + Some(Value::Object(child)) + ); + assert_eq!(vm.str(child).unwrap(), "owned by result"); + vm.release(dictionary).unwrap(); + assert_eq!(vm.object_count(), 0); +} + +fn echo(arguments: &[Value]) -> Result { + arguments + .first() + .cloned() + .ok_or(VmError::InvalidArgument("expected an argument")) +} + +#[test] +fn native_return_of_a_temporary_argument_is_retained_before_inputs_are_popped() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let function = vm + .allocate(EndorNativeFunctionObject::new("echo", echo)) + .unwrap(); + vm.dict_set(globals, text("echo"), Value::Object(function)) + .unwrap(); + vm.release(function).unwrap(); + let code = make_code( + &mut vm, + &[ + op(Opcode::LoadGlobal, 0), + simple(Opcode::PushNull), + op(Opcode::BuildMap, 0), + op(Opcode::Call, 1), + simple(Opcode::ReturnValue), + ], + vec![], + CodeLayout { + names: vec!["echo".into()], + ..CodeLayout::default() + }, + ); + let result = vm.execute(code, globals).unwrap(); + vm.release(code).unwrap(); + vm.release(globals).unwrap(); + assert_eq!(vm.object_count(), 1); + vm.release_value(result).unwrap(); + assert_eq!(vm.object_count(), 0); +} + +fn fail(_arguments: &[Value]) -> Result { + Err(VmError::InvalidArgument("host failure")) +} + +#[test] +fn native_failure_unwinds_temporary_roots_and_keeps_execution_locations() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let function = vm + .allocate(EndorNativeFunctionObject::new("fail", fail)) + .unwrap(); + vm.dict_set(globals, text("fail"), Value::Object(function)) + .unwrap(); + let code = EndorCode::from_instructions( + &[ + op(Opcode::LoadGlobal, 0), + simple(Opcode::PushNull), + op(Opcode::BuildMap, 0), + op(Opcode::Call, 1), + simple(Opcode::ReturnValue), + ], + vec![], + CodeLayout { + names: vec!["fail".into()], + ..CodeLayout::default() + }, + ) + .unwrap() + .with_location("outer", "demo.endor", 12); + let code = vm.new_code(code).unwrap(); + let before = vm.object_count(); + assert_eq!( + vm.execute(code, globals), + Err(VmError::InvalidArgument("host failure")) + ); + assert_eq!(vm.object_count(), before); + assert_eq!(vm.last_traceback()[0].function, "outer"); + assert_eq!(vm.last_traceback()[0].instruction_offset, 3); + assert_eq!(vm.last_traceback()[0].filename, "demo.endor"); + assert_eq!(vm.last_traceback()[0].first_line, 12); +} + +#[test] +fn recursion_uses_explicit_frames_and_unwinds_at_the_frame_limit() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let code = make_code( + &mut vm, + &[ + op(Opcode::LoadGlobal, 0), + simple(Opcode::PushNull), + op(Opcode::Call, 0), + simple(Opcode::ReturnValue), + ], + vec![], + CodeLayout { + names: vec!["recurse".into()], + ..CodeLayout::default() + }, + ); + let function = vm + .new_function("recurse", code, globals, vec![], vec![]) + .unwrap(); + vm.dict_set(globals, text("recurse"), Value::Object(function)) + .unwrap(); + let roots = vm.ref_count(function).unwrap(); + assert_eq!( + vm.call_with_limits( + function, + &[], + ExecutionLimits { + max_frames: 64, + max_instructions: 1000 + } + ), + Err(VmError::ExecutionLimit("frames")) + ); + assert_eq!(vm.last_traceback().len(), 64); + assert_eq!(vm.ref_count(function).unwrap(), roots); + vm.release(function).unwrap(); + vm.release(code).unwrap(); + vm.release(globals).unwrap(); + assert_eq!(vm.collect_cycles(), 3); +} + +#[test] +fn instruction_limit_stops_a_valid_infinite_loop_without_leaking_frames() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let code = make_code( + &mut vm, + &[op(Opcode::JumpBackward, 1)], + vec![], + CodeLayout::default(), + ); + let before = vm.object_count(); + assert_eq!( + vm.execute_with_limits( + code, + globals, + ExecutionLimits { + max_frames: 8, + max_instructions: 3 + } + ), + Err(VmError::ExecutionLimit("instructions")) + ); + assert_eq!(vm.object_count(), before); + assert_eq!(vm.last_traceback().len(), 1); +} + +#[test] +fn function_globals_shadow_builtins_and_capture_the_original_builtins_dictionary() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let builtins = vm.new_dict().unwrap(); + vm.dict_set(builtins, text("answer"), text("builtin")) + .unwrap(); + vm.dict_set(globals, text("__builtins__"), Value::Object(builtins)) + .unwrap(); + let code = make_code( + &mut vm, + &[op(Opcode::LoadGlobal, 0), simple(Opcode::ReturnValue)], + vec![], + CodeLayout { + names: vec!["answer".into()], + ..CodeLayout::default() + }, + ); + let function = vm + .new_function("read", code, globals, vec![], vec![]) + .unwrap(); + let replacement = vm.new_dict().unwrap(); + vm.dict_set(globals, text("__builtins__"), Value::Object(replacement)) + .unwrap(); + vm.release(builtins).unwrap(); + assert_eq!(vm.call(function, &[]).unwrap(), text("builtin")); + vm.dict_set(globals, text("answer"), text("global")) + .unwrap(); + assert_eq!(vm.call(function, &[]).unwrap(), text("global")); +} + +#[test] +fn shared_free_variable_cells_are_read_written_and_deleted_by_bytecode() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let cell = vm.new_cell(Some(text("first"))).unwrap(); + let read_code = make_code( + &mut vm, + &[op(Opcode::LoadDeref, 0), simple(Opcode::ReturnValue)], + vec![], + CodeLayout { + free_variables: vec!["x".into()], + ..CodeLayout::default() + }, + ); + let write_code = make_code( + &mut vm, + &[ + op(Opcode::LoadFast, 0), + op(Opcode::StoreDeref, 1), + op(Opcode::LoadConst, 0), + simple(Opcode::ReturnValue), + ], + vec![Value::Nil], + CodeLayout { + parameters: vec!["value".into()], + locals: vec!["value".into()], + free_variables: vec!["x".into()], + ..CodeLayout::default() + }, + ); + let clear_code = make_code( + &mut vm, + &[ + op(Opcode::DeleteDeref, 0), + op(Opcode::LoadConst, 0), + simple(Opcode::ReturnValue), + ], + vec![Value::Nil], + CodeLayout { + free_variables: vec!["x".into()], + ..CodeLayout::default() + }, + ); + let read = vm + .new_function("read", read_code, globals, vec![], vec![cell]) + .unwrap(); + let write = vm + .new_function("write", write_code, globals, vec![], vec![cell]) + .unwrap(); + let clear = vm + .new_function("clear", clear_code, globals, vec![], vec![cell]) + .unwrap(); + assert_eq!(vm.call(read, &[]).unwrap(), text("first")); + vm.call(write, &[text("updated")]).unwrap(); + assert_eq!(vm.call(read, &[]).unwrap(), text("updated")); + vm.call(clear, &[]).unwrap(); + assert_eq!(vm.call(read, &[]), Err(VmError::UnboundLocal("x".into()))); +} + +#[test] +fn captured_argument_cells_keep_their_contents_after_the_frame_returns() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let code = make_code( + &mut vm, + &[op(Opcode::LoadClosure, 0), simple(Opcode::ReturnValue)], + vec![], + CodeLayout { + parameters: vec!["x".into()], + locals: vec!["x".into()], + cell_variables: vec!["x".into()], + ..CodeLayout::default() + }, + ); + let function = vm + .new_function("capture", code, globals, vec![], vec![]) + .unwrap(); + let value = vm.new_string("captured").unwrap(); + let returned = vm.call(function, &[Value::Object(value)]).unwrap(); + vm.release(value).unwrap(); + let Value::Object(cell) = returned else { + panic!("expected cell"); + }; + assert_eq!(vm.cell_get(cell).unwrap(), Value::Object(value)); + assert_eq!(vm.str(value).unwrap(), "captured"); + vm.release(cell).unwrap(); + assert!(matches!(vm.object(value), Err(VmError::InvalidHandle))); +} + +#[test] +fn attribute_operations_and_stack_copies_transfer_roots_without_leaks() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let code = make_code( + &mut vm, + &[ + op(Opcode::LoadFast, 1), + op(Opcode::LoadFast, 0), + op(Opcode::StoreAttr, 0), + op(Opcode::LoadFast, 0), + op(Opcode::LoadAttr, 0), + op(Opcode::Copy, 1), + op(Opcode::Swap, 2), + simple(Opcode::PopTop), + simple(Opcode::ReturnValue), + ], + vec![], + CodeLayout { + parameters: vec!["record".into(), "value".into()], + locals: vec!["record".into(), "value".into()], + names: vec!["x".into()], + ..CodeLayout::default() + }, + ); + let function = vm + .new_function("assign", code, globals, vec![], vec![]) + .unwrap(); + let record = vm.new_object().unwrap(); + let value = vm.new_string("value").unwrap(); + let returned = vm + .call(function, &[Value::Object(record), Value::Object(value)]) + .unwrap(); + assert_eq!(vm.ref_count(value).unwrap(), 3); // host, attribute, return value + vm.release_value(returned).unwrap(); + vm.release(record).unwrap(); + assert_eq!(vm.ref_count(value).unwrap(), 1); +} + +#[test] +fn comparison_truth_and_forward_conditionals_use_runtime_values() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let code = make_code( + &mut vm, + &[ + op(Opcode::LoadConst, 0), + op(Opcode::LoadConst, 1), + Instruction::compare(CompareOp::Lt), + simple(Opcode::ToBool), + simple(Opcode::UnaryNot), + op(Opcode::PopJumpIfTrue, 2), + op(Opcode::LoadConst, 2), + simple(Opcode::ReturnValue), + op(Opcode::LoadConst, 3), + simple(Opcode::ReturnValue), + ], + vec![text("a"), text("b"), text("less"), text("other")], + CodeLayout::default(), + ); + assert_eq!(vm.execute(code, globals).unwrap(), text("less")); +} + +#[test] +fn unbound_names_deleted_locals_and_invalid_null_values_fail_cleanly() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let missing = make_code( + &mut vm, + &[op(Opcode::LoadGlobal, 0), simple(Opcode::ReturnValue)], + vec![], + CodeLayout { + names: vec!["missing".into()], + ..CodeLayout::default() + }, + ); + assert_eq!( + vm.execute(missing, globals), + Err(VmError::NameNotFound("missing".into())) + ); + let deleted = make_code( + &mut vm, + &[ + op(Opcode::LoadConst, 0), + op(Opcode::StoreFast, 0), + op(Opcode::DeleteFast, 0), + op(Opcode::LoadFast, 0), + simple(Opcode::ReturnValue), + ], + vec![Value::Nil], + CodeLayout { + locals: vec!["x".into()], + ..CodeLayout::default() + }, + ); + assert_eq!( + vm.execute(deleted, globals), + Err(VmError::UnboundLocal("x".into())) + ); + let null = make_code( + &mut vm, + &[simple(Opcode::PushNull), simple(Opcode::ReturnValue)], + vec![], + CodeLayout::default(), + ); + assert!(matches!( + vm.execute(null, globals), + Err(VmError::InvalidBytecode { .. }) + )); + let success = return_constant(&mut vm, Value::Nil); + assert_eq!(vm.execute(success, globals).unwrap(), Value::Nil); + assert!(vm.last_traceback().is_empty()); +} + +#[test] +fn failed_map_construction_cleans_both_partial_map_and_temporary_input() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let code = make_code( + &mut vm, + &[ + op(Opcode::LoadConst, 0), + op(Opcode::BuildMap, 0), + op(Opcode::BuildMap, 1), + simple(Opcode::ReturnValue), + ], + vec![Value::Number(1_i64.into())], + CodeLayout::default(), + ); + let before = vm.object_count(); + assert_eq!( + vm.execute(code, globals), + Err(VmError::UnsupportedOperation("numeric hash")) + ); + assert_eq!(vm.object_count(), before); +} + +#[test] +fn implicit_receiver_slot_is_distinct_from_the_null_call_marker() { + let mut vm = Vm::new().unwrap(); + let globals = vm.new_dict().unwrap(); + let function = vm + .allocate(EndorNativeFunctionObject::new("echo", echo)) + .unwrap(); + let code = make_code( + &mut vm, + &[ + op(Opcode::LoadConst, 0), + op(Opcode::LoadConst, 1), + op(Opcode::Call, 0), + simple(Opcode::ReturnValue), + ], + vec![Value::Object(function), Value::Nil], + CodeLayout::default(), + ); + // Nil is an actual implicit argument here, not the internal NULL marker. + assert_eq!(vm.execute(code, globals).unwrap(), Value::Nil); +} diff --git a/docs/architecture.md b/docs/architecture.md index bc70beb..393e7d6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -4,7 +4,7 @@ 当前任务是 Endor 的虚拟机实现层。`Vm::new()` 只创建空堆，不加载语言根类，不分配 Object／Type，也没有类创建、继承或元类 API。`EndorObject` 是公共运行时结构，不能与语言根类混为一谈。 -`EndorObject` 保存 `ObjectHeader`、`Box`、托管属性和载荷引用快照。对象头仅保存身份、总强引用数和宿主外部强引用数。操作通过 Rust trait 的分发机制选择，不需要 Python type 系统；kind_name 只是调试标签，as_any 仅供原生 Rust 扩展识别自身的数据结构。 +`EndorObject` 保存 `ObjectHeader`、`Box`、托管属性和载荷引用快照。对象头仅保存身份、总强引用数和宿主／执行根强引用数。操作通过 Rust trait 的分发机制选择，不需要 Python type 系统；kind_name 只是调试标签，as_any 仅供原生 Rust 扩展识别自身的数据结构。 ### 分配与句柄 @@ -14,7 +14,7 @@ allocate/new_object 返回一个外部强引用。ObjectRef 是可复制的借用句柄；复制或 Value::clone 不 retain。宿主通过 retain/release 显式持有与释放。头部总计数等于外部计数加来自托管属性及载荷引用快照的入边数。 -属性写入先验证目标、被引用对象和计数溢出，再增加新引用、替换属性、释放旧引用；同目标替换不会短暂释放对象。属性读取和原生调用返回借用值，宿主需要跨越所有者生命周期使用时应先 retain。没有外部引用时，release 返回 NoOwnedReference，不能误释放内部属性引用。release 不代表删除任意对象。 +属性写入先验证目标、被引用对象和计数溢出，再增加新引用、替换属性、释放旧引用；同目标替换不会短暂释放对象。属性读取返回借用值，宿主需要跨越所有者生命周期使用时应先 retain。call／execute 的对象结果持有一份外部引用，宿主使用后须 release_value。没有外部引用时，release 返回 NoOwnedReference，不能误释放内部属性引用。release 不代表删除任意对象。 载荷通过 trace 报告自己持有的引用；分配时验证全部边并记录快照，相同引用出现两次就计数两次。分配失败不改变已存在对象的计数。自定义载荷不能自行修改分配后的引用；内置字典和 Cell 通过 VM 事务性替换载荷与快照更新引用，先保留新边，再释放旧边。对象本身的 Box 和身份不变。扩展未声明的借用句柄不受 GC 保护，访问或返回时仍需通过 VM 验证。 @@ -30,11 +30,15 @@ VM 销毁时 Box 的 Rust 所有权会清理全部剩余载荷，即使其计数 ObjectPayload 定义 repr、str、is_true、hash、compare、call 及属性许可。默认表示带诊断名和身份，默认 str 复用 repr，默认真值为 true，默认哈希基于身份，默认 Eq／Ne 按身份，排序和调用默认报 UnsupportedOperation。compare 返回 None 时尝试右侧反向操作，再使用身份回退；没有 Python 继承优先级规则。 -Record 支持托管属性，NativeFunction 包装 Rust 函数指针。原生调用先验证参数句柄，再执行回调，再验证结果句柄。回调只计算返回值，不能修改或分配当前 VM；对象结果是借用句柄。自定义载荷可覆盖操作，若提供内容相等语义，必须保持哈希一致，或显式拒绝哈希。错误通过 Result 返回，不是语言异常机制。 +Record 支持托管属性，NativeFunction 包装 Rust 函数指针。原生调用先验证参数句柄，再执行回调，再验证结果句柄。回调只计算返回值，不能修改或分配当前 VM；回调的原始对象结果是借用句柄，Vm::call 会验证并 retain 后作为持有式结果返回。自定义载荷可覆盖操作，若提供内容相等语义，必须保持哈希一致，或显式拒绝哈希。错误通过 Result 返回，不是语言异常机制。 Value 中标量和字符串不强制堆分配；数字统一为 EndorNumber，同时可装箱为 EndorNumberObject。整数和精确小数通过文本保留任意长度数字，分数保留分子分母，浮点保留 IEEE 位，复数保存两个实数表示。当前不实现数值运算、跨表示数值相等或哈希，数字暂不能当字典键。Rust 值相等只比较表示，不是语言数字相等。 -EndorDictObject 保留插入顺序，当前线性查询；EndorCellObject 提供共享可变槽。EndorFunctionObject 保存代码数据、全局字典、默认参数和闭包 Cell；NativeFunction 的正式名称是 EndorNativeFunctionObject。EndorBaseExceptionObject 保存异常数据及链，不建立类型继承或异常执行流程。EndorStringObject 使用 UTF-8，按 Unicode 标量值访问字符。更详细的接口和限制见 [基本对象](endor-builtins.md) 与 [基础对象](endor-object.md)。 +EndorDictObject 保留插入顺序，当前线性查询；EndorCellObject 提供共享可变槽。EndorFunctionObject 保存共享代码对象句柄、全局／内建字典、默认参数和闭包 Cell；NativeFunction 的正式名称是 EndorNativeFunctionObject。EndorBaseExceptionObject 保存异常数据及链，不建立类型继承或异常执行流程。EndorStringObject 使用 UTF-8，按 Unicode 标量值访问字符。更详细的接口和限制见 [基本对象](endor-builtins.md) 与 [基础对象](endor-object.md)。 + +## 指令、执行帧与求值循环 + +小型 wordcode 采用两字节 opcode/oparg 代码单元和 EXTENDED_ARG。代码对象持有验证后的指令、常量、名称表和局部槽布局，验证控制流汇合的栈高度并计算最大求值栈深度。函数共享代码对象；执行帧保存指令位置、调用者索引、本地／Cell／自由变量槽和求值栈。语言调用切换显式帧，不递归进入 Rust 求值器。返回和报错时清理根引用，移交的返回值继续存活。详见 [字节码 VM](bytecode-vm.md)。 ## 宿主与持久化 @@ -52,4 +56,4 @@ PostgreSQL 最多 5 条连接，获取／建立连接超时 5 秒，启动真实 已决定：EndorObject 实现层基础结构、稳定 Box 分配、代际句柄、外部／内部引用计数、手动全堆循环回收、Rust 操作分发、存储属于宿主。 -未决定：语言根类、Python type／元类体系、继承、字节码格式、调用栈、可变 VM 回调上下文、语言异常与终结器、外部对象协议、共享可变 VM、自动／分代 GC、堆序列化和浏览器通信。不得在交接中把这些当成已实现契约。 +未决定：语言根类、Python type／元类体系、继承、可变 VM 回调上下文、语言异常与终结器、外部对象协议、共享可变 VM、自动／分代 GC、堆序列化和浏览器通信。不得在交接中把这些当成已实现契约。 diff --git a/docs/bytecode-vm.md b/docs/bytecode-vm.md new file mode 100644 index 0000000..81089aa --- /dev/null +++ b/docs/bytecode-vm.md @@ -0,0 +1,134 @@ +# 小型字节码 VM + +本阶段完成指令与代码对象、内部执行帧、取指执行循环，并接通语言函数。参考 CPython 3.14 的职责划分和栈式执行设计；不实现 lexer/parser，不读取 Python `.pyc`，不承诺 CPython 操作码编号、ABI 或完整语言兼容性。 + +## 与 CPython 的对应 + +| 职责 | CPython 3.14 | Endor | +| --- | --- | --- | +| 指令定义与解码 | opcode/oparg、`_Py_CODEUNIT`、`EXTENDED_ARG` | `Opcode`、`Instruction`、`DecodedInstruction` | +| 保存可执行代码 | `PyCodeObject` | 堆中的 `EndorCodeObject`，持有不可变 `EndorCode` | +| 一次调用的状态 | `_PyInterpreterFrame` | `EndorFrame` | +| 执行入口 | `PyEval_EvalCode`、函数调用入口 | `Vm::execute`、`Vm::call` | +| 帧求值 | `_PyEval_EvalFrameDefault` | `eval_frames` 的取指／分派循环 | + +`Vm::new()` 仍从空堆启动，按宿主需求创建代码、全局字典和函数。没有自动加载 Python type、语言根类或默认 print。 + +## 指令：要做什么 + +`Instruction` 是一个操作码与 u32 参数。实际 wordcode 每个代码单元为 `[opcode: u8, argument: u8]`，先操作码、后参数，与机器端序无关。大于 255 的参数在前面使用最多三个 `EXTENDED_ARG` 单元补足高位。单个逻辑指令因此可以占多个代码单元。 + +`Instruction::encode_all` 接受逻辑指令并自动生成扩展参数。`EndorCode::with_layout` 也可以接收原始字节序列，先解码并验证再使用。 + +所有执行位置和跳转距离以**两字节代码单元**计数，包含扩展参数单元；不是字节偏移，也不一定等于逻辑指令下标。前跳从当前逻辑指令之后的位置加距离；后跳从该位置减距离。目标必须是完整逻辑指令的开头，不能跳进扩展参数中间。 + +首版有 28 个可执行操作码和一个由编解码器处理的 EXTENDED_ARG： + +| 指令组 | 内容 | +| --- | --- | +| 常量与局部变量 | `LoadConst`、`LoadFast`、`StoreFast`、`DeleteFast` | +| 名称与属性 | `LoadGlobal`、`StoreGlobal`、`DeleteGlobal`、`LoadAttr`、`StoreAttr` | +| 栈操作 | `Nop`、`PopTop`、`Copy`、`Swap`、`PushNull` | +| 调用与返回 | `Call`、`ReturnValue` | +| 控制流 | `JumpForward`、`JumpBackward`、`PopJumpIfFalse`、`PopJumpIfTrue` | +| 真值与比较 | `ToBool`、`UnaryNot`、`CompareOp` | +| 闭包 | `LoadDeref`、`StoreDeref`、`DeleteDeref`、`LoadClosure` | +| 容器 | `BuildMap` | + +没有把 Number 转为 f64 来补运算。数字仍以无损表示为主；真假可以根据是否为零判断，数值比较和算术尚未定义。文本可作内容比较，普通对象使用现有比较接口。 + +代码构造时检查：未知操作码、奇数字节长度、悬空／过长扩展参数、非法索引、错误 Cell 槽、无效跳转、栈下溢、控制流汇合的栈高度不一致，以及可达路径跌出代码末尾。返回时栈必须只剩一个返回值。验证同时计算最大求值栈深度。NULL 调用标记的实际用法在执行时继续检查，它不能作为语言值返回。 + +## 代码对象：保存静态信息 + +`EndorCode` 保存 wordcode、解码结果、常量、名称表、局部变量布局、参数与自由变量信息、最大栈深度、代码名和源码位置元信息。它是不可变 Rust 数据。 + +`vm.new_code(code)` 把数据放进 `EndorCodeObject`，分配一个受管理的堆对象。代码对象持有常量中的对象引用，并参与引用计数和循环回收；函数持有代码对象句柄，所以多个函数可共享同一代码对象，代码常量也只由这一对象统一持有。 + +`CodeLayout` 规定： + +1. `locals` 的开头必须与 `parameters` 一致，目前只支持位置参数。 +2. `cell_variables` 必须来自本地变量名；捕获的局部变量仍占原来的本地槽位，但槽内保存 Cell。 +3. `free_variables` 的 Cell 槽紧接所有本地槽位；不得与本地名称重复。 +4. `names` 提供全局名称和属性名称的索引表。 + +函数通过 `vm.new_function(name, code_ref, globals, defaults, closure)` 创建。工厂检查代码与全局字典种类、默认参数数量和闭包 Cell 数量，并从 globals 的可选 `__builtins__` 字典捕获内建环境。全局查找先查 globals，缺失再查捕获的 builtins。 + +## 执行帧：这一次运行到哪里 + +`EndorFrame` 不是堆中的语言对象，没有 EndorObject 头。它类似 CPython 的内部 `_PyInterpreterFrame`；以后可另加可见的帧包装，不把反射对象强加给每次调用。 + +帧保存函数及代码引用、共享可执行数据、globals／builtins、调用者帧索引、当前与下一条指令位置、本地／Cell／自由变量槽，以及求值栈。Rust 使用安全容器表达这些逻辑区域，没有复制 CPython 的 C 指针和柔性数组物理布局。 + +帧初始化时绑定位置参数及尾部默认参数，未绑定局部槽为空，捕获的局部槽转为 Cell，并放入闭包提供的自由变量 Cell。读取空槽或已删除变量会报错。Cell 初始化在建帧时完成，首版无需单独的 MAKE_CELL／COPY_FREE_VARS 启动指令。 + +## 执行循环与函数调用 + +```text +Vm::execute(code, globals) + 创建临时函数 + → Vm::call + → 准备第一个 EndorFrame + → 读取当前指令 + → match opcode 执行操作 + → 推进或跳转到下一位置 + → CALL 切入新帧 / RETURN_VALUE 恢复调用者 +``` + +语言函数互调使用一个显式帧向量，不递归调用 Rust 求值器。调用者已推进的指令位置就是返回后继续的位置，`previous` 用向量索引连接调用者。NativeFunction 则通过已有的 Rust 回调入口执行。 + +普通函数调用采用 `[callable, NULL, arg0, ...]` 栈布局，CALL 参数是显式位置参数个数。NULL 是内部标记，与 `Value::Nil` 不同；如果第二个位置是实际值，就把它作为隐式接收者加在参数前。尚未实现属性描述符或自动方法绑定。 + +## 对象引用与返回值 + +每个帧持有函数的一份执行根，函数继续保活代码、全局字典、内建字典、默认值及闭包。局部槽和求值栈中的每个对象值也各自持有执行根，重复出现分别计数。执行根复用现有外部强引用计数，所以循环回收能保留它们。 + +装载借用值时先 retain，弹栈或清空槽时 release；槽之间移动直接转移持有。调用先准备并保活被调函数的参数，再清理调用者的输入。返回值从当前帧移交给调用者栈或宿主，帧清理不会释放这一份结果。 + +**API 变化：**`Vm::call` 现在接收 `&mut self`，语言函数和原生函数统一返回持有的一份结果。返回 `Value::Object` 后，宿主完成使用时调用 `vm.release_value(result)`，或取出句柄后 `release`。原生回调本身仍接收借用参数、返回借用值；VM 会验证并 retain 回调的结果，之后才弹出输入参数。即使回调返回的是刚在字节码里创建的临时参数，也不会在清理输入时失效。 + +`get_attr`、`dict_get`、`dict_items`、`cell_get` 仍返回借用内容，不属于上述调用返回值变化。 + +执行出错时，当前尚未消费的栈值仍由帧持有；所有活动帧按从内到外顺序清理。已完成的全局字典／属性写入不会回滚，但未完成操作和临时执行根会清理。字典构造失败时也会清理尚未返回的临时字典。 + +## 最小示例 + +```rust +use aster_core::{CodeLayout, EndorCode, Instruction, Opcode, Value, Vm}; + +let mut vm = Vm::new()?; +let globals = vm.new_dict()?; +let code = EndorCode::from_instructions( + &[ + Instruction::new(Opcode::LoadConst, 0), + Instruction::simple(Opcode::ReturnValue), + ], + vec![Value::Text("Hello Endor".into())], + CodeLayout::default(), +)?; +let code = vm.new_code(code)?; +let result = vm.execute(code, globals)?; +assert_eq!(result, Value::Text("Hello Endor".into())); +vm.release_value(result)?; +vm.release(code)?; +vm.release(globals)?; +# Ok::<(), aster_core::VmError>(()) +``` + +该示例直接提供指令，不表示已经支持 Endor 或 Python 源码解析。函数和嵌套函数目前由宿主工厂创建，尚无 MAKE_FUNCTION 指令；后续编译器接入时再补函数定义所需的操作。 + +## 限额、诊断与边界 + +`ExecutionLimits` 默认限制 1000 个活动帧，指令配额为 `usize::MAX`，正常使用不设置有限的指令预算。`execute_with_limits`／`call_with_limits` 可显式指定限额，所有嵌套语言调用共享计步器；扩展参数不单独计步。可选指令预算是 Endor 的宿主运行边界，不是 CPython 默认行为，也不会抢占正在执行的宿主回调。 + +运行错误仍返回 `VmError`。`last_traceback()` 另提供从外到内的函数名、文件、代码首行和出错指令偏移；它保存独立元数据，不保活已经清理的帧。首行是代码声明的 first_line，不是由行号表推导出的实际错误源码行。成功的新一次调用会清空旧记录。尚无语言异常对象传播、raise／try／except 或完整源码行号表。 + +暂未实现：lexer/parser、源码编译器、关键字／可变参数、生成器／协程、迭代器专用指令、数值算术、MAKE_FUNCTION、自动方法绑定、指令特化和 JIT。 + +## 验证 + +`crates/core/tests/execution.rs` 覆盖 wordcode、控制流校验、变量、循环、嵌套调用与默认参数、闭包、容器和返回值所有权、错误清理及帧／指令限额；已有 builtins 测试已迁移共享代码对象和持有式调用返回值。lib.rs 另有可供远端 rustdoc 执行的例子。 + +本地仅格式化和静态审阅，不构建、不执行测试、不下载依赖。远端 CI 执行工作区检查和双数据库验证；按用户要求不主动跟踪 CI／部署结果，不能把未查询的结果标为通过。 + +参考：[CPython 解释器设计](https://github.com/python/cpython/blob/v3.14.0/InternalDocs/interpreter.md)、[执行帧设计](https://github.com/python/cpython/blob/v3.14.0/InternalDocs/frames.md)、[帧结构](https://github.com/python/cpython/blob/v3.14.0/Include/internal/pycore_interpframe_structs.h)、[指令定义](https://github.com/python/cpython/blob/v3.14.0/Python/bytecodes.c)、[求值入口与执行器](https://github.com/python/cpython/blob/v3.14.0/Python/ceval.c)。 diff --git a/docs/development.md b/docs/development.md index 2423667..c55c567 100644 --- a/docs/development.md +++ b/docs/development.md @@ -8,9 +8,9 @@ 用户已澄清：当前实现 Endor 的 VM 运行时，不实现 Python type 系统，也不定义语言根类。旧的 Class／Type、create_class、create_instance、root_class 和 type_of API 已移除。公共结构和操作接口在 object.rs，分配、引用计数和循环回收在 heap.rs，对外检查入口在 vm.rs。详见 endor-object.md。 -关键所有权规则：分配给一个外部强引用；句柄拷贝与 Value 克隆只产生借用；额外持有使用 retain。托管属性和 trace 声明的载荷边分别计数。get_attr／call／dict_get／cell_get 返回借用值。长链释放使用迭代队列；循环回收以外部引用为根。内置字典和 Cell 已使用 VM 管理的载荷替换协议：先验证、保留新边，再更新载荷及快照，最后释放旧边；自定义载荷不能只改 trace 的返回结果。 +关键所有权规则：分配给一个外部强引用；句柄拷贝与 Value 克隆只产生借用；额外持有使用 retain。托管属性和 trace 声明的载荷边分别计数。get_attr／dict_get／cell_get 返回借用值；call／execute 的对象结果现在持有一份强引用，用后 release_value。执行帧中的每个对象槽也持有根，返回时转移结果，失败时逐帧清理。长链释放使用迭代队列；循环回收以外部引用为根。内置字典和 Cell 已使用 VM 管理的载荷替换协议：先验证、保留新边，再更新载荷及快照，最后释放旧边；自定义载荷不能只改 trace 的返回结果。 -基础载荷见 builtins.rs，统一无损 Number 见 number.rs，容器 API 和函数／异常关联校验见 vm_builtins.rs。功能与未实现边界详见 endor-builtins.md。新增 builtins 测试交由远端 CI 执行，不在本机运行；数字只完成表示，语言函数只完成环境数据，异常只完成实例数据，不能在交接里称执行器或数值引擎已完成。 +基础载荷见 builtins.rs，统一无损 Number 见 number.rs，容器 API 和函数／异常关联校验见 vm_builtins.rs。功能与未实现边界详见 endor-builtins.md。新增 builtins 测试交由远端 CI 执行，不在本机运行；数字只完成表示，异常仍为实例数据；语言函数已经接入小型 wordcode 执行器。指令在 instruction.rs，代码验证在 code.rs，执行帧在 frame.rs，执行循环在 eval.rs；详见 bytecode-vm.md。不能将当前子集称为完整 Python 执行器或数值引擎。 CLI bootstrap 演示两个原生运行时对象并显式释放；HTTP /bootstrap 返回 object_model=EndorObject、type_system=false、heap_objects=0。修改宿主表示时同步 scripts/smoke.py。 @@ -22,15 +22,15 @@ CLI bootstrap 演示两个原生运行时对象并显式释放；HTTP /bootstrap ## 下一阶段建议 -1. 为执行栈与宿主持有值建立明确的强引用根管理，扩展原生调用上下文，补对应生命周期测试。 -2. 在 core 内构建可测的最小执行循环，先用手写指令，继续不做解析器和渲染器。 +1. 在已完成的执行根管理上扩展宿主调用上下文；保持结果所有权一致，并继续补远端生命周期测试。 +2. 基于已完成的指令、代码对象、执行帧和取指循环，按需求补函数定义、迭代器等指令，再讨论源码前端；不自动开始解析器或渲染器。 3. 按实际分配负载设计 GC 触发、资源限额与暂停策略；现有手动循环回收不等于分代或增量 GC。 4. 浏览器能力与外部对象桥接使用原生载荷；保持 core 无网络或宿主库依赖。 5. VM 快照需要另行定义持久身份与格式，不能序列化当前进程句柄。语言类型、根类和继承以后再确定。 ## Git 状态与后续远程 -origin 为用户提供的 `ssh://git@gitea.entroia.com:22222/Zayden-Jung/endorphin-aster-vm.git`。仓库级 author 和隔离 SSH 命令已配置；不得复制其他身份或猜测 owner。发布 CI/CD 需要提交、推送并跟踪远端结果；遵守 AGENTS.md 的授权及中文逐文件报告要求。初始开发阶段记录与后续部署记录在 validation.md 中分开说明。 +origin 为用户提供的 `ssh://git@gitea.entroia.com:22222/Zayden-Jung/endorphin-aster-vm.git`。仓库级 author 和隔离 SSH 命令已配置；不得复制其他身份或猜测 owner。发布 CI/CD 按任务授权提交和推送；用户已要求不主动跟踪 CI／部署结果，有问题由用户反馈。遵守 AGENTS.md 的授权及中文逐文件报告要求。初始开发阶段记录与后续部署记录在 validation.md 中分开说明。 ## 远端 CI 与部署 diff --git a/docs/endor-builtins.md b/docs/endor-builtins.md index 5fdeb39..518e4f1 100644 --- a/docs/endor-builtins.md +++ b/docs/endor-builtins.md @@ -1,12 +1,12 @@ # Endor 基本对象与统一 Number -本次增加的是虚拟机实现层的数据结构与宿主 API，不增加 Python type、语言根类、解析器或执行器。它们通过 `ObjectPayload` 接入现有 `EndorObject`，共同使用对象头、稳定 Box、代际句柄、引用计数和循环回收。不是每种对象重新实现一套堆管理。 +本次增加的是虚拟机实现层的数据结构与宿主 API，不增加 Python type、语言根类或解析器；后续已接入 [小型字节码执行器](bytecode-vm.md)。它们通过 `ObjectPayload` 接入现有 `EndorObject`，共同使用对象头、稳定 Box、代际句柄、引用计数和循环回收。不是每种对象重新实现一套堆管理。 ## 对应关系 | CPython 中的结构 | Endor 对应结构 | 当前实现 | | --- | --- | --- | -| `PyFunctionObject` | `EndorFunctionObject` | 名称、代码数据、全局字典、默认参数、闭包 Cell；验证和保活关联对象 | +| `PyFunctionObject` | `EndorFunctionObject` | 名称、共享代码对象、全局／内建字典、默认参数和闭包 Cell；由 Vm::call 执行 | | `PyDictObject` | `EndorDictObject` | 保留插入顺序的键值容器，VM 管理查询、写入、删除和清空 | | `PyCFunctionObject` | `EndorNativeFunctionObject` | Rust 宿主函数指针、名称与调用入口；旧名 NativeFunction 保留为别名 | | `PyBaseExceptionObject` | `EndorBaseExceptionObject` | 异常名称、参数、cause、context、上下文抑制、备注与回溯位置 | @@ -14,7 +14,7 @@ | `PyLongObject`、`PyFloatObject` 等 | `EndorNumberObject` / `EndorNumber` | 一个对外数字接口，内部保留不同数字的无损表示 | | `PyCellObject` | `EndorCellObject` | 可被多个闭包共享的可变槽；区分空槽与 Nil | -`EndorCode` 是普通 Rust 代码数据结构，不是另一个堆对象。保存不透明的字节序列、常量、位置参数名和自由变量名。尚未定义指令集，所以字节序列不是 Python 字节码，也没有可执行性保证。 +`EndorCode` 保存经过验证的 Endor wordcode 和静态元数据，`EndorCodeObject` 将它放进受管理的堆；函数持有代码对象句柄并可共享同一份代码。指令格式、执行帧与入口见 [字节码 VM](bytecode-vm.md)。不是 CPython 字节码格式。 ## Function 与 NativeFunction @@ -35,11 +35,13 @@ fn echo(arguments: &[Value]) -> Result { let mut vm = Vm::new()?; let function = vm.allocate(EndorNativeFunctionObject::new("echo", echo))?; let value = vm.call(function, &[Value::Text("hello".into())])?; +assert_eq!(value, Value::Text("hello".into())); +vm.release_value(value)?; vm.release(function)?; # Ok::<(), VmError>(()) ``` -Native 函数现在可调用；语言函数的 `call` 明确返回 `UnsupportedOperation("language function execution")`。不以调用一个 Rust 回调来冒充语言函数执行。Native 回调仍接收借用参数、返回借用对象值，不能在回调里修改当前 VM。完整调用帧、参数绑定及执行器是后续工作。 +Native 函数和语言函数均通过 `Vm::call(&mut self, ...)` 调用。语言函数用显式帧绑定位置参数、默认值和闭包，再由取指循环执行。Native 回调仍接收借用参数、返回借用对象值，不能在回调里修改当前 VM；VM 会将对象结果 retain 后返回，因此宿主需对 call／execute 的对象结果配对 release_value。 ## Number：优先表示，不先做重计算 @@ -75,7 +77,7 @@ let value = Value::Number(amount); 字典使用 `Vec<(Value, Value)>` 保留插入顺序，查询目前为 O(n)，没有假装已经实现 CPython 哈希表性能。键须通过哈希能力检查；支持 Nil、Bool、文本及可哈希的对象。堆字符串与直接文本按内容匹配；覆盖已有键保留首次插入的键对象和位置。可变字典、Cell 与尚未定义数值哈希的数字不能当键。 -`dict_items`、`dict_get`、`cell_get` 返回借用内容。宿主需要跨越删除、覆盖或容器释放继续持有一个对象时，应先 `retain`。删除字典条目不返回一个可能已经被回收的借用对象，而是返回是否删除成功。 +`dict_items`、`dict_get`、`cell_get` 返回借用内容；这与现在返回持有式结果的 call／execute 不同。宿主需要跨越删除、覆盖或容器释放继续持有一个对象时，应先 `retain`。删除字典条目不返回一个可能已经被回收的借用对象，而是返回是否删除成功。 字典键和值、Cell 内容、函数环境与闭包、异常参数与因果链都作为强引用参与 GC。可变容器不暴露内部可变引用：VM 先构造新载荷，验证并保留所有新边，再替换载荷和引用快照，最后释放旧边。对象自身的 Box、身份、属性和外部引用数不变。非法／跨 VM 句柄使更新失败，原值及引用计数不变。同对象替换不会在中间短暂释放它，自环、字典与 Cell 互相引用、函数与全局字典的环都能由现有循环回收处理。 @@ -110,7 +112,7 @@ vm.release(cell)?; # Ok::<(), aster_core::VmError>(()) ``` -当前已实现 Cell 共享与保活的数据基础；上面的 Python 闭包示例说明语义目标，不表示 Endor 已能解析和执行它。 +当前已能通过字节码读取、写入和删除共享 Cell，并创建捕获参数的 Cell。上面的 Python 闭包示例说明语义目标；Endor 尚无 Python 源码解析器，函数对象目前由宿主工厂建立。 ## 字符串与异常的准确含义 @@ -118,7 +120,7 @@ vm.release(cell)?; Python 语言层所有异常的顶级基类叫 `BaseException`，其下除了普通 `Exception` 分支，还有 `SystemExit`、`KeyboardInterrupt` 等。`PyBaseExceptionObject` 是这些异常的公共 C 实例数据布局，不能把这个结构名直接当成语言类名。 -Endor 当前用 `EndorBaseExceptionObject` 保存公共数据，名称只是错误标签；尚无异常继承和按类匹配机制，也没有 raise / try / except 执行流程。回溯保存独立的位置记录，不持有尚不存在的调用帧。宿主 API 的 `VmError` 仍表示操作失败，不会自动分配语言异常或把全部错误强制变成堆对象。 +Endor 当前用 `EndorBaseExceptionObject` 保存公共数据，名称只是错误标签；尚无异常继承和按类匹配机制，也没有 raise / try / except 执行流程。异常数据中的回溯保存独立位置记录；执行器另通过 last_traceback 提供已经清理的执行帧位置元信息。宿主 API 的 `VmError` 仍表示操作失败，不会自动分配语言异常或把全部错误强制变成堆对象。 来源：[CPython 函数对象](https://docs.python.org/3.14/c-api/function.html)、[Cell 对象](https://docs.python.org/3.14/c-api/cell.html)、[异常说明](https://docs.python.org/3.14/library/exceptions.html)、[Unicode 对象](https://docs.python.org/3.14/c-api/unicode.html)。实现不承诺 CPython ABI 或默认 Python 兼容性。 diff --git a/docs/endor-object.md b/docs/endor-object.md index aa9ee72..d08aec2 100644 --- a/docs/endor-object.md +++ b/docs/endor-object.md @@ -23,7 +23,7 @@ EndorObject payload_edges 分配时记录的载荷引用快照 ``` -`ObjectHeader` 不保存语言类型指针。`identity` 是 ObjectRef，用于识别同一个对象。`strong_refs` 是所有强引用的总数，`external_refs` 是宿主持有数，也是循环回收的根来源。 +`ObjectHeader` 不保存语言类型指针。`identity` 是 ObjectRef，用于识别同一个对象。`strong_refs` 是所有强引用的总数，`external_refs` 统计宿主持有和执行帧持有的根引用，也是循环回收的根来源。 `payload` 是 `Box`。Box 保存原生数据，dyn trait 让运行时通过统一接口调用不同 Rust 实现。Rust 会为 trait 分发生成所需机制；不需要创建一个 Python 式的 Type 对象。`kind_name` 只用于诊断，`as_any` 只用于原生 Rust 扩展判断自己能否处理另一份载荷。 @@ -82,7 +82,7 @@ ObjectPayload 提供以下默认行为，载荷可以逐项覆盖： Record 使用身份语义并开启属性。NativeFunction 保存名称和 Rust 函数指针，覆盖 repr 和 call。原生函数收到 Value 参数并返回 Result；VM 在调用前后检查参数和返回值中的对象句柄。 -原生调用当前不能修改 VM 或直接在该 VM 中分配新对象；对象参数和对象返回值是借用句柄。这个接口足够表达标量计算和已有对象返回。函数数据与闭包 Cell 已具备，语言执行栈、函数执行帧和字节码执行尚未实现。 +原生回调当前不能修改 VM 或直接在该 VM 中分配新对象；其参数和原始返回对象值是借用句柄。Vm::call 会验证并 retain 对象结果，对外统一返回一份持有式结果。语言函数已通过 [wordcode 执行器](bytecode-vm.md) 使用显式帧、参数槽、闭包和求值栈执行；call／execute 的结果完成使用后须 release_value。 compare 的反向规则是 Lt 对应 Gt、Le 对应 Ge，Eq／Ne 保持不变。它不实现 Python 的子类型优先规则。内容比较由扩展定义；如果两个对象内容相等，它们必须给出相同哈希，或者明确拒绝哈希。 @@ -107,6 +107,6 @@ compare 的反向规则是 Lt 对应 Gt、Le 对应 Ge，Eq／Ne 保持不变。 已实现：统一对象结构、稳定地址、代际句柄、身份、显式引用计数、托管属性、原生操作覆盖、比较与哈希、原生调用、资源清理和显式循环回收。 -尚未实现：Python type 系统、语言根类、继承和描述符、语言执行器、可变 VM 调用上下文、自动／分代／增量 GC、语言终结器／复活、资源限额和 VM 快照。数据库仍然只保存宿主元数据。 +尚未实现：Python type 系统、语言根类、继承和描述符、源码解析与编译器、可变 VM 调用上下文、自动／分代／增量 GC、语言终结器／复活、完整资源限额和 VM 快照；执行器已有帧数和指令数限额。数据库仍然只保存宿主元数据。 实际测试结果以本次提交对应的远端 Gitea CI 为准，不在本地构建、测试或下载依赖。 diff --git a/docs/validation.md b/docs/validation.md index cfe3961..ec271cc 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -61,3 +61,11 @@ 用户明确当前目标是实现层公共对象，不需要 Python type 系统。移除语言 Object／Type 引导与类工厂，新增 EndorObject 原生操作、稳定 Box、代际槽位、外部／内部引用计数、迭代释放及显式循环回收；CLI／HTTP 冒烟改为运行时对象演示与空堆信息。 本次针对身份与地址稳定、持有与借用、属性替换与删除、错误写入原子性、长链释放、可达环保护、不可达环清理、载荷边计数、反向比较、原生调用错误和 VM 销毁资源释放编写检查。提交时未取得本版远端结果，以对应 Gitea CI 为准。没有在本地运行构建、测试、冒烟，也没有下载依赖。旧版 #162 已成功，但不代表本次修正通过。 + +## 小型字节码 VM（2026-10-18） + +新增 wordcode 指令与代码校验、共享代码对象、内部执行帧及显式帧切换求值器。语言函数已接入执行入口；call／execute 的对象返回值统一持有一份根引用，用后须 release_value。 + +新增 20 项执行测试，覆盖扩展参数、非法代码与控制流、变量及循环、嵌套调用和默认参数、闭包 Cell、容器与属性、返回值存活、报错清理、帧数与指令限额；已有 builtins 测试迁移到共享代码对象及持有式返回值，crate 文档新增直接执行指令示例。 + +本地仅运行 rustfmt 格式化和 git diff --check，并静态审阅代码与文档；未编译、未执行测试或冒烟、未下载依赖。远端仍由现有 ces-control CI 执行检查与双数据库测试。按用户要求不主动跟踪 CI／部署状态，本记录不宣称当前版本已通过运行验证。