diff --git a/dev-cli/container/Dockerfile b/dev-cli/container/Dockerfile index d0dcfb529..920c31d5f 100644 --- a/dev-cli/container/Dockerfile +++ b/dev-cli/container/Dockerfile @@ -3,7 +3,7 @@ LABEL maintainer="tom wilson " # The working directory used by the base image is /src, so we can mount volumes to there # to expose files on the host to the ao container -# +# # https://github.com/emscripten-core/emsdk/blob/9b0db91883452051aca8deddc932363aab29060b/docker/Dockerfile#L120 RUN apt-get update --fix-missing -qq -y @@ -53,12 +53,12 @@ RUN cp -r /lua-${LUA_VERSION} /lua-${LUA_VERSION}-32 # And, re-compile lua with "generic WASM" RUN cd /lua-${LUA_VERSION} && \ make clean && \ - make generic CC='emcc -s WASM=1 -s MEMORY64=1 -s SUPPORT_LONGJMP=1' + make generic CC='emcc -s WASM=1 -s MEMORY64=1 -s SUPPORT_LONGJMP=1' # And, re-compile lua with "generic WASM 32-bit" RUN cd /lua-${LUA_VERSION}-32 && \ make clean && \ - make generic CC='emcc -s WASM=1 -s SUPPORT_LONGJMP=1' + make generic CC='emcc -s WASM=1 -s SUPPORT_LONGJMP=1' ############################# @@ -115,7 +115,7 @@ ENV NM='/emsdk/upstream/bin/llvm-nm' ########################################### # We first create a directory for the node impls to be placed # and dependencies installed -# +# # By running npm link, we allow any commands exposed by # the node module to be ran globally within the container RUN mkdir -p /opt/node @@ -124,3 +124,21 @@ RUN cd /opt/node && \ npm install --omit="dev" && \ npm link + +########################################### +### Rust and WASM toolchains ### +########################################### + +# Set environment variables +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:/usr/local/rustup/bin:$PATH + +# Install Rust, components, and cbindgen in a single step +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly && \ + rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu && \ + cargo install cbindgen && \ + rustc --version && cargo --version && cbindgen --version && rustup show + +# Make build directory +RUN mkdir /opt/aorustlib diff --git a/dev-cli/container/src/ao-build-module b/dev-cli/container/src/ao-build-module index 71ac2502d..715efef5a 100644 --- a/dev-cli/container/src/ao-build-module +++ b/dev-cli/container/src/ao-build-module @@ -37,7 +37,7 @@ def determine_language(): # raise Exception('Multiple languages detected in the module') if len(langs) == 0: raise Exception('No language detected in the module') - + if langs.count('lua') == 0 and langs.count('c') >= 1: lang = 'c' elif langs.count('lua') == 0 and langs.count('rust') >= 1: @@ -51,7 +51,7 @@ def determine_language(): def main(): # Load the definition file definition = Definition('/opt/definition.yml') - + # Load the config file config = Config('/src/config.yml') @@ -67,11 +67,11 @@ def main(): c_source_files = [] injected_lua_files = [] dependency_libraries = [] - + # Inject c files into c_program if language is c if(language == 'c'): c_program = inject_c_files(definition, c_program, c_source_files, link_libraries) - + # Inject rust files into c_program if language is rust if(language == 'rust'): c_program = inject_rust_files(definition, c_program) @@ -79,14 +79,14 @@ def main(): # Inject lua files into c_program always to load lua files and replace placeholders if(language == 'lua'): c_program = inject_lua_files(definition, c_program, injected_lua_files) - + # Load all libraries c_program = load_libraries(config, definition, c_program, link_libraries, c_source_files, injected_lua_files,dependency_libraries) - + # Inject the lua files into the c program c_program = c_program.replace( '__INJECT_LUA_FILES__', '\n'.join(injected_lua_files)) - + #Generate compile target file debug_print('Start to generate complie.c') @@ -97,25 +97,25 @@ def main(): debug_print('Start to compile as WASM') # Setup the compile command - cmd = ['emcc', '-O3', + cmd = ['emcc', '-O3', '-g2', # '-s', 'MAIN_MODULE', # This is required to load dynamic libraries at runtime '-s', 'ASYNCIFY=1', '-s', 'STACK_SIZE=' + str(config.stack_size), '-s', 'ASYNCIFY_STACK_SIZE=' + str(config.stack_size), - '-s', 'ALLOW_MEMORY_GROWTH=1', - '-s', 'INITIAL_MEMORY=' + str(config.initial_memory), - '-s', 'MAXIMUM_MEMORY=' + str(config.maximum_memory), - '-s', 'WASM=1', + '-s', 'ALLOW_MEMORY_GROWTH=1', + '-s', 'INITIAL_MEMORY=' + str(config.initial_memory), + '-s', 'MAXIMUM_MEMORY=' + str(config.maximum_memory), + '-s', 'WASM=1', '-s', 'MODULARIZE', - # '-s', 'FILESYSTEM=0', - '-s', 'DETERMINISTIC=1', - '-s', 'NODERAWFS=0', + # '-s', 'FILESYSTEM=0', + '-s', 'DETERMINISTIC=1', + '-s', 'NODERAWFS=0', '-s', 'FORCE_FILESYSTEM=1', '-msimd128', '--pre-js', '/opt/pre.js' ] - + # If the target is 64 bit, add the MEMORY64 flag and link against 64 bit libraries if config.target == 64: cmd.extend(['-sMEMORY64=1']) @@ -128,7 +128,7 @@ def main(): # Link Rust library if(language == 'rust'): - cmd.extend(['-L/opt/aorustlib', '-l:libaorust.a']) + cmd.extend(['-I/opt/aorustlib', '-L/opt/aorustlib', '-laorust']) cmd.extend(['-s','ASSERTIONS=1']) cmd.extend(definition.get_extra_args()) @@ -138,7 +138,7 @@ def main(): # Add all c source files if there are any for f in c_source_files: cmd.append(quote(f)) - + # Add the compile file and link libraries cmd.extend(['/tmp/compile.c']) cmd.extend([quote(v.filepath) for v in link_libraries]) @@ -157,8 +157,12 @@ def main(): # add metering library # meter_cmd = ['node', '/opt/node/apply-metering.cjs'] # shell_exec(*meter_cmd) + if config.keep_js is False: - shell_exec(*['rm', os.path.join(os.getcwd(), 'process.js')]) + process_js_path = os.path.join(os.getcwd(), 'process.js') + # For rust case, js code is not exists + if os.path.exists(process_js_path): + shell_exec(*['rm', process_js_path]) if __name__ == '__main__': main() diff --git a/dev-cli/container/src/ao_module_lib/languages/rust.py b/dev-cli/container/src/ao_module_lib/languages/rust.py index 5593c0f27..57e9fd4d3 100644 --- a/dev-cli/container/src/ao_module_lib/languages/rust.py +++ b/dev-cli/container/src/ao_module_lib/languages/rust.py @@ -9,31 +9,34 @@ def inject_rust_files(definition: Definition, c_program: str): c_header_files = [] - rust_source_files = [] + # rust_source_files = [] - c_header_files += glob.glob('/src/**/*.h', recursive=True) + c_header_files += glob.glob('/src/**/*.h', recursive=True) c_header_files += glob.glob('/src/**/*.hpp', recursive=True) inc = [] for header in c_header_files: c_program = '#include "{}"\n'.format(header) + c_program - # c_program = 'const char *process_handle(const char *arg_0, const char *arg_1);\n' + c_program + c_program = 'const char *process_handle(const char *arg_0, const char *arg_1);\n' + c_program c_program = c_program.replace('__FUNCTION_DECLARATIONS__', definition.make_c_function_delarations()) + c_program = c_program.replace('__LUA_BASE__', "") + c_program = c_program.replace('__LUA_MAIN__', "") - rust_source_files += glob.glob('/src/**/*.rs', recursive=True) - rust_src_dir = os.path.join(RUST_DIR, "src") + # rust_source_files += glob.glob('/src/**/*.rs', recursive=True) + # rust_src_dir = os.path.join(RUST_DIR, "src") - for file in rust_source_files: - if os.path.isfile(file): - shutil.copy2(file, os.path.join(rust_src_dir)) + # for file in rust_source_files: + # if os.path.isfile(file): + # shutil.copy2(file, os.path.join(rust_src_dir)) - prev_dir = os.getcwd() - os.chdir(RUST_DIR) +# prev_dir = os.getcwd() +# os.chdir(RUST_DIR) + # build rust code at '/src' os.environ["RUSTFLAGS"] = "--cfg=web_sys_unstable_apis --Z wasm_c_abi=spec" cmd = [ 'cargo', @@ -46,19 +49,20 @@ def inject_rust_files(definition: Definition, c_program: str): ] shell_exec(*cmd) - rust_lib = glob.glob('/opt/aorustlib/**/release/*.a', recursive=True)[0] + rust_lib = glob.glob('/src/target/wasm64-unknown-unknown/release/*.a', recursive=True)[0] rust_lib = shutil.copy2(rust_lib, RUST_DIR) cmd = [ - 'cbindgen', - '--config', + 'cbindgen', + '--config', 'cbindgen.toml', - '--crate', - 'aorust', - '--output', + '--crate', + 'aorust', + '--output', 'aorust.h' ] shell_exec(*cmd) - c_program = '#include "{}"\n'.format('/opt/aorustlib/aorust.h') + c_program - os.chdir(prev_dir) + rust_header = shutil.copy2('/src/aorust.h', RUST_DIR) + c_program = '#include "{}"\n'.format('aorust.h') + c_program +# os.chdir(prev_dir) return c_program diff --git a/dev-cli/src/starters/rust/.gitignore b/dev-cli/src/starters/rust/.gitignore new file mode 100644 index 000000000..f300cd9fb --- /dev/null +++ b/dev-cli/src/starters/rust/.gitignore @@ -0,0 +1,11 @@ +/target/ +Cargo.lock +**/*.log +**/*.tmp +*.swp +*.bak +*.bk +*.tmp +*.old +aorust.h +process.wasm diff --git a/dev-cli/src/starters/rust/Cargo.toml b/dev-cli/src/starters/rust/Cargo.toml new file mode 100644 index 000000000..3b5833d98 --- /dev/null +++ b/dev-cli/src/starters/rust/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "aorust" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["staticlib"] + +[dependencies] +serde = "1.0" +serde_json = "1.0" + diff --git a/dev-cli/src/starters/rust/Makefile b/dev-cli/src/starters/rust/Makefile new file mode 100644 index 000000000..d663da2e1 --- /dev/null +++ b/dev-cli/src/starters/rust/Makefile @@ -0,0 +1,15 @@ +all:build bindgen + +build: + RUSTFLAGS="--cfg=web_sys_unstable_apis --Z wasm_c_abi=spec" \ + cargo +nightly build \ + -Zbuild-std=std,panic_unwind,panic_abort \ + --target=wasm64-unknown-unknown \ + --release + +bindgen: + cbindgen --config cbindgen.toml --crate aorust --output aorust.h + +clean: + cargo clean + rm aorust.h diff --git a/dev-cli/src/starters/rust/cbindgen.toml b/dev-cli/src/starters/rust/cbindgen.toml new file mode 100644 index 000000000..863aad3de --- /dev/null +++ b/dev-cli/src/starters/rust/cbindgen.toml @@ -0,0 +1,7 @@ +language = "C" + +[defines] +MY_DEFINE = "1" + +[export] +include = ["process_handler"] diff --git a/dev-cli/src/starters/rust/src/lib.rs b/dev-cli/src/starters/rust/src/lib.rs new file mode 100644 index 000000000..c00d75cba --- /dev/null +++ b/dev-cli/src/starters/rust/src/lib.rs @@ -0,0 +1,72 @@ +pub mod libao; +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; + +/// Processes the input `msg` and `env` strings, performs some operations, +/// and returns the result as a C-compatible string. +/// +/// # Safety +/// - The caller must ensure that the `msg` and `env` pointers are valid, null-terminated C strings. +/// - The caller is responsible for freeing the returned string using `free_c_string`. +/// +/// # Arguments +/// - `msg`: A pointer to a null-terminated C string representing the message. +/// - `env`: A pointer to a null-terminated C string representing the environment. +/// +/// # Returns +/// - A pointer to a null-terminated C string containing the result. The caller must not modify +/// or attempt to free this pointer. +#[no_mangle] +pub extern "C" fn process_handle(msg: *const c_char, env: *const c_char) -> *const c_char { + // Convert the C strings to Rust strings + let msg = unsafe { CStr::from_ptr(msg).to_str().unwrap_or("Invalid msg") }; + let env = unsafe { CStr::from_ptr(env).to_str().unwrap_or("Invalid env") }; + + // Call the Rust handler function to process the inputs + let ret = handler(msg, env); + + // Convert the Rust string result into a CString + let c_response = CString::new(ret).unwrap(); + + // Return the raw pointer to the C-compatible string + // The caller is now responsible for freeing this memory + c_response.into_raw() +} + +pub fn handler(msg: &str, env: &str) -> String { + let mut ao = libao::AO::new(); + ao.init(env); + ao.log("Normalize"); + let norm = ao.normalize(msg); + ao.log(&msg); + let send = &ao.send(&norm); + println!("Send: {}", send); + "{\"ok\": true,\"response\":{\"Output\":\"Success\"},\"Memory\":50000000}".to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::{CStr, CString}; + + #[test] + fn test_process_handle() { + let msg_json = r#"{"key": "value"}"#; + let env_json = r#"{"env": "test"}"#; + + let c_msg = CString::new(msg_json).expect("Failed to create CString for msg"); + let c_env = CString::new(env_json).expect("Failed to create CString for env"); + + let c_result = process_handle(c_msg.as_ptr(), c_env.as_ptr()); + + assert!(!c_result.is_null(), "Returned pointer is null"); + + let result_str = unsafe { CStr::from_ptr(c_result).to_string_lossy().into_owned() }; + + let expected_result = r#"{"ok": true,"response":{"Output":"Success"},"Memory":50000000}"#; + assert_eq!( + result_str, expected_result, + "Output does not match expected result" + ); + } +} diff --git a/dev-cli/src/starters/rust/src/libao.rs b/dev-cli/src/starters/rust/src/libao.rs new file mode 100644 index 000000000..5be006ee4 --- /dev/null +++ b/dev-cli/src/starters/rust/src/libao.rs @@ -0,0 +1,257 @@ +use serde_json::{json, Map, Value}; + +#[derive(Default)] +pub struct AO { + version: String, + module: Option, + id: Option, + authorities: Option>, + outbox: Value, + ref_count: usize, +} + +impl AO { + pub fn new() -> AO { + AO { + version: "0.0.4".to_string(), + module: None, + id: None, + authorities: None, + outbox: json!({ + "Output": [], + "Messages": [], + "Spawns": [], + "Assignments": [], + "Error": null + }), + ref_count: 0, + } + } + + pub fn init(&mut self, env: &str) { + // Parse the environment JSON string into a Value (serde_json's equivalent of cJSON) + let env_json: Value = serde_json::from_str(env).unwrap(); + + // Retrieve the "Process" object from the parsed JSON + if let Some(process) = env_json.get("Process") { + // If id is None, retrieve it from the "Process" object + if self.id.is_none() { + self.id = process + .get("Id") + .and_then(Value::as_str) + .map(str::to_string); + } + + // Retrieve the "Tags" array from the "Process" object + if let Some(tags) = process.get("Tags").and_then(Value::as_array) { + // Search for the "Module" tag within "Tags" + if self.module.is_none() { + for tag in tags { + if let Some(name) = tag.get("name").and_then(Value::as_str) { + if name == "Module" { + self.module = + tag.get("value").and_then(Value::as_str).map(str::to_string); + break; + } + } + } + } + + // Initialize authorities as an empty vector if it's None + if self.authorities.is_none() { + self.authorities = Some(Vec::new()); + for tag in tags { + if let Some(name) = tag.get("name").and_then(Value::as_str) { + if name == "Authority" { + if let Some(value) = tag.get("value").and_then(Value::as_str) { + self.authorities.as_mut().unwrap().push(value.to_string()); + } + } + } + } + } + } + } + self.clear_outbox(); + } + + pub fn normalize(&self, msg: &str) -> String { + let msg_json: Value = serde_json::from_str(msg).unwrap(); + serde_json::to_string(&msg_json).unwrap() + } + + pub fn sanitize(&self, msg: &str) -> String { + let mut msg_json: Value = serde_json::from_str(msg).unwrap(); + let mut new_message = Map::new(); + + // Iterate over each key in the JSON object, excluding non-forwardable tags + for (key, value) in msg_json.as_object_mut().unwrap().iter_mut() { + if !self.is_string_in_array(&key, &self.non_forwardable_tags()) { + new_message.insert(key.clone(), value.clone()); + } + } + + serde_json::to_string(&new_message).unwrap() + } + + pub fn log(&mut self, msg: &str) { + let output = self.outbox.get_mut("Output").unwrap(); + if let Some(arr) = output.as_array_mut() { + arr.push(Value::String(msg.to_string())); + } + } + + pub fn send(&mut self, msg: &str) -> String { + let msg_json: Value = serde_json::from_str(msg).unwrap(); + self.ref_count += 1; + let padded_ref = format!("{:032}", self.ref_count); + + let mut message = Map::new(); + self.add_string_to_json_if_exists(&msg_json, &mut message, "Target"); + self.add_string_to_json_if_exists(&msg_json, &mut message, "Data"); + self.add_string_to_json_if_exists(&msg_json, &mut message, "Anchor"); + let default = vec![]; + let tags = msg_json + .get("Tags") + .and_then(Value::as_array) + .unwrap_or(&default); + let mut tags_array = Vec::new(); + self.add_tag_to_array(&mut tags_array, "Data-Protocol", "ao"); + self.add_tag_to_array(&mut tags_array, "Variant", "aoc.TN.1"); + self.add_tag_to_array(&mut tags_array, "Type", "Message"); + self.add_tag_to_array(&mut tags_array, "Reference", &padded_ref); + + for tag in tags { + if let Some(name) = tag.get("name").and_then(Value::as_str) { + if !self.is_string_in_array(name, &self.persistent_tags()) { + self.add_tag_to_array( + &mut tags_array, + name, + tag.get("value").and_then(Value::as_str).unwrap(), + ); + } + } + } + + message.insert("Tags".to_string(), Value::Array(tags_array)); + + let messages = self + .outbox + .get_mut("Messages") + .unwrap() + .as_array_mut() + .unwrap(); + messages.push(Value::Object(message.clone())); + + serde_json::to_string(&message).unwrap() + } + + pub fn assign(&mut self, assignment: &str) { + let assignment_json: Value = serde_json::from_str(assignment).unwrap(); + let assignments = self + .outbox + .get_mut("Assignments") + .unwrap() + .as_array_mut() + .unwrap(); + assignments.push(assignment_json); + } + + pub fn is_trusted(&self, msg: &str) -> bool { + let msg_json: Value = serde_json::from_str(msg).unwrap(); + + if let Some(authorities) = &self.authorities { + for authority in authorities { + if let Some(from) = msg_json.get("From").and_then(Value::as_str) { + if from == authority { + return true; + } + } + + if let Some(owner) = msg_json.get("Owner").and_then(Value::as_str) { + if owner == authority { + return true; + } + } + } + false + } else { + true + } + } + + pub fn result(&self, result: &str) -> String { + let result_json: Value = serde_json::from_str(result).unwrap(); + if let Some(error) = result_json.get("Error") { + if !error.is_null() { + return serde_json::to_string(&error).unwrap(); + } + } + + let output = result_json.get("Output").unwrap_or(&self.outbox["Output"]); + let mut result_out = Map::new(); + result_out.insert("Output".to_string(), output.clone()); + + serde_json::to_string(&result_out).unwrap() + } + + pub fn print(&self) { + println!("Version: {}", self.version); + println!("Module: {}", self.module.clone().unwrap_or_default()); + println!("Ref: {}", self.ref_count); + println!("Id: {}", self.id.clone().unwrap_or_default()); + println!( + "Authorities: {}", + serde_json::to_string(&self.authorities).unwrap() + ); + println!("Outbox: {}", serde_json::to_string(&self.outbox).unwrap()); + } + + pub fn clear_outbox(&mut self) { + self.outbox = json!({ + "Output": [], + "Messages": [], + "Spawns": [], + "Assignments": [], + "Error": null + }); + } + + pub fn add_tag_to_array(&self, tags_array: &mut Vec, name: &str, value: &str) { + tags_array.push(json!({ "name": name, "value": value })); + } + + pub fn is_string_in_array(&self, s: &str, array: &[&str]) -> bool { + array.contains(&s) + } + + pub fn non_forwardable_tags(&self) -> Vec<&str> { + vec![ + "Data-Protocol", + "Variant", + "From-Process", + "From-Module", + "Type", + "From", + "Owner", + "Anchor", + "Target", + "Tags", + ] + } + + pub fn persistent_tags(&self) -> Vec<&str> { + vec!["Target", "Data", "Anchor", "Tags", "From"] + } + + pub fn add_string_to_json_if_exists( + &self, + msg_json: &Value, + message: &mut Map, + key: &str, + ) { + if let Some(val) = msg_json.get(key) { + message.insert(key.to_string(), val.clone()); + } + } +}