FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Allow stdlib freeze by palaviv · Pull Request #1159 · RustPython/RustPython · GitHub

Repository navigation

Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension .lock  (1) .rs  (5) .toml  (3) All 3 file types selected
Only manifest files
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ path = "./benchmarks/bench.rs"
[features]
default = ["rustpython-vm/use-proc-macro-hack"]
flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
freeze-stdlib = ["rustpython-vm/freeze-stdlib"]

[dependencies]
log="0.4.1"
Expand Down
1 change: 1 addition & 0 deletions derive/Cargo.toml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,4 @@ rustpython-compiler = { path = "../compiler", version = "0.1.0" }
rustpython-bytecode = { path = "../bytecode", version = "0.1.0" }
bincode = "1.1"
proc-macro-hack = { version = "0.5", optional = true }
maplit = "1.0"
110 changes: 93 additions & 17 deletions derive/src/compile_bytecode.rs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,17 @@ use proc_macro2::{Span, TokenStream as TokenStream2};
use quote::quote;
use rustpython_bytecode::bytecode::CodeObject;
use rustpython_compiler::compile;
use std::collections::HashMap;
use std::env;
use std::fs;
use std::path::PathBuf;
use syn::parse::{Parse, ParseStream, Result as ParseResult};
use syn::{self, parse2, Lit, LitByteStr, Meta, Token};
use syn::{self, parse2, Lit, LitByteStr, LitStr, Meta, Token};

enum CompilationSourceKind {
File(PathBuf),
SourceCode(String),
Dir(PathBuf),
}

struct CompilationSource {
Expand All @@ -36,14 +38,22 @@ struct CompilationSource {
}

impl CompilationSource {
fn compile(self, mode: &compile::Mode, module_name: String) -> Result<CodeObject, Diagnostic> {
let compile = |source| {
compile::compile(source, mode, module_name, 0).map_err(|err| {
Diagnostic::spans_error(self.span, format!("Compile error: {}", err))
})
};

match &self.kind {
fn compile_string(
&self,
source: &str,
mode: &compile::Mode,
module_name: String,
) -> Result<CodeObject, Diagnostic> {
compile::compile(source, mode, module_name, 0)
.map_err(|err| Diagnostic::spans_error(self.span, format!("Compile error: {}", err)))
}

fn compile(
&self,
mode: &compile::Mode,
module_name: String,
) -> Result<HashMap<String, CodeObject>, Diagnostic> {
Ok(match &self.kind {
CompilationSourceKind::File(rel_path) => {
let mut path = PathBuf::from(
env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"),
Expand All @@ -55,10 +65,59 @@ impl CompilationSource {
format!("Error reading file {:?}: {}", path, err),
)
})?;
compile(&source)
hashmap! {module_name.clone() => self.compile_string(&source, mode, module_name.clone())?}
}
CompilationSourceKind::SourceCode(code) => {
hashmap! {module_name.clone() => self.compile_string(code, mode, module_name.clone())?}
}
CompilationSourceKind::Dir(rel_path) => {
let mut path = PathBuf::from(
env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"),
);
path.push(rel_path);
self.compile_dir(&path, String::new(), mode)?
}
})
}

fn compile_dir(
&self,
path: &PathBuf,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality
Suggested change
path: &PathBuf,
path: &Path,

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

Why is this better?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

PathBuf vs Path is akin to String vs str - an owned, allocated buffer vs a slice. Like you should (almost) always prefer &str over &String, so should you for &Path over &PathBuf.

parent: String,
mode: &compile::Mode,
) -> Result<HashMap<String, CodeObject>, Diagnostic> {
let mut code_map = HashMap::new();
let paths = fs::read_dir(&path).map_err(|err| {
Diagnostic::spans_error(self.span, format!("Error listing dir {:?}: {}", path, err))
})?;
for path in paths {
let path = path.map_err(|err| {
Diagnostic::spans_error(self.span, format!("Failed to list file: {}", err))
})?;
let path = path.path();
let file_name = path.file_name().unwrap().to_str().unwrap();
if path.is_dir() {
code_map.extend(self.compile_dir(
&path,
format!("{}{}.", parent, file_name),
mode,
)?);
} else if file_name.ends_with(".py") {
let source = fs::read_to_string(&path).map_err(|err| {
Diagnostic::spans_error(
self.span,
format!("Error reading file {:?}: {}", path, err),
)
})?;
let file_name_splitte: Vec<&str> = file_name.splitn(2, ".").collect();
let module_name = format!("{}{}", parent, file_name_splitte[0]);
code_map.insert(
module_name.clone(),
self.compile_string(&source, mode, module_name)?,
);
}
CompilationSourceKind::SourceCode(code) => compile(code),
}
Ok(code_map)
}
}

Expand All @@ -69,7 +128,7 @@ struct PyCompileInput {
}

impl PyCompileInput {
fn compile(&self) -> Result<CodeObject, Diagnostic> {
fn compile(&self) -> Result<HashMap<String, CodeObject>, Diagnostic> {
let mut module_name = None;
let mut mode = None;
let mut source: Option<CompilationSource> = None;
Expand Down Expand Up @@ -122,6 +181,16 @@ impl PyCompileInput {
kind: CompilationSourceKind::File(path),
span: extract_spans(&name_value).unwrap(),
});
} else if name_value.ident == "dir" {
assert_source_empty(&source)?;
let path = match &name_value.lit {
Lit::Str(s) => PathBuf::from(s.value()),
_ => bail_span!(name_value.lit, "source must be a string"),
};
source = Some(CompilationSource {
kind: CompilationSourceKind::Dir(path),
span: extract_spans(&name_value).unwrap(),
});
}
}
}
Expand Down Expand Up @@ -154,16 +223,23 @@ impl Parse for PyCompileInput {
pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Diagnostic> {
let input: PyCompileInput = parse2(input)?;

let code_obj = input.compile()?;
let code_map = input.compile()?;

let bytes = bincode::serialize(&code_obj).expect("Failed to serialize");
let bytes = LitByteStr::new(&bytes, Span::call_site());
let modules = code_map.iter().map(|(module_name, code_obj)| {
let module_name = LitStr::new(&module_name, Span::call_site());
let bytes = bincode::serialize(&code_obj).expect("Failed to serialize");
let bytes = LitByteStr::new(&bytes, Span::call_site());
quote! { #module_name.into() => bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(#bytes)
.expect("Deserializing CodeObject failed") }
});

let output = quote! {
({
use ::rustpython_vm::__exports::bincode;
bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(#bytes)
.expect("Deserializing CodeObject failed")
use ::rustpython_vm::__exports::hashmap;
hashmap! {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

You should probably add hashmap to __exports and use it here.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

What is the purpose of the __exports? This is my first time using procedural macro

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

I think I understand what __exports do. Added hashmap

#(#modules),*
}
})
};

Expand Down
3 changes: 3 additions & 0 deletions derive/src/lib.rs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@

extern crate proc_macro;

#[macro_use]
extern crate maplit;

#[macro_use]
mod error;
mod compile_bytecode;
Expand Down
1 change: 1 addition & 0 deletions vm/Cargo.toml
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ default = ["rustpython-parser", "rustpython-compiler", "use-proc-macro-hack"]
vm-tracing-logging = []
flame-it = ["flame", "flamer"]
use-proc-macro-hack = ["proc-macro-hack", "rustpython-derive/proc-macro-hack"]
freeze-stdlib = []

[dependencies]
# Crypto:
Expand Down
32 changes: 19 additions & 13 deletions vm/src/frozen.rs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,24 @@ use crate::bytecode::CodeObject;
use std::collections::HashMap;

pub fn get_module_inits() -> HashMap<String, CodeObject> {
hashmap! {
"__hello__".into() => py_compile_bytecode!(
source = "initialized = True; print(\"Hello world!\")\n",
module_name = "__hello__",
),
"_frozen_importlib".into() => py_compile_bytecode!(
file = "Lib/_bootstrap.py",
module_name = "_frozen_importlib",
),
"_frozen_importlib_external".into() => py_compile_bytecode!(
file = "Lib/_bootstrap_external.py",
module_name = "_frozen_importlib_external",
),
let mut modules = HashMap::new();
modules.extend(py_compile_bytecode!(
source = "initialized = True; print(\"Hello world!\")\n",
module_name = "__hello__",
));
modules.extend(py_compile_bytecode!(
file = "Lib/_bootstrap.py",
module_name = "_frozen_importlib",
));
modules.extend(py_compile_bytecode!(
file = "Lib/_bootstrap_external.py",
module_name = "_frozen_importlib_external",
));

#[cfg(feature = "freeze-stdlib")]
{
modules.extend(py_compile_bytecode!(dir = "../Lib/",));
}

modules
}
1 change: 1 addition & 0 deletions vm/src/lib.rs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,5 @@ pub use rustpython_bytecode::*;
#[doc(hidden)]
pub mod __exports {
pub use bincode;
pub use maplit::hashmap;
}
8 changes: 4 additions & 4 deletions wasm/lib/src/browser_module.rs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -381,8 +381,8 @@ pub fn setup_browser_module(vm: &VirtualMachine) {
vm.stdlib_inits
.borrow_mut()
.insert("_browser".to_string(), Box::new(make_module));
vm.frozen.borrow_mut().insert(
"browser".to_string(),
py_compile_bytecode!(file = "src/browser.py"),
);
vm.frozen.borrow_mut().extend(py_compile_bytecode!(
file = "src/browser.py",
module_name = "browser",
));
}

Back | FazBrowse Home | New Git URL