RustPython/examples/parse_folder.rs at main · RustPython/RustPython · GitHub

Repository navigation

History

135 lines (121 loc) · 3.69 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

/// This an example usage of the rustpython_parser crate.

/// This program crawls over a directory of python files and

/// tries to parse them into an abstract syntax tree (AST)

///

/// example usage:

/// $ RUST_LOG=info cargo run --release parse_folder /usr/lib/python3.7

extern crate env_logger;

#[macro_use]

extern crate log;

use core::time::Duration;

use ruff_python_parser::parse_module;

use rustpython_compiler::ast;

use std::{

path::{Path, PathBuf},

time::Instant,

};

fn main() {

env_logger::init();

let folder: PathBuf = std::env::args_os()

.nth(1)

.expect("please pass a path argument")

.into();

if folder.exists() && folder.is_dir() {

println!("Parsing folder of python code: {folder:?}");

let t1 = Instant::now();

let parsed_files = parse_folder(&folder).unwrap();

let t2 = Instant::now();

let results = ScanResult {

t1,

t2,

parsed_files,

};

statistics(results);

} else {

println!("{folder:?} is not a folder.");

}

fn parse_folder(path: &Path) -> std::io::Result<Vec<ParsedFile>> {

let mut res = vec![];

info!("Parsing folder of python code: {path:?}");

for entry in path.read_dir()? {

debug!("Entry: {entry:?}");

let entry = entry?;

let metadata = entry.metadata()?;

let path = entry.path();

if metadata.is_dir() {

res.extend(parse_folder(&path)?);

}

if metadata.is_file() && path.extension().and_then(|s| s.to_str()) == Some("py") {

let parsed_file = parse_python_file(&path);

match &parsed_file.result {

Ok(_) => {}

Err(y) => error!("Error in file {path:?} {y:?}"),

}

res.push(parsed_file);

}

Ok(res)

}

fn parse_python_file(filename: &Path) -> ParsedFile {

info!("Parsing file {filename:?}");

match std::fs::read_to_string(filename) {

Err(e) => ParsedFile {

num_lines: 0,

result: Err(e.to_string()),

Ok(source) => {

let num_lines = source.lines().count();

let result = parse_module(&source)

.map(|x| x.into_suite())

.map_err(|e| e.to_string());

ParsedFile { num_lines, result }

}

fn statistics(results: ScanResult) {

// println!("Processed {:?} files", res.len());

println!("Scanned a total of {} files", results.parsed_files.len());

let total: usize = results.parsed_files.len();

let total_lines: usize = results.parsed_files.iter().map(|p| p.num_lines).sum();

let failed = results

.parsed_files

.iter()

.filter(|p| p.result.is_err())

.count();

let passed = results

.parsed_files

.iter()

.filter(|p| p.result.is_ok())

.count();

println!("Passed: {passed} Failed: {failed} Total: {total}");

println!(

"That is {} % success rate.",

(passed as f64 * 100.0) / total as f64

);

let duration = results.t2 - results.t1;

println!("Total time spend: {duration:?}");

println!(

"Processed {} files. That's {} files/second",

total,

rate(total, duration)

);

println!(

"Processed {} lines of python code. That's {} lines/second",

total_lines,

rate(total_lines, duration)

);

}

fn rate(counter: usize, duration: Duration) -> f64 {

(counter * 1_000_000) as f64 / duration.as_micros() as f64

}

struct ScanResult {

t1: Instant,

t2: Instant,

parsed_files: Vec<ParsedFile>,

}

struct ParsedFile {

num_lines: usize,

result: ParseResult,

}

type ParseResult = Result<ast::Suite, String>;

RustPython/examples/parse_folder.rs at main · RustPython/RustPython · GitHub

Repository navigation

Expand file tree

Breadcrumbs

parse_folder.rs

Latest commit

History

Breadcrumbs

parse_folder.rs

File metadata and controls