FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Add files for CK · lyriccoder/temp_python@e742fc5 · GitHub

/ temp_python Public

Commit e742fc5

Browse files
Anonym
committed
Add files for CK
1 parent f9c2263 commit e742fc5

2 files changed

Lines changed: 369 additions & 0 deletions

File tree

‎collect_dataset.py‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
import pandas as pd
2+
import subprocess
3+
import os
4+
import re
5+
import json
6+
from pathlib import Path
7+
from tqdm import tqdm
8+
from concurrent.futures import TimeoutError
9+
import argparse
10+
import os
11+
import multiprocessing
12+
import traceback
13+
from cchardet import detect
14+
import sys
15+
from multiprocessing import Pool
16+
from program_slicing.graph.parse import tree_sitter_ast_java
17+
from program_slicing.graph.parse import tree_sitter_parsers
18+
from program_slicing.graph.cdg import ControlDependenceGraph
19+
from program_slicing.graph.statement import Statement, StatementType
20+
from program_slicing.graph.point import Point
21+
import csv
22+
from collections import defaultdict, OrderedDict
23+
from program_graphs.ddg.ddg import mk_ddg
24+
from program_graphs.cfg.parser.java.parser import mk_cfg
25+
from program_graphs.cfg import CFG
26+
from program_graphs.adg import parse_java
27+
from typing import List, Tuple, Set, Optional
28+
import networkx as nx
29+
import numpy as np
30+
31+
CycloComplexity = int
32+
33+
34+
sys.setrecursionlimit(100000)
35+
36+
def __traverse(root):
37+
yield root
38+
if root.children:
39+
for child in root.children:
40+
for result in __traverse(child):
41+
yield result
42+
43+
def detect_encoding_of_file(filename: str):
44+
with open(filename, 'rb') as target_file:
45+
return detect_encoding_of_data(target_file.read())
46+
47+
48+
def detect_encoding_of_data(data: bytes):
49+
return detect(data)['encoding']
50+
51+
52+
def read_text_with_autodetected_encoding(filename: str):
53+
with open(filename, 'rb') as target_file:
54+
data = target_file.read()
55+
56+
if not data:
57+
return '' # In case of empty file, return empty string
58+
59+
encoding = detect_encoding_of_data(data) or 'utf-8'
60+
return data.decode(encoding)
61+
62+
63+
def cc(source_code: str) -> CycloComplexity:
64+
adg = parse_java(source_code)
65+
#cfg = parse_cfg(source_code)
66+
cfg = adg.to_cfg()
67+
E = len(cfg.edges())
68+
N = len(cfg.nodes())
69+
P = nx.number_weakly_connected_components(cfg)
70+
return E - N + 2 * P
71+
72+
73+
def get_json_with_tokens(groupby_objects):
74+
filepath, group_item, dataset_dir = groupby_objects
75+
total_test_list = []
76+
ignored_file = None
77+
#print('begin ', filepath)
78+
try:
79+
for _, item in group_item.iterrows():
80+
#print(f'{_}, item {item}')
81+
data_dir = item.data_dir
82+
fullpath = Path(dataset_dir) / data_dir.replace('/', '', 1) / str(filepath)
83+
#print(f'fullpath {fullpath} exists? {fullpath.exists()}')
84+
#fullpath = Path(dataset_dir) / str(filepath)
85+
source_code = read_text_with_autodetected_encoding(str(fullpath))
86+
bytes_size = fullpath.stat().st_size
87+
mb_size = bytes_size >> 20
88+
file_dict = {'filename': fullpath.resolve(), 'bytes': bytes_size, 'kb': bytes_size >> 10, 'mb': mb_size}
89+
if mb_size > 0:
90+
ignored_file = fullpath
91+
total_test_list.append(file_dict)
92+
return total_test_list, ignored_file
93+
source_code_bytes = bytes(source_code, "utf8")
94+
ast = tree_sitter_ast_java.parse(source_code).root_node
95+
methods = {}
96+
comments_dicts = []
97+
for node in __traverse(ast):
98+
if node.type == 'method_declaration':
99+
method_name_in_file = source_code_bytes[node.child_by_field_name('name').start_byte:node.child_by_field_name('name').end_byte].decode('utf-8')
100+
methods[tuple([method_name_in_file, node.start_point[0]])] = node
101+
#print(fullpath, method_name_in_file, node.start_point[0])
102+
if node.type in {"line_comment", "block_comment", "comment"}:
103+
comment = source_code_bytes[node.start_byte:node.end_byte].decode('utf-8')
104+
if comment.lower().find('generated') > -1:
105+
#print(f'class {fullpath}; comment {comment}')
106+
comments_dicts.append({'line': node.start_point[0]})
107+
method_name_in_config = item['methodname']
108+
method_startline = item['startline']
109+
found_method = methods.get((method_name_in_config, method_startline))
110+
if found_method:
111+
#print(f'Found {method_name_in_file}')
112+
method_code = source_code_bytes[found_method.start_byte:found_method.end_byte].decode('utf-8')
113+
#print(f'{method_code}')
114+
cyclo_complexity = cc(method_code)
115+
counts = defaultdict(int)
116+
for x in ['for_statement', 'decimal_integer_literal', 'character_literal',
117+
'generated_comment', 'while_statement', 'if_statement', 'variable_declarator', 'lines_n',
118+
'local_variable_declaration', 'switch_expression', 'case']:
119+
counts[x] = 0
120+
counts['cyclo_complexity'] = cyclo_complexity
121+
counts['method_name'] = method_name_in_config
122+
for node in __traverse(found_method):
123+
#print(node)
124+
#if node.type == 'method_declaration':
125+
#method_name_in_file = source_code_bytes[node.start_byte:node.end_byte].decode('utf-8')
126+
#print(method_name_in_file)
127+
#cyclo_complexity = cc(method_name_in_file)
128+
#ccs.append(cyclo_complexity)
129+
#methods.append((method_name_in_file, node.start_point[0], node))
130+
131+
if node.type in {"if_statement"}:
132+
counts[node.type] += 1
133+
if node.type in {"while_statement"}:
134+
counts[node.type] += 1
135+
if node.type in {"for_statement"}:
136+
counts[node.type] += 1
137+
if node.type in {"local_variable_declaration"}:
138+
counts[node.type] += 1
139+
if node.type in {"variable_declarator"}:
140+
counts[node.type] += 1
141+
if node.type in {"method_invocations"}:
142+
counts[node.type] += 1
143+
if node.type in {"character_literal"}:
144+
counts[node.type] += 1
145+
if node.type in {"switch_expression"}:
146+
counts[node.type] += 1
147+
if node.type in {"case"}:
148+
counts[node.type] += 1
149+
if node.type in {"decimal_integer_literal"}:
150+
#qq = source_code_bytes[node.child_by_field_name('name').start_byte:node.child_by_field_name('name').end_byte].decode('utf-8')
151+
#print(qq)
152+
counts[node.type] += 1
153+
d = {**counts, **{'generated_comment': comments_dicts}, **file_dict}
154+
#d['cyclo_complexity'] = np.mean(ccs)
155+
d['lines_n'] = len([x for x in source_code.split('\n') if x.strip()])
156+
total_test_list.append(d)
157+
#else:
158+
#print(f'Method {method_name_in_config} not found in {fullpath}')
159+
160+
#except FileNotFoundError as e:
161+
#print(f'Cannot find file {fullpath}')
162+
#pass
163+
except Exception as e:
164+
print(traceback.format_exc())
165+
print(f'Cannot read/open/parse file {fullpath} {str(e)}')
166+
167+
#print('end ', filepath)
168+
return total_test_list, ignored_file
169+
170+
171+
if __name__ == '__main__':
172+
parser = argparse.ArgumentParser()
173+
parser.add_argument('--input', '-i', type=str, required=True)
174+
parser.add_argument('--dir', '-d', type=str, required=True)
175+
parser.add_argument('--output', '-o', type=str, required=True)
176+
args = parser.parse_args()
177+
df = pd.read_csv(args.input, sep=',', encoding='utf-8')
178+
before_size = df.shape[0]
179+
print(f'df before {before_size}')
180+
df = df[~df['filepath'].str.contains('zxiaofan__JDK')]
181+
perc = (df.shape[0]/float(before_size)) * 100
182+
print(f'df after {df.shape[0]}, remained {perc}% items')
183+
cpu_count = multiprocessing.cpu_count()
184+
#cpu_count = 1
185+
count = 0
186+
manager = multiprocessing.Manager()
187+
lst = manager.list()
188+
groupby_objects = [(filepath, group_item, args.dir) for filepath, group_item in df.groupby('filepath')]
189+
files = []
190+
print(f'CPU : {cpu_count}')
191+
fields=['filename','lines_n', 'for_statement', 'decimal_integer_literal', 'character_literal', 'generated_comment', 'while_statement', 'if_statement',
192+
'variable_declarator', 'cyclo_complexity', 'lines', 'local_variable_declaration', 'switch_expression', 'case', 'kb', 'bytes', 'mb', 'method_name']
193+
if Path(args.output).exists():
194+
Path(args.output).unlink()
195+
if Path('ignored_files.csv').exists():
196+
Path('ignored_files.csv').unlink()
197+
rows_n = 0
198+
pool = Pool(cpu_count)
199+
with open('ignored_files.csv', "a", newline='\n') as output_ignored:
200+
writer_ignored = csv.DictWriter(output_ignored, fieldnames=['filename', 'bytes', 'kb', 'mb'])
201+
writer_ignored.writeheader()
202+
with open(args.output, "a", newline='\n') as output:
203+
writer = csv.DictWriter(output, fieldnames=fields)
204+
writer.writeheader()
205+
res_p = list(tqdm(pool.imap(get_json_with_tokens, groupby_objects), total=len(groupby_objects)))
206+
if res_p:
207+
for j in res_p:
208+
file_ignored = j[-1]
209+
results = j[0]
210+
211+
if file_ignored is None:
212+
#print('file_ignored ', file_ignored)
213+
try:
214+
if results:
215+
#count += 1
216+
for j in results:
217+
rows_n += 1
218+
#print('####################################\n', j, '\n###################################')
219+
lst.append(j)
220+
if (rows_n % 500) == 0:
221+
sys.stdout.flush()
222+
for x in lst:
223+
writer.writerow(x)
224+
output.flush()
225+
lst[:] = []
226+
except Exception as e:
227+
print(f'{str(e)}')
228+
else:
229+
#print('file_ignored2 ', results)
230+
print('file_ignored ', file_ignored)
231+
for m in results:
232+
#print('file_ignored ', m)
233+
writer_ignored.writerow(m)
234+
output_ignored.flush()
235+
for x in lst:
236+
writer.writerow(x)
237+
print(f'Finished, items {rows_n}')

‎draw_graph.py‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import pandas as pd
2+
import subprocess
3+
import os
4+
import re
5+
import json
6+
from pathlib import Path
7+
from tqdm import tqdm
8+
from concurrent.futures import TimeoutError
9+
import argparse
10+
import os
11+
import multiprocessing
12+
import traceback
13+
from cchardet import detect
14+
import sys
15+
from multiprocessing import Pool
16+
from program_slicing.graph.parse import tree_sitter_ast_java
17+
from program_slicing.graph.parse import tree_sitter_parsers
18+
from program_slicing.graph.cdg import ControlDependenceGraph
19+
from program_slicing.graph.statement import Statement, StatementType
20+
from program_slicing.graph.point import Point
21+
import csv
22+
from collections import defaultdict, OrderedDict
23+
from program_graphs.ddg.ddg import mk_ddg
24+
from program_graphs.cfg.parser.java.parser import mk_cfg
25+
from program_graphs.cfg import CFG
26+
from program_graphs.adg import parse_java
27+
from typing import List, Tuple, Set, Optional
28+
import networkx as nx
29+
import numpy as np
30+
import networkx as nx
31+
import matplotlib.pyplot as plt
32+
from networkx.drawing.nx_agraph import graphviz_layout
33+
34+
35+
CycloComplexity = int
36+
37+
sys.setrecursionlimit(100000)
38+
39+
m_code = '''
40+
protected Expression transformClosureExpression ( ClosureExpression ce)
41+
{
42+
boolean oldInClosure = inClosure;
43+
inClosure = true;
44+
Parameter[] paras = ce.getParameters();
45+
if (paras != null) {
46+
for (Parameter para : paras) {
47+
ClassNode t = para.getType();
48+
resolveOrFail(t, ce);
49+
visitAnnotations(para);
50+
if (para.hasInitialExpression()) {
51+
Object initialVal = para.getInitialExpression();
52+
if (initialVal instanceof Expression) {
53+
para.setInitialExpression(transform((Expression) initialVal));
54+
}
55+
}
56+
visitAnnotations(para);
57+
}
58+
}
59+
Statement code = ce.getCode();
60+
if (code != null) code.visit(this);
61+
inClosure = oldInClosure;
62+
return ce;
63+
}
64+
'''
65+
m_code = ''' private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
66+
try {
67+
int i = 0;
68+
}
69+
finally {
70+
switch (te.getMessage()) {
71+
case "Unable to commit: transaction marked for rollback":
72+
// don't log as error, this happens if there's a ConcurrentUpdateException
73+
// at transaction end inside VCS
74+
isRollbackDuringCommit = true;
75+
// $FALL-THROUGH$
76+
case "Unable to commit: Transaction timeout":
77+
// don't log either
78+
log.debug(msg, e);
79+
break;
80+
default:
81+
log.error(msg, e);
82+
}
83+
throw new TransactionRuntimeException(e.getMessage(), e);
84+
}
85+
86+
}
87+
'''
88+
#with open('/hdd/emaslov/pmd-bin-6.46.0/large_disk/method_name_data/code2vec/java-large/training/apache__hive/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java') as f:
89+
#m_code = f.read()
90+
adg = parse_java(m_code)
91+
ast = adg.to_ast()
92+
nodes = [ast.nodes[x].get('ast_node') for x in ast if ast.nodes[x].get('ast_node')]
93+
#print(nodes)
94+
print([x for x in nodes if x.type == 'throw_statement'])
95+
ddg = adg.to_ddg()
96+
97+
G=nx.Graph()
98+
egde_labels = {}
99+
# Add nodes and edges
100+
for x, y in ddg.edges:
101+
#print(name, type(name))
102+
x_node = ast.nodes[x].get('ast_node')
103+
y_node = ast.nodes[y].get('ast_node')
104+
#print(node, type(node))
105+
x_label = ddg._node_to_label(x)
106+
y_label = ddg._node_to_label(y)
107+
egde_labels[x] = x_label + f'; start_pos={x_node.start_point}'
108+
egde_labels[y] = y_label + f'; start_pos={y_node.start_point}'
109+
#print(x_label, y_label, type(x_label), type(y_label))
110+
G.add_edge(x, y)
111+
112+
113+
print(G)
114+
plt.figure(figsize=(20,14))
115+
# <matplotlib.figure.Figure object at 0x7f1b65ea5e80>
116+
pos = nx.nx_pydot.graphviz_layout(G)
117+
nx.draw(G, pos = pos, \
118+
node_size=1200, node_color='lightblue', linewidths=0.25, \
119+
font_size=10, font_weight='bold', with_labels=False)
120+
nx.draw_networkx_labels(G, pos, egde_labels, font_size=16)
121+
plt.savefig('labels.png')
122+
#plt.show() ## plot2.png attached
123+
#nx.draw(G, pos=graphviz_layout(G), with_labels = True)
124+
comp = list(nx.connected_components(G))
125+
126+
#pdot = nx.nx_pydot.to_pydot(G)
127+
#print(pdot)
128+
#print(ddg.edges)
129+
#print(dir(ddg))
130+
#print(nx.connected_components(ddg.to_undirected()))
131+
#nx.nx_pydot.to_pydot(ddg.to_undirected())
132+
#print(ddg)

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL