FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Initial commit. · aboutcode-org/commoncode@8768022 · GitHub

/ commoncode Public

Commit 8768022

Browse files
pombredanne
committed
Initial commit.
0 parents  commit 8768022

111 files changed

Lines changed: 6118 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/commoncode/PSF.LICENSE‎

Lines changed: 635 additions & 0 deletions
Large diffs are not rendered by default.

‎src/commoncode/__init__.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#
2+
# Copyright (c) 2015 nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
4+
# The ScanCode software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode require an acknowledgment.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# When you publish or redistribute any data created with ScanCode or any ScanCode
16+
# derivative work, you must accompany this data with the following acknowledgment:
17+
#
18+
# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
19+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
20+
# ScanCode should be considered or used as legal advice. Consult an Attorney
21+
# for any legal advice.
22+
# ScanCode is a free software code scanning tool from nexB Inc. and others.
23+
# Visit https://github.com/nexB/scancode-toolkit/ for support and download.
24+
25+
26+
# set re and fnmatch _MAXCACHE to 1M to cache regex compiled aggressively
27+
# their default is 100 and many utilities and libraries use a lot of regex
28+
29+
import re
30+
31+
remax = getattr(re, '_MAXCACHE', 0)
32+
if remax < 1000000:
33+
setattr(re, '_MAXCACHE', 1000000)
34+
del remax
35+
36+
import fnmatch
37+
38+
fnmatchmax = getattr(fnmatch, '_MAXCACHE', 0)
39+
if fnmatchmax < 1000000:
40+
setattr(fnmatch, '_MAXCACHE', 1000000)
41+
del fnmatchmax
42+
del re

‎src/commoncode/codec.py‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
#
2+
# Copyright (c) 2015 nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
4+
# The ScanCode software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode require an acknowledgment.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# When you publish or redistribute any data created with ScanCode or any ScanCode
16+
# derivative work, you must accompany this data with the following acknowledgment:
17+
#
18+
# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
19+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
20+
# ScanCode should be considered or used as legal advice. Consult an Attorney
21+
# for any legal advice.
22+
# ScanCode is a free software code scanning tool from nexB Inc. and others.
23+
# Visit https://github.com/nexB/scancode-toolkit/ for support and download.
24+
25+
from __future__ import absolute_import, print_function
26+
27+
"""
28+
Numbers to bytes or strings and URLs coder/decoders.
29+
"""
30+
31+
padding = '/'
32+
33+
b85_symbols = ('0123456789'
34+
'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
35+
'abcdefghijklmnopqrstuvwxyz'
36+
'!#$%&()*+-;<=>?@^_`{|}~')
37+
38+
len_b85_symbols = len(b85_symbols)
39+
40+
41+
def to_base_n(num, base):
42+
"""
43+
Convert `num` number to a string representing this number in base `base`
44+
where base <= 85.
45+
46+
Use recursion for progressive encoding.
47+
"""
48+
# ensure that base is within bounds
49+
assert base >= 2 and base <= len_b85_symbols
50+
if num == 0:
51+
return '0'
52+
# recurse with a floor division to encode from left to right
53+
based = to_base_n(num // base, base)
54+
# remove leading zeroes resulting from floor-based encoding
55+
stripped = based.lstrip('0')
56+
# pick the symbol in the symbol table using a modulo
57+
encoded = b85_symbols[num % base]
58+
return stripped + encoded
59+
60+
61+
MAXLEN = len(to_base_n(pow(2, 32) - 1, 85))
62+
63+
64+
def to_base85(num):
65+
"""
66+
Convert `num` number to a string representing this number in base 85,
67+
padded as needed.
68+
69+
The character set to encode 85 base85 digits is defined to be:
70+
'0'..'9', 'A'..'Z', 'a'..'z', '!', '#', '$', '%', '&', '(',
71+
')', '*', '+', '-', ';', '<', '=', '>', '?', '@', '^', '_',
72+
'`', '{', '|', '}', and '~'.
73+
74+
From http://www.faqs.org/rfcs/rfc1924.html
75+
76+
See also http://en.wikipedia.org/wiki/Base_85 for the rationale for Base
77+
85. Git also uses https://github.com/git/git/blob/master/base85.c
78+
"""
79+
encoded = to_base_n(num, 85)
80+
# add padding
81+
elen = len(encoded)
82+
if elen < MAXLEN:
83+
encoded = encoded + (padding * (MAXLEN - (elen)))
84+
return encoded
85+
86+
87+
def to_base10(s, b=36):
88+
"""
89+
Convert a string s representing a number in base b back to an integer.
90+
"""
91+
assert b <= len(b85_symbols) and b >= 2, ('Base must be in range(2, %d)'
92+
% (len(b85_symbols)))
93+
# strip padding
94+
s = s.replace(padding, '')
95+
96+
base10_num = 0
97+
i = len(s) - 1
98+
for digit in s:
99+
base10_num += b85_symbols.index(digit) * pow(b, i)
100+
i -= 1
101+
return base10_num
102+
103+
104+
def num_to_bin(num):
105+
"""
106+
Convert a `num` integer or long to a binary string byte-ordered such that
107+
the least significant bytes are at the beginning of the string (aka. little
108+
endian).
109+
110+
NOTE: The code below does not use struct for conversions to handle
111+
arbitrary long binary strings (such as a SHA512 digest) and convert that
112+
safely to a long: using structs does not work easily for this.
113+
"""
114+
binstr = []
115+
while num > 0:
116+
# add the least significant byte value
117+
binstr.append(chr(num & 0xFF))
118+
# shift the next byte to least significant and repeat
119+
num = num >> 8
120+
121+
# reverse the list now to the most significant
122+
# byte is at the start of ths string to speed decoding
123+
return ''.join(reversed(binstr))
124+
125+
126+
def bin_to_num(binstr):
127+
"""
128+
Convert a little endian byte-ordered binary string to an integer or long.
129+
"""
130+
# this will cast to long as needed
131+
num = 0
132+
for charac in binstr:
133+
# the most significant byte is a the start of the string so we multiply
134+
# that value by 256 (e.g. <<8) and add the value of the current byte,
135+
# then move to next byte in the string and repeat
136+
num = (num << 8) + ord(charac)
137+
return num
138+
139+
140+
from base64 import standard_b64decode as stddecode
141+
from base64 import urlsafe_b64encode as b64encode
142+
143+
144+
def urlsafe_b64encode(s):
145+
"""
146+
Encode a binary string to a url safe base64 encoding.
147+
"""
148+
return b64encode(s)
149+
150+
151+
def urlsafe_b64decode(b64):
152+
"""
153+
Decode a url safe base64-encoded string.
154+
Note that we use stddecode to work around a bug in the standard library.
155+
"""
156+
b = b64.replace('-', '+').replace('_', '/')
157+
return stddecode(b)
158+
159+
160+
def _encode(num):
161+
"""
162+
Encode a number (int or long) in url safe base64.
163+
Used in simhash5
164+
"""
165+
return b64encode(num_to_bin(num))

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL