FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
cecli/cecli/decoding.py at main · cecli-dev/cecli · GitHub
cecli-dev
cecli
Repository navigation
Code
Issues
11
(11)
Pull requests
8
(8)
Actions
Projects
Security and quality
Insights
Expand file tree
Breadcrumbs
cecli
/
cecli
/
decoding.py
Copy path
More file actions
More file actions
Latest commit
History
History
History
115 lines (87 loc) · 4.09 KB
Breadcrumbs
cecli
/
cecli
/
decoding.py
Copy path
File metadata and controls
115 lines (87 loc) · 4.09 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
"""
Encoding-aware file operations with automatic encoding detection for reading
and configurable encoding for writing.
Provides safe_open() as a drop-in replacement for built-in open() that handles
encoding intelligently on reads, smart_read() for direct file reading with
fallback encoding detection via charset_normalizer, and safe_write() for
writing files with a configurable default encoding.
The module-level DEFAULT_ENCODING variable can be overridden at runtime
(e.g., from main.py) to set the preferred encoding for write operations.
"""
import
charset_normalizer
# Global encoding setting used by safe_open() and safe_write() for write
# operations. Can be overridden at runtime from main.py or elsewhere.
DEFAULT_ENCODING
=
"utf-8"
def
smart_read
(
filepath
):
"""
Read a text file, automatically detecting the encoding if not UTF-8.
Attempts a fast UTF-8 decode first using Python's optimized native
decoder. Falls back to charset_normalizer for legacy encodings (e.g.
GBK, Latin-1, Shift-JIS) when a UnicodeDecodeError is raised.
Args:
filepath: Path to the file to read.
Returns:
The file contents as a string.
Raises:
ValueError: If the encoding cannot be determined.
FileNotFoundError: If the file does not exist.
"""
try
:
# FAST PATH: Try standard UTF-8 first using Python's optimized
# native decoder — this handles the vast majority of modern files.
with
open
(
filepath
,
"r"
,
encoding
=
"utf-8"
)
as
f
:
return
f
.
read
()
except
UnicodeDecodeError
:
# SLOW PATH: A decode error means this is likely a legacy encoding.
# Let the library read the raw bytes and guess the correct encoding.
results
=
charset_normalizer
.
from_path
(
filepath
)
match
=
results
.
best
()
if
match
is
None
:
raise
ValueError
(
f"Could not determine text encoding for
{
filepath
}
"
)
return
str
(
match
)
def
safe_write
(
filepath
,
content
,
encoding
=
None
):
"""
Write string content to a file using the specified or default encoding.
Args:
filepath: Path to the file to write.
content: The string content to write.
encoding: Encoding to use. Defaults to the module-level
DEFAULT_ENCODING if not provided.
Returns:
The content that was written (for convenience / chaining).
"""
if
encoding
is
None
:
encoding
=
DEFAULT_ENCODING
with
open
(
filepath
,
"w"
,
encoding
=
encoding
)
as
f
:
f
.
write
(
content
)
return
content
def
safe_open
(
filepath
,
mode
=
"r"
,
encoding
=
None
,
**
kwargs
):
"""
Open a file with intelligent encoding handling.
"""
# Binary modes — pass through to built-in open() unchanged.
if
"b"
in
mode
:
return
open
(
filepath
,
mode
,
**
kwargs
)
# If the user explicitly provided an encoding, respect it unconditionally.
if
encoding
is
not
None
:
return
open
(
filepath
,
mode
,
encoding
=
encoding
,
**
kwargs
)
# Write / append modes (with no explicit encoding) — use DEFAULT_ENCODING.
# Note: We don't check for "r+" here because if a file is being read,
# we must detect its existing encoding first so we don't corrupt it.
if
"w"
in
mode
or
"a"
in
mode
or
"x"
in
mode
:
return
open
(
filepath
,
mode
,
encoding
=
DEFAULT_ENCODING
,
**
kwargs
)
# Read modes (and read/write "r+" modes) — Auto-detect the encoding.
try
:
# FAST PATH: Try UTF-8 first by opening and reading a tiny chunk
with
open
(
filepath
,
mode
,
encoding
=
"utf-8"
,
**
kwargs
)
as
f
:
f
.
read
(
1
)
# Test if it raises a DecodeError
# If we got here, UTF-8 is valid. Return a fresh file object.
return
open
(
filepath
,
mode
,
encoding
=
"utf-8"
,
**
kwargs
)
except
UnicodeDecodeError
:
# SLOW PATH: Detect encoding using charset_normalizer
results
=
charset_normalizer
.
from_path
(
filepath
)
match
=
results
.
best
()
if
match
is
None
:
raise
ValueError
(
f"Could not determine text encoding for
{
filepath
}
"
)
# Return a real file object using the detected encoding
return
open
(
filepath
,
mode
,
encoding
=
match
.
encoding
,
**
kwargs
)
Back
|
FazBrowse Home
|
New Git URL