cpython/Lib/shlex.py at main · python/cpython · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

"""A lexical analyzer class for simple shell-like syntaxes."""

# Module and documentation by Eric S. Raymond, 21 Dec 1998

# Input stacking and error message cleanup added by ESR, March 2000

# push_source() and pop_source() made explicit by ESR, January 2001.

# Posix compliance, split(), string arguments, and

# iterator interface by Gustavo Niemeyer, April 2003.

# changes to tokenize more like Posix shells by Vinay Sajip, July 2016.

import sys

from io import StringIO

__all__ = ["shlex", "split", "quote", "join"]

class shlex:

"A lexical analyzer class for simple shell-like syntaxes."

def __init__(self, instream=None, infile=None, posix=False,

punctuation_chars=False):

from collections import deque # deferred import for performance

if isinstance(instream, str):

instream = StringIO(instream)

if instream is not None:

self.instream = instream

self.infile = infile

else:

self.instream = sys.stdin

self.infile = None

self.posix = posix

if posix:

self.eof = None

else:

self.eof = ''

self.commenters = '#'

self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'

'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')

if self.posix:

self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'

'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')

self.whitespace = ' \t\r\n'

self.whitespace_split = False

self.quotes = '\'"'

self.escape = '\\'

self.escapedquotes = '"'

self.state = ' '

self.pushback = deque()

self.lineno = 1

self.debug = 0

self.token = ''

self.filestack = deque()

self.source = None

if not punctuation_chars:

punctuation_chars = ''

elif punctuation_chars is True:

punctuation_chars = '();<>|&'

self._punctuation_chars = punctuation_chars

if punctuation_chars:

# _pushback_chars is a push back queue used by lookahead logic

self._pushback_chars = deque()

# these chars added because allowed in file names, args, wildcards

self.wordchars += '~-./*?='

#remove any punctuation chars from wordchars

t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))

self.wordchars = self.wordchars.translate(t)

@property

def punctuation_chars(self):

return self._punctuation_chars

def push_token(self, tok):

"Push a token onto the stack popped by the get_token method"

if self.debug >= 1:

print("shlex: pushing token " + repr(tok))

self.pushback.appendleft(tok)

def push_source(self, newstream, newfile=None):

"Push an input source onto the lexer's input source stack."

if isinstance(newstream, str):

newstream = StringIO(newstream)

self.filestack.appendleft((self.infile, self.instream, self.lineno))

self.infile = newfile

self.instream = newstream

self.lineno = 1

if self.debug:

if newfile is not None:

print('shlex: pushing to file %s' % (self.infile,))

else:

print('shlex: pushing to stream %s' % (self.instream,))

def pop_source(self):

"Pop the input source stack."

self.instream.close()

(self.infile, self.instream, self.lineno) = self.filestack.popleft()

if self.debug:

print('shlex: popping to %s, line %d' \

% (self.instream, self.lineno))

self.state = ' '

def get_token(self):

"Get a token from the input stream (or from stack if it's nonempty)"

if self.pushback:

tok = self.pushback.popleft()

if self.debug >= 1:

print("shlex: popping token " + repr(tok))

return tok

# No pushback. Get a token.

raw = self.read_token()

# Handle inclusions

if self.source is not None:

while raw == self.source:

spec = self.sourcehook(self.read_token())

if spec:

(newfile, newstream) = spec

self.push_source(newstream, newfile)

raw = self.get_token()

# Maybe we got EOF instead?

while raw == self.eof:

if not self.filestack:

return self.eof

else:

self.pop_source()

raw = self.get_token()

# Neither inclusion nor EOF

if self.debug >= 1:

if raw != self.eof:

print("shlex: token=" + repr(raw))

else:

print("shlex: token=EOF")

return raw

def read_token(self):

quoted = False

escapedstate = ' '

while True:

if self.punctuation_chars and self._pushback_chars:

nextchar = self._pushback_chars.pop()

else:

nextchar = self.instream.read(1)

if nextchar == '\n':

self.lineno += 1

if self.debug >= 3:

print("shlex: in state %r I see character: %r" % (self.state,

nextchar))

if self.state is None:

self.token = '' # past end of file

break

elif self.state == ' ':

if not nextchar:

self.state = None # end of file

break

elif nextchar in self.whitespace:

if self.debug >= 2:

print("shlex: I see whitespace in whitespace state")

if self.token or (self.posix and quoted):

break # emit current token

else:

continue

elif nextchar in self.commenters:

self.instream.readline()

self.lineno += 1

elif self.posix and nextchar in self.escape:

escapedstate = 'a'

self.state = nextchar

elif nextchar in self.wordchars:

self.token = nextchar

self.state = 'a'

elif nextchar in self.punctuation_chars:

self.token = nextchar

self.state = 'c'

elif nextchar in self.quotes:

if not self.posix:

self.token = nextchar

self.state = nextchar

elif self.whitespace_split:

self.token = nextchar

self.state = 'a'

else:

self.token = nextchar

if self.token or (self.posix and quoted):

break # emit current token

else:

continue

elif self.state in self.quotes:

quoted = True

if not nextchar: # end of file

if self.debug >= 2:

print("shlex: I see EOF in quotes state")

# XXX what error should be raised here?

raise ValueError("No closing quotation")

if nextchar == self.state:

if not self.posix:

self.token += nextchar

self.state = ' '

break

else:

self.state = 'a'

elif (self.posix and nextchar in self.escape and self.state

in self.escapedquotes):

escapedstate = self.state

self.state = nextchar

else:

self.token += nextchar

elif self.state in self.escape:

if not nextchar: # end of file

if self.debug >= 2:

print("shlex: I see EOF in escape state")

# XXX what error should be raised here?

raise ValueError("No escaped character")

# In posix shells, only the quote itself or the escape

# character may be escaped within quotes.

if (escapedstate in self.quotes and

nextchar != self.state and nextchar != escapedstate):

self.token += self.state

self.token += nextchar

self.state = escapedstate

elif self.state in ('a', 'c'):

if not nextchar:

self.state = None # end of file

break

elif nextchar in self.whitespace:

if self.debug >= 2:

print("shlex: I see whitespace in word state")

self.state = ' '

if self.token or (self.posix and quoted):

break # emit current token

else:

continue

elif nextchar in self.commenters:

self.instream.readline()

self.lineno += 1

if self.posix:

self.state = ' '

if self.token or (self.posix and quoted):

break # emit current token

else:

continue

elif self.state == 'c':

if nextchar in self.punctuation_chars:

self.token += nextchar

else:

if nextchar not in self.whitespace:

self._pushback_chars.append(nextchar)

self.state = ' '

break

elif self.posix and nextchar in self.quotes:

self.state = nextchar

elif self.posix and nextchar in self.escape:

escapedstate = 'a'

self.state = nextchar

elif (nextchar in self.wordchars or nextchar in self.quotes

or (self.whitespace_split and

nextchar not in self.punctuation_chars)):

self.token += nextchar

else:

if self.punctuation_chars:

self._pushback_chars.append(nextchar)

else:

self.pushback.appendleft(nextchar)

if self.debug >= 2:

print("shlex: I see punctuation in word state")

self.state = ' '

if self.token or (self.posix and quoted):

break # emit current token

else:

continue

result = self.token

self.token = ''

if self.posix and not quoted and result == '':

result = None

if self.debug > 1:

if result:

print("shlex: raw token=" + repr(result))

else:

print("shlex: raw token=EOF")

return result

def sourcehook(self, newfile):

"Hook called on a filename to be sourced."

import os.path

if newfile[0] == '"':

newfile = newfile[1:-1]

# This implements cpp-like semantics for relative-path inclusion.

if isinstance(self.infile, str) and not os.path.isabs(newfile):

newfile = os.path.join(os.path.dirname(self.infile), newfile)

return (newfile, open(newfile, "r"))

def error_leader(self, infile=None, lineno=None):

"Emit a C-compiler-like, Emacs-friendly error-message leader."

if infile is None:

infile = self.infile

if lineno is None:

lineno = self.lineno

return "\"%s\", line %d: " % (infile, lineno)

def __iter__(self):

return self

def __next__(self):

token = self.get_token()

if token == self.eof:

raise StopIteration

return token

def split(s, comments=False, posix=True):

"""Split the string *s* using shell-like syntax."""

if s is None:

raise ValueError("s argument must not be None")

lex = shlex(s, posix=posix)

lex.whitespace_split = True

if not comments:

lex.commenters = ''

return list(lex)

def join(split_command):

"""Return a shell-escaped string from *split_command*."""

return ' '.join(quote(arg) for arg in split_command)

def quote(s, *, force=False):

"""Return a shell-escaped version of the string *s*.

If *force* is *True*, then *s* is unconditionally quoted,

even if it is already safe for a shell without being quoted.

"""

if not s:

return "''"

if not isinstance(s, str):

raise TypeError(f"expected string object, got {type(s).__name__!r}")

# Use bytes.translate() for performance

safe_chars = (b'%+,-./0123456789:=@'

b'ABCDEFGHIJKLMNOPQRSTUVWXYZ_'

b'abcdefghijklmnopqrstuvwxyz')

# No quoting is needed if we are not forcing quoting

# and `s` is an ASCII string consisting only of `safe_chars`.

if (not force

and s.isascii() and not s.encode().translate(None, delete=safe_chars)):

return s

# use single quotes, and put single quotes into double quotes

# the string $'b is then quoted as '$'"'"'b'

return "'" + s.replace("'", "'\"'\"'") + "'"

def _print_tokens(lexer):

while tt := lexer.get_token():

print("Token: " + repr(tt))

if __name__ == '__main__':

if len(sys.argv) == 1:

_print_tokens(shlex())

else:

fn = sys.argv[1]

with open(fn) as f:

_print_tokens(shlex(f, fn))

cpython/Lib/shlex.py at main · python/cpython · GitHub

Repository navigation

Expand file tree

Breadcrumbs

shlex.py

Latest commit

History

Breadcrumbs

shlex.py

File metadata and controls