--- An object that loads chunks of strings on demand compatible with a subset
-- of the string API suitable for parsing.
--
-- Project URL: https://github.com/gilzoide/stringstream-lua
--
-- Example:
-- local stringstream = require 'stringstream'
--
-- -- Streams may be created with callable objects (functions or tables/userdata
-- -- with __call metamethod) like the ones `load` expects, or file-like objects
-- -- that contain a `read` method, like open files.
-- local stream = assert(stringstream.new(io.stdin))
--
-- -- Alternatively, `stringstream.open(filename, ...)` may be used to open a file
-- -- by name in read mode and create a stringstream from it.
-- --
-- -- local stream = assert(stringstream.open("README.md"))
--
-- -- Now just call the supported string methods =D
-- while true do
-- local token, advance = stream:match("(%S+)()")
-- if not token then break end
-- -- ... do something with token
-- print('TOKEN', token)
-- stream = stream:sub(advance)
-- end
--
-- @module stringstream
-- @license [Public Domain](https://unlicense.org/)
-- Lua 5.1+ compatibility
local unpack = table.unpack or unpack
--- Internal Stream state, caching chunks
local stream_state = {}
stream_state.__index = stream_state
function stream_state.new(read, ...)
local state = setmetatable({
read = read,
contents = {},
first_loaded_chunk = 1,
last_loaded_chunk = 0,
loaded_length = 0,
read_args = { ... },
}, stream_state)
state:load_next()
return state
end
function stream_state:calculate_chunk_index(from_chunk, starting_index)
-- @warning: only call after checking `load_if_needed`
local chunk_start = from_chunk
local chunk = self.contents[chunk_start]
while starting_index > #chunk do
chunk_start = chunk_start + 1
starting_index = starting_index - #chunk
chunk = self.contents[chunk_start]
end
return chunk_start, starting_index
end
function stream_state:load_if_needed(from_chunk, starting_index)
assert(from_chunk >= self.first_loaded_chunk, "Seeking stream backwards is not supported")
-- TODO: after thorough testing, remove this next assertion
assert(from_chunk = self.first_loaded_chunk, "Trying to access unloaded chunks")
local content = table.concat(self.contents, '', chunk, self.last_loaded_chunk)
--print(' !!! STRING', chunk, starting_index, end_index, self.contents[chunk] ~= nil, self.contents[self.last_loaded_chunk] ~= nil)
return content:sub(starting_index, end_index)
end
function stream_state:__len()
return self.loaded_length
end
function stream_state:__tostring()
return self:string_from(self.first_loaded_chunk, 1)
end
--- String Stream module
local stringstream = {}
--- Default read size of streams created with file objects.
--
-- Change this to configure defaults module-wise.
-- Default value is 1024.
stringstream.readsize = 1024
--- Default maximum number of bytes that `stringstream:find` may load.
--
-- Change this to configure defaults module-wise.
-- Default value is `math.huge`, that is, load until stream is completely consumed, if necessary.
--
-- @see stringstream:find
stringstream.max_find_lookahead = math.huge
local function iscallable(v)
if type(v) == 'function' then
return true
else
local mt = getmetatable(v)
return mt and mt.__call
end
end
local function wrap_stream(stream, max_find_lookahead, chunk, starting_index)
return setmetatable({
stream = stream,
max_find_lookahead = max_find_lookahead or stringstream.max_find_lookahead,
chunk = chunk or stream.first_loaded_chunk,
starting_index = starting_index or 1,
}, stringstream)
end
--- Create a new stringstream from callable object or file-like object.
--
-- Both functions and tables or userdata with a `__call` metamethod are
-- accepted as callables. Just like `load`, each call must return a string that
-- concatenates with previous results and a return of an empty string or a falsey
-- value signals the end of the stream.
--
-- File-like objects are tables or userdata with a `read` method.
-- Chunks will be read calling the `read` method as in `object:read(...)`.
--
-- @usage
-- -- stream that reads chunks of 1024 bytes from opened file
-- -- and retries `find` as many times as necessary
-- stream = stringstream.new(io.open('file'))
--
-- -- stream that reads lines from stdin and fails a search after
-- -- having more than 4096 bytes loaded
-- stream = stringstream.new(io.stdin, { max_find_lookahead = 4096 }, '*l')
--
-- -- stream that generates digits from 1 to 10
-- local gen_numbers = coroutine.wrap(function()
-- for i = 1, 10 do
-- coroutine.yield(tostring(i))
-- end
-- end)
-- stream = stringstream.new(gen_numbers)
--
-- @tparam ?function|table|userdata callable_or_file Callable or file-like object which chunks will be read from
-- @param[opt] options Table of stream options. Currently the only supported option is `max_find_lookahead`.
-- @param[opt] ... Arguments to be forwarded to the reading function. If `callable_or_file` is a file and no extra arguments are passed, `stringstream.readsize` will be used
--
-- @return[1] stringstream object
-- @return[2] nil
-- @return[2] error message
function stringstream.new(callable_or_file, options, ...)
local read, should_forward_default_chunksize
if iscallable(callable_or_file) then
read = callable_or_file
else
local read_type = type(callable_or_file)
if read_type == 'table' or read_type == 'userdata' then
if not callable_or_file.read then
return nil, 'Argument is not a file-like object, no "read" method found'
end
should_forward_default_chunksize = select('#', ...) == 0 and io.type(callable_or_file) == 'file'
read = function(...) return callable_or_file:read(...) end
else
return nil, string.format("Expected callable or file-like object, found %q", read_type)
end
end
local stream
if should_forward_default_chunksize then
stream = stream_state.new(read, stringstream.readsize)
else
stream = stream_state.new(read, ...)
end
return wrap_stream(stream, options and options.max_find_lookahead, 1, 1)
end
--- Creates a new stream from file.
--
-- Opens the file with `io.open` in read mode and forward parameters to
-- `stringstream.new`.
--
-- @see stringstream.new
--
-- @usage
-- -- stream that reads chunks of 1024 bytes from 'file.txt'
-- -- and loads the entire contents on a call to `find`, if necessary
-- stream = stringstream.open('file.txt')
--
-- -- stream that reads lines from 'file.txt'
-- -- and loads the entire contents on a call to `find`, if necessary
-- stream = stringstream.open('file.txt', nil, '*l')
--
-- @tparam string filename Name of the file to open
-- @param[opt] ... Arguments forwarded to `stringstream:new`
--
-- @return[1] stringstream object
-- @treturn[2] nil
-- @treturn[2] string Error message
-- @treturn[2] number Error code on `io.open` failure
function stringstream.open(filename, ...)
local file, err, code = io.open(filename, 'r')
if not file then
return nil, err, code
end
return stringstream.new(file, ...)
end
--- Returns a substring or a new view into stream.
--
-- If both `i` and `j` are passed, returns the substring that starts at `i`
-- and continues until `j` or until the end of stream.
-- Otherwise, returns a new stringstream object with starting index `i`.
--
-- @raise If either `i` or `j` are non-positive, as reading from the end of
-- stream is not supported.
--
-- @tparam number i Starting index
-- @tparam[opt] number j Starting index
--
-- @treturn[1] stringstream If `j` is not passed and `i` is not past the end of the stream
-- @treturn[2] string Empty if `j` is not passed in and `i` is past the end of the stream
-- @treturn[3] string Substring if `j` is passed in
function stringstream:sub(i, j)
assert(i and i > 0, "Calling stringstream.sub with non-positive index is not supported!")
if not j then
local starting_index = self.starting_index + i - 1
if self.stream:load_if_needed(self.chunk, starting_index) then
return wrap_stream(self.stream, self.max_find_lookahead, self.stream:calculate_chunk_index(self.chunk, starting_index))
else
return ''
end
else
assert(j > 0, "Calling stringstream.sub with negative index is not supported!")
if j < i then
return ''
end
local starting_index = self.starting_index + i - 1
local ending_index = self.starting_index + j - 1
self.stream:load_if_needed(self.chunk, ending_index)
return self.stream:string_from(self.chunk, starting_index, ending_index)
end
end
--- Try finding `pattern` on loaded contents.
--
-- Upon failure or if match spans the entire loaded content, loads new chunks,
-- bailing out after having more than `max_find_lookahead` bytes loaded
-- counting from `init`, returning `nil` afterwards.
--
-- Notice that the default value for `max_find_lookahead` makes the whole stream be
-- loaded in case of failures or big matches. This is to have consistent output
-- between stringstream and the string API by default.
--
-- @raise If `init` is a negative number.
--
-- @see string:find
--
-- @tparam string pattern Pattern string to search
-- @tparam[opt] number init Where to start the search from
-- @param[opt] plain If truthy, turns off the pattern matching facilities
--
-- @treturn[1] number Starting index of found pattern
-- @treturn[1] number Ending index of found pattern
-- @treturn[1] ... Aditional captures of found pattern
-- @treturn[2] fail If pattern is not found
function stringstream:find(pattern, init, plain)
init = init or 1
assert(init >= 0, "Calling stringstream.find with negative index is not supported!")
local stream, chunk, starting_index = self.stream, self.chunk, self.starting_index
local loaded_length = stream:load_if_needed(chunk, starting_index + init - 1)
while loaded_length < self.max_find_lookahead do
local text = stream:string_from(chunk, starting_index)
local results = { text:find(pattern, init, plain) }
if results[1] and results[2] < #text then
return unpack(results)
-- TODO: check if `init` may be updated with `results[1]` in case a
-- match was found, but may be longer, to reduce search space
-- (check if there are cases where this would return wrong results)
end
local extra_contents = stream:load_next()
if not extra_contents then
return unpack(results)
end
loaded_length = loaded_length + #extra_contents
end
return nil
end
--- Try matching `pattern` on loaded contents.
--
-- Uses `stringstream:find` for searching, so the same caveats apply.
--
-- @raise If `init` is a negative number.
--
-- @see string:match
-- @see stringstream:find
--
-- @tparam string pattern Pattern string to search
-- @tparam[opt] number init Where to start the search from
--
-- @treturn[1] ... Captures of the found pattern
-- @treturn[2] string The whole match, if pattern specifies no captures
-- @treturn[3] fail If pattern is not found
function stringstream:match(pattern, init)
assert((not init) or init >= 0, "Calling stringstream.match with negative index is not supported!")
local find_results = { self:find(pattern, init, false) }
if find_results[1] and not find_results[3] then
return self:sub(find_results[1], find_results[2])
else
return unpack(find_results, 3)
end
end
--- Returns an iterator function that, each time it is called, returns the next
-- match from loaded contents.
--
-- Uses `stringstream:find` for searching, so the same caveats apply.
--
-- @raise If `init` is a negative number.
--
-- @see string:gmatch
-- @see stringstream:match
--
-- @tparam string pattern Pattern string to search
-- @tparam[opt] number init Where to start the search from
--
-- @treturn function Iterator function over matches
function stringstream:gmatch(pattern, init)
assert((not init) or init >= 0, "Calling stringstream.gmatch with negative index is not supported!")
-- Don't interpreted '^' as an anchor, just like `string.gmatch` does
if pattern:sub(1, 1) == '^' then
pattern = '%' .. pattern
end
local function iterate()
local init = init
while true do
local find_results = { self:find(pattern, init, false) }
if find_results[1] then
--print('GMATCH', unpack(find_results))
if not find_results[3] then
coroutine.yield(self:sub(find_results[1], find_results[2]))
else
coroutine.yield(unpack(find_results, 3))
end
init = find_results[2] + 1
else
return nil
end
end
end
return coroutine.wrap(iterate)
end
--- Returns the current loaded content string.
--
-- Be careful that it almost never reflects the entire content string.
function stringstream:__tostring()
return self.stream:string_from(self.chunk, self.starting_index)
end
--- Returns the current loaded content length.
--
-- Be careful that it almost never reflects the entire content length.
function stringstream:__len()
return self.stream.loaded_length - (self.starting_index - 1)
end
--- Alias for `stringstream:__len`
-- @function stringstream.len
stringstream.__index = {
sub = stringstream.sub,
find = stringstream.find,
match = stringstream.match,
gmatch = stringstream.gmatch,
len = stringstream.__len,
__len = stringstream.__len,
}
--- Module version.
stringstream._VERSION = '0.2.0'
return stringstream