/* GNUPLOT - scanner.c */

/*[
 * Copyright 1986 - 1993, 1998, 2004   Thomas Williams, Colin Kelley
 *
 * Permission to use, copy, and distribute this software and its
 * documentation for any purpose with or without fee is hereby granted,
 * provided that the above copyright notice appear in all copies and
 * that both that copyright notice and this permission notice appear
 * in supporting documentation.
 *
 * Permission to modify the software is granted, but not the right to
 * distribute the complete modified source code.  Modifications are to
 * be distributed as patches to the released version.  Permission to
 * distribute binaries produced by compiling modified sources is granted,
 * provided you
 *   1. distribute the corresponding source modifications from the
 *    released version in the form of a patch file along with the binaries,
 *   2. add special version identification to distinguish your version
 *    in addition to the base release version number,
 *   3. provide your name and address as the primary contact for the
 *    support of your modified version, and
 *   4. retain our contact information in regard to use of the base
 *    software.
 * Permission to distribute the released version of the source code along
 * with corresponding source modifications in the form of a patch file is
 * granted with same provisions 2 through 4 for binary distributions.
 *
 * This software is provided "as is" without express or implied warranty
 * to the extent permitted by applicable law.
]*/

#include "scanner.h"

#include "alloc.h"
#include "command.h"
#include "util.h"

int curly_brace_count;

static int get_num(char str[]);
static void substitute(char **strp, size_t *str_lenp, int current);

#define isident(c) (isalnum((unsigned char)(c)) || (c) == '_' || ALLOWED_8BITVAR(c))

#define LBRACE '{'
#define RBRACE '}'

#define APPEND_TOKEN {token[t_num].length++; current++;}

static int t_num;		/* number of token I'm working on */

TBOOLEAN
legal_identifier(char *p)
{
    if (!p || !(*p) || isdigit((unsigned char)*p))
	return FALSE;
    while (*p) {
	if (!isident(*p))
	    return FALSE;
	p++;
    }
    return TRUE;
}

/*
 * scanner() breaks expression[] into lexical units, storing them in token[].
 *   The total number of tokens found is returned as the function
 *   value.  Scanning will stop when '\0' is found in expression[], or
 *   when token[] is full.  extend_input_line() is called to extend
 *   expression array if needed.
 *
 *       Scanning is performed by following rules:
 *
 *      Current char    token should contain
 *     -------------    -----------------------
 *      1.  alpha,_     all following alpha-numerics
 *      2.  digit       0 or more following digits, 0 or 1 decimal point,
 *                              0 or more digits, 0 or 1 'e' or 'E',
 *                              0 or more digits.
 *      3.  ^,+,-,/     only current char
 *          %,~,(,)
 *          [,],;,:,
 *          ?,comma
 *          $           
 *      4.  &,|,=,*     current char; also next if next is same
 *      5.  !,       current char; also next if next is =
 *      6.  ", '        all chars up until matching quote
 *      7.  #           this token cuts off scanning of the line (DFK).
 *      8.  `           (command substitution: all characters through the
 *                      matching backtic are replaced by the output of
 *                      the contained command, then scanning is restarted.)
 * EAM Jan 2010:	Bugfix. No rule covered an initial period. This caused
 *			string concatenation to fail for variables whose first
 *			character is 'E' or 'e'.  Now we add a 9th rule:
 *	9.  .		A period may be a token by itself (string concatenation)
 *			or the start of a decimal number continuing with a digit
 *
 *                      white space between tokens is ignored
 */
int
scanner(char **expressionp, size_t *expressionlenp)
{
    int current;	/* index of current char in expression[] */
    char *expression = *expressionp;
    int quote;
    char brace;

    curly_brace_count = 0;

    for (current = t_num = 0; expression[current] != NUL; current++) {
	if (t_num + 1 >= token_table_size) {
	    /* leave space for dummy end token */
	    extend_token_table();
	}
	if (isspace((unsigned char) expression[current]))
	    continue;		/* skip the whitespace */
	token[t_num].start_index = current;
	token[t_num].length = 1;
	token[t_num].is_token = TRUE;	/* to start with... */

	if (expression[current] == '`') {
	    substitute(expressionp, expressionlenp, current);
	    expression = *expressionp;	/* expression might have moved */
	    current--;
	    continue;
	}
	/* allow _ to be the first character of an identifier */
	/* allow 8bit characters in identifiers */
	if (isalpha((unsigned char)expression[current])
	    || (expression[current] == '_')
	    || ALLOWED_8BITVAR(expression[current])) {
		while (isident(expression[current + 1]))
		    APPEND_TOKEN;

	} else if (isdigit((unsigned char) expression[current])) {
	    token[t_num].is_token = FALSE;
	    token[t_num].length = get_num(&expression[current]);
	    current += (token[t_num].length - 1);

	} else if (expression[current] == '.') {
	    /* Rule 9 */
	    if (isdigit((unsigned char)expression[current+1])) {
		token[t_num].is_token = FALSE;
		token[t_num].length = get_num(&expression[current]);
		current += (token[t_num].length - 1);
	    } /* do nothing if the . is a token by itself */

	} else if (expression[current] == LBRACE) {
	    int partial;
	    token[t_num].is_token = FALSE;
	    token[t_num].l_val.type = CMPLX;
	    partial = sscanf(&expression[++current], "%lf , %lf %c",
			&token[t_num].l_val.v.cmplx_val.real,
			&token[t_num].l_val.v.cmplx_val.imag,
			&brace);

	    if (partial