Re: Integer Basic Tokenization

[pausch@saaf.se (Paul Schlyter)]

In article <3AB1D3FB.8CE2B306@swbell.net>,
Rubywand  <rubywand@swbell.net> wrote:
 
> Don't think I've ever come across a token listing for INT BASIC.
 
Then let me present you to one.  Note that $01 is used to represent
the end of an Integer Basic line.  Also note that since this is C
source code, "\"" means a string containing only one quote (") char.
 
In Applesoft it's about equally simple to convert tokens to ASCII and
ASCII to tokens.  But in Integer Basic it is much simpler to convert
tokens to ASCII than to convert ASCII to tokens, since several tokens
have the same ASCII representation.  This will make tokenization much
more complex since which token e.g. a comma or a start parentheses
should be translated to depends on the context where it appears.  One
simple example: in Applesoft, a plus always translates to the same
token, but in Integer Basic a uniary plus will become $35 in
tokenized form, while a binary plus will become $12 in tokenized
form.  When tokenizing an ASCII string, one must therefore parse a
statement to determine whether a particular plus is a unary plus, a
binary plus, or a syntax error.
 
 
#define REM_TOKEN   0x5D
#define UNARY_PLUS  0x35
#define UNARY_MINUS 0x36
#define QUOTE_START 0x28
#define QUOTE_END   0x29
static char *itoken[128] =
{
    /* $00-$0F */
    "HIMEM:","<$01>", "_",     " : ",
    "LOAD",  "SAVE",  "CON",   "RUN",    /* Direct commands */
    "RUN",   "DEL",   ",",     "NEW",
    "CLR",   "AUTO",  ",",     "MAN",
 
    /* $10-$1F */
    "HIMEM:","LOMEM:","+",     "-",     /* Binary ops */
    "*",     "/",     "=",     "#",
    ">=",    ">",     "<=",    "<>",
    "<",     "AND",   "OR",    "MOD",
 
    /* $20-$2F */
    "^",     "+",     "(",     ",",
    "THEN",  "THEN",  ",",     ",",
    "\"",    "\"",    "(",     "!",
    "!",     "(",     "PEEK",  "RND",
 
    /* $30-$3F */
    "SGN",   "ABS",   "PDL",   "RNDX",
    "(",     "+",     "-",     "NOT",      /* Unary ops */
    "(",     "=",     "#",     "LEN(",
    "ASC(",  "SCRN(", ",",     "(",
 
    /* $40-$4F */
    "$",     "$",     "(",     ",",
    ",",     ";",     ";",     ";",
    ",",     ",",     ",",     "TEXT",  /* Statements */
    "GR",    "CALL",  "DIM",   "DIM",
 
    /* $50-$5F */
    "TAB",   "END",   "INPUT", "INPUT",
    "INPUT", "FOR",   "=",     "TO",
    "STEP",  "NEXT",  ",",     "RETURN",
    "GOSUB", "REM",   "LET",   "GOTO",
 
    /* $60-$6F */
    "IF",    "PRINT", "PRINT", "PRINT",
    "POKE",  ",",     "COLOR=","PLOT",
    ",",     "HLIN",  ",",     "AT",
    "VLIN",  ",",     "AT",    "VTAB",
 
    /* $70-$7F */
    "=",     "=",     ")",     ")",
    "LIST",  ",",     "LIST",  "POP",
    "NODSP", "DSP",  "NOTRACE","DSP",
    "DSP",   "TRACE", "PR#",   "IN#",
};
-- 
----------------------------------------------------------------
Paul Schlyter,  Swedish Amateur Astronomer's Society (SAAF)
Grev Turegatan 40,  S-114 38 Stockholm,  SWEDEN
e-mail:  pausch at saaf dot se   or    paul.schlyter at ausys dot se
WWW:     http://hotel04.ausys.se/pausch    http://welcome.to/pausch