[scn_base.h] Basic Token Separation


#include "symbols.h"
#include "scn_io.h"



The module [scn_base] implements the scan stream interface based on a given STYX-konform scanner definition. ( see [scn_gen] )

The scan stream is reentrant and allows multiple scan operations at a time.


Macros


Token definition flags ( see Scn_dfnToken )

#define SCN_FLG_IgnoreToken 1
#define SCN_FLG_IgnoreCase  2


Token preprocessing result values

#define SCN_FLG_RescanNone     0
#define SCN_FLG_RescanString   1
#define SCN_FLG_RescanBString  2
#define SCN_FLG_RescanFile     3
#define SCN_FLG_RescanBFile    4
#define SCN_FLG_RescanExtFile  5
#define SCN_FLG_RescanExtBFile 6
#define SCN_FLG_RescanChar     7
#define SCN_FLG_RescanChrItr   8
#define SCN_FLG_RescanBChrItr  9



Stream of tokens


Scn_Stream Stream onto a scanner

Scn_Stream_Itr Character iterator onto a scan stream

   Type of token preprocessing function
   RC = SCN_FLG_RescanChrItr   & valid cMacVal --> rescan character iterator cMacVal
                                                   ( frees character iterator )
   RC = SCN_FLG_RescanBChrItr  & valid cMacVal --> rescan binary char iterator cMacVal
                                                   ( frees character iterator )
   RC = SCN_FLG_RescanString   & valid cMacVal --> rescan string cMacVal
   RC = SCN_FLG_RescanBString  & valid cMacVal --> rescan binary string cMacVal
   RC = SCN_FLG_RescanBFile    & valid cMacVal --> rescan binary file cMacVal
   RC = SCN_FLG_RescanExtFile  & valid cMacVal --> rescan file cMacVal,
                                                   with main stream read function
   RC = SCN_FLG_RescanExtBFile & valid cMacVal --> rescan binary file cMacVal,
                                                   with main stream read function
   RC = SCN_FLG_RescanChar     & valid cMacVal --> rescan character cMacVal
                                                   ( single byte or ucs4 )
   RC = SCN_FLG_RescanNone     & valid cMacVal --> replace cTokVal with cMacVal
                                                   ( single byte or utf-8 )
   ELSE                                        --> ignore cTokVal


typedef int (*Scn_PreMacFun)
            (
              Scn_Stream pStream, c_string cTokNam,
              c_string cTokVal, symbol* cMacVal
            );



Accessing scanner definition

short Scn_check_Token(Scn_T scn, c_string s)
whether string 's' is a token in scanner 'scn';
result = token number ( > 0 ) or 0

short Scn_check_WCToken(Scn_T scn, wc_string s)
whether wide string 's' is a token in scanner 'scn';
result = token number ( > 0 ) or 0

int Scn_tokens(Scn_T scn)
number of tokens defined in scanner 'scn'
c_string Scn_tokid(Scn_T scn, int i)
name of the i-th token definied in scanner 'scn'
c_byte Scn_tokFlags(Scn_T scn, int i)
flags of the i-th token definied in scanner 'scn'


Character iterator definition

Scn_Stream_Itr Stream_Itr_new
               (
                 int     f_getc (StdCPtr file),
                 void    f_close(StdCPtr file),
                 c_string  f_wc2mb(wc_string wc),
                 int     f_seek (StdCPtr file, long offset, int origin),
                 StdCPtr file,
                 c_string  fileid
               )
creates and initializes a character iterator
onto the open character source 'file'
'f_getc' : next character
'f_close': closes character source
'f_wc2mb': converts wide to multibyte character
'f_seek' : positions character source
'fileid' : character source identifier

void Stream_Itr_free(Scn_Stream_Itr itr)
frees character iterator 'itr'


Scan stream definition

Scn_Stream Stream_bgn(Scn_T scn, Scn_Stream_Itr itr)
creates and initializes a scan stream
onto scanner 'scn' and character iterator 'itr'

void Stream_close(Scn_Stream t)
closes character source i.e. file of scan stream 't'
void Stream_free(Scn_Stream t)
frees scan stream 't'
void Stream_premac_set(Scn_Stream t, Scn_PreMacFun cPreMac)
adds 'cPreMac' to scan stream 't'
for token preprocessing & macro expansion

void Stream_add_ctxval(Scn_Stream t, Any_T id, Any_T val)
adds the context pair 'id' |--> 'val' to scan stream 't';
updates existing entries

void Stream_concat_other(Scn_Stream t)
activates other token concatenation in scan stream 't'
void Stream_binmode_set(Scn_Stream t)
activates binary mode in scan stream 't'
( deactivates other token concatenation )

Scn_Stream Stream_line
           (
             Scn_T scn, StdCPtr line, int (*cGet)(StdCPtr line), c_string id
           )
#define STREAM_STDIN(scn) \
	Stream_line(scn,stdin,(int (*)(StdCPtr f))fgetc,"<stdin>")
creates a scan stream
onto scanner 'scn' and character stream 'line'
'cGet': next character
'id' : character stream identifier

Scn_Stream Stream_file(Scn_T scn, c_string EnvVar, c_string FileName, c_string Ext)
creates a scan stream
onto scanner 'scn' and file [$'EnvVar'/]'FileName''Ext'

Scn_Stream Stream_extfile
           (
             Scn_T scn, c_string EnvVar, c_string FileName, c_string Ext,
             int (*cGet)(FILE* f)
           )
creates a scan stream
onto scanner 'scn' and file [$'EnvVar'/]'FileName''Ext'
with external next character function 'cGet'

Scn_Stream Stream_bstring(Scn_T scn, c_bstring bstr)
creates a scan stream onto scanner 'scn' and binary string 'bstr'
void Stream_bstring_set(Scn_Stream stream, c_bstring bstr)
re-initializes scan stream 'stream' with binary string 'bstr'
Scn_Stream Stream_string(Scn_T scn, c_string Text)
creates a scan stream onto scanner 'scn' and string 'Text'
void Stream_string_set(Scn_Stream stream, c_string Text)
re-initializes scan stream 'stream' with string 'Text'
void Stream_string_insert(Scn_Stream stream, c_string Text)
inserts string 'Text' at current position in scan stream 'stream'


Defining token numbers

void Stream_defEofId(Scn_Stream t, short tid)
defines 'tid' as EOF token number in scan stream 't'
void Stream_defErrId(Scn_Stream t, short tid)
defines 'tid' as error token number in scan stream 't'
void Stream_defTokId(Scn_Stream t, c_string Name, short tid)
defines 'tid' as number for token 'Name' in scan stream 't'
void Stream_defKeyId(Scn_Stream t, c_string Text, short tid)
defines 'tid' as number for keyword 'Text' in scan stream 't'
void Stream_defWCKeyId(Scn_Stream t, wc_string Text, short tid)
defines 'tid' as number for wide keyword 'Text' in scan stream 't'


Scan stream operations

void Stream_next(Scn_Stream t)
separates the next token in scan stream 't';
That's the workhorse operation.



Accessing current token

short Stream_ctid(Scn_Stream t)
token number of the current token in scan stream 't'
c_bool Stream_cica(Scn_Stream t)
whether the current token in scan stream 't'
has the attribute 'ignore case'

symbol Stream_csym(Scn_Stream t)
symbol of the current token in scan stream 't'
c_string Stream_cbuf(Scn_Stream t)
string of the current token in scan stream 't' or NULL
wc_string Stream_wcbuf(Scn_Stream t)
wide string of the current token in scan stream 't' or NULL
long Stream_clen(Scn_Stream t)
length of the current token in scan stream 't'
symbol Stream_cfil(Scn_Stream t)
source identifier of the current token in scan stream 't'
long  Stream_clin(Scn_Stream t)
first line number of the current token in scan stream 't'
long  Stream_ccol(Scn_Stream t)
first column number of the current token in scan stream 't'


Accessing scan stream definition

Scn_T Stream_scn(Scn_Stream t)
scanner definition of scan stream 't'
c_bool Stream_binmode(Scn_Stream t)
whether current scan stream 't' has binary mode
c_bool Stream_unicode(Scn_Stream t)
whether current scan stream 't' is unicode-based
Any_T Stream_get_ctxval(Scn_Stream t, Any_T id)
value of context variable 'id' in scan stream 't' or NULL