tokenize.c

Go to the documentation of this file.

#include "buffer.h"
#include "getln.h"
#include "mess822.h"
#include "logmsg.h"
#include "exit.h"
 
#define WHO "tokenize"
 
static void nomem()
{
  logmsg(WHO,111,FATAL,"out of memory");
}
 
void put(char *buf,int len)
{
  char ch;
 
  while (len) {
    ch = *buf;
    if (ch == '\n') ch = 0;
    buffer_put(buffer_1,&ch,1);
    ++buf;
    --len;
  }
}
 
stralloc line = {0};
int match;
 
stralloc tokens = {0};
 
int main()
{
  int i;
  int j;
 
  for (;;) {
    if (getln(buffer_0,&line,&match,'\n') == -1)
      logmsg(WHO,111,FATAL,"unable to read input: ");
    if (!line.len) break;
    if (match) --line.len;
 
    for (i = 0; i < line.len; ++i)
      if (line.s[i] == 0)
          line.s[i] = '\n';
    if (!stralloc_0(&line)) nomem();
 
    if (!mess822_token(&tokens,line.s)) nomem();
 
    for (j = i = 0; j < tokens.len; ++j)
      if (!tokens.s[j]) {
        if (tokens.s[i] == ' ')
          buffer_puts(buffer_1,"space\n");
        else if (tokens.s[i] == '\t')
         buffer_puts(buffer_1,"tab\n");
        else if (tokens.s[i] == '|') {
          buffer_puts(buffer_1,"string {");
          put(tokens.s + i + 1,j - i - 1);
          buffer_puts(buffer_1,"}\n");
        } else if (tokens.s[i] == '(') {
          buffer_puts(buffer_1,"comment {");
          put(tokens.s + i + 1,j - i - 1);
          buffer_puts(buffer_1,"}\n");
        } else {
          buffer_puts(buffer_1,"special ");
          put(tokens.s + i,j - i);
          buffer_puts(buffer_1,"\n");
        }
       i = j + 1;
    }
 
    buffer_puts(buffer_1,"\n");
    if (!match) break;
  }
 
  buffer_flush(buffer_1);
  _exit(0);
}