idxthread.c

Go to the documentation of this file.

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "error.h"
#include "alloc.h"
#include "str.h"
#include "stralloc.h"
#include "logmsg.h"
#include "lock.h"
#include "open.h"
#include "getln.h"
#include "scan.h"
#include "byte.h"
#include "idx.h"
#include "errtxt.h"
#include "buffer.h"
#include "fmt.h"
#include "readwrite.h"
//#include "makehash.h"
#include "lockfile.h"
#include "date2yyyymm.h"
#include "getconf.h"
 
#define WHO "idxthread"
#define HASHLEN 20

 
#define DATENO 100
static stralloc line = {0};            /* primary input */
static stralloc authline = {0};        /* second line of primary input */
static stralloc dummyind = {0};
 
static buffer bi;
static char indexbuf[1024];
 
static char strnum[FMT_ULONG];
 
static struct stat st;
                   /* if no data, these may be the entire table, so  need to be static */
static subentry sdummy;
static authentry adummy;
 
static int fdlock;
 
/* NOTE: These do NOT prevent double locking */
 
static void die_nomem() { logmsg(WHO,111,FATAL,ERR_NOMEM); }
static void lockup() { fdlock = lockfile("lock"); }
static void unlock() { close(fdlock); }

 
static void newsub(subentry *psubt,const char *subject,unsigned int sublen,unsigned long msg)
{
  const char *cpfrom;
  char *cpto;
  unsigned int cpno;
 
  psubt->higher = (subentry *) 0;
  psubt->lower = (subentry *) 0;
  psubt->firstmsg = msg;
  psubt->lastmsg = msg;
  psubt->msginthread = 1;
  if (!(psubt->sub = alloc ((sublen) * sizeof(char))))
    die_nomem();
  cpto = psubt->sub;
  cpno = sublen;
  cpfrom = subject;
  while (cpno--) *(cpto++) = *(cpfrom++);
  psubt->sublen = sublen;
}

 
static void newauth(authentry *pautht,const char *author,unsigned int authlen,unsigned long msg)
{
  const char *cpfrom;
  char *cpto;
  unsigned int cpno;
 
  pautht->higher = (subentry *) 0;
  pautht->lower = (subentry *) 0;
  pautht->firstmsg = msg;
  if (!(pautht->auth = alloc ((authlen) * sizeof(char))))
    die_nomem();
  cpto = pautht->auth;
  cpno = authlen;
  cpfrom = author;
  while (cpno--) 
    *(cpto++) = *(cpfrom++);
  pautht->authlen = authlen;
}
 
static void init_dummy(void)
{
  unsigned int i;
 
  if (!stralloc_ready(&dummyind,HASHLEN + 1)) die_nomem();
  for (i = 0; i< HASHLEN; i++)
    dummyind.s[i] = 'a';
  dummyind.len = HASHLEN;
  if (!stralloc_append(&dummyind," ")) die_nomem();
}

 
void idx_mkthreads(msgentry **pmsgtable,subentry **psubtable,
                   authentry **pauthtable, dateentry **pdatetable,
                   unsigned long msg_from,unsigned long msg_to,
                   unsigned long msg_latest,int locked)
{
  unsigned long idxlatest;             /* need to lock for this (last) index file */
  unsigned long msg;                   /* current msg number */
  unsigned long endmsg;                /* max msg in this idx file */
  unsigned long tmpmsg;                /* index entry's msg number */
  unsigned long idx;                   /* current index file no */
  unsigned long idxto;                 /* index containing end of range */
  unsigned long ulmrange;              /* total # of messages in range */
  char *subject;                       /* subject on line */
  unsigned int sublen;                 /* length of subject */
  char *auth;
  unsigned int authlen;
  unsigned int pos,posa;
  unsigned long submax;                /* max subject num in subtable */
  subentry *psubnext;                  /* points to next entry in subtable */
  subentry *psubt;                     /* points to entry in subtable */
  authentry *pauthnext;                /* points to next entry in authtable */
  authentry *pautht;                   /* points to entry in authtable */
  int fd;                              /* index file handle */
  int flagmissingindex;                /* current index file is missing */
  int flagauth;                        /* read index entry has author info */
  int hasauth;                         /* current msg's entry has author info */
  msgentry *pmsgt;
  int res;
  int match;
  unsigned int datepos,datemax;
  unsigned int datetablesize,datetableunit;
  unsigned int lastdate = 0;
  unsigned int thisdate;
  msgentry *x, *y;
 
  /* a few unnecessary sanity checks */
 
  if (msg_to > msg_latest)
    msg_to = msg_latest;
  if (msg_to < msg_from)
    logmsg(WHO,100,FATAL,"Program error: bad range in idx_mkthreads");
  ulmrange = msg_to - msg_from + 1;
  if (!(*pmsgtable = (msgentry *) alloc(ulmrange * sizeof(msgentry))))
        die_nomem();
  y = *pmsgtable;
  x = y + ulmrange;                    /* clear */
  while (--x >= y) {
    x->subnum = 0;
    x->authnum = 0;
    x->date = 0;
  }
 
  /* max entries - acceptable waste for now */
 
  if (!(*psubtable = (subentry *) alloc((ulmrange + 1) * sizeof(subentry))))
        die_nomem();
 
  if (!(*pauthtable = (authentry *) alloc((ulmrange + 1) * sizeof(authentry))))
        die_nomem();
 
  datetableunit = DATENO * sizeof(dateentry);
  datetablesize = datetableunit;
 
  if (!(*pdatetable = (dateentry *) alloc(datetablesize)))
        die_nomem();
 
  datepos = 0;
  datemax = DATENO - 2;                /* entry 0 and end marker */
  lastdate = 0;
 
  idxlatest = msg_latest / 100;
  idxto = msg_to / 100;
  submax = 0;
  psubnext = *psubtable;               /* dummy node to get tree going. Basically, */
  psubt = &sdummy;                     /* assure that subject > psubt-sub and that */
  init_dummy();                        /* below ok unless HASHLEN > 40 */
  psubt->sub = (char *)"                                       ";
  psubt->sublen = 40;                  /* there is something to hold psubt->higher */
  psubt->higher = (subentry *) 0;
  psubt->lower = (subentry *) 0;
  pauthnext = *pauthtable;
  pautht = &adummy;
  pautht->auth = psubt->sub;
  pautht->authlen = psubt->sublen;
  pautht->higher = (authentry *) 0;
  pautht->lower = (authentry *) 0;
 
  for (idx = msg_from / 100; idx <= idxto; idx++) {   /* make index file name */
    if (!stralloc_copys(&line,"archive/")) die_nomem();
    if (!stralloc_catb(&line,strnum,fmt_ulong(strnum,idx))) die_nomem();
    if (!stralloc_cats(&line,"/index")) die_nomem();
    if (!stralloc_0(&line)) die_nomem();
    if (!locked && idx == idxlatest)
      lockup();
    flagmissingindex = 0;
    fd = open_read(line.s);
    if (fd == -1) {
      if (errno == ENOENT) { /* this means the index is not here  but the lists is supposedly indexed */
        flagmissingindex = 1;
      } else
        logmsg(WHO,111,FATAL,B(ERR_OPEN,line.s));
    } else
      buffer_init(&bi,buffer_unixread,fd,indexbuf,sizeof(indexbuf));
 
    msg = 100L * idx;                  /* current msg# */
    endmsg = msg + 99L;                /* max msg in this index */
    if (!msg) msg = 1L;                /* for start to make msg > tmpmsg */
    tmpmsg = 0L;                       /* msg number of read index line */
    if (endmsg > msg_to)               /* skip non-asked for subjects */
      endmsg = msg_to;
    for (; msg <= endmsg; msg++) {
      if (!flagmissingindex && (msg > tmpmsg)) {
        flagauth = 0;
        if (getln(&bi,&line,&match,'\n') == -1)
          logmsg(WHO,111,FATAL,B(ERR_READ,"index"));
        if (!match)
          flagmissingindex = 1;
        else {
          pos = scan_ulong(line.s,&tmpmsg);
          if (line.s[pos++] == ':') {
            if (getln(&bi,&authline,&match,'\n') == -1)
              logmsg(WHO,111,FATAL,B(ERR_READ,"index"));
            if (!match)
              flagmissingindex = 1;
            else {
              flagauth = 1;
        }
            pos++;
          }
        }
      }
      if (msg < msg_from)              /* Nothing before start of range */
        continue;
      if (msg == tmpmsg) {
        subject = line.s + pos;
        sublen = line.len - pos;
        if (sublen <= HASHLEN)
          logmsg(WHO,100,FATAL,ERR_BAD_INDEX);
        hasauth = flagauth;
      } else {
        subject = dummyind.s;
        sublen = dummyind.len;
        hasauth = 0;
      }
      for (;;) {                       /* search among already known subjects */
        res = str_diffn(psubt->sub,subject,HASHLEN);
        if (res < 0) {
          if (psubt->higher)
            psubt = psubt->higher;
          else {
            newsub(psubnext,subject,sublen,msg);
            psubt->higher = psubnext;
            psubt = psubnext;
            psubnext++;
            break;
          }
        } else if (res > 0) {
          if (psubt->lower)
            psubt = psubt->lower;
          else {
            newsub(psubnext,subject,sublen,msg);
            psubt->lower = psubnext;
            psubt = psubnext;
            psubnext++;
            break;
          }
        } else {
          psubt->lastmsg = msg;
          (psubt->msginthread)++;      /* one more message in thread */
          break;
        }
      }
 
      /* first subnum =1 (=0 is empty for thread) */
 
      pmsgt = *pmsgtable + msg - msg_from;
      pmsgt->subnum = (unsigned int) (psubt - *psubtable + 1);
      pmsgt->date = lastdate;
      if (hasauth) {
        pos = 0;
        while (authline.s[pos] && authline.s[pos] != ' ') 
          pos++;
        if (authline.s[++pos]) {
          thisdate = date2yyyymm(authline.s + pos);
          if (thisdate) pmsgt->date = thisdate;
          if (pmsgt->date > lastdate) {
            lastdate = pmsgt->date;
            if (datepos >= datemax) {            /* more space */
              datemax += DATENO;
              if (!alloc_re((void**)pdatetable,datetablesize,datetablesize+datetableunit))
                die_nomem();
            }
            (*pdatetable)[datepos].msg = msg;    /* first msg this mo */
            (*pdatetable)[datepos].date = lastdate;
            datepos++;
          }
          posa = byte_chr(authline.s,authline.len,';');
          if (authline.len > posa + HASHLEN + 1 && authline.s[pos + 1] != ' ') {
                    /* old: "; auth", new: ";hash auth" */
            auth = authline.s + posa + 1;
            authlen = authline.len - posa - 1;
          } else {
            auth = dummyind.s;
            authlen = dummyind.len;
          }
        }
 
        /* allright! Same procedure, but for author */
 
        for (;;) {    /* search among already known authors */
          res = str_diffn(pautht->auth,auth,HASHLEN);
          if (res < 0) {
            if (pautht->higher)
              pautht = pautht->higher;
            else {
              newauth(pauthnext,auth,authlen,msg);
              pautht->higher = pauthnext;
              pautht = pauthnext;
              pauthnext++;
              break;
            }
          } else if (res > 0) {
            if (pautht->lower)
              pautht = pautht->lower;
            else {
              newauth(pauthnext,auth,authlen,msg);
              pautht->lower = pauthnext;
              pautht = pauthnext;
              pauthnext++;
              break;
            }
          } else {
            break;
          }
        }                              /* link from message to this author */
        pmsgt->authnum = (unsigned int) (pautht - *pauthtable + 1);
        pautht = *pauthtable;
      }
 
      psubt = *psubtable;              /* setup psubt. Done here rather than before */
                                       /* the for loop, so that we can start off the dummy node. */
    }
    if (fd != -1)
      close(fd);
    if (!locked && idx == idxlatest)
      unlock();                        /* 'locked' refers to locked before calling */
  }
  psubnext->sub = (char *) 0;          /* end of table marker */
  pauthnext->auth = (char *) 0;        /* end of table marker */
  (*pdatetable)[datepos].msg = msg_to + 1;
  (*pdatetable)[datepos].date = lastdate + 1;
}

 
void idx_mkthread(msgentry **pmsgtable,subentry **psubtable,authentry **pauthtable,
          unsigned long msg_from,unsigned long msg_to,unsigned long msg_master,unsigned long msg_latest,int locked)
{
  subentry *psubt;                     /* points to last entry in subtable */
  int ffound;                          /* msg subject was found in subtable */
  unsigned long idxlatest;             /* need to lock for this (last) index file */
  unsigned long idxto;                 /* index for last msg in range */
  unsigned long idx;                   /* current index file no */
  unsigned long msg;                   /* index entry's msg number */
  unsigned long ulmrange;              /* total # of messages in range */
  int flagauth;                        /* there is author info */
  int firstfound = 1;                  /* = 1 until first message in thread found */
  int res;                             /* comparison result */
  char *auth;
  unsigned int authlen;
  authentry *pauthnext;                /* points to next entry in authtable */
  authentry *pautht;                   /* points to entry in authtable */
  unsigned int pos;
  int fd;                              /* index file handle */
  int match;
  msgentry *pmsgt;
  msgentry *x,*y;
 
  if ((ulmrange = msg_to - msg_from +1) <= 0)
    logmsg(WHO,100,FATAL,"Program error: bad range in idx_mkthreads");
  if (!(*pmsgtable = (msgentry *) alloc(ulmrange * sizeof(msgentry))))
    die_nomem();
  y = *pmsgtable;
  x = y + ulmrange;
  while (--x >= y) {
    x->subnum = 0;
    x->authnum = 0;
    x->date = 0;
  }
 
  if (!(*psubtable = (subentry *) alloc(2 * sizeof(subentry))))
    die_nomem();
 
  if (!(*pauthtable = (authentry *) alloc((ulmrange + 1) * sizeof(authentry))))
    die_nomem();
 
  pauthnext = *pauthtable;
  pautht = &adummy;
  init_dummy();
  pautht->auth = (char*)"                     ";
  pautht->authlen = 21;
  pautht->higher = (authentry *) 0;
  pautht->lower = (authentry *) 0;
  idxlatest = msg_latest / 100;
  idxto = msg_to / 100;
  idx = msg_master / 100;              /* index for master subject */
 
  /* Get master subject */
 
  if (!stralloc_copys(&line,"archive/")) die_nomem();
  if (!stralloc_catb(&line,strnum,fmt_ulong(strnum,idx))) die_nomem();
  if (!stralloc_cats(&line,"/index")) die_nomem();
  if (!stralloc_0(&line)) die_nomem();
  ffound = 0;
  if (!locked && idx == idxlatest)
    lockup();
  fd = open_read(line.s);
  psubt = *psubtable;
  if (fd == -1) {
    if (errno != ENOENT)
      logmsg(WHO,111,FATAL,B(ERR_OPEN,line.s));
    else
      logmsg(WHO,111,FATAL,ERR_NOINDEX);              /* temp - admin can fix! */
  } else {
    buffer_init(&bi,buffer_unixread,fd,indexbuf,sizeof(indexbuf));
 
    for(;;) {
      if (getln(&bi,&line,&match,'\n') == -1)
          logmsg(WHO,111,FATAL,B(ERR_OPEN,"index"));
      if (!match)
        break;
      pos=scan_ulong(line.s,&msg);
      if (line.s[pos++] == ':') {                     /* marker for author info */
        pos++;
        flagauth = 1;
      } else
        flagauth = 0;
      if (msg == msg_master) {
        newsub(psubt,line.s + pos,line.len - pos,msg);/* need to update msg later! */
        ffound = 1;
        break;
      }
      if (flagauth) {                                 /* skip author line */
        if (getln(&bi,&line,&match,'\n') == -1)
          logmsg(WHO,111,FATAL,B(ERR_OPEN,"index"));
      if (!match)
        break;
      }
    }
    close(fd);
  }
  if (!locked && idx == idxlatest)
    unlock();
  if (!ffound)
    logmsg(WHO,100,FATAL,ERR_NOINDEX);
 
  for (idx = msg_from / 100; idx <= idxto; idx++) {   /* make index file name */
    if (!stralloc_copys(&line,"archive/")) die_nomem();
    if (!stralloc_catb(&line,strnum,fmt_ulong(strnum,idx))) die_nomem();
    if (!stralloc_cats(&line,"/index")) die_nomem();
    if (!stralloc_0(&line)) die_nomem();
    if (!locked && idx == idxlatest)
      lockup();
 
    fd = open_read(line.s);
 
    if (fd == -1) {
      if (errno != ENOENT)
        logmsg(WHO,111,FATAL,B(ERR_OPEN,line.s));
    } else {
      buffer_init(&bi,buffer_unixread,fd,indexbuf,sizeof(indexbuf));
 
      for(;;) {
        if (getln(&bi,&line,&match,'\n') == -1)
          logmsg(WHO,111,FATAL,B(ERR_READ,"index"));
        if (!match)
          break;
        pos=scan_ulong(line.s,&msg);
        if (line.s[pos++] == ':') {
          pos++;
          flagauth = 1;
          if (getln(&bi,&authline,&match,'\n') == -1)
            logmsg(WHO,111,FATAL,B(ERR_READ,"index"));
          if (!match)
            break;
        } else
          flagauth = 0;
 
        if (msg < msg_from) continue;                 /* Nothing before start of range */
        if (msg > msg_to) break;                      /* Don't do anything after range */
 
        if (!str_diffn(psubt->sub,line.s + pos,HASHLEN)) {
          pmsgt = *pmsgtable + msg - msg_from;
          if (firstfound) {                           /* update to first message with this subj */
            psubt->firstmsg = msg;
            firstfound = 0;
          }
          psubt->lastmsg = msg;
          pmsgt->subnum = 1;
          if (flagauth) {
            if (*authline.s)
              pmsgt->date = date2yyyymm(authline.s + 1);
            pos = byte_chr(authline.s,authline.len,';');
            if (authline.len > pos + HASHLEN + 1 && authline.s[pos+1] != ' ') {
                                                      /* old: "; auth", new: ";hash auth" */
              auth = authline.s + pos + 1;
              authlen = authline.len - pos - 1;
            } else {
              auth = dummyind.s;
              authlen = dummyind.len;
            }
 
            for (;;) {                                /* search among already known authors */
              res = str_diffn(pautht->auth,auth,HASHLEN);
              if (res < 0) {
                if (pautht->higher)
                  pautht = pautht->higher;
                else {
                  newauth(pauthnext,auth,authlen,msg);
                  pautht->higher = pauthnext;
                  pautht = pauthnext;
                  pauthnext++;
                  break;
                }
              } else if (res > 0) {
                if (pautht->lower)
                  pautht = pautht->lower;
                else {
                  newauth(pauthnext,auth,authlen,msg);
                  pautht->lower = pauthnext;
                  pautht = pauthnext;
                  pauthnext++;
                  break;
                }
              } else {
                  break;
              }
            }                                         /* link from message to this author */
 
            pmsgt->authnum = (unsigned int) (pautht - *pauthtable + 1);
            pautht = *pauthtable;
          }
        }
      }
      close(fd);
    }                                                 // finished reading
 
    if (!locked && idx == idxlatest) unlock();
  }                                                   // outer for loop done
  ++psubt;
  psubt->sub = (char *) 0;                            /* end of table marker */
  pauthnext->auth = (char *) 0;                       /* end of table marker */
}

 
void idx_mklist(msgentry **pmsgtable,subentry **psubtable,authentry **pauthtable,
                unsigned long msg_from,unsigned long msg_to)
{
  unsigned long ulmrange;
  msgentry *x,*y;
  subentry *psubt;
  authentry *pautht;
 
  if ((ulmrange = msg_to - msg_from +1) <= 0)
    logmsg(WHO,111,FATAL,"bad range in idx_mkthreads");
 
  if (!(*pmsgtable = (msgentry *) alloc(ulmrange * sizeof(msgentry))))
    die_nomem();
 
  y = *pmsgtable;
  x = y + ulmrange;
  while (--x >= y) {
    x->subnum = 1;
    x->authnum = 0;
    x->date = 0;
  }
 
  if (!(*psubtable = (subentry *) alloc(2 * sizeof(subentry))))
    die_nomem();
  psubt = *psubtable;
  newsub(psubt,dummyind.s,dummyind.len,msg_from);
  psubt->lastmsg = msg_to;
  ++psubt;
  psubt->sub = (char *) 0;
  if (!(*pauthtable = (authentry *) alloc(sizeof(authentry))))
    die_nomem();                       /* nodata. Avoid dangling ptr. */
  pautht = *pauthtable;
  pautht->auth = 0;                    /* tells app that there are no author data */
  pautht->higher = (authentry *) 0;
  pautht->lower = (authentry *) 0;
}

 
void idx_destroythread(msgentry *msgtable,subentry *subtable,authentry *authtable)
{
  subentry *psubt;
  authentry *pautht;
 
  psubt = subtable;                    /* free subjects */
  while (psubt->sub) {
    alloc_free(psubt->sub);
    psubt++;
  }
 
  pautht = authtable;                  /* free authors */
  while (pautht->auth) {
    alloc_free(pautht->auth);
    pautht++;
  }
 
  alloc_free(subtable);                /* free subtable */
  alloc_free(authtable);               /* free authtable */
  alloc_free(msgtable);                /* free msgtable */
  subtable = (subentry *) 0;           /* kill pointers */
  authtable = (authentry *) 0;
  msgtable = (msgentry *) 0;
}