src/list.c

/* [<][>]
[^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following functions.
  1. newsgroupUnlockWrite
  2. newsgroupUnlockRead
  3. newsgroupLockRead
  4. newsgroupFreqMove
  5. newsgroupFreqAdj
  6. newsgroupLockWrite
  7. newsgroupFindAddLock
  8. find_list
  9. scanDirForLoAndHi
  10. list_merge
  11. build_overview_fmt
  12. fetch_list
  13. getHi
  14. getLo
  15. print_active
  16. print_type
  17. do_list
  18. CMDlist
  19. updateDaemon

/* $Id: list.c,v 1.5 2002/03/26 11:18:35 proff Exp $ */

#include "nglobal.h"
#include "acc.h"
#include "group.h"
#include "ipc.h"
#include "ll.h"
#include "xover.h"
#include "filesystem.h"

#include "nlist.h"

EXPORT int volatile UpdateDaemonPid = 0;

EXPORT struct newsgroup_index *Ni;

/*
 * race conditions here - locking is definately not atomic, but will
 * hopefully suffice. pointers to nodes are modified atomicly as one
 * can in C, and only by one writer
 */

#define LOCK_TIMEOUT_CYCLES 1000000

EXPORT void newsgroupUnlockWrite(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
        if (--n->write_locks < 0)
                n->write_locks = 0;
}

EXPORT void newsgroupUnlockRead(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
        if (--n->read_locks<0)
                n->read_locks=0;
}

EXPORT void newsgroupLockRead(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
        int i;
        for (i=0;;i++)
        {
                if (n->write_locks == 0)
                        break;
                
                if (i>LOCK_TIMEOUT_CYCLES)
                {
                        logw (("newsgroup->write_lock reset"));
                        n->write_locks = 0;
                        break;
                }
        }
        n->read_locks++;
}

/*
 * unlinks FROM and moves it before TO
 */

static void newsgroupFreqMove(struct newsgroup *from, struct newsgroup *to)
/* [<][>][^][v][top][bottom][index][help] */
{
        if (from->freq.prev)
                from->freq.prev->next = from->freq.next;
        else
                Ni->newsgroup_freq_head = from->freq.next;
        if (from->freq.next)
                from->freq.next->freq.prev = from->freq.prev;
        else
                Ni->newsgroup_freq_tail = from->freq.prev;
        /* unlinked, we can safely now modify from */
        from->freq.prev = to->freq.prev;
        from->freq.next = to;
        if (to->freq.prev)
                to->freq.prev->freq.next = from;
        else
                Ni->newsgroup_freq_head = from;
        to->freq.prev = from;
}

EXPORT void newsgroupFreqAdj(struct newsgroup *n, int change)
/* [<][>][^][v][top][bottom][index][help] */
{
        int nv = n->freq.weight + change;
        struct newsgroup *
p;
        if (change>0)
        {
                for (p = n; p->freq.prev && p->freq.prev->freq.weight < nv; p=p->freq.prev) ;
                if (p!=n)
                        newsgroupFreqMove(n, p);
        } else
        {
                for (p = n; p->next && p->next->freq.weight > nv; p=p->freq.next) ;
                if (p!=n)
                        newsgroupFreqMove(n, p->next? p->next: p);
        }
}
        

EXPORT void newsgroupLockWrite(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
        int wl=0, rl=0;
        for (;;)
        {
                if (n->read_locks != 0)
                        rl++;
                else
                        break;
                if (n->write_locks != 0)
                        wl++;
                else
                        break;
                
                if (wl==LOCK_TIMEOUT_CYCLES)
                {
                        logw (("newsgroup->write_lock reset"));
                        n->write_locks = 0;
                        wl=0;
                }
                if (rl==LOCK_TIMEOUT_CYCLES)
                {
                        logw (("newsgroup->read_lock reset"));
                        n->read_locks = 0;
                        rl=0;
                }
        }
        n->write_locks++;
}


/*
 * there are race conditions in here. We try to minimise them to acceptable levels with
 * spin locks, but ultimately should be using some kind of atomic test/set locking system.
 *
 * fortunately, due to the fact there is only one writer that changes node pointers,
 * we should be safe even if we lose a race for data within a node.
 *
 * if the group is found, then the node is returned in a read locked condition.
 * the caller must unlock the node when finished
 *
 * if lock_write = TRUE and node = NULL and we found a node matching s, then the
 * group will be returned with write_locks=1 rather than read_locks++
 */

EXPORT struct newsgroup *newsgroupFindAddLock(char *s, struct newsgroup *node, bool lock_write)
/* [<][>][^][v][top][bottom][index][help] */
{
        unsigned long h;
        struct newsgroup *p, *p2;
        int r;
        h=strHash(0, s)%MAX_NEWSGROUPS_HASH;
        if (node)
                newsgroupLockWrite(node);
        p=Ni->newsgroup_hash[h];
        p2 = NULL;
        if (!p)
        {
                if (!node)
                        return NULL;
                Ni->newsgroup_hash[h] = node;
                node->left = node->right = node->next = NULL;
                goto addit;
        }
        do
        {
                if (p2)
                        newsgroupUnlockWrite(p2);
                newsgroupLockWrite(p);
                r=strcasecmp(s, p->group); /* tri-state */
                if (r==0)
                {
                        if (!lock_write)
                        {
                                newsgroupUnlockWrite(p);
                                newsgroupLockRead(p);
                        }
                        return p;
                }
                p2 = p;
                p = (r==1)? p->right: p->left;
        } while (p);
        if (node)
        {
                node->left = node->right = node->next = NULL;
                if (r == 1)
                    p2->right = node;
                else
                    p2->left = node;
addit:
                if (Ni->newsgroup_head)
                {
                        node->freq.prev = Ni->newsgroup_freq_tail;
                        Ni->newsgroup_tail->next = Ni->newsgroup_freq_tail->freq.next = node;
                }
                else
                {
                        node->freq.prev = NULL;
                        Ni->newsgroup_head = Ni->newsgroup_freq_head = node;
                }
                Ni->newsgroup_tail = Ni->newsgroup_freq_tail = node;
                
                newsgroupUnlockWrite(node);
        }
        if (p2)
                newsgroupUnlockWrite(p2);
        return NULL;
}

static struct list_s *find_list (struct list_s *l, char *s)
/* [<][>][^][v][top][bottom][index][help] */
{
        for (; l->name; l++)
                if (strCaseEq (l->name, s))
                        return l;
        return NULL;
}

static void scanDirForLoAndHi(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
        DIR *dp;
        char dir[MAX_PATH];
        char *group = Sstrdup(n->group);
        char *p;
        struct dirent *ep;
        int i;
        int lo = 0, hi = 0;

        strExchange(group, '.', '/');
        sprintf(dir, "%.127s/%.127s/%.255s", con->cacheDir, n->server_cfg->host, group);
        free(group);

        if (!(dp = opendir(dir)))
                return;

        while ((ep = readdir(dp)))
        {
                p = ep->d_name;
                i = strspn(p, "01234567890");
                if (i == 0 || p[i])
                        continue;
                i = strToi(p);
                if (i < lo || lo == 0)
                        lo = i;
                if (i > hi)
                        hi = i;
        }

        closedir(dp);

        newsgroupLockWrite(n);
        n->hi = hi;
        n->lo = lo;
        newsgroupUnlockWrite(n);
}

static bool list_merge(struct server_cfg *scfg, char *buf, int len, time_t tim, struct list_s *list)
/* [<][>][^][v][top][bottom][index][help] */
{
        struct authent *auth;
        struct group_cfg *group_cfg;
        struct server_cfg *server_cfg = NULL;
        char sbuf[128];
        struct newsgroup *n;
        bool f_new_node;
        char *p;
        char *group;
        int hi, lo;

        strStripEOL(buf);
        for (p = buf; *p && !isspace (*p); p++) ;
        if (!*p)
                return FALSE;
        *p++ = '\0';
        if (*p == '\0')
                return FALSE;   /* msnews.microsoft.com for instance returns */
                                /* a few hundred newsgroups without descriptions */
                                /* so it's probably better not to saturate syslog */
                                /* with warnings about descriptionless groups */
        group = buf;
        if (!safeGroup(group))
        {
                logl (("listmerge() ignored %s->%s->%s: bogus chracters in group name", scfg->host, list->name, group));
                return FALSE;
        }
        /* check for an all hosts deny (strip) on this group */
        auth = authorise (NULL, group);
        if (auth && auth->strip)
        {
                logl (("listmerge() ignored %s->%s->%s: auth->strip", scfg->host, list->name, group));
                return FALSE;
        }
        /*
         * which server manages this group?
         */
        for (group_cfg = GroupList; group_cfg; group_cfg = group_cfg->next)
        {
                if (matchExp (group_cfg->group_pat, group, 1, 0))
                        server_cfg = group_cfg->server_cfg;
        }
        /*
         * is it the current server?
         */
        if (server_cfg != scfg)
        {
                loglc (("listmerge() ignored %s->%s->%s: group not bound to this server", scfg->host, list->name, group));
                return FALSE;
        }
        /*
         * do we already have a node for this group?
         */
        n = newsgroupFindAddLock(group, NULL, TRUE);
        if (!n)
        {
                if (list->type != l_active)
                {
                        if (!con->listPermitLonelyness)
                        {
                                logl (("listmerge() ignored %s->%s->%s: lonely entry", scfg->host, list->name, group));
                                return FALSE;
                        }
                }
                n = XMcalloc (1, sizeof *n);
                n->group = XMstrdup (group);
                n->server_cfg = server_cfg;
                scanDirForLoAndHi(n);
                f_new_node = TRUE;
        } else
                f_new_node = FALSE;
        /*
         * update the node
         */
        n->last_rebuild = tim;
        switch (list->type)
        {
        case l_active:
                sscanf (p, "%d %d %c", &hi, &lo, &n->moderation);
                setGroup (n, n->msgs, lo, hi);
                if (!(n->flags&NF_ACTIVE))
                    {
                        n->flags|=NF_ACTIVE;
                        Stats->list_stats[list->type].entries++;
                        Stats->list_stats[list->type].len+=len;
                    }
                break;
        case l_active_times:
                if (!n->moderation)     /* no/bad active information */
                        break;
                *sbuf='\0';
                {
                unsigned int i=0;
                sscanf (p, "%u %127[^\r\n]", &i, sbuf);
                n->creation_time = i;
                }
                if (!n->creator)
                    n->creator = XMstrdup(sbuf);
                if (!(n->flags&NF_ACTIVE_TIMES))
                    {
                        n->flags|=NF_ACTIVE_TIMES;
                        Stats->list_stats[list->type].entries++;
                        Stats->list_stats[list->type].len+=len;
                    }
                break;
        case l_newsgroups:
                if (!n->moderation)     /* no/bad active information */
                        break;
                
                if (!n->desc)
                {
                        *sbuf='\0';
                        sscanf (p, " %127[^\r\n]", sbuf);
                        n->desc = XMstrdup(sbuf);
                        if (!(n->flags&NF_NEWSGROUPS))
                            {
                                n->flags|=NF_NEWSGROUPS;
                                Stats->list_stats[list->type].entries++;
                                Stats->list_stats[list->type].len+=len;
                            }
                }
                break;
        default:
                break;
        }
        if (f_new_node)
        {
                if (list->type == l_active && !con->listPermitLonelyness && n->hi_server < con->listActiveThreshold)
                {
                        logl (("listmerge() ignored %s->%s->%s: highest article %d < active threshold %d",
                               scfg->host, list->name, group, n->hi_server, con->listActiveThreshold));
                        XMfree (n);
                        return FALSE;
                }
                newsgroupFindAddLock(buf, n, FALSE);
        }
        else
                newsgroupUnlockWrite(n);
        return TRUE;
}

static bool build_overview_fmt(struct server_cfg *scfg, struct list_cfg *lcfg)
/* [<][>][^][v][top][bottom][index][help] */
{
    char buf[MAX_LINE];
    struct strList *fmt=NULL;
    int lines;
    int bytes;
    int cc;
    for (lines=bytes=0; (cc = Cfget(scfg, buf, sizeof buf)) && !EL(buf); lines++, bytes+=cc)
        {
            char *p;
            strStripEOL(buf);
            p = strchr (buf, ':');
            if (!p)
                {
                    logwn (("missing ':' in overview.fmt field from '%s'", scfg->host));
                    continue;
                }
            fmt = strListAdd (fmt, buf);
        }
    if (cc<1 || !fmt)
        {
            if (fmt)
                strListFree(fmt);
            lcfg->rebuild_fail = time(NULL);
            return FALSE;
        }
    fmt = fmt->head;
    overviewFmtGen(scfg, fmt, NULL);
    strListFree(fmt);
    lcfg->bytes = bytes;
    lcfg->lines = lines;
    lcfg->entries = lines;
    lcfg->rebuild_good = time(NULL);
    return TRUE;
}

static bool fetch_list (struct server_cfg *scfg, struct list_s *list)
/* [<][>][^][v][top][bottom][index][help] */
{
        char buf[MAX_LINE];
        int bytes;
        int lines;
        int entries;
        int cc;
        time_t ti;
        struct list_cfg *lcfg = &scfg->share->list[list->type];

        settaskinfo("merging \"%s\" <- %s", list->name, scfg->host);
        log (("checking server %s for '%s'", scfg->host, list->name));

        if (list->type == l_active)
                Cfemitf(scfg, "list\r\n");
        else
                Cfemitf(scfg, "list %s\r\n", list->name);
        Cfflush(scfg);
        if (!Cfget (scfg, buf, sizeof buf))
                goto dropped;
        if (strToi(buf) != NNTP_LIST_FOLLOWS_VAL)
        {
                strStripEOL(buf);
                logwn (("refused list %s on %s: '%.128s'", list->name, scfg->host, buf));
                lcfg->rebuild_refused = time(NULL);
                return FALSE;
        }
        log (("parsing '%s' from %s", list->name, scfg->host));
        if (list->type == l_overview_fmt)
            {
                if (!build_overview_fmt(scfg, lcfg))
                    goto dropped;
                return TRUE;
            }
        ti = time(NULL);
        for (entries=bytes=lines=0; (cc=Cfget(scfg, buf, sizeof buf)) && !EL (buf); bytes+=cc, lines++)
            {
                if (lines%100)          /* minimise system calls */
                    ti = time(NULL);
                if (list_merge(scfg, buf, cc, ti, list))
                    entries++;
            }
        if (cc<1)
            {
            dropped:
                lcfg->rebuild_fail = time(NULL);
                logw (("%s dropped connection during rebuild of %s", scfg->host, list->name));
                return FALSE;
            }
        lcfg->rebuild_good = time(NULL);
        lcfg->entries = entries;
        lcfg->lines = lines;
        lcfg->bytes = bytes;
        return TRUE;
}

EXPORT int getHi(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
        return MAX(n->hi, n->hi_server);
}

EXPORT int getLo(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
        int lo=0;
        if (n->lo>0)
                lo = n->lo;
        if (n->lo_server>0)
        {
                if (lo>0)
                        lo = MIN(lo, n->lo_server);
                else
                        lo = n->lo_server;
        }
        return lo;
}

static int print_active(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
        if (!n->moderation)
                return 0;
        return emitf ("%s %d %d %c\r\n", n->group, getHi(n), getLo(n), n->moderation);
}

static int print_type(struct newsgroup *n, struct list_s *list)
/* [<][>][^][v][top][bottom][index][help] */
{

    int bytes = 0;
    switch (list->type)
        {
        case l_active:
            bytes+=print_active(n);
            break;
        case l_active_times:
            if (n->creation_time && n->creator)
                bytes+=emitf ("%s %lu %s\r\n", n->group, n->creation_time, n->creator);
            break;
        case l_xgtitle:
        case l_newsgroups:
            if (n->desc)
                bytes+=emitf ("%s\t%s\r\n", n->group, n->desc);
            break;
        default:
            break;
        }
    return bytes;
}

static bool do_list (struct list_s *list, char *group_pat)
/* [<][>][^][v][top][bottom][index][help] */
{
        FILE *fp;
        char buf[MAX_LINE];
        struct newsgroup *n;
        int bytes = 0;
        /*
         * get list from file (only "subscriptions" at the moment)
         */
        if (list->file)
        {
                char f[MAX_FILE];
                sprintf(f, "%.127s/list.%.127s", con->cacheDir, list->name);
                fp=fopen(f, "r");
                if (!fp)
                {
                        loge (("couldn't open %s", f));
                        if (list->type == l_subscriptions)
                                emitrn ("503 No list of subscriptions available");
                        else
                                emitrn (NNTP_DONTHAVEIT);
                        return FALSE;
                }
                emitrn (NNTP_LIST_FOLLOWS);
                while (fgets (buf, sizeof (buf), fp))
                {
                        strMakeEOLrn (buf);
                        if (!group_pat)
                                emit (buf);
                        else
                        {
                                char *p;
                                char c='\0';
                                for (p = buf; *p && !isspace (*p); p++) ;
                                if (*p)
                                {
                                        c=*p;
                                        *p='\0';
                                        if (matchExp (group_pat, buf, 1, 0))
                                        {
                                                *p=c;
                                                emit(buf);
                                        }
                                } else
                                        emit(buf);
                        }
                }
                goto end;
        }
        if (list->type == l_xgtitle)
                emitrn (NNTP_XGTITLE_OK);
        else
                emitrn (NNTP_LIST_FOLLOWS);
        /* overview.fmt */
        if (list->type == l_overview_fmt)
        {
                struct strList *l = overviewFmt;
                logd (("sending overview.fmt"));
                for (; l; l=l->next)
                        bytes+=emitf ("%s\r\n",l->data);
                goto end;
        }
        /* exact group wildmat*/
        if (group_pat && !strpbrk(group_pat, "*?[]"))
        {
                if ((n = newsgroupFindAddLock(group_pat, NULL, FALSE)))
                {
                        print_type(n, list);
                        newsgroupUnlockRead(n);
                }
                goto end;
        }
        /* no wildmat or pattern wildmat */
        for (n=Ni->newsgroup_head; n; n=n->next)
            {
                    newsgroupLockRead(n);
                    if ((!group_pat || match (group_pat, n->group, 1, 0)) &&
                        (!con->listSecurity || authGroup(n->group, FALSE)))
                        bytes += print_type (n, list);
                    newsgroupUnlockRead(n);
            }
 end:
        emitrn (".");
        return TRUE;
}

EXPORT struct list_s lists[] =
{
        {l_overview_fmt, "overview.fmt", TRUE, FALSE},
        {l_active, "active", TRUE, FALSE},
        {l_active_times, "active.times", TRUE, FALSE},
        {l_newsgroups, "newsgroups", TRUE, FALSE},
        {l_subscriptions, "subscriptions", FALSE, TRUE},
        {l_xgtitle, "xgtitle", FALSE, FALSE},
        {l_other, NULL, FALSE, FALSE}
};

EXPORT bool CMDlist (char *args)
/* [<][>][^][v][top][bottom][index][help] */
{
        struct list_s *list;
        char what[MAX_GROUP]="";
        char group_pat[MAX_GROUP]="";

        sscanf (args, "%*s %254[^ \t\r\n] %254[^ \t\r\n]", what, group_pat);
        list = find_list (lists, (*what) ? what : "active");
        if (!list)
        {
                emitrn (NNTP_DONTHAVEIT);
                return FALSE;
        }
        Stats->list_stats[list->type].cache_stats.requests++;
        return do_list (list, (*group_pat)? group_pat: NULL);
}

EXPORT void updateDaemon (bool force)
/* [<][>][^][v][top][bottom][index][help] */
{
        static time_t last_time;
        struct list_s *list;
        struct server_cfg *scfg;
        struct stats st;
        int pid = 0;
        time_t tim;
        sigset_t myset;

        if (Task->ti_state != nc_master)
                return;
        tim = time(NULL);
        if (UpdateDaemonPid || (!force && tim - last_time < con->minUpdateDelay))
                return;
        last_time = tim;
        sigemptyset(&myset);
        sigaddset(&myset, SIGCHLD);
        sigprocmask (SIG_BLOCK, &myset, NULL);
        pid = make_vm_proc(nc_update, -1, "update");
        if (pid < 0)
        {
                sigprocmask (SIG_UNBLOCK, &myset, NULL);
                return;
        }
        if (pid > 0)
        {
                UpdateDaemonPid = pid;
                sigprocmask (SIG_UNBLOCK, &myset, NULL);
                saveStats (con->statsFile);
                return;
        }

        /*
         * Resetting of some list-stats in case the newsgroup list could
         * not be recovered from file -- hugo --
         */

        if (f_cleanSlate) {
            for (list=lists; list->name; list++) {
                Stats->list_stats [list->type].entries = 0 ;
                Stats->list_stats [list->type].len = 0 ;
            }
        }

        sigprocmask (SIG_UNBLOCK, &myset, NULL);
        while (HoldForksUpdate) {}
        memset (&st, 0, sizeof st);
        ModeReader = TRUE;
        for (scfg = ServerList; scfg; scfg=scfg->next)
        {
                for (list=lists; list->name; list++)
                {
                        struct list_cfg *l;
                        int timeout=0; /* stop warnings */
                        tim = time(NULL);
                        if (!list->auto_update)
                                continue;
                        switch (list->type)
                        {
                        case l_active:
                                timeout = scfg->active_timeout;
                                break;
                        case l_active_times:
                                timeout = scfg->active_times_timeout;
                                break;
                        case l_newsgroups:
                                timeout = scfg->newsgroups_timeout;
                                break;
                        case l_overview_fmt:
                                timeout = scfg->overview_fmt_timeout;
                                break;
                        default:
                                assert("PC" == "never here");
                                break;
                        }
                        l = &scfg->share->list[list->type];
                        if (l->rebuild_good && tim - l->rebuild_good < timeout)
                            continue;
                        if ((l->rebuild_fail > l->rebuild_good && tim - l->rebuild_fail < con->minUpdateFailDelay) ||
                            (l->rebuild_refused > l->rebuild_good && tim - l->rebuild_refused < con->minUpdateRefusedDelay))
                            continue;
                        if (!fetch_list(scfg, list) && list->type == l_overview_fmt && !scfg->share->overview_fmt)
                            overviewFmtGen(scfg, con->overviewFmtBozo, NULL);

                }
        }
        retire_vm_proc (0);
        NOTREACHED;
}

/* [<][>][^][v][top][bottom][index][help] */