src/list.c
/* [<][>][^][v][top][bottom][index][help] */
DEFINITIONS
This source file includes following functions.
- newsgroupUnlockWrite
- newsgroupUnlockRead
- newsgroupLockRead
- newsgroupFreqMove
- newsgroupFreqAdj
- newsgroupLockWrite
- newsgroupFindAddLock
- find_list
- scanDirForLoAndHi
- list_merge
- build_overview_fmt
- fetch_list
- getHi
- getLo
- print_active
- print_type
- do_list
- CMDlist
- updateDaemon
/* $Id: list.c,v 1.5 2002/03/26 11:18:35 proff Exp $ */
#include "nglobal.h"
#include "acc.h"
#include "group.h"
#include "ipc.h"
#include "ll.h"
#include "xover.h"
#include "filesystem.h"
#include "nlist.h"
EXPORT int volatile UpdateDaemonPid = 0;
EXPORT struct newsgroup_index *Ni;
/*
* race conditions here - locking is definately not atomic, but will
* hopefully suffice. pointers to nodes are modified atomicly as one
* can in C, and only by one writer
*/
#define LOCK_TIMEOUT_CYCLES 1000000
EXPORT void newsgroupUnlockWrite(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
if (--n->write_locks < 0)
n->write_locks = 0;
}
EXPORT void newsgroupUnlockRead(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
if (--n->read_locks<0)
n->read_locks=0;
}
EXPORT void newsgroupLockRead(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
int i;
for (i=0;;i++)
{
if (n->write_locks == 0)
break;
if (i>LOCK_TIMEOUT_CYCLES)
{
logw (("newsgroup->write_lock reset"));
n->write_locks = 0;
break;
}
}
n->read_locks++;
}
/*
* unlinks FROM and moves it before TO
*/
static void newsgroupFreqMove(struct newsgroup *from, struct newsgroup *to)
/* [<][>][^][v][top][bottom][index][help] */
{
if (from->freq.prev)
from->freq.prev->next = from->freq.next;
else
Ni->newsgroup_freq_head = from->freq.next;
if (from->freq.next)
from->freq.next->freq.prev = from->freq.prev;
else
Ni->newsgroup_freq_tail = from->freq.prev;
/* unlinked, we can safely now modify from */
from->freq.prev = to->freq.prev;
from->freq.next = to;
if (to->freq.prev)
to->freq.prev->freq.next = from;
else
Ni->newsgroup_freq_head = from;
to->freq.prev = from;
}
EXPORT void newsgroupFreqAdj(struct newsgroup *n, int change)
/* [<][>][^][v][top][bottom][index][help] */
{
int nv = n->freq.weight + change;
struct newsgroup *
p;
if (change>0)
{
for (p = n; p->freq.prev && p->freq.prev->freq.weight < nv; p=p->freq.prev) ;
if (p!=n)
newsgroupFreqMove(n, p);
} else
{
for (p = n; p->next && p->next->freq.weight > nv; p=p->freq.next) ;
if (p!=n)
newsgroupFreqMove(n, p->next? p->next: p);
}
}
EXPORT void newsgroupLockWrite(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
int wl=0, rl=0;
for (;;)
{
if (n->read_locks != 0)
rl++;
else
break;
if (n->write_locks != 0)
wl++;
else
break;
if (wl==LOCK_TIMEOUT_CYCLES)
{
logw (("newsgroup->write_lock reset"));
n->write_locks = 0;
wl=0;
}
if (rl==LOCK_TIMEOUT_CYCLES)
{
logw (("newsgroup->read_lock reset"));
n->read_locks = 0;
rl=0;
}
}
n->write_locks++;
}
/*
* there are race conditions in here. We try to minimise them to acceptable levels with
* spin locks, but ultimately should be using some kind of atomic test/set locking system.
*
* fortunately, due to the fact there is only one writer that changes node pointers,
* we should be safe even if we lose a race for data within a node.
*
* if the group is found, then the node is returned in a read locked condition.
* the caller must unlock the node when finished
*
* if lock_write = TRUE and node = NULL and we found a node matching s, then the
* group will be returned with write_locks=1 rather than read_locks++
*/
EXPORT struct newsgroup *newsgroupFindAddLock(char *s, struct newsgroup *node, bool lock_write)
/* [<][>][^][v][top][bottom][index][help] */
{
unsigned long h;
struct newsgroup *p, *p2;
int r;
h=strHash(0, s)%MAX_NEWSGROUPS_HASH;
if (node)
newsgroupLockWrite(node);
p=Ni->newsgroup_hash[h];
p2 = NULL;
if (!p)
{
if (!node)
return NULL;
Ni->newsgroup_hash[h] = node;
node->left = node->right = node->next = NULL;
goto addit;
}
do
{
if (p2)
newsgroupUnlockWrite(p2);
newsgroupLockWrite(p);
r=strcasecmp(s, p->group); /* tri-state */
if (r==0)
{
if (!lock_write)
{
newsgroupUnlockWrite(p);
newsgroupLockRead(p);
}
return p;
}
p2 = p;
p = (r==1)? p->right: p->left;
} while (p);
if (node)
{
node->left = node->right = node->next = NULL;
if (r == 1)
p2->right = node;
else
p2->left = node;
addit:
if (Ni->newsgroup_head)
{
node->freq.prev = Ni->newsgroup_freq_tail;
Ni->newsgroup_tail->next = Ni->newsgroup_freq_tail->freq.next = node;
}
else
{
node->freq.prev = NULL;
Ni->newsgroup_head = Ni->newsgroup_freq_head = node;
}
Ni->newsgroup_tail = Ni->newsgroup_freq_tail = node;
newsgroupUnlockWrite(node);
}
if (p2)
newsgroupUnlockWrite(p2);
return NULL;
}
static struct list_s *find_list (struct list_s *l, char *s)
/* [<][>][^][v][top][bottom][index][help] */
{
for (; l->name; l++)
if (strCaseEq (l->name, s))
return l;
return NULL;
}
static void scanDirForLoAndHi(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
DIR *dp;
char dir[MAX_PATH];
char *group = Sstrdup(n->group);
char *p;
struct dirent *ep;
int i;
int lo = 0, hi = 0;
strExchange(group, '.', '/');
sprintf(dir, "%.127s/%.127s/%.255s", con->cacheDir, n->server_cfg->host, group);
free(group);
if (!(dp = opendir(dir)))
return;
while ((ep = readdir(dp)))
{
p = ep->d_name;
i = strspn(p, "01234567890");
if (i == 0 || p[i])
continue;
i = strToi(p);
if (i < lo || lo == 0)
lo = i;
if (i > hi)
hi = i;
}
closedir(dp);
newsgroupLockWrite(n);
n->hi = hi;
n->lo = lo;
newsgroupUnlockWrite(n);
}
static bool list_merge(struct server_cfg *scfg, char *buf, int len, time_t tim, struct list_s *list)
/* [<][>][^][v][top][bottom][index][help] */
{
struct authent *auth;
struct group_cfg *group_cfg;
struct server_cfg *server_cfg = NULL;
char sbuf[128];
struct newsgroup *n;
bool f_new_node;
char *p;
char *group;
int hi, lo;
strStripEOL(buf);
for (p = buf; *p && !isspace (*p); p++) ;
if (!*p)
return FALSE;
*p++ = '\0';
if (*p == '\0')
return FALSE; /* msnews.microsoft.com for instance returns */
/* a few hundred newsgroups without descriptions */
/* so it's probably better not to saturate syslog */
/* with warnings about descriptionless groups */
group = buf;
if (!safeGroup(group))
{
logl (("listmerge() ignored %s->%s->%s: bogus chracters in group name", scfg->host, list->name, group));
return FALSE;
}
/* check for an all hosts deny (strip) on this group */
auth = authorise (NULL, group);
if (auth && auth->strip)
{
logl (("listmerge() ignored %s->%s->%s: auth->strip", scfg->host, list->name, group));
return FALSE;
}
/*
* which server manages this group?
*/
for (group_cfg = GroupList; group_cfg; group_cfg = group_cfg->next)
{
if (matchExp (group_cfg->group_pat, group, 1, 0))
server_cfg = group_cfg->server_cfg;
}
/*
* is it the current server?
*/
if (server_cfg != scfg)
{
loglc (("listmerge() ignored %s->%s->%s: group not bound to this server", scfg->host, list->name, group));
return FALSE;
}
/*
* do we already have a node for this group?
*/
n = newsgroupFindAddLock(group, NULL, TRUE);
if (!n)
{
if (list->type != l_active)
{
if (!con->listPermitLonelyness)
{
logl (("listmerge() ignored %s->%s->%s: lonely entry", scfg->host, list->name, group));
return FALSE;
}
}
n = XMcalloc (1, sizeof *n);
n->group = XMstrdup (group);
n->server_cfg = server_cfg;
scanDirForLoAndHi(n);
f_new_node = TRUE;
} else
f_new_node = FALSE;
/*
* update the node
*/
n->last_rebuild = tim;
switch (list->type)
{
case l_active:
sscanf (p, "%d %d %c", &hi, &lo, &n->moderation);
setGroup (n, n->msgs, lo, hi);
if (!(n->flags&NF_ACTIVE))
{
n->flags|=NF_ACTIVE;
Stats->list_stats[list->type].entries++;
Stats->list_stats[list->type].len+=len;
}
break;
case l_active_times:
if (!n->moderation) /* no/bad active information */
break;
*sbuf='\0';
{
unsigned int i=0;
sscanf (p, "%u %127[^\r\n]", &i, sbuf);
n->creation_time = i;
}
if (!n->creator)
n->creator = XMstrdup(sbuf);
if (!(n->flags&NF_ACTIVE_TIMES))
{
n->flags|=NF_ACTIVE_TIMES;
Stats->list_stats[list->type].entries++;
Stats->list_stats[list->type].len+=len;
}
break;
case l_newsgroups:
if (!n->moderation) /* no/bad active information */
break;
if (!n->desc)
{
*sbuf='\0';
sscanf (p, " %127[^\r\n]", sbuf);
n->desc = XMstrdup(sbuf);
if (!(n->flags&NF_NEWSGROUPS))
{
n->flags|=NF_NEWSGROUPS;
Stats->list_stats[list->type].entries++;
Stats->list_stats[list->type].len+=len;
}
}
break;
default:
break;
}
if (f_new_node)
{
if (list->type == l_active && !con->listPermitLonelyness && n->hi_server < con->listActiveThreshold)
{
logl (("listmerge() ignored %s->%s->%s: highest article %d < active threshold %d",
scfg->host, list->name, group, n->hi_server, con->listActiveThreshold));
XMfree (n);
return FALSE;
}
newsgroupFindAddLock(buf, n, FALSE);
}
else
newsgroupUnlockWrite(n);
return TRUE;
}
static bool build_overview_fmt(struct server_cfg *scfg, struct list_cfg *lcfg)
/* [<][>][^][v][top][bottom][index][help] */
{
char buf[MAX_LINE];
struct strList *fmt=NULL;
int lines;
int bytes;
int cc;
for (lines=bytes=0; (cc = Cfget(scfg, buf, sizeof buf)) && !EL(buf); lines++, bytes+=cc)
{
char *p;
strStripEOL(buf);
p = strchr (buf, ':');
if (!p)
{
logwn (("missing ':' in overview.fmt field from '%s'", scfg->host));
continue;
}
fmt = strListAdd (fmt, buf);
}
if (cc<1 || !fmt)
{
if (fmt)
strListFree(fmt);
lcfg->rebuild_fail = time(NULL);
return FALSE;
}
fmt = fmt->head;
overviewFmtGen(scfg, fmt, NULL);
strListFree(fmt);
lcfg->bytes = bytes;
lcfg->lines = lines;
lcfg->entries = lines;
lcfg->rebuild_good = time(NULL);
return TRUE;
}
static bool fetch_list (struct server_cfg *scfg, struct list_s *list)
/* [<][>][^][v][top][bottom][index][help] */
{
char buf[MAX_LINE];
int bytes;
int lines;
int entries;
int cc;
time_t ti;
struct list_cfg *lcfg = &scfg->share->list[list->type];
settaskinfo("merging \"%s\" <- %s", list->name, scfg->host);
log (("checking server %s for '%s'", scfg->host, list->name));
if (list->type == l_active)
Cfemitf(scfg, "list\r\n");
else
Cfemitf(scfg, "list %s\r\n", list->name);
Cfflush(scfg);
if (!Cfget (scfg, buf, sizeof buf))
goto dropped;
if (strToi(buf) != NNTP_LIST_FOLLOWS_VAL)
{
strStripEOL(buf);
logwn (("refused list %s on %s: '%.128s'", list->name, scfg->host, buf));
lcfg->rebuild_refused = time(NULL);
return FALSE;
}
log (("parsing '%s' from %s", list->name, scfg->host));
if (list->type == l_overview_fmt)
{
if (!build_overview_fmt(scfg, lcfg))
goto dropped;
return TRUE;
}
ti = time(NULL);
for (entries=bytes=lines=0; (cc=Cfget(scfg, buf, sizeof buf)) && !EL (buf); bytes+=cc, lines++)
{
if (lines%100) /* minimise system calls */
ti = time(NULL);
if (list_merge(scfg, buf, cc, ti, list))
entries++;
}
if (cc<1)
{
dropped:
lcfg->rebuild_fail = time(NULL);
logw (("%s dropped connection during rebuild of %s", scfg->host, list->name));
return FALSE;
}
lcfg->rebuild_good = time(NULL);
lcfg->entries = entries;
lcfg->lines = lines;
lcfg->bytes = bytes;
return TRUE;
}
EXPORT int getHi(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
return MAX(n->hi, n->hi_server);
}
EXPORT int getLo(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
int lo=0;
if (n->lo>0)
lo = n->lo;
if (n->lo_server>0)
{
if (lo>0)
lo = MIN(lo, n->lo_server);
else
lo = n->lo_server;
}
return lo;
}
static int print_active(struct newsgroup *n)
/* [<][>][^][v][top][bottom][index][help] */
{
if (!n->moderation)
return 0;
return emitf ("%s %d %d %c\r\n", n->group, getHi(n), getLo(n), n->moderation);
}
static int print_type(struct newsgroup *n, struct list_s *list)
/* [<][>][^][v][top][bottom][index][help] */
{
int bytes = 0;
switch (list->type)
{
case l_active:
bytes+=print_active(n);
break;
case l_active_times:
if (n->creation_time && n->creator)
bytes+=emitf ("%s %lu %s\r\n", n->group, n->creation_time, n->creator);
break;
case l_xgtitle:
case l_newsgroups:
if (n->desc)
bytes+=emitf ("%s\t%s\r\n", n->group, n->desc);
break;
default:
break;
}
return bytes;
}
static bool do_list (struct list_s *list, char *group_pat)
/* [<][>][^][v][top][bottom][index][help] */
{
FILE *fp;
char buf[MAX_LINE];
struct newsgroup *n;
int bytes = 0;
/*
* get list from file (only "subscriptions" at the moment)
*/
if (list->file)
{
char f[MAX_FILE];
sprintf(f, "%.127s/list.%.127s", con->cacheDir, list->name);
fp=fopen(f, "r");
if (!fp)
{
loge (("couldn't open %s", f));
if (list->type == l_subscriptions)
emitrn ("503 No list of subscriptions available");
else
emitrn (NNTP_DONTHAVEIT);
return FALSE;
}
emitrn (NNTP_LIST_FOLLOWS);
while (fgets (buf, sizeof (buf), fp))
{
strMakeEOLrn (buf);
if (!group_pat)
emit (buf);
else
{
char *p;
char c='\0';
for (p = buf; *p && !isspace (*p); p++) ;
if (*p)
{
c=*p;
*p='\0';
if (matchExp (group_pat, buf, 1, 0))
{
*p=c;
emit(buf);
}
} else
emit(buf);
}
}
goto end;
}
if (list->type == l_xgtitle)
emitrn (NNTP_XGTITLE_OK);
else
emitrn (NNTP_LIST_FOLLOWS);
/* overview.fmt */
if (list->type == l_overview_fmt)
{
struct strList *l = overviewFmt;
logd (("sending overview.fmt"));
for (; l; l=l->next)
bytes+=emitf ("%s\r\n",l->data);
goto end;
}
/* exact group wildmat*/
if (group_pat && !strpbrk(group_pat, "*?[]"))
{
if ((n = newsgroupFindAddLock(group_pat, NULL, FALSE)))
{
print_type(n, list);
newsgroupUnlockRead(n);
}
goto end;
}
/* no wildmat or pattern wildmat */
for (n=Ni->newsgroup_head; n; n=n->next)
{
newsgroupLockRead(n);
if ((!group_pat || match (group_pat, n->group, 1, 0)) &&
(!con->listSecurity || authGroup(n->group, FALSE)))
bytes += print_type (n, list);
newsgroupUnlockRead(n);
}
end:
emitrn (".");
return TRUE;
}
EXPORT struct list_s lists[] =
{
{l_overview_fmt, "overview.fmt", TRUE, FALSE},
{l_active, "active", TRUE, FALSE},
{l_active_times, "active.times", TRUE, FALSE},
{l_newsgroups, "newsgroups", TRUE, FALSE},
{l_subscriptions, "subscriptions", FALSE, TRUE},
{l_xgtitle, "xgtitle", FALSE, FALSE},
{l_other, NULL, FALSE, FALSE}
};
EXPORT bool CMDlist (char *args)
/* [<][>][^][v][top][bottom][index][help] */
{
struct list_s *list;
char what[MAX_GROUP]="";
char group_pat[MAX_GROUP]="";
sscanf (args, "%*s %254[^ \t\r\n] %254[^ \t\r\n]", what, group_pat);
list = find_list (lists, (*what) ? what : "active");
if (!list)
{
emitrn (NNTP_DONTHAVEIT);
return FALSE;
}
Stats->list_stats[list->type].cache_stats.requests++;
return do_list (list, (*group_pat)? group_pat: NULL);
}
EXPORT void updateDaemon (bool force)
/* [<][>][^][v][top][bottom][index][help] */
{
static time_t last_time;
struct list_s *list;
struct server_cfg *scfg;
struct stats st;
int pid = 0;
time_t tim;
sigset_t myset;
if (Task->ti_state != nc_master)
return;
tim = time(NULL);
if (UpdateDaemonPid || (!force && tim - last_time < con->minUpdateDelay))
return;
last_time = tim;
sigemptyset(&myset);
sigaddset(&myset, SIGCHLD);
sigprocmask (SIG_BLOCK, &myset, NULL);
pid = make_vm_proc(nc_update, -1, "update");
if (pid < 0)
{
sigprocmask (SIG_UNBLOCK, &myset, NULL);
return;
}
if (pid > 0)
{
UpdateDaemonPid = pid;
sigprocmask (SIG_UNBLOCK, &myset, NULL);
saveStats (con->statsFile);
return;
}
/*
* Resetting of some list-stats in case the newsgroup list could
* not be recovered from file -- hugo --
*/
if (f_cleanSlate) {
for (list=lists; list->name; list++) {
Stats->list_stats [list->type].entries = 0 ;
Stats->list_stats [list->type].len = 0 ;
}
}
sigprocmask (SIG_UNBLOCK, &myset, NULL);
while (HoldForksUpdate) {}
memset (&st, 0, sizeof st);
ModeReader = TRUE;
for (scfg = ServerList; scfg; scfg=scfg->next)
{
for (list=lists; list->name; list++)
{
struct list_cfg *l;
int timeout=0; /* stop warnings */
tim = time(NULL);
if (!list->auto_update)
continue;
switch (list->type)
{
case l_active:
timeout = scfg->active_timeout;
break;
case l_active_times:
timeout = scfg->active_times_timeout;
break;
case l_newsgroups:
timeout = scfg->newsgroups_timeout;
break;
case l_overview_fmt:
timeout = scfg->overview_fmt_timeout;
break;
default:
assert("PC" == "never here");
break;
}
l = &scfg->share->list[list->type];
if (l->rebuild_good && tim - l->rebuild_good < timeout)
continue;
if ((l->rebuild_fail > l->rebuild_good && tim - l->rebuild_fail < con->minUpdateFailDelay) ||
(l->rebuild_refused > l->rebuild_good && tim - l->rebuild_refused < con->minUpdateRefusedDelay))
continue;
if (!fetch_list(scfg, list) && list->type == l_overview_fmt && !scfg->share->overview_fmt)
overviewFmtGen(scfg, con->overviewFmtBozo, NULL);
}
}
retire_vm_proc (0);
NOTREACHED;
}