src/xover.c
/* [<][>][^][v][top][bottom][index][help] */
DEFINITIONS
This source file includes following functions.
- overviewFmtGen
- xrefRewrite
- xrefRewriteCopy
- xover_extract_xhdr
- xoverIsFilt
- filter_header
- xover_nocem
- xover_filter
- xover_translate
- xf_find
- xf_write_one
- xf_check_buf
- xf_acquire_reader
- xf_release_reader
- xf_acquire_writer
- xf_release_writer
- xf_close
- xf_add
- xf_destroy_all
- xf_release_all
- xover_emit_xhdr
- xf_grab_outbuf
- xf_open
- xover_input
- xover_output
- xover_io
- xover_finish
- xover_process
- CMDxover
- isFieldInXover
- CMDxhdr
/* $Id: xover.c,v 1.4 2002/03/29 09:50:58 proff Exp $
* $Copyright$
*/
#include "nglobal.h"
#include "network.h"
#include "acc.h"
#include "article.h"
#include "filter.h"
#include "group.h"
#include "history.h"
#include "lock.h"
#include "reg.h"
#include "xover.h"
/*
* we used to store all the xovers in the hashed history file db but this
* caused the drive heads to thrash (lots of prefetch/sequential/random
* access issues here)
*
* now we store xovers corresponding to a certain range (mod 512)
* of article numbers as nnnn.xover. this
* equates to around 200k of disk space (assuming full range). we also
* prepend an array index of 512*32 bits which forms a pointer to
* each xover in the xover file and the length of each xover (so we can
* optimise reads.. useless if mmaped) 20 bits of pointer and 12 bits of
* length (max length = 4096, longer xovers are insane and are discarded).
*
* if the average xover length in an xover file isnt <2048 then nasty things
* start to happen. I've never seen an xover >1500 bytes long, but theoretically
* massively crossposted articles with Xref's may exceed it. this is why
* we permit upto 4096 bytes for each xover, but expect the average to easily
* be <2048 (average seems to be 200 to 1000 depending on group)
*
* this method lets the file system sequentialise the xover data,
* which we may be storing in a non-sequential manner.
*
* we also handle all the server and client io in an async manner, to
* maximise throughput and TCP packet size. RTT with the server was a problem
* before if the the cache had a lot of small "holes".
*
* the pointers in the index file are stored in network byte order, so
* the xover databases should be portable accross different endian's. e.g
* the one database being used by several machines via nfs. note that in
* most cases nntpcache can pull its data out of the file-system and transmit
* it over the network faster than nfs.
*
* as for xhdr's, when going via a cached server, they are converted to
* xovers, if that header is in the overview.fmt for that server
*/
#define MI XOVER_INDEX_SIZE
#define C4toINT (x) (((x)[3]>>24)|((x)[2]>>16)|((x)[1]>>8)|(x)[0])
#define UC unsigned char
#define INTtoC4 (y,x) (x)[3]=(UC)((y)>>24)&0xff;(x)[2]=(UC)((y)>>16)&0xff;(x)[1]=(UC)((y)>>8)&0xff;(x)[0]=(UC)((y)&0xff)
#undef UC
#define FILT (CurrentGroupXoverIsFilt)
/*
* we cache the first part of every xover file (the index) as well as the
* file descriptor's involved.
*/
static struct xover_file
{
struct xover_file *next, *prev;
int id;
char *name;
char *dir;
char *buf;
char *outbuf;
int outbuf_len;
int outbuf_used;
int size;
int fd;
bool reader;
bool writer;
time_t mtime;
} *xover_file_head = NULL, *xover_file_tail = NULL;
static int xover_file_nodes = 0; /* number in cache */
static enum xover_state
{
listening, sending, receiving, dead
} xover_server_state, xover_client_state;
static fd_set fdset;
static int outstanding = 0;
EXPORT struct strList *overviewFmtGen(struct server_cfg *scfg, struct strList *l, n_u32 *hashp)
/* [<][>][^][v][top][bottom][index][help] */
{
struct strList *list=NULL;
int fmt_len=0, fmt_entries=0;
struct strList *xref = NULL;
n_u32 hash;
for (hash = 0xadeadfed; l; l = l->next)
{
list=strXMListAdd(list, l->data); /* place into shared memory */
hash=strHash(hash, l->data);
fmt_len += strlen(l->data) + 2;
fmt_entries++;
if (scfg && !xref && strnCaseEq(l->data, "Xref:", 5))
xref = l;
}
if (list)
{
list = list->head;
if (scfg && (!scfg->share->overview_fmt_hash || scfg->share->overview_fmt_hash != hash))
{
struct strList *t;
t = scfg->share->overview_fmt;
scfg->share->overview_fmt = list->head;
scfg->share->overview_fmt_xref = xref;
scfg->share->overview_fmt_hash = hash;
scfg->share->overview_fmt_xref_full = xref? strnCaseEq(xref->data, "Xref:full", 9): FALSE;
if (t)
strXMListFree(t);
}
}
if (hashp)
*hashp = hash;
return list;
}
/*
* XrefRewrite
*
* Xref (s): is terminated by one of \0 \n \r \t
*
* returns length of new xref
*/
EXPORT int xrefRewrite(struct server_cfg *scfg, char *s, bool full)
/* [<][>][^][v][top][bottom][index][help] */
{
char *p, *o, *s_orig=s;
SKIPSPACE(s);
if (full)
{
SKIPNOWHITE(s); /* Xref: */
SKIPSPACE(s);
}
/* we don't modify the hostname field - it may actually be
* useful to more intelligent newsreaders
*/
SKIPNOSPACE(s); /* hostname */
SKIPSPACE(s);
for (o=s; *s && *s!='\t' && *s!='\r' && *s!='\n';)
{
bool keep;
char c;
p=s;
while (*s && *s!=':')
s++;
if (p==s)
break;
c=*s;
*s='\0';
keep = getServerGroup(p) == scfg ? TRUE : FALSE;
*s = c;
SKIPNOWHITE(s);
SKIPSPACE(s);
if (keep)
{
if (o!=p)
{
int len=s-p;
memmove(o, p, len);
o+=len;
} else
o=s;
}
}
if (s!=o)
memset(o, ' ', s-o);
return s_orig-o;
}
/*
* as above, but strip and copy into dst. caller is responsibly for
* allocating a correct sized dst. this version is optimised for
* xover_translate.
* returns length of chars in dst
*/
EXPORT int xrefRewriteCopy(struct server_cfg *scfg, char *s, char *dst, bool full)
/* [<][>][^][v][top][bottom][index][help] */
{
char *p;
char *odst = dst;
while (*s == ' ')
*dst++=*s++;
if (full)
{
while (*s && !isspace(*s))
*dst++=*s++; /* Xref: */
while (*s == ' ')
*dst++=*s++;
}
/* we don't modify the hostname field - it may actually be
* useful to more intelligent newsreaders
*/
while (*s && !isspace(*s))
*dst++=*s++; /* hostname */
while (*s == ' ')
*dst++=*s++;
for (; *s && !isspace(*s);)
{
bool keep;
char c;
p=s;
while (*s && *s!=':')
*dst++=*s++; /* alt.suicide:1234 */
c=*s;
*s='\0';
keep = getServerGroup(p) == scfg ? TRUE : FALSE;
*s=c;
if (keep)
{
while (*s && !isspace(*s))
*dst++=*s++;
} else
{
dst -= s-p;
while (*s && !isspace(*s))
s++;
}
while (*s == ' ')
*dst++=*s++;
}
*dst='\0';
return dst-odst;
}
/*
* returns pointer to start of header in xover or null.
*/
static char *xover_extract_xhdr (char *xover, struct strList *i_fmt, char *header)
/* [<][>][^][v][top][bottom][index][help] */
{
for (; i_fmt; i_fmt = i_fmt->next)
{
if (strCaseEq (i_fmt->data, header))
return xover;
xover = strchr (xover, '\t');
if (!xover)
break;
xover++;
}
return NULL;
}
EXPORT bool xoverIsFilt(struct authent *auth)
/* [<][>][^][v][top][bottom][index][help] */
{
struct filter *f;
struct filter_chain *fc;
if (!auth || !con->xoverFilters)
return FALSE;
for (fc=auth->filter_chain; fc; fc=fc->next)
{
for (f=fc->filter; f; f=f->next)
{
if (f->scope == sc_header)
return TRUE;
}
}
return FALSE;
}
static char *filter_header (char *header, char *text)
/* [<][>][^][v][top][bottom][index][help] */
{
struct filter *f;
struct filter_chain *fc;
int score;
for (fc=CurrentGroupAuth->filter_chain; fc; fc=fc->next)
{
score = 0;
for (f=fc->filter; f; f=f->next)
{
if (f->scope != sc_header ||
!strCaseEq(header, f->scope_header))
continue;
#ifdef USE_REGEX
if (nn_regexec(&f->preg, text, strlen(text), 0, 0, 0)==0)
#else
if (matchExp(f->pat, text, f->ignore_case, 0))
#endif
{
score+=f->weight;
switch (f->weight_op)
{
case '*':
score*=f->weight;
break;
case '/':
score/=f->weight;
break;
default:
break;
}
}
if (score >= 10000)
return fc->name;
if (score <=-10000)
return NULL;
}
if (score>=100)
return fc->name;
}
return NULL;
}
static char *xover_nocem (char *xover, struct strList *i_fmt)
/* [<][>][^][v][top][bottom][index][help] */
{
char *ret=NULL;
for (; i_fmt; i_fmt = i_fmt->next)
{
char *xover2 = strchr (xover, '\t');
if (!strCaseEq (i_fmt->data, "Message-ID:"))
{
if (!xover2)
return NULL;
xover = xover2+1;
continue;
}
if (xover2)
*xover2 = '\0';
if (xover[0] == '<' &&
xover2[-1] == '>')
{
char *p;
xover2[-1] = '\0';
p = hisGet(xover+1);
if (p && strCaseEq (p, SPAM))
ret = SPAM;
xover2[-1] = '>';
}
else
logd (("badly formed msgid '%s' seen in xover_nocem ()", xover));
xover = xover2;
if (!xover)
break;
*xover = '\t';
if (ret)
return ret;
xover++;
}
return ret;
}
static char *xover_filter (char *xover, struct strList *i_fmt)
/* [<][>][^][v][top][bottom][index][help] */
{
char *ret=NULL;
for (; i_fmt; i_fmt = i_fmt->next)
{
char *xover2 = strchr (xover, '\t');
if (xover2)
*xover2 = '\0';
ret = filter_header(i_fmt->data, xover);
xover = xover2;
if (!xover)
break;
*xover = '\t';
if (ret)
return ret;
xover++;
}
return ret;
}
/*
* convert one xover format into another (xover is minus leading artnum\t)
* the input xover needs to have EOL stripped.
*/
static char *xover_translate (char *xover, struct server_cfg *cf)
/* [<][>][^][v][top][bottom][index][help] */
{
static char buf[MAX_XOVER];
struct strList *i_fmt = cf->share->overview_fmt, *o_fmt = overviewFmt;
char *o = buf;
int n = 0;
for (; o_fmt; o_fmt = o_fmt->next, n++)
{
/*
* round robin for efficiency with non-diverging data sets.
*/
struct strList *i_fmt_orig = i_fmt;
do
{
if (strCaseEq (i_fmt->data, o_fmt->data))
{
i_fmt = i_fmt->next;
break;
}
i_fmt = i_fmt->next;
if (!i_fmt)
i_fmt = cf->share->overview_fmt;
n++;
} while (i_fmt != i_fmt_orig);
/*
* TODO: merge below into round robin
*/
if (i_fmt != i_fmt_orig)
{
char *i = xover;
int tabs = 0;
if (!i_fmt)
i_fmt = i_fmt_orig;
do
{
if (n == tabs++)
{
if (cf->share->overview_fmt_xref == i_fmt)
{
o+=xrefRewriteCopy(cf, i, o, cf->share->overview_fmt_xref_full);
} else
{
while (*i && *i != '\t')
*o++ = *i++;
}
break;
}
} while (*i && (i = strchr (i, '\t')) && i++);
}
*o++ = '\t';
}
*--o = '\0';
return buf;
}
/*
* find node
*/
static struct xover_file *xf_find (int i)
/* [<][>][^][v][top][bottom][index][help] */
{
/*
* we work backwards, as newer nodes are
* close to the end of the cache
*/
struct xover_file *xf = xover_file_tail;
for (; xf; xf = xf->prev)
{
if (xf->id == i && strEq (xf->dir, CurrentDir))
return xf;
}
return NULL;
}
static bool xf_write_one (struct xover_file *xf)
/* [<][>][^][v][top][bottom][index][help] */
{
if (lseek (xf->fd, 0, SEEK_SET) != 0 ||
write (xf->fd, xf->buf, 4 * MI) != 4 * MI ||
(xf->outbuf &&
(lseek (xf->fd, 0, SEEK_END) < 4 * MI ||
write (xf->fd, xf->outbuf, xf->outbuf_used) != xf->outbuf_used)))
{
loge (("unable to write/lseek '%s/%s' correctly (...unlinking)", xf->dir, xf->name));
unlink (xf->name);
return FALSE;
}
xf->mtime = time (NULL);
return TRUE;
}
static bool xf_check_buf (struct xover_file *xf, bool readAll)
/* [<][>][^][v][top][bottom][index][help] */
{
struct stat st;
int size;
if (fstat (xf->fd, &st) < 0)
{
loge (("unable to fstat '%s/%s'", xf->dir, xf->name));
return FALSE;
}
if (st.st_size < MI * 4)
{
loge (("short file, %d bytes '%s/%s'", (int)st.st_size, xf->dir, xf->name));
return FALSE;
}
xf->size = st.st_size;
if (!xf->buf || st.st_mtime > xf->mtime)
{
if (lseek (xf->fd, 0, SEEK_SET) != 0)
{
loge (("couldn't lseek to start of '%s/%s'", xf->dir, xf->name));
return FALSE;
}
if (readAll)
size = xf->size;
else
size = MI * 4;
xf->buf = Srealloc (xf->buf, size);
if (read (xf->fd, xf->buf, size) != size)
{
loge (("couldn't read all of %d bytes from '%s/%s'", size, xf->dir, xf->name));
free (xf->buf);
xf->buf = 0;
return FALSE;
}
} else
{
if (!readAll)
return TRUE;
if (lseek (xf->fd, MI * 4, SEEK_SET) != MI * 4)
{
loge (("couldn't lseek to %d in '%s/%s'", MI * 4, xf->dir, xf->name));
return FALSE;
}
xf->buf = Srealloc (xf->buf, xf->size);
if (read (xf->fd, xf->buf + MI * 4, xf->size - MI * 4) != xf->size - MI * 4)
{
loge (("couldn't read all of %d bytes from '%s/%s'", xf->size - MI * 4, xf->dir, xf->name));
free (xf->buf);
xf->buf = 0;
return FALSE;
}
}
xf->mtime = st.st_mtime;
return TRUE;
}
#if 0 /* not needed at the moment */
static bool xf_acquire_reader (struct xover_file *xf, bool readAll)
/* [<][>][^][v][top][bottom][index][help] */
{
if (xf->fd >= 0 && !xf->reader)
if (locksh (xf->fd) == 0)
if (!(xf->reader = xf_check_buf (xf, readAll)))
lockun (xf->fd);
return xf->reader;
}
#endif
static void xf_release_reader (struct xover_file *xf)
/* [<][>][^][v][top][bottom][index][help] */
{
if (xf->fd >= 0 && xf->reader)
lockun (xf->fd);
xf->reader = FALSE;
}
static bool xf_acquire_writer (struct xover_file *xf, bool readAll)
/* [<][>][^][v][top][bottom][index][help] */
{
if (xf->reader)
xf_release_reader (xf);
if (xf->fd >= 0 && !xf->writer)
if (lockex (xf->fd) == 0)
if (!(xf->writer = xf_check_buf (xf, readAll)));
lockun (xf->fd);
return xf->writer;
}
static bool xf_release_writer (struct xover_file *xf)
/* [<][>][^][v][top][bottom][index][help] */
{
bool res = TRUE;
if (xf->fd > 0 && xf->writer && xf->buf)
res = xf_write_one (xf);
if (xf->outbuf)
{
free (xf->outbuf);
xf->outbuf = NULL;
xf->outbuf_used = 0;
}
if (xf->fd > 0 && xf->writer)
lockun (xf->fd);
xf->writer = FALSE;
return res;
}
static void xf_close (struct xover_file *xf)
/* [<][>][^][v][top][bottom][index][help] */
{
xf_release_reader (xf);
xf_release_writer (xf);
if (xf->name)
{
free (xf->name);
xf->name = NULL;
}
if (xf->dir)
{
free (xf->dir);
xf->dir = NULL;
}
if (xf->buf)
{
free (xf->buf);
xf->buf = NULL;
}
if (xf->fd > 0)
{
close (xf->fd);
xf->fd = -1;
}
xf->id = -1;
}
/*
* add node. we presume con->MaxXoverNodes > 1
*/
static struct xover_file *xf_add (int i)
/* [<][>][^][v][top][bottom][index][help] */
{
struct xover_file *xf;
if (xover_file_nodes>=con->maxXoverNodes)
{
/*
* the cache is full. close the oldest node
* and take over it's data space with the
* new node. our linked list is now a
* circular stack (i.e avoid malloc/free)
*/
xf_close (xover_file_head);
xover_file_tail->next = xf = xover_file_head;
xf->next->prev = NULL;
xover_file_head = xf->next;
memset (xf, 0, sizeof *xf);
} else
{
xf = Scalloc (1, sizeof *xf);
if (xover_file_tail)
xover_file_tail->next = xf;
else
xover_file_head = xf;
xover_file_nodes++;
}
xf->prev = xover_file_tail;
xover_file_tail = xf;
xf->id = i;
return xf;
}
#if 0
static void xf_destroy_all ()
/* [<][>][^][v][top][bottom][index][help] */
{
struct xover_file *xf = xover_file_head;
while (xf)
{
struct xover_file *xtmp;
if (xf->id >= 0)
xf_close (xf);
xtmp = xf->next;
free (xf);
xf = xtmp;
}
xover_file = xover_file_tail = NULL;
}
#endif
static void xf_release_all ()
/* [<][>][^][v][top][bottom][index][help] */
{
struct xover_file *xf = xover_file_tail;
for (; xf; xf = xf->prev)
if (!xf_release_writer (xf))
xf_close (xf);
}
static int xover_emit_xhdr (char *xover, int xn, struct strList *i_fmt, char *xhdr)
/* [<][>][^][v][top][bottom][index][help] */
{
int len;
char *hdr=xover_extract_xhdr(xover, i_fmt, xhdr);
len=emitf("%d ", xn);
if (hdr)
{
char *tab = strchr(hdr, '\t');
if (tab)
*tab = '\0';
len+=emitrn(hdr);
if (tab)
*tab = '\t';
} else
len+=emitrn("(none)");
return len;
}
static void xf_grab_outbuf (struct xover_file *xf, int len)
/* [<][>][^][v][top][bottom][index][help] */
{
#define GRAB_SIZE 32768
if (!xf->outbuf)
{
xf->outbuf = Smalloc (GRAB_SIZE);
xf->outbuf_len = GRAB_SIZE;
} else if (xf->outbuf_used + len > xf->outbuf_len)
xf->outbuf = Srealloc (xf->outbuf, (xf->outbuf_len += GRAB_SIZE));
#undef GRAB_SIZE
}
static struct xover_file *xf_open (int index)
/* [<][>][^][v][top][bottom][index][help] */
{
int fd;
char fn[64];
struct stat st;
struct xover_file *xf;
sprintf (fn, "%d_xover", index * MI);
fd = open (fn, O_RDWR | O_CREAT, 0664);
if (fd < 0)
{
loge (("unable to open/create '%s/%s'", CurrentDir, fn));
return NULL;
}
xf = xf_add (index);
xf->fd = fd;
xf->name = Sstrdup (fn);
xf->dir = Sstrdup (CurrentDir);
if (lockex (xf->fd) != 0)
{
loge (("unable to lockex '%s/%s' (...unlinking)", xf->dir, xf->name));
unlink (xf->name);
xf_close (xf);
return NULL;
}
if (fstat (xf->fd, &st) != 0)
{
loge (("unable to fstat '%s/%s' (...unlinking)", xf->dir, xf->name));
unlink (xf->name);
lockun (xf->fd);
xf_close (xf);
return NULL;
}
if (st.st_size < MI * 4 && ftruncate (xf->fd, MI * 4) != 0)
{
loge (("unable to truncate '%s/%s' (...unlinking)", xf->dir, xf->name));
unlink (xf->name);
lockun (xf->fd);
xf_close (xf);
return NULL;
}
lockun (xf->fd);
return xf;
}
static bool xover_input (char *xhdr, int min, int max)
/* [<][>][^][v][top][bottom][index][help] */
{
int len;
int response;
char *p;
struct xover_file *xf;
int t;
n_u32 *idx;
n_u32 i32;
char xover[MAX_XOVER + 20];
bool f_translated;
if (!(len=Cget (xover, sizeof xover)))
{
xover_server_state = dead;
outstanding = 0;
CurrentScfg->share->xover_fail++;
return FALSE;
}
if (EL (xover))
{
xover_server_state = listening;
outstanding--;
CurrentScfg->share->xover_good++;
return TRUE;
}
response = strToi (xover);
if (xover_server_state == listening)
{
if (response != NNTP_OVERVIEW_FOLLOWS_VAL)
{
outstanding--;
if (xover_client_state == listening)
{
emit (xover);
return FALSE;
} else
return TRUE; /* not sure this is correct behavior */
}
xover_server_state = sending;
return TRUE;
} /* must be an xover then .. */
if (response<min || response>max)
return TRUE; /* not what we asked for */
if (xover_client_state == listening)
{
emitrn (xhdr? NNTP_HEAD_FOLLOWS: NNTP_OVERVIEW_FOLLOWS);
xover_client_state = receiving;
}
/* skip artnum */
p = strchr (xover, '\t');
if (!p)
{
logw (("xover %d in %s contains no fields", response, CurrentGroup));
return TRUE;
}
++p;
strnStripEOL (xover, len);
if (overviewFmt_hash != CurrentGroupScfg->share->overview_fmt_hash)
{
p = xover_translate (p, CurrentGroupScfg);
f_translated = TRUE;
} else
{
f_translated = FALSE;
if (CurrentGroupScfg->share->overview_fmt_xref)
{
char *t;
struct strList *f=CurrentGroupScfg->share->overview_fmt;
for (t=p; f && t; f=f->next, t=strchr(t, '\t'))
{
if (*++t != '\t' && f == CurrentGroupScfg->share->overview_fmt_xref)
{
xrefRewrite(CurrentGroupScfg, t, CurrentGroupScfg->share->overview_fmt_xref_full);
break;
}
}
}
}
t = response / MI;
if (!(xf = xf_find (t)))
{
if (!(xf = xf_open (t)))
goto auth_xover;
#if 0
/* This is now always done in xover_process. */
/*
* set failed flag in index for xover region asked.
* we then knock em out one by one as xovers come in
*
* the bit map per xover cache file initially looks like this:
*
* 0 min max 511
* | | | |
* uuuuuuuuuuuuuffffffffffffffffffffffffffffffffffffuuuuuuuuuuuuuuuu
* | | |
* unknown failed unknown
*
* after the xover responses come in, we transform to:
*
* min cached max
* | | |
* uuuuuuuuuuuuuff******f******************f********uuuuuuuuuuuuuuuu
* | \______|_________________/ |
* | | |
* unknown failed/expired unknown
*
* keep in mind that min and max may be outside 0-511 making the
* current xover bitmap only a "window" into the bitmaps tickled
* by the request
*
* the next time an xover is requested on the region:
*
* u - will be fetched from the server
* f - will be skipped
* * - will be pulled from the cache
*
* note that nocem detected spams are marked as f
*/
base = (min<t*MI)? 0: min%MI;
memset (xf->buf+base*4, 0xff, (MIN(MI, max+1-t*MI)-base)*4);
#endif
}
if (!xf_acquire_writer (xf, FALSE))
goto auth_xover;
len = strlen (p) + 1;
idx = (n_u32 *) xf->buf;
i32 = idx[response % MI];
if (CurrentGroupNocem)
{
char *blocker = xover_nocem(p, f_translated? overviewFmt: CurrentGroupScfg->share->overview_fmt);
if (blocker)
{
logd (("nocem filter %s blocked xover of %s/%d", blocker, CurrentDir, response));
idx[response % MI] = 0xffffffff;
CS->nocem_blocked++;
return TRUE;
}
}
if (!CurrentGroupNode->lo_xover || CurrentGroupNode->lo_xover > response)
CurrentGroupNode->lo_xover = response;
if (CurrentGroupNode->hi_xover < response)
CurrentGroupNode->hi_xover = response;
if (i32 == 0 || i32 == 0xffffffff) /* only if we don't have it already */
{
idx[response % MI] = htonl ((len << 20) | (((xf->size > 0) ? xf->size : MI * 4) + xf->outbuf_used));
xf_grab_outbuf(xf, len);
memcpy (xf->outbuf + xf->outbuf_used, p, len);
xf->outbuf_used += len;
}
auth_xover:
{
char *blocker;
if (FILT && (blocker = xover_filter(p, f_translated? overviewFmt: CurrentGroupScfg->share->overview_fmt)))
{
logd (("content filter %s blocked xover of %s/%d", blocker, CurrentDir, response));
CS->filter_blocked++;
return TRUE;
}
if (f_translated)
{
if (xhdr)
xover_emit_xhdr(p, response, overviewFmt, xhdr);
else
emitf ("%d\t%s\r\n", response, p);
} else
{
if (xhdr)
xover_emit_xhdr(p, response, CurrentGroupScfg->share->overview_fmt, xhdr);
else
emitrn (xover);
}}
return TRUE;
}
static void xover_output (int min, int max)
/* [<][>][^][v][top][bottom][index][help] */
{
Cemitf("xover %d-%d\r\n", min, max);
Cflush ();
outstanding++;
}
static bool xover_io (int min, int max, char *xhdr, int orig_min, int orig_max)
/* [<][>][^][v][top][bottom][index][help] */
{
while (min || outstanding>0)
{
if (xover_server_state != dead && (!min || Smore (CurrentGroupScfg->fd)))
{
if (xover_input (xhdr, orig_min, orig_max) && Smore (CurrentGroupScfg->fd))
continue;
}
/*
* here, not above, as it is possible output will block due to input not
* being cleared
*/
if (min)
{
xover_output (min, max);
return TRUE;
}
if (outstanding>0) /* TODO: work out when we have to wait on a socket */
usleep (5000); /* doesn't use SIGALRM */
}
if (xover_client_state == listening)
emitrn (xhdr? NNTP_HEAD_FOLLOWS: NNTP_OVERVIEW_FOLLOWS);
emitrn (".");
return TRUE;
}
static bool xover_finish (char *xhdr, int orig_min, int orig_max)
/* [<][>][^][v][top][bottom][index][help] */
{
bool ret;
ret = xover_io (0, 0, xhdr, orig_min, orig_max); /* finish all pending i/o */
xf_release_all ();
return ret;
}
static bool xover_process (int min, int max, char *xhdr)
/* [<][>][^][v][top][bottom][index][help] */
{
int colmin = min, colmax, colmax_high=0;
int xmin = min / MI, xmax = max / MI;
int xi;
FD_ZERO (&fdset);
xover_client_state = xover_server_state = listening;
outstanding = 0;
for (xi = xmin; xi <= xmax; xi++)
{
char xover[20 + MAX_XOVER];
int start, end;
n_u32 *idx;
char *xover_buf = NULL; /* not null when entire xover db file in incore */
int n;
int xn;
struct xover_file *xf;
if (!(xf = xf_find (xi)) && !(xf = xf_open (xi)))
continue;
/*
* we have two techniques depending on how much of
* the xover file is required. if we require more than 16
* xovers then we read the whole xover file into memory
* else we merely lseek() to the required positions.
* this appears to be more efficient than mmap or readv.
*/
start = (xi == xmin) ? min % MI : 0;
end = (xi == xmax) ? max % MI : MI - 1; /* upto and including */
if (!xf_acquire_writer (xf, end - start >= 16))
goto bad;
if (end - start >= 16)
xover_buf = xf->buf + MI * 4;
idx = (n_u32 *) (xf->buf);
for (n = start, xn = start + MI * xi, colmax = 0; n <= end; n++, xn++)
{
char *blocker;
int offset;
int len;
char *xov;
n_u32 i = idx[n];
/*
* if we couldn't get it before, don't try now
*/
if (i == 0xffffffff)
{
if (colmax == 0)
colmin = xn+1;
continue;
}
if (i==0)
{
idx[n] = 0xffffffff; /* set failed flag, xover_input will judge */
colmax = xn;
continue;
}
if (colmax != 0)
{
xover_io (colmin, colmax, xhdr, min, max);
colmax_high=colmax;
colmax = 0;
}
i = ntohl (i);
colmin = xn + 1;
offset = i & 0xfffff;
len = (i >> 20) & 0xfff;
if (xover_client_state == listening)
{
emitrn (xhdr? NNTP_HEAD_FOLLOWS: NNTP_OVERVIEW_FOLLOWS);
xover_client_state = receiving;
}
if (xover_buf)
{
if (offset >= xf->size)
{
loge (("bad xover offset in '%s/%s' (%d >= %d) (...unlinked)", xf->dir, xf->name, offset, xf->size));
goto bad;
}
xov = xover_buf + offset - 4 * MI;
} else
{
if (lseek (xf->fd, offset, SEEK_SET) != offset)
{
loge (("lseek failed for '%s/%s' (...unlinked)", xf->dir, xf->name));
goto bad;
}
if (read (xf->fd, xover, len) != len)
{
loge (("only read part of '%s/%s' (...unlinked)", xf->dir, xf->name));
goto bad;
}
xov = xover;
}
if (CurrentGroupNode->lo_xover > xn)
CurrentGroupNode->lo_xover = xn;
if (CurrentGroupNode->hi_xover < xn)
CurrentGroupNode->hi_xover = xn;
if (FILT && (blocker = xover_filter(xov, overviewFmt)))
{
logd (("content filter %s blocked xover of '%s/%d'", blocker, CurrentDir, xn));
CS->filter_blocked++;
}
else
{
if (xhdr)
xover_emit_xhdr (xov, xn, overviewFmt, xhdr);
else
emitf ("%d\t%s\r\n", xn, xov);
}
}
if (xover_buf)
Srealloc (xf->buf, MI * 4); /* keep only the index in the cache */
xf_release_reader (xf);
continue;
bad:
logw (("unlinking %s dir %s cwd %s", xf->name, xf->dir, CurrentDir));
unlink (xf->name);
xf_close (xf);
continue;
}
if (colmax_high < max && colmin <= max)
xover_io (colmin, max, xhdr, min, max);
xover_finish (xhdr, min, max);
return TRUE;
}
EXPORT bool CMDxover (char *args)
/* [<][>][^][v][top][bottom][index][help] */
{
int min=0, max = 0;
sscanf (args, "%*s %d-%d", &min, &max);
if (min<0 || max < 0 || (max > 0 && min > max))
{
emitrn (NNTP_SYNTAX_USE);
return FALSE;
}
CS->by_artnum++;
if (!*CurrentGroup || !setGD())
{
emitrn (NNTP_NOTINGROUP);
return FALSE;
}
if (!CurrentGroupScfg->share->overview_fmt)
{
emitf("%d no overview.fmt for this server\r\n", NNTP_DONTHAVEIT_VAL);
return FALSE;
}
if (CurrentGroupScfg->xover_timeout > 0)
{
if (min == 0 && max == 0)
{
min = max = CurrentGroupArtNum;
} else
{
int nmax = getHi(CurrentGroupNode);
int nmin = getLo(CurrentGroupNode);
if (min<nmin)
min = nmin;
if (max<1 || max >nmax)
max = strchr (args, '-')? nmax: min;
}
xover_process (min, max, NULL);
} else
{
char line[MAX_LINE];
Cemit (args);
Cflush ();
if (!Cget (line, sizeof line))
{
emitrn (NNTP_SERVERDOWN);
return FALSE;
}
emit (line);
if (strToi (line) == NNTP_OVERVIEW_FOLLOWS_VAL)
{
int len;
while ((len=Cget (line, sizeof line)) && !EL(line))
{
char *blocker;
char *p=strchr(line, '\t');
if (!p)
{
emit (line);
continue;
}
if (CurrentGroupNocem)
{
blocker = xover_nocem(p+1, CurrentGroupScfg->share->overview_fmt);
if (blocker)
{
logd (("nocem filter %s blocked xover of %s/%d", blocker, CurrentDir, strToi(line)));
CS->nocem_blocked++;
continue;
}
}
if (FILT && (blocker = xover_filter(p+1, CurrentGroupScfg->share->overview_fmt)))
{
logd (("content filter %s blocked xover of %s/%d", blocker, CurrentDir, strToi(line)));
CS->filter_blocked++;
continue;
}
if (overviewFmt_hash != CurrentGroupScfg->share->overview_fmt_hash)
{
*p++ = '\0'; /* kill tab */
strStripEOL(p);
p = xover_translate (p, CurrentGroupScfg);
emitf("%s\t%s\r\n", line, p);
} else
emit(line);
}
emitrn(".");
}
}
return TRUE;
}
static bool isFieldInXover(char *field)
/* [<][>][^][v][top][bottom][index][help] */
{
struct strList *o_fmt;
for (o_fmt = overviewFmt->head; o_fmt; o_fmt = o_fmt->next)
if (strCaseEq (o_fmt->data, field))
return TRUE;
return FALSE;
}
EXPORT bool CMDxhdr (char *args)
/* [<][>][^][v][top][bottom][index][help] */
{
int min, max = 0;
char msgid[MAX_MSGID];
char buf[MAX_LINE];
char header[MAX_HEADER];
char field[MAX_HEADER];
int len;
if (sscanf (args, "%*s %127s", header) != 1)
{
emitrn (NNTP_SYNTAX_USE);
bad:
return FALSE;
}
if (strSnip(args, 0, "<", ">\r\n", msgid, sizeof msgid) < 1)
{
sscanf (args, "%*s %*s %d-%d", &min, &max);
if (min<0 || max < 0 || (max > 0 && min > max) || (max == 0 && min ==0))
goto bad;
}
if (!*msgid && (!*CurrentGroup || !setGD()))
{
emitrn (NNTP_NOTINGROUP);
goto bad;
}
sprintf(field, "%.126s:", header);
if (*msgid) /* TODO: cache xhdr header <msgid> */
{
struct server_cfg *osrvrs = ServerList;
struct server_cfg *cf;
int n;
CS->by_msgid++;
Cemit (args);
Cflush ();
if (con->maxMsgIDsearch<1 || !Cget (buf, sizeof buf))
{
CurrentScfg->share->xhdr_fail++;
emitrn (NNTP_SERVERTEMPDOWN);
return FALSE;
}
if (strToi (buf) == NNTP_HEAD_FOLLOWS_VAL)
{
emit (buf);
while ((len=Cget (buf, sizeof buf)) && !EL (buf))
{
if (FILT)
{
char *p;
char *blocker;
for (p=buf; *p && *p!=' ' && *p!='\t'; p++) ;
if (*p && *++p && (blocker = filter_header(header, p)))
{
logd (("content filter %s blocked xhdr of <%s>", blocker, msgid));
CS->filter_blocked++;
continue;
}
}
emit (buf);
}
emitrn(".");
CurrentScfg->share->xhdr_good++;
return TRUE;
}
/*
* we are doing a get by <art-id> and current server didn't have it,
* so loop through all servers until
* we find one that does have it and make it the current server
*/
for (n=0, osrvrs = ServerList; osrvrs && n<con->maxMsgIDsearch; osrvrs = osrvrs->next)
{
if (CurrentScfg==osrvrs)
continue;
n++;
cf = attachServer (osrvrs);
if (!cf)
continue;
Cfemit (cf, args);
Cfflush (cf);
if (!(len=Cfget (cf, buf, sizeof (buf))))
continue;
if (strToi (buf) != NNTP_HEAD_FOLLOWS_VAL)
continue;
emit(buf);
CurrentScfg = cf;
while ((len=Cget (buf, sizeof buf)) && !EL (buf))
{
if (FILT)
{
char *p;
char *blocker;
for (p=buf; *p && *p!=' ' && *p!='\t'; p++) ;
if (*p && *++p && (blocker = filter_header(header, p)))
{
logd (("content filter %s blocked xhdr of <%s>", blocker, msgid));
CS->filter_blocked++;
continue;
}
}
emit (buf);
}
CurrentScfg->share->xhdr_good++;
emitrn(".");
break;
}
if (!osrvrs) /* no servers had <msgid> */
emitrn (NNTP_DONTHAVEIT);
return TRUE;
}
CS->by_artnum++;
if (CurrentScfg->xover_timeout < 1 || !isFieldInXover(field) || !CurrentScfg->share->overview_fmt)
{
Cemit (args);
Cflush ();
if (!Cget (buf, sizeof buf))
{
CurrentScfg->share->xhdr_fail++;
emitrn (NNTP_SERVERDOWN);
return FALSE;
}
emit (buf);
if (strToi(buf) != NNTP_HEAD_FOLLOWS_VAL)
{
CurrentScfg->share->xhdr_fail++;
return FALSE;
}
while ((len=Cget (buf, sizeof buf)) && !EL (buf))
{
if (FILT)
{
char *p;
char *blocker;
for (p=buf; *p && *p!=' ' && *p!='\t'; p++) ;
if (*p && *++p && (blocker = filter_header(header, p)))
{
logd (("content filter %s blocked xhdr of %s/%d", blocker, CurrentDir, strToi(buf)));
CS->filter_blocked++;
continue;
}
}
emit (buf);
}
CurrentScfg->share->xhdr_good++;
emitrn(".");
return TRUE;
}
/* cached and not <msgid>. convert to xovers */
if (min == 0 && max == 0)
{
min = max = CurrentGroupArtNum;
} else
{
int nmax = getHi(CurrentGroupNode);
int nmin = getLo(CurrentGroupNode);
if (min<nmin)
min = nmin;
if (max<1 || max >nmax)
max = strchr (args, '-')? nmax: min;
}
xover_process (min, max, field);
return TRUE;
}