00001
00005 #include "system.h"
00006
00007 #if defined(WITH_SQLITE)
00008 #include <sqlite3.h>
00009 #ifdef __LCLINT__
00010
00011 extern const char *sqlite3_errmsg(sqlite3 *db)
00012 ;
00013 extern int sqlite3_open(
00014 const char *filename,
00015 sqlite3 **ppDb
00016 )
00017 ;
00018 extern int sqlite3_exec(
00019 sqlite3 *db,
00020 const char *sql,
00021 int (*callback)(void*,int,char**,char**),
00022 void *,
00023 char **errmsg
00024 )
00025 ;
00026 extern int sqlite3_prepare(
00027 sqlite3 *db,
00028 const char *zSql,
00029 int nByte,
00030 sqlite3_stmt **ppStmt,
00031 const char **pzTail
00032 )
00033 ;
00034 extern int sqlite3_reset(sqlite3_stmt *pStmt)
00035 ;
00036 extern int sqlite3_step(sqlite3_stmt *pStmt)
00037 ;
00038 extern int sqlite3_finalize( sqlite3_stmt *pStmt)
00039 ;
00040 extern int sqlite3_close(sqlite3 * db)
00041 ;
00042
00043 #endif
00044 #endif
00045
00046 #include <rpmio_internal.h>
00047 #include <fts.h>
00048 #include <argv.h>
00049 #include <mire.h>
00050 #include <poptIO.h>
00051
00052 #include <rpmtypes.h>
00053 #include <rpmtag.h>
00054 #include <pkgio.h>
00055 #include <rpmts.h>
00056
00057 #include "debug.h"
00058
00059
00060
00061
00062
00063
00064
00065 static int _repo_debug;
00066
00067 typedef struct rpmrepo_s * rpmrepo;
00068 typedef struct rpmrfile_s * rpmrfile;
00069
00073 struct rpmrfile_s {
00074
00075 const char * type;
00076
00077 const char * xml_init;
00078
00079 const char * xml_qfmt;
00080
00081 const char * xml_fini;
00082
00083 const char ** sql_init;
00084
00085 const char * sql_qfmt;
00086 #ifdef NOTYET
00087
00088 const char ** sql_fini;
00089 #endif
00090
00091 const char * yaml_init;
00092
00093 const char * yaml_qfmt;
00094
00095 const char * yaml_fini;
00096
00097 const char * Packages_init;
00098
00099 const char * Packages_qfmt;
00100
00101 const char * Packages_fini;
00102
00103 const char * Sources_init;
00104
00105 const char * Sources_qfmt;
00106
00107 const char * Sources_fini;
00108
00109 FD_t fd;
00110 #if defined(WITH_SQLITE)
00111 sqlite3 * sqldb;
00112 #endif
00113
00114 const char * digest;
00115
00116 const char * Zdigest;
00117 time_t ctime;
00118 };
00119
00123 struct rpmrepo_s {
00124 int quiet;
00125 int verbose;
00126 int dryrun;
00127
00128 ARGV_t exclude_patterns;
00129
00130 miRE excludeMire;
00131 int nexcludes;
00132
00133 ARGV_t include_patterns;
00134
00135 miRE includeMire;
00136 int nincludes;
00137
00138 const char * basedir;
00139
00140 const char * baseurl;
00141 #ifdef NOTYET
00142
00143 const char * groupfile;
00144 #endif
00145 int split;
00146 #if defined(WITH_SQLITE)
00147 int database;
00148 #endif
00149 int pretty;
00150 int checkts;
00151
00152 const char * outputdir;
00153
00154 int nofollow;
00155
00156 ARGV_t manifests;
00157
00158
00159 const char * tempdir;
00160
00161 const char * finaldir;
00162
00163 const char * olddir;
00164
00165 time_t mdtimestamp;
00166
00167 int uniquemdfilenames;
00168
00169
00170 rpmts ts;
00171
00172 ARGV_t pkglist;
00173 unsigned current;
00174 unsigned pkgcount;
00175
00176
00177 ARGV_t directories;
00178 int ftsoptions;
00179 uint32_t pkgalgo;
00180 uint32_t algo;
00181 int compression;
00182
00183 const char * markup;
00184
00185 const char * suffix;
00186
00187 const char * wmode;
00188
00189 struct rpmrfile_s primary;
00190 struct rpmrfile_s filelists;
00191 struct rpmrfile_s other;
00192 struct rpmrfile_s repomd;
00193
00194 };
00195
00196
00197 static const char primary_xml_init[] =
00198 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
00199 "<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"0\">\n";
00200
00201 static const char primary_xml_fini[] = "</metadata>\n";
00202
00203
00204 static const char filelists_xml_init[] =
00205 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
00206 "<filelists xmlns=\"http://linux.duke.edu/metadata/filelists\" packages=\"0\">\n";
00207
00208 static const char filelists_xml_fini[] = "</filelists>\n";
00209
00210
00211 static const char other_xml_init[] =
00212 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
00213 "<otherdata xmlns=\"http://linux.duke.edu/metadata/other\" packages=\"0\">\n";
00214
00215 static const char other_xml_fini[] = "</otherdata>\n";
00216
00217
00218 static const char repomd_xml_init[] = "\
00219 <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
00220 <repomd xmlns=\"http://linux.duke.edu/metadata/repo\">\n";
00221
00222 static const char repomd_xml_fini[] = "</repomd>\n";
00223
00224
00225
00226 static const char primary_xml_qfmt[] =
00227 #include "yum_primary_xml"
00228 ;
00229
00230
00231 static const char filelists_xml_qfmt[] =
00232 #include "yum_filelists_xml"
00233 ;
00234
00235
00236 static const char other_xml_qfmt[] =
00237 #include "yum_other_xml"
00238 ;
00239
00240
00241 static const char primary_yaml_qfmt[] =
00242 #include "wnh_primary_yaml"
00243 ;
00244
00245
00246 static const char filelists_yaml_qfmt[] =
00247 #include "wnh_filelists_yaml"
00248 ;
00249
00250
00251 static const char other_yaml_qfmt[] =
00252 #include "wnh_other_yaml"
00253 ;
00254
00255
00256 static const char Packages_qfmt[] =
00257 #include "deb_Packages"
00258 ;
00259
00260
00261 static const char Sources_qfmt[] =
00262 #include "deb_Sources"
00263 ;
00264
00265
00266
00267 static const char *primary_sql_init[] = {
00268 "PRAGMA synchronous = \"OFF\";",
00269 "pragma locking_mode = \"EXCLUSIVE\";",
00270 "CREATE TABLE conflicts ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
00271 "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);",
00272 "CREATE TABLE files ( pkgKey INTEGER, name TEXT, type TEXT );",
00273 "CREATE TABLE obsoletes ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
00274 "CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT, name TEXT, arch TEXT, version TEXT, epoch TEXT, release TEXT, summary TEXT, description TEXT, url TEXT, time_file INTEGER, time_build INTEGER, rpm_license TEXT, rpm_vendor TEXT, rpm_group TEXT, rpm_buildhost TEXT, rpm_sourcerpm TEXT, rpm_header_start INTEGER, rpm_header_end INTEGER, rpm_packager TEXT, size_package INTEGER, size_installed INTEGER, size_archive INTEGER, location_href TEXT, location_base TEXT, checksum_type TEXT);",
00275 "CREATE TABLE provides ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
00276 "CREATE TABLE requires ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
00277 "CREATE INDEX filenames ON files (name);",
00278 "CREATE INDEX packageId ON packages (pkgId);",
00279 "CREATE INDEX packagename ON packages (name);",
00280 "CREATE INDEX pkgconflicts on conflicts (pkgKey);",
00281 "CREATE INDEX pkgobsoletes on obsoletes (pkgKey);",
00282 "CREATE INDEX pkgprovides on provides (pkgKey);",
00283 "CREATE INDEX pkgrequires on requires (pkgKey);",
00284 "CREATE INDEX providesname ON provides (name);",
00285 "CREATE INDEX requiresname ON requires (name);",
00286 "CREATE TRIGGER removals AFTER DELETE ON packages\
00287 \n BEGIN\n\
00288 \n DELETE FROM files WHERE pkgKey = old.pkgKey;\
00289 \n DELETE FROM requires WHERE pkgKey = old.pkgKey;\
00290 \n DELETE FROM provides WHERE pkgKey = old.pkgKey;\
00291 \n DELETE FROM conflicts WHERE pkgKey = old.pkgKey;\
00292 \n DELETE FROM obsoletes WHERE pkgKey = old.pkgKey;\
00293 \n END;",
00294 "INSERT into db_info values (9, 'direct_create');",
00295 NULL
00296 };
00297
00298
00299
00300 static const char *filelists_sql_init[] = {
00301 "PRAGMA synchronous = \"OFF\";",
00302 "pragma locking_mode = \"EXCLUSIVE\";",
00303 "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);",
00304 "CREATE TABLE filelist ( pkgKey INTEGER, name TEXT, type TEXT );",
00305 "CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);",
00306 "CREATE INDEX filelistnames ON filelist (name);",
00307 "CREATE INDEX keyfile ON filelist (pkgKey);",
00308 "CREATE INDEX pkgId ON packages (pkgId);",
00309 "CREATE TRIGGER remove_filelist AFTER DELETE ON packages\
00310 \n BEGIN\
00311 \n DELETE FROM filelist WHERE pkgKey = old.pkgKey;\
00312 \n END;",
00313 "INSERT into db_info values (9, 'direct_create');",
00314 NULL
00315 };
00316
00317
00318
00319 static const char *other_sql_init[] = {
00320 "PRAGMA synchronous = \"OFF\";",
00321 "pragma locking_mode = \"EXCLUSIVE\";",
00322 "CREATE TABLE changelog ( pkgKey INTEGER, author TEXT, date INTEGER, changelog TEXT);",
00323 "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);",
00324 "CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);",
00325 "CREATE INDEX keychange ON changelog (pkgKey);",
00326 "CREATE INDEX pkgId ON packages (pkgId);",
00327 "CREATE TRIGGER remove_changelogs AFTER DELETE ON packages\
00328 \n BEGIN\
00329 \n DELETE FROM changelog WHERE pkgKey = old.pkgKey;\
00330 \n END;",
00331 "INSERT into db_info values (9, 'direct_create');",
00332 NULL
00333 };
00334
00335
00336
00337
00338
00339
00340
00341
00342
00343
00344
00345
00346
00347
00348
00349
00350
00351
00352
00353
00354
00355
00356
00357
00358
00359
00360
00361
00362
00363
00364
00365
00366
00367
00368
00369
00370
00371
00372
00373
00374
00375
00376
00377
00378
00379
00380
00381
00382
00383
00384
00385
00386
00387
00388
00389
00390
00391
00392 static const char primary_sql_qfmt[] =
00393 #include "yum_primary_sqlite"
00394 ;
00395
00396
00397
00398
00399
00400
00401
00402
00403 static const char filelists_sql_qfmt[] =
00404 #include "yum_filelists_sqlite"
00405 ;
00406
00407
00408
00409
00410
00411
00412
00413
00414
00415 static const char other_sql_qfmt[] =
00416 #include "yum_other_sqlite"
00417 ;
00418
00419
00420
00421 static struct rpmrepo_s __rpmrepo = {
00422 .pretty = 1,
00423 #if defined(WITH_SQLITE)
00424 .database = 0,
00425 #endif
00426 .tempdir = ".repodata",
00427 .finaldir = "repodata",
00428 .olddir = ".olddata",
00429 .markup = ".xml",
00430 .pkgalgo = PGPHASHALGO_SHA1,
00431 .algo = PGPHASHALGO_SHA1,
00432 .primary = {
00433 .type = "primary",
00434 .xml_init= primary_xml_init,
00435 .xml_qfmt= primary_xml_qfmt,
00436 .xml_fini= primary_xml_fini,
00437 .sql_init= primary_sql_init,
00438 .sql_qfmt= primary_sql_qfmt,
00439 #ifdef NOTYET
00440 .sql_fini= NULL,
00441 #endif
00442 .yaml_init= NULL,
00443 .yaml_qfmt= primary_yaml_qfmt,
00444 .yaml_fini= NULL,
00445 .Packages_init= NULL,
00446 .Packages_qfmt= NULL,
00447 .Packages_fini= NULL,
00448 .Sources_init= NULL,
00449 .Sources_qfmt= NULL,
00450 .Sources_fini= NULL
00451 },
00452 .filelists = {
00453 .type = "filelists",
00454 .xml_init= filelists_xml_init,
00455 .xml_qfmt= filelists_xml_qfmt,
00456 .xml_fini= filelists_xml_fini,
00457 .sql_init= filelists_sql_init,
00458 .sql_qfmt= filelists_sql_qfmt,
00459 #ifdef NOTYET
00460 .sql_fini= NULL,
00461 #endif
00462 .yaml_init= NULL,
00463 .yaml_qfmt= filelists_yaml_qfmt,
00464 .yaml_fini= NULL,
00465 .Packages_init= NULL,
00466 .Packages_qfmt= NULL,
00467 .Packages_fini= NULL,
00468 .Sources_init= NULL,
00469 .Sources_qfmt= NULL,
00470 .Sources_fini= NULL
00471 },
00472 .other = {
00473 .type = "other",
00474 .xml_init= other_xml_init,
00475 .xml_qfmt= other_xml_qfmt,
00476 .xml_fini= other_xml_fini,
00477 .sql_init= other_sql_init,
00478 .sql_qfmt= other_sql_qfmt,
00479 #ifdef NOTYET
00480 .sql_fini= NULL,
00481 #endif
00482 .yaml_init= NULL,
00483 .yaml_qfmt= other_yaml_qfmt,
00484 .yaml_fini= NULL,
00485 .Packages_init= NULL,
00486 .Packages_qfmt= NULL,
00487 .Packages_fini= NULL,
00488 .Sources_init= NULL,
00489 .Sources_qfmt= NULL,
00490 .Sources_fini= NULL
00491 },
00492 .repomd = {
00493 .type = "repomd",
00494 .xml_init= repomd_xml_init,
00495 .xml_qfmt= NULL,
00496 .xml_fini= repomd_xml_fini,
00497 .sql_init= NULL,
00498 .sql_qfmt= NULL,
00499 #ifdef NOTYET
00500 .sql_fini= NULL,
00501 #endif
00502 .yaml_init= NULL,
00503 .yaml_qfmt= NULL,
00504 .yaml_fini= NULL,
00505 .Packages_init= NULL,
00506 .Packages_qfmt= Packages_qfmt,
00507 .Packages_fini= NULL,
00508 .Sources_init= NULL,
00509 .Sources_qfmt= Sources_qfmt,
00510 .Sources_fini= NULL
00511 }
00512 };
00513
00514
00515
00516 static rpmrepo _rpmrepo = &__rpmrepo;
00517
00518
00524
00525 static void
00526 repo_error(int lvl, const char *fmt, ...)
00527
00528
00529 {
00530 va_list ap;
00531
00532 va_start(ap, fmt);
00533 (void) fflush(NULL);
00534 (void) fprintf(stderr, "%s: ", __progname);
00535 (void) vfprintf(stderr, fmt, ap);
00536 va_end (ap);
00537 (void) fprintf(stderr, "\n");
00538 if (lvl)
00539 exit(EXIT_FAILURE);
00540 }
00541
00549 static void repoProgress( rpmrepo repo,
00550 const char * item, int current, int total)
00551
00552
00553 {
00554 static size_t ncols = 80 - 1;
00555 const char * bn = (item != NULL ? strrchr(item, '/') : NULL);
00556 size_t nb;
00557
00558 if (bn != NULL)
00559 bn++;
00560 else
00561 bn = item;
00562 nb = fprintf(stdout, "\r%s: %d/%d", __progname, current, total);
00563 if (bn != NULL)
00564 nb += fprintf(stdout, " - %s", bn);
00565 nb--;
00566 if (nb < ncols)
00567 fprintf(stdout, "%*s", (int)(ncols - nb), "");
00568 ncols = nb;
00569 (void) fflush(stdout);
00570 }
00571
00577 static int rpmioExists(const char * fn, struct stat * st)
00578
00579
00580 {
00581 return (Stat(fn, st) == 0);
00582 }
00583
00589 static time_t rpmioCtime(const char * fn)
00590
00591
00592 {
00593 struct stat sb;
00594 time_t stctime = 0;
00595
00596 if (rpmioExists(fn, &sb))
00597 stctime = sb.st_ctime;
00598 return stctime;
00599 }
00600
00606
00607 static const char * repoRealpath(const char * lpath)
00608
00609
00610 {
00611
00612 const char *rpath = Realpath(lpath, NULL);
00613 if (rpath == NULL) {
00614 char fullpath[MAXPATHLEN];
00615 rpath = Realpath(lpath, fullpath);
00616 if (rpath != NULL)
00617 rpath = xstrdup(rpath);
00618 }
00619 return rpath;
00620 }
00621
00622
00629 static int repoMkdir(rpmrepo repo, const char * dn)
00630
00631
00632 {
00633 const char * dnurl = rpmGetPath(repo->outputdir, "/", dn, NULL);
00634
00635 int ut = urlPath(dnurl, &dn);
00636
00637 int rc = 0;;
00638
00639
00640 if (ut == URL_IS_UNKNOWN)
00641 rc = rpmioMkpath(dn, 0755, (uid_t)-1, (gid_t)-1);
00642 else
00643 rc = (Mkdir(dnurl, 0755) == 0 || errno == EEXIST ? 0 : -1);
00644 if (rc)
00645 repo_error(0, _("Cannot create/verify %s: %s"), dnurl, strerror(errno));
00646 dnurl = _free(dnurl);
00647 return rc;
00648 }
00649
00657 static const char * repoGetPath(rpmrepo repo, const char * dir,
00658 const char * type, int compress)
00659
00660
00661 {
00662 return rpmGetPath(repo->outputdir, "/", dir, "/", type,
00663 (repo->markup != NULL ? repo->markup : ""),
00664 (repo->suffix != NULL && compress ? repo->suffix : ""), NULL);
00665 }
00666
00672 static int repoTestSetupDirs(rpmrepo repo)
00673
00674
00675 {
00676 const char ** directories = repo->directories;
00677 struct stat sb, *st = &sb;
00678 const char * dn;
00679 const char * fn;
00680 int rc = 0;
00681
00682
00683
00684 if (directories != NULL)
00685 while ((dn = *directories++) != NULL) {
00686 if (!rpmioExists(dn, st) || !S_ISDIR(st->st_mode)) {
00687 repo_error(0, _("Directory %s must exist"), dn);
00688 rc = 1;
00689 }
00690 }
00691
00692
00693 if (!rpmioExists(repo->outputdir, st)) {
00694 repo_error(0, _("Directory %s does not exist."), repo->outputdir);
00695 rc = 1;
00696 }
00697 if (Access(repo->outputdir, W_OK)) {
00698 repo_error(0, _("Directory %s must be writable."), repo->outputdir);
00699 rc = 1;
00700 }
00701
00702 if (repoMkdir(repo, repo->tempdir)
00703 || repoMkdir(repo, repo->finaldir))
00704 rc = 1;
00705
00706 dn = rpmGetPath(repo->outputdir, "/", repo->olddir, NULL);
00707 if (rpmioExists(dn, st)) {
00708 repo_error(0, _("Old data directory exists, please remove: %s"), dn);
00709 rc = 1;
00710 }
00711 dn = _free(dn);
00712
00713 {
00714 static const char * dirs[] = { ".repodata", "repodata", NULL };
00715
00716 static const char * types[] =
00717 { "primary", "filelists", "other", "repomd", NULL };
00718 const char ** dirp, ** typep;
00719 for (dirp = dirs; *dirp != NULL; dirp++) {
00720 for (typep = types; *typep != NULL; typep++) {
00721 fn = repoGetPath(repo, *dirp, *typep, strcmp(*typep, "repomd"));
00722 if (rpmioExists(fn, st)) {
00723 if (Access(fn, W_OK)) {
00724 repo_error(0, _("Path must be writable: %s"), fn);
00725 rc = 1;
00726 } else
00727 if (repo->checkts && st->st_ctime > repo->mdtimestamp)
00728 repo->mdtimestamp = st->st_ctime;
00729 }
00730 fn = _free(fn);
00731 }
00732 }
00733 }
00734
00735 #ifdef NOTYET
00736 if (repo->groupfile != NULL) {
00737 if (repo->split || repo->groupfile[0] != '/') {
00738 fn = rpmGetPath(repo->package_dir, "/", repo->groupfile, NULL);
00739 repo->groupfile = _free(repo->groupfile);
00740 repo->groupfile = fn;
00741 fn = NULL;
00742 }
00743 if (!rpmioExists(repo->groupfile, st)) {
00744 repo_error(0, _("groupfile %s cannot be found."), repo->groupfile);
00745 rc = 1;
00746 }
00747 }
00748 #endif
00749 return rc;
00750 }
00751
00758 static int chkSuffix(const char * fn, const char * suffix)
00759
00760 {
00761 size_t flen = strlen(fn);
00762 size_t slen = strlen(suffix);
00763 return (flen > slen && !strcmp(fn + flen - slen, suffix));
00764 }
00765
00773
00774 static const char ** repoGetFileList(rpmrepo repo, const char *roots[],
00775 const char * ext)
00776
00777
00778 {
00779 const char ** pkglist = NULL;
00780 FTS * t;
00781 FTSENT * p;
00782 int xx;
00783
00784 if ((t = Fts_open((char *const *)roots, repo->ftsoptions, NULL)) == NULL)
00785 repo_error(1, _("Fts_open: %s"), strerror(errno));
00786
00787 while ((p = Fts_read(t)) != NULL) {
00788 #ifdef NOTYET
00789 const char * fts_name = p->fts_name;
00790 size_t fts_namelen = p->fts_namelen;
00791
00792
00793 if (p->fts_level == 0 && fts_namelen == 0) {
00794 fts_name = ".";
00795 fts_namelen = sizeof(".") - 1;
00796 }
00797 #endif
00798
00799
00800
00801
00802 if (mireApply(repo->excludeMire, repo->nexcludes, p->fts_name, 0, -1) >= 0)
00803 continue;
00804 if (mireApply(repo->includeMire, repo->nincludes, p->fts_name, 0, +1) < 0)
00805 continue;
00806
00807
00808 switch (p->fts_info) {
00809 case FTS_D:
00810 case FTS_DP:
00811 default:
00812 continue;
00813 break;
00814 case FTS_SL:
00815 if (repo->nofollow)
00816 continue;
00817
00818 break;
00819 case FTS_F:
00820
00821 if (chkSuffix(p->fts_name, ext))
00822 xx = argvAdd(&pkglist, p->fts_path);
00823 break;
00824 }
00825 }
00826
00827 (void) Fts_close(t);
00828
00829 if (_repo_debug)
00830 argvPrint("pkglist", pkglist, NULL);
00831
00832 return pkglist;
00833 }
00834
00840 static int repoCheckTimeStamps(rpmrepo repo)
00841
00842
00843 {
00844 int rc = 0;
00845
00846 if (repo->checkts) {
00847 const char ** pkg;
00848
00849 if (repo->pkglist != NULL)
00850 for (pkg = repo->pkglist; *pkg != NULL ; pkg++) {
00851 struct stat sb, *st = &sb;
00852 if (!rpmioExists(*pkg, st)) {
00853 repo_error(0, _("cannot get to file: %s"), *pkg);
00854 rc = 1;
00855 } else if (st->st_ctime > repo->mdtimestamp)
00856 rc = 1;
00857 }
00858 } else
00859 rc = 1;
00860
00861 return rc;
00862 }
00863
00870 static int rfileXMLWrite(rpmrfile rfile, const char * spew)
00871
00872
00873 {
00874 size_t nspew = (spew != NULL ? strlen(spew) : 0);
00875
00876 size_t nb = (nspew > 0 ? Fwrite(spew, 1, nspew, rfile->fd) : 0);
00877
00878 int rc = 0;
00879 if (nspew != nb) {
00880 repo_error(0, _("Fwrite failed: expected write %u != %u bytes: %s\n"),
00881 (unsigned)nspew, (unsigned)nb, Fstrerror(rfile->fd));
00882 rc = 1;
00883 }
00884 spew = _free(spew);
00885 return rc;
00886 }
00887
00894 static int repoFclose(rpmrepo repo, FD_t fd)
00895
00896 {
00897 int rc = 0;
00898
00899 if (fd != NULL) {
00900 if (repo->ts != NULL) {
00901 (void) rpmswAdd(rpmtsOp(repo->ts, RPMTS_OP_UNCOMPRESS),
00902 fdstat_op(fd, FDSTAT_READ));
00903 (void) rpmswAdd(rpmtsOp(repo->ts, RPMTS_OP_DIGEST),
00904 fdstat_op(fd, FDSTAT_DIGEST));
00905 }
00906 rc = Fclose(fd);
00907 }
00908 return rc;
00909 }
00910
00917 static int repoOpenMDFile(const rpmrepo repo, rpmrfile rfile)
00918
00919
00920 {
00921 const char * spew = rfile->xml_init;
00922 size_t nspew = strlen(spew);
00923 const char * fn = repoGetPath(repo, repo->tempdir, rfile->type, 1);
00924 const char * tail;
00925 size_t nb;
00926 int rc = 0;
00927
00928 rfile->fd = Fopen(fn, repo->wmode);
00929 assert(rfile->fd != NULL);
00930
00931 if (repo->algo != PGPHASHALGO_NONE)
00932 fdInitDigest(rfile->fd, repo->algo, 0);
00933
00934 if ((tail = strstr(spew, " packages=\"0\">\n")) != NULL)
00935 nspew -= strlen(tail);
00936
00937 nb = Fwrite(spew, 1, nspew, rfile->fd);
00938
00939 if (tail != NULL) {
00940 char buf[64];
00941 size_t tnb = snprintf(buf, sizeof(buf), " packages=\"%d\">\n",
00942 repo->pkgcount);
00943 nspew += tnb;
00944 nb += Fwrite(buf, 1, tnb, rfile->fd);
00945 }
00946 if (nspew != nb) {
00947 repo_error(0, _("Fwrite failed: expected write %u != %u bytes: %s\n"),
00948 (unsigned)nspew, (unsigned)nb, Fstrerror(rfile->fd));
00949 rc = 1;
00950 }
00951
00952 fn = _free(fn);
00953
00954 #if defined(WITH_SQLITE)
00955 if (repo->database) {
00956 const char ** stmt;
00957 int xx;
00958 fn = rpmGetPath(repo->outputdir, "/", repo->tempdir, "/",
00959 rfile->type, ".sqlite", NULL);
00960 if ((xx = sqlite3_open(fn, &rfile->sqldb)) != SQLITE_OK)
00961 repo_error(1, "sqlite3_open(%s): %s", fn, sqlite3_errmsg(rfile->sqldb));
00962 for (stmt = rfile->sql_init; *stmt != NULL; stmt++) {
00963 char * msg;
00964 xx = sqlite3_exec(rfile->sqldb, *stmt, NULL, NULL, &msg);
00965 if (xx != SQLITE_OK)
00966 repo_error(1, "sqlite3_exec(%s, \"%s\"): %s\n", fn, *stmt,
00967 (msg != NULL ? msg : "failed"));
00968 }
00969 fn = _free(fn);
00970 }
00971 #endif
00972
00973 return rc;
00974 }
00975
00982 static Header repoReadHeader(rpmrepo repo, const char * path)
00983
00984
00985 {
00986
00987 FD_t fd = Fopen(path, "r.ufdio");
00988 Header h = NULL;
00989
00990 if (fd != NULL) {
00991 uint32_t algo = repo->pkgalgo;
00992 rpmRC rpmrc;
00993
00994 if (algo != PGPHASHALGO_NONE)
00995 fdInitDigest(fd, algo, 0);
00996
00997
00998 rpmrc = rpmReadPackageFile(repo->ts, fd, path, &h);
00999 if (algo != PGPHASHALGO_NONE) {
01000 char buffer[32 * BUFSIZ];
01001 size_t nb = sizeof(buffer);
01002 size_t nr;
01003 while ((nr = Fread(buffer, sizeof(buffer[0]), nb, fd)) == nb)
01004 {};
01005 if (Ferror(fd)) {
01006 fprintf(stderr, _("%s: Fread(%s) failed: %s\n"),
01007 __progname, path, Fstrerror(fd));
01008 rpmrc = RPMRC_FAIL;
01009 } else {
01010 static int asAscii = 1;
01011 const char *digest = NULL;
01012 fdFiniDigest(fd, algo, &digest, NULL, asAscii);
01013 (void) headerSetDigest(h, digest);
01014 digest = _free(digest);
01015 }
01016 }
01017
01018 (void) Fclose(fd);
01019
01020 switch (rpmrc) {
01021 case RPMRC_NOTFOUND:
01022 case RPMRC_FAIL:
01023 default:
01024 (void)headerFree(h);
01025 h = NULL;
01026 break;
01027 case RPMRC_NOTTRUSTED:
01028 case RPMRC_NOKEY:
01029 case RPMRC_OK:
01030 if (repo->baseurl)
01031 (void) headerSetBaseURL(h, repo->baseurl);
01032 break;
01033 }
01034 }
01035 return h;
01036 }
01037
01044 static const char * rfileHeaderSprintf(Header h, const char * qfmt)
01045
01046
01047 {
01048 const char * msg = NULL;
01049 const char * s = headerSprintf(h, qfmt, NULL, NULL, &msg);
01050 if (s == NULL)
01051 repo_error(1, _("headerSprintf(%s): %s"), qfmt, msg);
01052 assert(s != NULL);
01053 return s;
01054 }
01055
01056 #if defined(WITH_SQLITE)
01057
01062 static int rfileSQL(rpmrfile rfile, const char * msg, int rc)
01063
01064
01065 {
01066 if (rc != SQLITE_OK || _repo_debug)
01067 repo_error(0, "sqlite3_%s(%s): %s", msg, rfile->type,
01068 sqlite3_errmsg(rfile->sqldb));
01069 return rc;
01070 }
01071
01077 static int rfileSQLStep(rpmrfile rfile, sqlite3_stmt * stmt)
01078
01079
01080 {
01081 int loop = 1;
01082 int rc = 0;
01083 int xx;
01084
01085
01086 while (loop) {
01087 rc = sqlite3_step(stmt);
01088 switch (rc) {
01089 default:
01090 rc = rfileSQL(rfile, "step", rc);
01091
01092 case SQLITE_DONE:
01093 loop = 0;
01094 break;
01095 }
01096 }
01097
01098
01099 xx = rfileSQL(rfile, "reset",
01100 sqlite3_reset(stmt));
01101
01102 return rc;
01103 }
01104
01111 static const char * rfileHeaderSprintfHack(Header h, const char * qfmt)
01112
01113
01114 {
01115 static const char mark[] = "'XXX'";
01116 static size_t nmark = sizeof("'XXX'") - 1;
01117 const char * msg = NULL;
01118 char * s = (char *) headerSprintf(h, qfmt, NULL, NULL, &msg);
01119 char * f, * fe;
01120 int nsubs = 0;
01121
01122 if (s == NULL)
01123 repo_error(1, _("headerSprintf(%s): %s"), qfmt, msg);
01124 assert(s != NULL);
01125
01126
01127
01128 for (f = s; *f != '\0' && (fe = strstr(f, "'XXX'")) != NULL; fe += nmark, f = fe)
01129 nsubs++;
01130
01131
01132 if (nsubs > 0) {
01133 char instance[64];
01134 int xx = snprintf(instance, sizeof(instance), "'%u'",
01135 (unsigned) headerGetInstance(h));
01136 size_t tlen = strlen(s) + nsubs * ((int)strlen(instance) - (int)nmark);
01137 char * t = xmalloc(tlen + 1);
01138 char * te = t;
01139
01140 xx = xx;
01141
01142 for (f = s; *f != '\0' && (fe = strstr(f, mark)) != NULL; fe += nmark, f = fe) {
01143 *fe = '\0';
01144 te = stpcpy( stpcpy(te, f), instance);
01145 }
01146
01147 if (*f != '\0')
01148 te = stpcpy(te, f);
01149 s = _free(s);
01150 s = t;
01151 }
01152
01153 return s;
01154 }
01155
01162 static int rfileSQLWrite(rpmrfile rfile, const char * cmd)
01163
01164
01165 {
01166 sqlite3_stmt * stmt;
01167 const char * tail;
01168 int xx;
01169
01170 xx = rfileSQL(rfile, "prepare",
01171 sqlite3_prepare(rfile->sqldb, cmd, (int)strlen(cmd), &stmt, &tail));
01172
01173 xx = rfileSQL(rfile, "reset",
01174 sqlite3_reset(stmt));
01175
01176 xx = rfileSQLStep(rfile, stmt);
01177
01178 xx = rfileSQL(rfile, "finalize",
01179 sqlite3_finalize(stmt));
01180
01181 cmd = _free(cmd);
01182
01183 return 0;
01184 }
01185 #endif
01186
01194 static int repoWriteMDFile(rpmrepo repo, rpmrfile rfile, Header h)
01195
01196
01197 {
01198 int rc = 0;
01199
01200 if (rfile->xml_qfmt != NULL) {
01201 if (rfileXMLWrite(rfile, rfileHeaderSprintf(h, rfile->xml_qfmt)))
01202 rc = 1;
01203 }
01204
01205 #if defined(WITH_SQLITE)
01206 if (repo->database) {
01207 if (rfileSQLWrite(rfile, rfileHeaderSprintfHack(h, rfile->sql_qfmt)))
01208 rc = 1;
01209 }
01210 #endif
01211
01212 return rc;
01213 }
01214
01221 static int repoWriteMetadataDocs(rpmrepo repo, const char ** pkglist)
01222
01223
01224 {
01225 const char * pkg;
01226 int rc = 0;
01227
01228 while ((pkg = *pkglist++) != NULL) {
01229 Header h = repoReadHeader(repo, pkg);
01230
01231 repo->current++;
01232 if (h == NULL) {
01233 #ifdef DYING
01234 repo_error(0, _("\nError %s: %s\n"), pkg, strerror(errno));
01235 continue;
01236 #else
01237 rc = 1;
01238 break;
01239 #endif
01240 }
01241 (void) headerSetInstance(h, (uint32_t)repo->current);
01242
01243 #ifdef NOTYET
01244
01245 reldir = (pkgpath != NULL ? pkgpath : rpmGetPath(repo->basedir, "/", repo->directories[0], NULL));
01246 self.primaryfile.write(po.do_primary_xml_dump(reldir, baseurl=repo->baseurl))
01247 self.flfile.write(po.do_filelists_xml_dump())
01248 self.otherfile.write(po.do_other_xml_dump())
01249 #endif
01250 if (repoWriteMDFile(repo, &repo->primary, h)
01251 || repoWriteMDFile(repo, &repo->filelists, h)
01252 || repoWriteMDFile(repo, &repo->other, h))
01253 rc = 1;
01254
01255 (void)headerFree(h);
01256 h = NULL;
01257 if (rc) break;
01258
01259 if (!repo->quiet) {
01260 if (repo->verbose)
01261 repo_error(0, "%d/%d - %s", repo->current, repo->pkgcount, pkg);
01262 else
01263 repoProgress(repo, pkg, repo->current, repo->pkgcount);
01264 }
01265 }
01266 return rc;
01267 }
01268
01273 static int repoRfileDigest(const rpmrepo repo, rpmrfile rfile,
01274 const char ** digestp)
01275
01276 {
01277 static int asAscii = 1;
01278 struct stat sb, *st = &sb;
01279 const char * fn = repoGetPath(repo, repo->tempdir, rfile->type, 1);
01280 const char * path = NULL;
01281 int ut = urlPath(fn, &path);
01282 FD_t fd = NULL;
01283 int rc = 1;
01284 int xx;
01285
01286 memset(st, 0, sizeof(*st));
01287 if (!rpmioExists(fn, st))
01288 goto exit;
01289 fd = Fopen(fn, "r.ufdio");
01290 if (fd == NULL || Ferror(fd))
01291 goto exit;
01292
01293 switch (ut) {
01294 case URL_IS_PATH:
01295 case URL_IS_UNKNOWN:
01296 #if defined(HAVE_MMAP)
01297 { void * mapped = (void *)-1;
01298
01299 if (st->st_size > 0)
01300 mapped = mmap(NULL, st->st_size, PROT_READ, MAP_SHARED, Fileno(fd), 0);
01301 if (mapped != (void *)-1) {
01302 rpmop op = rpmtsOp(repo->ts, RPMTS_OP_DIGEST);
01303 rpmtime_t tstamp = rpmswEnter(op, 0);
01304 DIGEST_CTX ctx = rpmDigestInit(repo->algo, RPMDIGEST_NONE);
01305 xx = rpmDigestUpdate(ctx, mapped, st->st_size);
01306 xx = rpmDigestFinal(ctx, digestp, NULL, asAscii);
01307 tstamp = rpmswExit(op, st->st_size);
01308 xx = munmap(mapped, st->st_size);
01309 break;
01310 }
01311 }
01312 #endif
01313 default:
01314 { char buf[64 * BUFSIZ];
01315 size_t nb;
01316 size_t fsize = 0;
01317
01318 fdInitDigest(fd, repo->algo, 0);
01319 while ((nb = Fread(buf, sizeof(buf[0]), sizeof(buf), fd)) > 0)
01320 fsize += nb;
01321 if (Ferror(fd))
01322 goto exit;
01323 fdFiniDigest(fd, repo->algo, digestp, NULL, asAscii);
01324 } break;
01325 }
01326
01327 rc = 0;
01328
01329 exit:
01330 if (fd)
01331 xx = repoFclose(repo, fd);
01332 fn = _free(fn);
01333 return rc;
01334 }
01335
01342 static int repoCloseMDFile(const rpmrepo repo, rpmrfile rfile)
01343
01344
01345 {
01346 static int asAscii = 1;
01347 char * xmlfn = xstrdup(fdGetOPath(rfile->fd));
01348 int rc = 0;
01349
01350 if (!repo->quiet)
01351 repo_error(0, _("Saving %s metadata"), basename(xmlfn));
01352
01353 if (rfileXMLWrite(rfile, xstrdup(rfile->xml_fini)))
01354 rc = 1;
01355
01356 if (repo->algo > 0)
01357 fdFiniDigest(rfile->fd, repo->algo, &rfile->digest, NULL, asAscii);
01358 else
01359 rfile->digest = xstrdup("");
01360
01361 (void) repoFclose(repo, rfile->fd);
01362 rfile->fd = NULL;
01363
01364
01365 rfile->Zdigest = NULL;
01366 (void) repoRfileDigest(repo, rfile, &rfile->Zdigest);
01367
01368 #if defined(WITH_SQLITE)
01369 if (repo->database && rfile->sqldb != NULL) {
01370 const char *dbfn = rpmGetPath(repo->outputdir, "/", repo->tempdir, "/",
01371 rfile->type, ".sqlite", NULL);
01372 int xx;
01373 if ((xx = sqlite3_close(rfile->sqldb)) != SQLITE_OK)
01374 repo_error(1, "sqlite3_close(%s): %s", dbfn, sqlite3_errmsg(rfile->sqldb));
01375 rfile->sqldb = NULL;
01376 dbfn = _free(dbfn);
01377 }
01378 #endif
01379
01380 rfile->ctime = rpmioCtime(xmlfn);
01381 xmlfn = _free(xmlfn);
01382
01383 return rc;
01384 }
01385
01391 static int repoDoPkgMetadata(rpmrepo repo)
01392
01393
01394 {
01395 int rc = 0;
01396
01397 repo->current = 0;
01398
01399 #ifdef NOTYET
01400 def _getFragmentUrl(self, url, fragment):
01401 import urlparse
01402 urlparse.uses_fragment.append('media')
01403 if not url:
01404 return url
01405 (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url)
01406 return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment)))
01407
01408 def doPkgMetadata(self):
01409 """all the heavy lifting for the package metadata"""
01410 if (argvCount(repo->directories) == 1) {
01411 MetaDataGenerator.doPkgMetadata(self)
01412 return
01413 }
01414
01415 ARGV_t roots = NULL;
01416 filematrix = {}
01417 for mydir in repo->directories {
01418 if (mydir[0] == '/')
01419 thisdir = xstrdup(mydir);
01420 else if (mydir[0] == '.' && mydir[1] == '.' && mydir[2] == '/')
01421 thisdir = Realpath(mydir, NULL);
01422 else
01423 thisdir = rpmGetPath(repo->basedir, "/", mydir, NULL);
01424
01425 xx = argvAdd(&roots, thisdir);
01426 thisdir = _free(thisdir);
01427
01428 filematrix[mydir] = repoGetFileList(repo, roots, '.rpm')
01429 self.trimRpms(filematrix[mydir])
01430 repo->pkgcount = argvCount(filematrix[mydir]);
01431 roots = argvFree(roots);
01432 }
01433
01434 mediano = 1;
01435 repo->baseurl = self._getFragmentUrl(repo->baseurl, mediano)
01436 #endif
01437
01438 if (repoOpenMDFile(repo, &repo->primary)
01439 || repoOpenMDFile(repo, &repo->filelists)
01440 || repoOpenMDFile(repo, &repo->other))
01441 rc = 1;
01442 if (rc) return rc;
01443
01444 #ifdef NOTYET
01445 for mydir in repo->directories {
01446 repo->baseurl = self._getFragmentUrl(repo->baseurl, mediano)
01447
01448 if (repoWriteMetadataDocs(repo, filematrix[mydir]))
01449 rc = 1;
01450 mediano++;
01451 }
01452 repo->baseurl = self._getFragmentUrl(repo->baseurl, 1)
01453 #else
01454 if (repoWriteMetadataDocs(repo, repo->pkglist))
01455 rc = 1;
01456 #endif
01457
01458 if (!repo->quiet)
01459 fprintf(stderr, "\n");
01460 if (repoCloseMDFile(repo, &repo->primary)
01461 || repoCloseMDFile(repo, &repo->filelists)
01462 || repoCloseMDFile(repo, &repo->other))
01463 rc = 1;
01464
01465 return rc;
01466 }
01467
01470 static const char *
01471 algo2tagname(uint32_t algo)
01472
01473 {
01474 const char * tagname = NULL;
01475
01476 switch (algo) {
01477 case PGPHASHALGO_NONE: tagname = "none"; break;
01478 case PGPHASHALGO_MD5: tagname = "md5"; break;
01479
01480 case PGPHASHALGO_SHA1: tagname = "sha"; break;
01481 case PGPHASHALGO_RIPEMD160: tagname = "rmd160"; break;
01482 case PGPHASHALGO_MD2: tagname = "md2"; break;
01483 case PGPHASHALGO_TIGER192: tagname = "tiger192"; break;
01484 case PGPHASHALGO_HAVAL_5_160: tagname = "haval160"; break;
01485 case PGPHASHALGO_SHA256: tagname = "sha256"; break;
01486 case PGPHASHALGO_SHA384: tagname = "sha384"; break;
01487 case PGPHASHALGO_SHA512: tagname = "sha512"; break;
01488 case PGPHASHALGO_MD4: tagname = "md4"; break;
01489 case PGPHASHALGO_RIPEMD128: tagname = "rmd128"; break;
01490 case PGPHASHALGO_CRC32: tagname = "crc32"; break;
01491 case PGPHASHALGO_ADLER32: tagname = "adler32"; break;
01492 case PGPHASHALGO_CRC64: tagname = "crc64"; break;
01493 case PGPHASHALGO_JLU32: tagname = "jlu32"; break;
01494 case PGPHASHALGO_SHA224: tagname = "sha224"; break;
01495 case PGPHASHALGO_RIPEMD256: tagname = "rmd256"; break;
01496 case PGPHASHALGO_RIPEMD320: tagname = "rmd320"; break;
01497 case PGPHASHALGO_SALSA10: tagname = "salsa10"; break;
01498 case PGPHASHALGO_SALSA20: tagname = "salsa20"; break;
01499 default: tagname = NULL; break;
01500 }
01501 return tagname;
01502 }
01503
01509 static const char * repoMDExpand(rpmrepo repo, rpmrfile rfile)
01510
01511
01512 {
01513 const char * spewalgo = algo2tagname(repo->algo);
01514 char spewtime[64];
01515
01516 (void) snprintf(spewtime, sizeof(spewtime), "%u", (unsigned)rfile->ctime);
01517 return rpmExpand("\
01518 <data type=\"", rfile->type, "\">\n\
01519 <checksum type=\"", spewalgo, "\">", rfile->Zdigest, "</checksum>\n\
01520 <timestamp>", spewtime, "</timestamp>\n\
01521 <open-checksum type=\"",spewalgo,"\">", rfile->digest, "</open-checksum>\n\
01522 <location href=\"", repo->finaldir, "/", rfile->type, (repo->markup != NULL ? repo->markup : ""), (repo->suffix != NULL ? repo->suffix : ""), "\"/>\n\
01523 </data>\n", NULL);
01524 }
01525
01531 static int repoDoRepoMetadata(rpmrepo repo)
01532
01533
01534 {
01535 rpmrfile rfile = &repo->repomd;
01536 const char * fn = repoGetPath(repo, repo->tempdir, rfile->type, 0);
01537 int rc = 0;
01538
01539 if ((rfile->fd = Fopen(fn, "w.ufdio")) != NULL) {
01540 if (rfileXMLWrite(rfile, xstrdup(rfile->xml_init))
01541 || rfileXMLWrite(rfile, repoMDExpand(repo, &repo->other))
01542 || rfileXMLWrite(rfile, repoMDExpand(repo, &repo->filelists))
01543 || rfileXMLWrite(rfile, repoMDExpand(repo, &repo->primary))
01544 || rfileXMLWrite(rfile, xstrdup(rfile->xml_fini)))
01545 rc = 1;
01546 (void) repoFclose(repo, rfile->fd);
01547 rfile->fd = NULL;
01548 }
01549
01550 fn = _free(fn);
01551 if (rc) return rc;
01552
01553 #ifdef NOTYET
01554 def doRepoMetadata(self):
01555 """wrapper to generate the repomd.xml file that stores the info on the other files"""
01556 const char * repopath =
01557 rpmGetPath(repo->outputdir, "/", repo->tempdir, NULL);
01558 repodoc = libxml2.newDoc("1.0")
01559 reporoot = repodoc.newChild(None, "repomd", None)
01560 repons = reporoot.newNs("http://linux.duke.edu/metadata/repo", None)
01561 reporoot.setNs(repons)
01562 repopath = rpmGetPath(repo->outputdir, "/", repo->tempdir, NULL);
01563 fn = repoGetPath(repo, repo->tempdir, repo->repomd.type, 1);
01564
01565 repoid = "garbageid";
01566
01567 if (repo->database) {
01568 if (!repo->quiet) repo_error(0, _("Generating sqlite DBs"));
01569 try:
01570 dbversion = str(sqlitecachec.DBVERSION)
01571 except AttributeError:
01572 dbversion = "9"
01573 rp = sqlitecachec.RepodataParserSqlite(repopath, repoid, None)
01574 }
01575
01576 { static const char * types[] =
01577 { "primary", "filelists", "other", NULL };
01578 const char ** typep;
01579 for (typep = types; *typep != NULL; typep++) {
01580 complete_path = repoGetPath(repo, repo->tempdir, *typep, 1);
01581
01582 zfo = _gzipOpen(complete_path)
01583 uncsum = misc.checksum(algo2tagname(repo->algo), zfo)
01584 zfo.close()
01585 csum = misc.checksum(algo2tagname(repo->algo), complete_path)
01586 (void) rpmioExists(complete_path, st)
01587 timestamp = os.stat(complete_path)[8]
01588
01589 db_csums = {}
01590 db_compressed_sums = {}
01591
01592 if (repo->database) {
01593 if (repo->verbose) {
01594 time_t now = time(NULL);
01595 repo_error(0, _("Starting %s db creation: %s"),
01596 *typep, ctime(&now));
01597 }
01598
01599 if (!strcmp(*typep, "primary"))
01600 rp.getPrimary(complete_path, csum)
01601 else if (!strcmp(*typep, "filelists"));
01602 rp.getFilelists(complete_path, csum)
01603 else if (!strcmp(*typep, "other"))
01604 rp.getOtherdata(complete_path, csum)
01605
01606 { const char * tmp_result_path =
01607 rpmGetPath(repo->outputdir, "/", repo->tempdir, "/",
01608 *typep, ".xml.gz.sqlite", NULL);
01609 const char * resultpath =
01610 rpmGetPath(repo->outputdir, "/", repo->tempdir, "/",
01611 *typep, ".sqlite", NULL);
01612
01613
01614 xx = Rename(tmp_result_path, resultpath);
01615 tmp_result_path = _free(tmp_result_path);
01616 result_compressed =
01617 rpmGetPath(repo->outputdir, "/", repo->tempdir, "/",
01618 *typep, ".sqlite.bz2", NULL);
01619 db_csums[*typep] = misc.checksum(algo2tagname(repo->algo), resultpath)
01620
01621
01622 bzipFile(resultpath, result_compressed)
01623
01624 db_compressed_sums[*typep] = misc.checksum(algo2tagname(repo->algo), result_compressed)
01625
01626 xx = Unlink(resultpath);
01627 resultpath = _free(resultpath);
01628 }
01629
01630 if (repo->uniquemdfilenames) {
01631 const char * csum_result_compressed =
01632 rpmGetPath(repo->outputdir, "/", repo->tempdir, "/",
01633 db_compressed_sums[*typep], "-", *typep, ".sqlite.bz2", NULL);
01634 xx = Rename(result_compressed, csum_result_compressed);
01635 result_compressed = _free(result_compressed);
01636 result_compressed = csum_result_compressed;
01637 }
01638
01639
01640 (void) rpmioExists(result_compressed, st)
01641 db_timestamp = os.stat(result_compressed)[8]
01642
01643
01644 db_data_type = rpmExpand(*typep, "_db", NULL);
01645 data = reporoot.newChild(None, 'data', None)
01646 data.newProp('type', db_data_type)
01647 location = data.newChild(None, 'location', None)
01648 if (repo->baseurl != NULL) {
01649 location.newProp('xml:base', repo->baseurl)
01650 }
01651
01652 location.newProp('href', rpmGetPath(repo->finaldir, "/", *typep, ".sqlite.bz2", NULL));
01653 checksum = data.newChild(None, 'checksum', db_compressed_sums[*typep])
01654 checksum.newProp('type', algo2tagname(repo->algo))
01655 db_tstamp = data.newChild(None, 'timestamp', str(db_timestamp))
01656 unchecksum = data.newChild(None, 'open-checksum', db_csums[*typep])
01657 unchecksum.newProp('type', algo2tagname(repo->algo))
01658 database_version = data.newChild(None, 'database_version', dbversion)
01659 if (repo->verbose) {
01660 time_t now = time(NULL);
01661 repo_error(0, _("Ending %s db creation: %s"),
01662 *typep, ctime(&now));
01663 }
01664 }
01665
01666 data = reporoot.newChild(None, 'data', None)
01667 data.newProp('type', *typep)
01668
01669 checksum = data.newChild(None, 'checksum', csum)
01670 checksum.newProp('type', algo2tagname(repo->algo))
01671 timestamp = data.newChild(None, 'timestamp', str(timestamp))
01672 unchecksum = data.newChild(None, 'open-checksum', uncsum)
01673 unchecksum.newProp('type', algo2tagname(repo->algo))
01674 location = data.newChild(None, 'location', None)
01675 if (repo->baseurl != NULL)
01676 location.newProp('xml:base', repo->baseurl)
01677 if (repo->uniquemdfilenames) {
01678 orig_file = repoGetPath(repo, repo->tempdir, *typep, strcmp(*typep, "repomd"));
01679 res_file = rpmExpand(csum, "-", *typep,
01680 (repo->markup ? repo->markup : ""),
01681 (repo->suffix && strcmp(*typep, "repomd") ? repo->suffix : ""), NULL);
01682 dest_file = rpmGetPath(repo->outputdir, "/", repo->tempdir, "/", res_file, NULL);
01683 xx = Rename(orig_file, dest_file);
01684
01685 } else
01686 res_file = rpmExpand(*typep,
01687 (repo->markup ? repo->markup : ""),
01688 (repo->suffix && strcmp(*typep, "repomd") ? repo->suffix : ""), NULL);
01689
01690 location.newProp('href', rpmGetPath(repo->finaldir, "/", res_file, NULL));
01691 }
01692 }
01693
01694 if (!repo->quiet && repo->database)
01695 repo_error(0, _("Sqlite DBs complete"));
01696
01697 if (repo->groupfile != NULL) {
01698 self.addArbitraryMetadata(repo->groupfile, 'group_gz', reporoot)
01699 self.addArbitraryMetadata(repo->groupfile, 'group', reporoot, compress=False)
01700 }
01701
01702
01703 try:
01704 repodoc.saveFormatFileEnc(fn, 'UTF-8', 1)
01705 except:
01706 repo_error(0, _("Error saving temp file for %s%s%s: %s"),
01707 rfile->type,
01708 (repo->markup ? repo->markup : ""),
01709 (repo->suffix && strcmp(*typep, "repomd") ? repo->suffix : ""),
01710 fn);
01711 repo_error(1, _("Could not save temp file: %s"), fn);
01712
01713 del repodoc
01714 #endif
01715
01716 return rc;
01717 }
01718
01724 static int repoDoFinalMove(rpmrepo repo)
01725
01726
01727 {
01728 const char * output_final_dir =
01729 rpmGetPath(repo->outputdir, "/", repo->finaldir, NULL);
01730 const char * output_old_dir =
01731 rpmGetPath(repo->outputdir, "/", repo->olddir, NULL);
01732 const char * oldfile;
01733 struct stat sb, *st = &sb;
01734 int xx;
01735
01736 if (rpmioExists(output_final_dir, st)) {
01737 if ((xx = Rename(output_final_dir, output_old_dir)) != 0)
01738 repo_error(1, _("Error moving final %s to old dir %s"),
01739 output_final_dir, output_old_dir);
01740 }
01741
01742 { const char * output_temp_dir =
01743 rpmGetPath(repo->outputdir, "/", repo->tempdir, NULL);
01744 if ((xx = Rename(output_temp_dir, output_final_dir)) != 0) {
01745 xx = Rename(output_old_dir, output_final_dir);
01746 repo_error(1, _("Error moving final metadata into place"));
01747 }
01748 output_temp_dir = _free(output_temp_dir);
01749 }
01750
01751 {
01752 static const char * types[] =
01753 { "primary", "filelists", "other", "repomd", "group", NULL };
01754 const char ** typep;
01755
01756 for (typep = types; *typep != NULL; typep++) {
01757 oldfile = rpmGetPath(output_old_dir, "/", *typep,
01758 (repo->markup != NULL ? repo->markup : ""),
01759 (repo->suffix != NULL && strcmp(*typep, "repomd")
01760 ? repo->suffix : ""), NULL);
01761 if (rpmioExists(oldfile, st)) {
01762 if (Unlink(oldfile))
01763 repo_error(1, _("Could not remove old metadata file: %s: %s"),
01764 oldfile, strerror(errno));
01765 }
01766 oldfile = _free(oldfile);
01767 }
01768 }
01769
01770 { DIR * dir = Opendir(output_old_dir);
01771 struct dirent * dp;
01772
01773 if (dir != NULL) {
01774 while ((dp = Readdir(dir)) != NULL) {
01775 const char * finalfile;
01776
01777 if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, ".."))
01778 continue;
01779
01780 finalfile = rpmGetPath(output_final_dir, "/", dp->d_name, NULL);
01781 oldfile = rpmGetPath(output_old_dir, "/", dp->d_name, NULL);
01782
01783 if (!strcmp(dp->d_name, "filelists.sqlite.bz2")
01784 || !strcmp(dp->d_name, "other.sqlite.bz2")
01785 || !strcmp(dp->d_name, "primary.sqlite.bz2"))
01786 {
01787 xx = Unlink(oldfile);
01788 oldfile = _free(oldfile);
01789 continue;
01790 }
01791
01792 if (rpmioExists(finalfile, st)) {
01793 if (!S_ISDIR(st->st_mode)) {
01794 if ((xx = Unlink(oldfile)) != 0)
01795 repo_error(1, _("Could not remove old metadata file: %s: %s"),
01796 oldfile, strerror(errno));
01797 }
01798 #ifdef NOTYET
01799 else {
01800 shutil.rmtree(oldfile)
01801 }
01802 #endif
01803 } else {
01804 if ((xx = Rename(oldfile, finalfile)) != 0) {
01805 repo_error(1, _("Could not restore old non-metadata file: %s -> %s: %s"),
01806 oldfile, finalfile, strerror(errno));
01807 }
01808 }
01809 oldfile = _free(oldfile);
01810 finalfile = _free(finalfile);
01811 }
01812 xx = Closedir(dir);
01813 }
01814 }
01815
01816 if ((xx = Rmdir(output_old_dir)) != 0) {
01817 repo_error(1, _("Could not remove old metadata dir: %s: %s"),
01818 repo->olddir, strerror(errno));
01819 }
01820 output_old_dir = _free(output_old_dir);
01821 output_final_dir = _free(output_final_dir);
01822
01823 return 0;
01824 }
01825
01826
01827
01830 static void repoArgCallback(poptContext con,
01831 enum poptCallbackReason reason,
01832 const struct poptOption * opt, const char * arg,
01833 void * data)
01834
01835
01836 {
01837 rpmrepo repo = _rpmrepo;
01838
01839
01840 if (opt->arg == NULL)
01841 switch (opt->val) {
01842
01843 case 'v':
01844 repo->verbose++;
01845 break;
01846 case '?':
01847 default:
01848 fprintf(stderr, _("%s: Unknown option -%c\n"), __progname, opt->val);
01849 poptPrintUsage(con, stderr, 0);
01850 exit(EXIT_FAILURE);
01851 break;
01852 }
01853 }
01854
01855
01856 static int compression = -1;
01857
01858
01859 static struct poptOption repoCompressionPoptTable[] = {
01860 { "uncompressed", '\0', POPT_ARG_VAL, &compression, 0,
01861 N_("don't compress"), NULL },
01862 { "gzip", 'Z', POPT_ARG_VAL, &compression, 1,
01863 N_("use gzip compression"), NULL },
01864 { "bzip2", '\0', POPT_ARG_VAL, &compression, 2,
01865 N_("use bzip2 compression"), NULL },
01866 { "lzma", '\0', POPT_ARG_VAL, &compression, 3,
01867 N_("use lzma compression"), NULL },
01868 { "xz", '\0', POPT_ARG_VAL, &compression, 4,
01869 N_("use xz compression"), NULL },
01870 POPT_TABLEEND
01871 };
01872
01873
01874 static struct poptOption optionsTable[] = {
01875
01876 { NULL, '\0', POPT_ARG_CALLBACK | POPT_CBFLAG_INC_DATA | POPT_CBFLAG_CONTINUE,
01877 repoArgCallback, 0, NULL, NULL },
01878
01879
01880 { "repodebug", '\0', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &_repo_debug, -1,
01881 N_("debug repo handling"), NULL },
01882
01883 { "quiet", 'q', POPT_ARG_VAL, &__rpmrepo.quiet, 0,
01884 N_("output nothing except for serious errors"), NULL },
01885 { "verbose", 'v', 0, NULL, (int)'v',
01886 N_("output more debugging info."), NULL },
01887 { "dryrun", '\0', POPT_ARG_VAL, &__rpmrepo.dryrun, 1,
01888 N_("sanity check arguments, don't create metadata"), NULL },
01889 { "excludes", 'x', POPT_ARG_ARGV, &__rpmrepo.exclude_patterns, 0,
01890 N_("glob PATTERN(s) to exclude"), N_("PATTERN") },
01891 { "includes", 'i', POPT_ARG_ARGV, &__rpmrepo.include_patterns, 0,
01892 N_("glob PATTERN(s) to include"), N_("PATTERN") },
01893 { "basedir", '\0', POPT_ARG_STRING|POPT_ARGFLAG_DOC_HIDDEN, &__rpmrepo.basedir, 0,
01894 N_("top level directory"), N_("DIR") },
01895 { "baseurl", 'u', POPT_ARG_STRING|POPT_ARGFLAG_DOC_HIDDEN, &__rpmrepo.baseurl, 0,
01896 N_("baseurl to append on all files"), N_("BASEURL") },
01897 #ifdef NOTYET
01898 { "groupfile", 'g', POPT_ARG_STRING|POPT_ARGFLAG_DOC_HIDDEN, &__rpmrepo.groupfile, 0,
01899 N_("path to groupfile to include in metadata"), N_("FILE") },
01900 #endif
01901 { "pretty", 'p', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__rpmrepo.pretty, 1,
01902 N_("make sure all xml generated is formatted"), NULL },
01903 { "checkts", 'C', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__rpmrepo.checkts, 1,
01904 N_("check timestamps on files vs the metadata to see if we need to update"), NULL },
01905 #if defined(WITH_SQLITE)
01906 { "database", 'd', POPT_ARG_VAL, &__rpmrepo.database, 1,
01907 N_("create sqlite3 database files"), NULL },
01908 #endif
01909 { "split", '\0', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__rpmrepo.split, 1,
01910 N_("generate split media"), NULL },
01911 { "pkglist", 'l', POPT_ARG_ARGV|POPT_ARGFLAG_DOC_HIDDEN, &__rpmrepo.manifests, 0,
01912 N_("use only the files listed in this file from the directory specified"), N_("FILE") },
01913 { "outputdir", 'o', POPT_ARG_STRING, &__rpmrepo.outputdir, 0,
01914 N_("<dir> = optional directory to output to"), N_("DIR") },
01915 { "skip-symlinks", 'S', POPT_ARG_VAL, &__rpmrepo.nofollow, 1,
01916 N_("ignore symlinks of packages"), NULL },
01917 { "unique-md-filenames", '\0', POPT_ARG_VAL|POPT_ARGFLAG_DOC_HIDDEN, &__rpmrepo.uniquemdfilenames, 1,
01918 N_("include the file's checksum in the filename, helps with proxies"), NULL },
01919
01920 { NULL, '\0', POPT_ARG_INCLUDE_TABLE, rpmioFtsPoptTable, 0,
01921 N_("Fts(3) traversal options:"), NULL },
01922
01923 #ifdef NOTYET
01924 { "version", '\0', 0, NULL, POPT_SHOWVERSION,
01925 N_("print the version"), NULL },
01926 #endif
01927
01928 { NULL, '\0', POPT_ARG_INCLUDE_TABLE, repoCompressionPoptTable, 0,
01929 N_("Available compressions:"), NULL },
01930
01931 { NULL, '\0', POPT_ARG_INCLUDE_TABLE, rpmioDigestPoptTable, 0,
01932 N_("Available digests:"), NULL },
01933
01934 { NULL, '\0', POPT_ARG_INCLUDE_TABLE, rpmioAllPoptTable, 0,
01935 N_("Common options for all rpmio executables:"),
01936 NULL },
01937
01938 POPT_AUTOALIAS
01939 POPT_AUTOHELP
01940 POPT_TABLEEND
01941
01942 };
01943
01944 int
01945 main(int argc, char *argv[])
01946
01947
01948
01949
01950 {
01951 rpmrepo repo = _rpmrepo;
01952 poptContext optCon;
01953 const char ** av = NULL;
01954 int ndirs = 0;
01955 int nfiles = 0;
01956 int rc = 1;
01957 int xx;
01958 int i;
01959
01960 #if !defined(__LCLINT__)
01961 __progname = "rpmrepo";
01962 #endif
01963
01964
01965 optCon = rpmioInit(argc, argv, optionsTable);
01966
01967 repo->ftsoptions = (rpmioFtsOpts ? rpmioFtsOpts : FTS_PHYSICAL);
01968 switch (repo->ftsoptions & (FTS_LOGICAL|FTS_PHYSICAL)) {
01969 case (FTS_LOGICAL|FTS_PHYSICAL):
01970 repo_error(1, "FTS_LOGICAL and FTS_PYSICAL are mutually exclusive");
01971 break;
01972 case 0:
01973 repo->ftsoptions |= FTS_PHYSICAL;
01974 break;
01975 }
01976
01977 repo->algo = (rpmioDigestHashAlgo >= 0 ? (rpmioDigestHashAlgo & 0xff) : PGPHASHALGO_SHA1);
01978
01979 repo->compression = (compression >= 0 ? compression : 1);
01980 switch (repo->compression) {
01981 case 0:
01982 repo->suffix = NULL;
01983 repo->wmode = "w.ufdio";
01984 break;
01985 default:
01986
01987 case 1:
01988 repo->suffix = ".gz";
01989 repo->wmode = "w9.gzdio";
01990 break;
01991 case 2:
01992 repo->suffix = ".bz2";
01993 repo->wmode = "w9.bzdio";
01994 break;
01995 case 3:
01996 repo->suffix = ".lzma";
01997 repo->wmode = "w.lzdio";
01998 break;
01999 case 4:
02000 repo->suffix = ".xz";
02001 repo->wmode = "w.xzdio";
02002 break;
02003 }
02004
02005 av = poptGetArgs(optCon);
02006 if (av == NULL || av[0] == NULL) {
02007 repo_error(0, _("Must specify path(s) to index."));
02008 poptPrintUsage(optCon, stderr, 0);
02009 goto exit;
02010 }
02011
02012 if (av != NULL)
02013 for (i = 0; av[i] != NULL; i++) {
02014 char fullpath[MAXPATHLEN];
02015 struct stat sb;
02016 const char * rpath;
02017 const char * lpath = NULL;
02018 int ut = urlPath(av[i], &lpath);
02019 size_t nb = (size_t)(lpath - av[i]);
02020 int isdir = (lpath[strlen(lpath)-1] == '/');
02021
02022
02023 if (lpath[0] != '/') {
02024 if ((rpath = repoRealpath(lpath)) == NULL)
02025 repo_error(1, _("Realpath(%s): %s"), lpath, strerror(errno));
02026 lpath = rpmGetPath(rpath, NULL);
02027 rpath = _free(rpath);
02028 } else
02029 lpath = rpmGetPath(lpath, NULL);
02030
02031
02032
02033 switch (ut) {
02034 case URL_IS_DASH:
02035 case URL_IS_UNKNOWN:
02036 rpath = lpath;
02037 lpath = NULL;
02038 break;
02039 default:
02040 assert(nb < sizeof(fullpath));
02041 strncpy(fullpath, av[i], nb);
02042 fullpath[nb] = '\0';
02043 rpath = rpmGenPath(fullpath, lpath, NULL);
02044 lpath = _free(lpath);
02045 break;
02046 }
02047
02048
02049 lpath = (isdir || (!Stat(rpath, &sb) && S_ISDIR(sb.st_mode))
02050 ? "/" : NULL);
02051 if (lpath != NULL) {
02052 lpath = rpmExpand(rpath, lpath, NULL);
02053 xx = argvAdd(&repo->directories, lpath);
02054 lpath = _free(lpath);
02055 ndirs++;
02056 } else {
02057 xx = argvAdd(&repo->pkglist, rpath);
02058 nfiles++;
02059 }
02060 rpath = _free(rpath);
02061 }
02062
02063 if (_repo_debug || repo->dryrun)
02064 argvPrint("repo->directories", repo->directories, NULL);
02065
02066 #ifdef NOTYET
02067 if (repo->basedir == NULL)
02068 repo->basedir = xstrdup(repo->directories[0]);
02069 #endif
02070
02071 if (repo->outputdir == NULL) {
02072 if (repo->directories != NULL && repo->directories[0] != NULL)
02073 repo->outputdir = xstrdup(repo->directories[0]);
02074 else {
02075 repo->outputdir = repoRealpath(".");
02076 if (repo->outputdir == NULL)
02077 repo_error(1, _("Realpath(%s): %s"), ".", strerror(errno));
02078 }
02079 }
02080
02081 if (repo->split && repo->checkts)
02082 repo_error(1, _("--split and --checkts options are mutually exclusive"));
02083
02084 #ifdef NOTYET
02085
02086 if (repo->manifests != NULL) {
02087 const char ** av = repo->manifests;
02088 const char * fn;
02089
02090 while ((fn = *av++) != NULL) {
02091
02092
02093
02094 xx = argvAdd(&repo->pkglist, fn);
02095 }
02096 }
02097 #endif
02098
02099
02100 if (mireLoadPatterns(RPMMIRE_GLOB, 0, repo->exclude_patterns, NULL,
02101 &repo->excludeMire, &repo->nexcludes))
02102 repo_error(1, _("Error loading exclude glob patterns."));
02103 if (mireLoadPatterns(RPMMIRE_GLOB, 0, repo->include_patterns, NULL,
02104 &repo->includeMire, &repo->nincludes))
02105 repo_error(1, _("Error loading include glob patterns."));
02106
02107
02108 if (repo->directories != NULL) {
02109 ARGV_t pkglist = repoGetFileList(repo, repo->directories, ".rpm");
02110 xx = argvAppend(&repo->pkglist, pkglist);
02111 pkglist = argvFree(pkglist);
02112 }
02113
02114
02115 xx = argvSort(repo->pkglist, NULL);
02116
02117 if (_repo_debug || repo->dryrun)
02118 argvPrint("repo->pkglist", repo->pkglist, NULL);
02119
02120 repo->pkgcount = argvCount(repo->pkglist);
02121
02122
02123 _rpmts_stats = _rpmsw_stats;
02124 repo->ts = rpmtsCreate();
02125
02126
02127 (void) rpmtsSetVSFlags(repo->ts, _RPMVSF_NOSIGNATURES);
02128
02129 rc = repoTestSetupDirs(repo);
02130
02131 if (rc || repo->dryrun)
02132 goto exit;
02133
02134 if (!repo->split) {
02135 rc = repoCheckTimeStamps(repo);
02136 if (rc == 0) {
02137 fprintf(stdout, _("repo is up to date\n"));
02138 goto exit;
02139 }
02140 }
02141
02142 if ((rc = repoDoPkgMetadata(repo)) != 0)
02143 goto exit;
02144 if ((rc = repoDoRepoMetadata(repo)) != 0)
02145 goto exit;
02146 if ((rc = repoDoFinalMove(repo)) != 0)
02147 goto exit;
02148
02149 exit:
02150 (void)rpmtsFree(repo->ts);
02151 repo->ts = NULL;
02152 repo->primary.digest = _free(repo->primary.digest);
02153 repo->primary.Zdigest = _free(repo->primary.Zdigest);
02154 repo->filelists.digest = _free(repo->filelists.digest);
02155 repo->filelists.Zdigest = _free(repo->filelists.Zdigest);
02156 repo->other.digest = _free(repo->other.digest);
02157 repo->other.Zdigest = _free(repo->other.Zdigest);
02158 repo->repomd.digest = _free(repo->repomd.digest);
02159 repo->repomd.Zdigest = _free(repo->repomd.Zdigest);
02160 repo->outputdir = _free(repo->outputdir);
02161 repo->pkglist = argvFree(repo->pkglist);
02162 repo->directories = argvFree(repo->directories);
02163 repo->manifests = argvFree(repo->manifests);
02164
02165 repo->excludeMire = mireFreeAll(repo->excludeMire, repo->nexcludes);
02166 repo->includeMire = mireFreeAll(repo->includeMire, repo->nincludes);
02167
02168 repo->exclude_patterns = argvFree(repo->exclude_patterns);
02169 repo->include_patterns = argvFree(repo->include_patterns);
02170
02171 tagClean(NULL);
02172 optCon = rpmioFini(optCon);
02173
02174 return rc;
02175 }