7 #if defined(WITH_DBSQL)
9 #elif defined(WITH_SQLITE)
13 extern const char *sqlite3_errmsg(sqlite3 *
db)
15 extern int sqlite3_open(
20 extern int sqlite3_exec(
23 int (*callback)(
void*,
int,
char**,
char**),
28 extern int sqlite3_prepare(
32 sqlite3_stmt **ppStmt,
36 extern int sqlite3_reset(sqlite3_stmt *pStmt)
38 extern int sqlite3_step(sqlite3_stmt *pStmt)
40 extern int sqlite3_finalize( sqlite3_stmt *pStmt)
42 extern int sqlite3_close(sqlite3 * db)
55 #define _RPMREPO_INTERNAL
68 #define REPODBG(_l) if (_rpmrepo_debug) fprintf _l
74 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
75 "<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"0\">\n";
81 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
82 "<filelists xmlns=\"http://linux.duke.edu/metadata/filelists\" packages=\"0\">\n";
88 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
89 "<otherdata xmlns=\"http://linux.duke.edu/metadata/other\" packages=\"0\">\n";
95 <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
96 <repomd xmlns=\"http://linux.duke.edu/metadata/repo\">\n";
103 #include "yum_primary_xml"
108 #include "yum_filelists_xml"
113 #include "yum_other_xml"
118 #include "wnh_primary_yaml"
123 #include "wnh_filelists_yaml"
128 #include "wnh_other_yaml"
133 #include "deb_Packages"
138 #include "deb_Sources"
144 "PRAGMA synchronous = \"OFF\";",
145 "pragma locking_mode = \"EXCLUSIVE\";",
146 "CREATE TABLE conflicts ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
147 "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);",
148 "CREATE TABLE files ( pkgKey INTEGER, name TEXT, type TEXT );",
149 "CREATE TABLE obsoletes ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
150 "CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT, name TEXT, arch TEXT, version TEXT, epoch TEXT, release TEXT, summary TEXT, description TEXT, url TEXT, time_file INTEGER, time_build INTEGER, rpm_license TEXT, rpm_vendor TEXT, rpm_group TEXT, rpm_buildhost TEXT, rpm_sourcerpm TEXT, rpm_header_start INTEGER, rpm_header_end INTEGER, rpm_packager TEXT, size_package INTEGER, size_installed INTEGER, size_archive INTEGER, location_href TEXT, location_base TEXT, checksum_type TEXT);",
151 "CREATE TABLE provides ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
152 "CREATE TABLE requires ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
153 "CREATE INDEX filenames ON files (name);",
154 "CREATE INDEX packageId ON packages (pkgId);",
155 "CREATE INDEX packagename ON packages (name);",
156 "CREATE INDEX pkgconflicts on conflicts (pkgKey);",
157 "CREATE INDEX pkgobsoletes on obsoletes (pkgKey);",
158 "CREATE INDEX pkgprovides on provides (pkgKey);",
159 "CREATE INDEX pkgrequires on requires (pkgKey);",
160 "CREATE INDEX providesname ON provides (name);",
161 "CREATE INDEX requiresname ON requires (name);",
162 "CREATE TRIGGER removals AFTER DELETE ON packages\
164 \n DELETE FROM files WHERE pkgKey = old.pkgKey;\
165 \n DELETE FROM requires WHERE pkgKey = old.pkgKey;\
166 \n DELETE FROM provides WHERE pkgKey = old.pkgKey;\
167 \n DELETE FROM conflicts WHERE pkgKey = old.pkgKey;\
168 \n DELETE FROM obsoletes WHERE pkgKey = old.pkgKey;\
170 "INSERT into db_info values (9, 'direct_create');",
177 "PRAGMA synchronous = \"OFF\";",
178 "pragma locking_mode = \"EXCLUSIVE\";",
179 "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);",
180 "CREATE TABLE filelist ( pkgKey INTEGER, name TEXT, type TEXT );",
181 "CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);",
182 "CREATE INDEX filelistnames ON filelist (name);",
183 "CREATE INDEX keyfile ON filelist (pkgKey);",
184 "CREATE INDEX pkgId ON packages (pkgId);",
185 "CREATE TRIGGER remove_filelist AFTER DELETE ON packages\
187 \n DELETE FROM filelist WHERE pkgKey = old.pkgKey;\
189 "INSERT into db_info values (9, 'direct_create');",
196 "PRAGMA synchronous = \"OFF\";",
197 "pragma locking_mode = \"EXCLUSIVE\";",
198 "CREATE TABLE changelog ( pkgKey INTEGER, author TEXT, date INTEGER, changelog TEXT);",
199 "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);",
200 "CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);",
201 "CREATE INDEX keychange ON changelog (pkgKey);",
202 "CREATE INDEX pkgId ON packages (pkgId);",
203 "CREATE TRIGGER remove_changelogs AFTER DELETE ON packages\
205 \n DELETE FROM changelog WHERE pkgKey = old.pkgKey;\
207 "INSERT into db_info values (9, 'direct_create');",
269 #include "yum_primary_sqlite"
280 #include "yum_filelists_sqlite"
292 #include "yum_other_sqlite"
299 .flags = REPO_FLAGS_PRETTY,
301 .tempdir =
".repodata",
302 .finaldir =
"repodata",
303 .olddir =
".olddata",
320 .Packages_init= NULL,
321 .Packages_qfmt= NULL,
322 .Packages_fini= NULL,
340 .Packages_init= NULL,
341 .Packages_qfmt= NULL,
342 .Packages_fini= NULL,
360 .Packages_init= NULL,
361 .Packages_qfmt= NULL,
362 .Packages_fini= NULL,
380 .Packages_init= NULL,
382 .Packages_fini= NULL,
404 return (
Stat(fn, st) == 0);
420 stctime = sb.st_ctime;
434 (void) vfprintf(stderr, fmt, ap);
436 (void) fprintf(stderr,
"\n");
450 const char *
type,
int compress)
454 return rpmGetPath(repo->outputdir,
"/", dir,
"/", type,
455 (repo->markup != NULL ? repo->markup :
""),
456 (repo->suffix != NULL && compress ? repo->suffix :
""), NULL);
467 const char * item,
int current,
int total)
471 static size_t ncols = 80 - 1;
472 const char * bn = (item != NULL ? strrchr(item,
'/') : NULL);
479 nb = fprintf(stdout,
"\r%s: %d/%d",
__progname, current, total);
481 nb += fprintf(stdout,
" - %s", bn);
484 fprintf(stdout,
"%*s", (
int)(ncols - nb),
"");
486 (void) fflush(stdout);
499 const char * dnurl =
rpmGetPath(repo->outputdir,
"/", dn, NULL);
509 rc = (
Mkdir(dnurl, 0755) == 0 ||
errno == EEXIST ? 0 : -1);
512 dnurl =
_free(dnurl);
519 const char *rpath =
Realpath(lpath, NULL);
533 const char ** directories = repo->directories;
534 struct stat sb, *st = &sb;
541 if (directories != NULL)
542 while ((dn = *directories++) != NULL) {
543 if (!
rpmioExists(dn, st) || !S_ISDIR(st->st_mode)) {
551 rpmrepoError(0,
_(
"Directory %s does not exist."), repo->outputdir);
555 rpmrepoError(0,
_(
"Directory %s must be writable."), repo->outputdir);
563 dn =
rpmGetPath(repo->outputdir,
"/", repo->olddir, NULL);
565 rpmrepoError(0,
_(
"Old data directory exists, please remove: %s"), dn);
571 static const char * dirs[] = {
".repodata",
"repodata", NULL };
573 static const char * types[] =
574 {
"primary",
"filelists",
"other",
"repomd", NULL };
575 const char ** dirp, ** typep;
576 for (dirp = dirs; *dirp != NULL; dirp++) {
577 for (typep = types; *typep != NULL; typep++) {
578 fn =
rpmrepoGetPath(repo, *dirp, *typep, strcmp(*typep,
"repomd"));
584 if (REPO_ISSET(CHECKTS) && st->st_ctime > repo->mdtimestamp)
585 repo->mdtimestamp = st->st_ctime;
593 if (repo->groupfile != NULL) {
594 if (REPO_ISSET(SPLIT) || repo->groupfile[0] !=
'/') {
595 fn =
rpmGetPath(repo->package_dir,
"/", repo->groupfile, NULL);
596 repo->groupfile =
_free(repo->groupfile);
597 repo->groupfile = fn;
601 rpmrepoError(0,
_(
"groupfile %s cannot be found."), repo->groupfile);
618 size_t flen = strlen(fn);
619 size_t slen = strlen(suffix);
620 return (flen > slen && !strcmp(fn + flen - slen, suffix));
626 const char ** pkglist = NULL;
631 if ((t =
Fts_open((
char *
const *)roots, repo->ftsoptions, NULL)) == NULL)
636 const char * fts_name = p->
fts_name;
640 if (p->
fts_level == 0 && fts_namelen == 0) {
642 fts_namelen =
sizeof(
".") - 1;
662 if (REPO_ISSET(NOFOLLOW))
686 if (REPO_ISSET(CHECKTS)) {
689 if (repo->pkglist != NULL)
690 for (pkg = repo->pkglist; *pkg != NULL ; pkg++) {
691 struct stat sb, *st = &sb;
695 }
else if (st->st_ctime > repo->mdtimestamp)
714 size_t nspew = (spew != NULL ? strlen(spew) : 0);
716 size_t nb = (nspew > 0 ?
Fwrite(spew, 1, nspew, rfile->fd) : 0);
720 rpmrepoError(0,
_(
"Fwrite failed: expected write %u != %u bytes: %s\n"),
721 (
unsigned)nspew, (
unsigned)nb,
Fstrerror(rfile->fd));
764 const char * spew = rfile->xml_init;
765 size_t nspew = strlen(spew);
766 const char * fn =
rpmrepoGetPath(repo, repo->tempdir, rfile->type, 1);
771 rfile->fd =
Fopen(fn, repo->wmode);
772 assert(rfile->fd != NULL);
777 if ((tail = strstr(spew,
" packages=\"0\">\n")) != NULL)
778 nspew -= strlen(tail);
780 nb =
Fwrite(spew, 1, nspew, rfile->fd);
784 size_t tnb =
snprintf(buf,
sizeof(buf),
" packages=\"%d\">\n",
787 nb +=
Fwrite(buf, 1, tnb, rfile->fd);
790 rpmrepoError(0,
_(
"Fwrite failed: expected write %u != %u bytes: %s\n"),
791 (
unsigned)nspew, (
unsigned)nb,
Fstrerror(rfile->fd));
797 #if defined(WITH_SQLITE)
798 if (REPO_ISSET(DATABASE)) {
801 fn =
rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
802 rfile->type,
".sqlite", NULL);
803 if ((xx = sqlite3_open(fn, &rfile->sqldb)) != SQLITE_OK)
804 rpmrepoError(1,
"sqlite3_open(%s): %s", fn, sqlite3_errmsg(rfile->sqldb));
805 for (stmt = rfile->sql_init; *stmt != NULL; stmt++) {
807 xx = sqlite3_exec(rfile->sqldb, *stmt, NULL, NULL, &msg);
809 rpmrepoError(1,
"sqlite3_exec(%s, \"%s\"): %s\n", fn, *stmt,
810 (msg != NULL ? msg :
"failed"));
819 #if defined(WITH_SQLITE)
827 static int rpmrfileSQL(
rpmrfile rfile,
const char * msg,
int rc)
831 if (rc != SQLITE_OK || _rpmrepo_debug)
833 sqlite3_errmsg(rfile->sqldb));
843 static int rpmrfileSQLStep(
rpmrfile rfile, sqlite3_stmt * stmt)
853 rc = sqlite3_step(stmt);
856 rc = rpmrfileSQL(rfile,
"step", rc);
865 xx = rpmrfileSQL(rfile,
"reset",
866 sqlite3_reset(stmt));
877 static int rpmrfileSQLWrite(
rpmrfile rfile,
const char *
cmd)
885 xx = rpmrfileSQL(rfile,
"prepare",
886 sqlite3_prepare(rfile->sqldb, cmd, (
int)strlen(cmd), &stmt, &tail));
888 xx = rpmrfileSQL(rfile,
"reset",
889 sqlite3_reset(stmt));
891 xx = rpmrfileSQLStep(rfile, stmt);
893 xx = rpmrfileSQL(rfile,
"finalize",
894 sqlite3_finalize(stmt));
907 const char ** digestp)
910 static int asAscii = 1;
911 struct stat sb, *st = &sb;
912 const char * fn =
rpmrepoGetPath(repo, repo->tempdir, rfile->type, 1);
913 const char * path = NULL;
919 memset(st, 0,
sizeof(*st));
922 fd =
Fopen(fn,
"r.ufdio");
923 if (fd == NULL ||
Ferror(fd))
929 #if defined(HAVE_MMAP)
930 {
void * mapped = (
void *)-1;
933 mapped = mmap(NULL, st->st_size, PROT_READ, MAP_SHARED,
Fileno(fd), 0);
934 if (mapped != (
void *)-1) {
946 xx = munmap(mapped, st->st_size);
952 {
char buf[64 * BUFSIZ];
957 while ((nb =
Fread(buf,
sizeof(buf[0]),
sizeof(buf), fd)) > 0)
984 static int asAscii = 1;
995 fdFiniDigest(rfile->fd, repo->algo, &rfile->digest, NULL, asAscii);
1003 rfile->Zdigest = NULL;
1006 #if defined(WITH_SQLITE)
1007 if (REPO_ISSET(DATABASE) && rfile->sqldb != NULL) {
1008 const char *dbfn =
rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1009 rfile->type,
".sqlite", NULL);
1011 if ((xx = sqlite3_close(rfile->sqldb)) != SQLITE_OK)
1012 rpmrepoError(1,
"sqlite3_close(%s): %s", dbfn, sqlite3_errmsg(rfile->sqldb));
1013 rfile->sqldb = NULL;
1019 xmlfn =
_free(xmlfn);
1030 const char * tagname = NULL;
1055 default: tagname = NULL;
break;
1073 (void)
snprintf(spewtime,
sizeof(spewtime),
"%u", (unsigned)rfile->ctime);
1075 <data type=\"", rfile->type,
"\">\n\
1076 <checksum type=\"", spewalgo,
"\">", rfile->Zdigest,
"</checksum>\n\
1077 <timestamp>", spewtime,
"</timestamp>\n\
1078 <open-checksum type=\"",spewalgo,
"\">", rfile->digest,
"</open-checksum>\n\
1079 <location href=\"", repo->finaldir,
"/", rfile->type, (repo->markup != NULL ? repo->markup :
""), (repo->suffix != NULL ? repo->suffix :
""),
"\"/>\n\
1086 const char * fn =
rpmrepoGetPath(repo, repo->tempdir, rfile->type, 0);
1089 if ((rfile->fd =
Fopen(fn,
"w.ufdio")) != NULL) {
1104 def doRepoMetadata(
self):
1105 """wrapper to generate the repomd.xml file that stores the info on the other files"""
1106 const char * repopath =
1107 rpmGetPath(repo->outputdir,
"/", repo->tempdir, NULL);
1108 repodoc = libxml2.newDoc(
"1.0")
1109 reporoot = repodoc.newChild(None,
"repomd", None)
1110 repons = reporoot.newNs(
"http://linux.duke.edu/metadata/repo", None)
1111 reporoot.setNs(repons)
1112 repopath =
rpmGetPath(repo->outputdir,
"/", repo->tempdir, NULL);
1115 repoid =
"garbageid";
1117 if (REPO_ISSET(DATABASE)) {
1118 if (!repo->quiet)
rpmrepoError(0,
_(
"Generating sqlite DBs"));
1120 dbversion =
str(sqlitecachec.DBVERSION)
1121 except AttributeError:
1123 rp = sqlitecachec.RepodataParserSqlite(repopath, repoid, None)
1126 {
static const char * types[] =
1127 {
"primary",
"filelists",
"other", NULL };
1128 const char ** typep;
1129 for (typep = types; *typep != NULL; typep++) {
1132 zfo = _gzipOpen(complete_path)
1135 csum = misc.checksum(
algo2tagname(repo->algo), complete_path)
1137 timestamp = os.stat(complete_path)[8]
1140 db_compressed_sums = {}
1142 if (REPO_ISSET(repo)) {
1143 if (repo->verbose) {
1144 time_t now =
time(NULL);
1146 *typep, ctime(&now));
1149 if (!strcmp(*typep,
"primary"))
1150 rp.getPrimary(complete_path, csum)
1151 else if (!strcmp(*typep,
"filelists"));
1152 rp.getFilelists(complete_path, csum)
1153 else if (!strcmp(*typep,
"other"))
1154 rp.getOtherdata(complete_path, csum)
1156 {
const char * tmp_result_path =
1157 rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1158 *typep,
".xml.gz.sqlite", NULL);
1159 const char * resultpath =
1160 rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1161 *typep,
".sqlite", NULL);
1164 xx =
Rename(tmp_result_path, resultpath);
1165 tmp_result_path =
_free(tmp_result_path);
1167 rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1168 *typep,
".sqlite.bz2", NULL);
1169 db_csums[*typep] = misc.checksum(
algo2tagname(repo->algo), resultpath)
1172 bzipFile(resultpath, result_compressed)
1174 db_compressed_sums[*typep] = misc.checksum(
algo2tagname(repo->algo), result_compressed)
1177 resultpath =
_free(resultpath);
1180 if (REPO_ISSET(UNIQUEMDFN)) {
1181 const char * csum_result_compressed =
1182 rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1183 db_compressed_sums[*typep],
"-", *typep,
".sqlite.bz2", NULL);
1184 xx =
Rename(result_compressed, csum_result_compressed);
1185 result_compressed =
_free(result_compressed);
1186 result_compressed = csum_result_compressed;
1191 db_timestamp = os.stat(result_compressed)[8]
1194 db_data_type =
rpmExpand(*typep,
"_db", NULL);
1195 data = reporoot.newChild(None,
'data', None)
1196 data.newProp(
'type', db_data_type)
1197 location =
data.newChild(None,
'location', None)
1198 if (repo->baseurl != NULL) {
1199 location.newProp(
'xml:base', repo->baseurl)
1202 location.newProp(
'href',
rpmGetPath(repo->finaldir,
"/", *typep,
".sqlite.bz2", NULL));
1203 checksum =
data.newChild(None,
'checksum', db_compressed_sums[*typep])
1205 db_tstamp =
data.newChild(None,
'timestamp',
str(db_timestamp))
1206 unchecksum =
data.newChild(None,
'open-checksum', db_csums[*typep])
1208 database_version =
data.newChild(None,
'database_version', dbversion)
1209 if (repo->verbose) {
1210 time_t now =
time(NULL);
1212 *typep, ctime(&now));
1216 data = reporoot.newChild(None,
'data', None)
1217 data.newProp(
'type', *typep)
1219 checksum =
data.newChild(None,
'checksum', csum)
1221 timestamp =
data.newChild(None,
'timestamp',
str(timestamp))
1222 unchecksum =
data.newChild(None,
'open-checksum', uncsum)
1224 location =
data.newChild(None,
'location', None)
1225 if (repo->baseurl != NULL)
1226 location.newProp(
'xml:base', repo->baseurl)
1227 if (REPO_ISSET(UNIQUEMDFN)) {
1228 orig_file =
rpmrepoGetPath(repo, repo->tempdir, *typep, strcmp(*typep,
"repomd"));
1230 (repo->markup ? repo->markup :
""),
1231 (repo->suffix && strcmp(*typep,
"repomd") ? repo->suffix :
""), NULL);
1232 dest_file =
rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/", res_file, NULL);
1233 xx =
Rename(orig_file, dest_file);
1237 (repo->markup ? repo->markup :
""),
1238 (repo->suffix && strcmp(*typep,
"repomd") ? repo->suffix :
""), NULL);
1240 location.newProp(
'href',
rpmGetPath(repo->finaldir,
"/", res_file, NULL));
1244 if (!repo->quiet && REPO_ISSET(DATABASE))
1247 if (repo->groupfile != NULL) {
1248 self.addArbitraryMetadata(repo->groupfile,
'group_gz', reporoot)
1249 self.addArbitraryMetadata(repo->groupfile,
'group', reporoot, compress=False)
1254 repodoc.saveFormatFileEnc(fn,
'UTF-8', 1)
1258 (repo->markup ? repo->markup :
""),
1259 (repo->suffix && strcmp(*typep,
"repomd") ? repo->suffix :
""),
1271 char * output_final_dir =
1272 rpmGetPath(repo->outputdir,
"/", repo->finaldir, NULL);
1273 char * output_old_dir =
1274 rpmGetPath(repo->outputdir,
"/", repo->olddir, NULL);
1275 struct stat sb, *st = &sb;
1279 if ((xx =
Rename(output_final_dir, output_old_dir)) != 0)
1281 output_final_dir, output_old_dir);
1284 {
const char * output_temp_dir =
1285 rpmGetPath(repo->outputdir,
"/", repo->tempdir, NULL);
1286 if ((xx =
Rename(output_temp_dir, output_final_dir)) != 0) {
1287 xx =
Rename(output_old_dir, output_final_dir);
1288 rpmrepoError(1,
_(
"Error moving final metadata into place"));
1290 output_temp_dir =
_free(output_temp_dir);
1295 char *
const _av[] = { output_old_dir, NULL };
1297 int (*_compar)(
const FTSENT **,
const FTSENT **) = NULL;
1302 while ((p =
Fts_read(t)) != NULL) {
1306 const char * nfn = NULL;
1312 if ((xx =
Rmdir(opath)) != 0)
1313 rpmrepoError(1,
_(
"Could not remove old metadata directory: %s: %s"),
1314 ofn, strerror(
errno));
1319 if ((xx =
Unlink(opath)) != 0)
1320 rpmrepoError(1,
_(
"Could not remove old metadata file: %s: %s"),
1321 ofn, strerror(
errno));
1326 nfn =
rpmGetPath(output_final_dir,
"/", obn, NULL);
1328 if ((xx =
Unlink(opath)) != 0)
1329 rpmrepoError(1,
_(
"Could not remove old metadata file: %s: %s"),
1330 ofn, strerror(
errno));
1332 if ((xx =
Rename(opath, nfn)) != 0)
1333 rpmrepoError(1,
_(
"Could not restore old non-metadata file: %s -> %s: %s"),
1334 ofn, nfn, strerror(
errno));
1341 if ((xx =
Unlink(opath)) != 0)
1342 rpmrepoError(1,
_(
"Could not remove old metadata symlink: %s: %s"),
1343 ofn, strerror(
errno));
1351 output_old_dir =
_free(output_old_dir);
1352 output_final_dir =
_free(output_final_dir);
1375 uint32_t algo = repo->pkgalgo;
1384 char buffer[32 * BUFSIZ];
1385 size_t nb =
sizeof(
buffer);
1387 while ((nr =
Fread(buffer,
sizeof(buffer[0]), nb, fd)) == nb)
1390 fprintf(stderr,
_(
"%s: Fread(%s) failed: %s\n"),
1394 static int asAscii = 1;
1395 const char *digest = NULL;
1398 digest =
_free(digest);
1433 const char * msg = NULL;
1441 #if defined(WITH_SQLITE)
1448 static const char * rfileHeaderSprintfHack(
Header h,
const char * qfmt)
1452 static const char mark[] =
"'XXX'";
1453 static size_t nmark =
sizeof(
"'XXX'") - 1;
1454 const char * msg = NULL;
1455 char * s = (
char *)
headerSprintf(h, qfmt, NULL, NULL, &msg);
1465 for (f = s; *f !=
'\0' && (fe = strstr(f,
"'XXX'")) != NULL; fe += nmark, f = fe)
1471 int xx =
snprintf(instance,
sizeof(instance),
"'%u'",
1473 size_t tlen = strlen(s) + nsubs * ((int)strlen(instance) - (int)nmark);
1479 for (f = s; *f !=
'\0' && (fe = strstr(f, mark)) != NULL; fe += nmark, f = fe) {
1507 if (rfile->xml_qfmt != NULL) {
1512 #if defined(WITH_SQLITE)
1513 if (REPO_ISSET(DATABASE)) {
1514 if (rpmrfileSQLWrite(rfile, rfileHeaderSprintfHack(h, rfile->sql_qfmt)))
1531 const char ** pkglist = repo->pkglist;
1536 while ((pkg = *pkglist++) != NULL) {
1549 reldir = (pkgpath != NULL ? pkgpath :
rpmGetPath(repo->basedir,
"/", repo->directories[0], NULL));
1550 self.primaryfile.write(po.do_primary_xml_dump(reldir, baseurl=repo->baseurl))
1551 self.flfile.write(po.do_filelists_xml_dump())
1552 self.otherfile.write(po.do_other_xml_dump())
1566 rpmrepoError(0,
"%d/%d - %s", repo->current, repo->pkgcount, pkg);
1581 def _getFragmentUrl(
self, url, fragment):
1583 urlparse.uses_fragment.append(
'media')
1586 (scheme, netloc, path,
query, fragid) = urlparse.urlsplit(url)
1587 return urlparse.urlunsplit((scheme, netloc, path,
query,
str(fragment)))
1589 def doPkgMetadata(
self):
1590 """all the heavy lifting for the package metadata"""
1591 if (
argvCount(repo->directories) == 1) {
1592 MetaDataGenerator.doPkgMetadata(
self)
1598 for mydir
in repo->directories {
1599 if (mydir[0] ==
'/')
1601 else if (mydir[0] ==
'.' && mydir[1] ==
'.' && mydir[2] ==
'/')
1604 thisdir =
rpmGetPath(repo->basedir,
"/", mydir, NULL);
1606 xx =
argvAdd(&roots, thisdir);
1607 thisdir =
_free(thisdir);
1610 self.trimRpms(filematrix[mydir])
1611 repo->pkgcount =
argvCount(filematrix[mydir]);
1616 repo->baseurl =
self._getFragmentUrl(repo->baseurl, mediano)
1626 for mydir
in repo->directories {
1627 repo->baseurl =
self._getFragmentUrl(repo->baseurl, mediano)
1633 repo->baseurl =
self._getFragmentUrl(repo->baseurl, 1)
1640 fprintf(stderr,
"\n");
1656 {
"uncompressed",
'\0', POPT_ARG_VAL, &
compression, 0,
1657 N_(
"don't compress"), NULL },
1659 N_(
"use gzip compression"), NULL },
1661 N_(
"use bzip2 compression"), NULL },
1663 N_(
"use lzma compression"), NULL },
1665 N_(
"use xz compression"), NULL },
1672 {
"quiet",
'q', POPT_ARG_VAL, &__repo.quiet, 0,
1673 N_(
"output nothing except for serious errors"), NULL },
1674 {
"verbose",
'v', 0, NULL, (int)
'v',
1675 N_(
"output more debugging info."), NULL },
1676 {
"dryrun",
'\0', POPT_BIT_SET, &__repo.flags, REPO_FLAGS_DRYRUN,
1677 N_(
"sanity check arguments, don't create metadata"), NULL },
1678 {
"excludes",
'x', POPT_ARG_ARGV, &__repo.exclude_patterns, 0,
1679 N_(
"glob PATTERN(s) to exclude"),
N_(
"PATTERN") },
1680 {
"includes",
'i', POPT_ARG_ARGV, &__repo.include_patterns, 0,
1681 N_(
"glob PATTERN(s) to include"),
N_(
"PATTERN") },
1682 {
"basedir",
'\0', POPT_ARG_STRING|POPT_ARGFLAG_DOC_HIDDEN, &__repo.basedir, 0,
1683 N_(
"top level directory"),
N_(
"DIR") },
1684 {
"baseurl",
'u', POPT_ARG_STRING|POPT_ARGFLAG_DOC_HIDDEN, &__repo.baseurl, 0,
1685 N_(
"baseurl to append on all files"),
N_(
"BASEURL") },
1687 {
"groupfile",
'g', POPT_ARG_STRING|POPT_ARGFLAG_DOC_HIDDEN, &__repo.groupfile, 0,
1688 N_(
"path to groupfile to include in metadata"),
N_(
"FILE") },
1690 {
"pretty",
'p', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__repo.flags, REPO_FLAGS_PRETTY,
1691 N_(
"make sure all xml generated is formatted"), NULL },
1692 {
"checkts",
'C', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__repo.flags, REPO_FLAGS_CHECKTS,
1693 N_(
"check timestamps on files vs the metadata to see if we need to update"), NULL },
1694 {
"database",
'd', POPT_BIT_SET, &__repo.flags, REPO_FLAGS_DATABASE,
1695 N_(
"create sqlite3 database files"), NULL },
1696 {
"split",
'\0', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__repo.flags, REPO_FLAGS_SPLIT,
1697 N_(
"generate split media"), NULL },
1698 {
"pkglist",
'l', POPT_ARG_ARGV|POPT_ARGFLAG_DOC_HIDDEN, &__repo.manifests, 0,
1699 N_(
"use only the files listed in this file from the directory specified"),
N_(
"FILE") },
1700 {
"outputdir",
'o', POPT_ARG_STRING, &__repo.outputdir, 0,
1701 N_(
"<dir> = optional directory to output to"),
N_(
"DIR") },
1702 {
"skip-symlinks",
'S', POPT_BIT_SET, &__repo.flags, REPO_FLAGS_NOFOLLOW,
1703 N_(
"ignore symlinks of packages"), NULL },
1704 {
"unique-md-filenames",
'\0', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__repo.flags, REPO_FLAGS_UNIQUEMDFN,
1705 N_(
"include the file's checksum in the filename, helps with proxies"), NULL },
1715 N_(
"Repository options:"), NULL },
1718 N_(
"Fts(3) traversal options:"), NULL },
1721 N_(
"Available compressions:"), NULL },
1724 N_(
"Available digests:"), NULL },
1727 N_(
"Common options for all rpmio executables:"),
1738 void *use = repo->_item.use;
1739 void *
pool = repo->_item.pool;
1741 poptContext con =
rpmioInit(ac, av, rpmrepoOptionsTable);
1747 repo->_item.use = use;
1748 repo->_item.pool = pool;
1761 rpmrepoError(1,
"FTS_LOGICAL and FTS_PYSICAL are mutually exclusive");
1771 repo->compression = (compression >= 0 ? compression : 1);
1772 switch (repo->compression) {
1774 repo->suffix = NULL;
1775 repo->wmode =
"w.ufdio";
1780 repo->suffix =
".gz";
1781 repo->wmode =
"w9.gzdio";
1784 repo->suffix =
".bz2";
1785 repo->wmode =
"w9.bzdio";
1788 repo->suffix =
".lzma";
1789 repo->wmode =
"w.lzdio";
1792 repo->suffix =
".xz";
1793 repo->wmode =
"w.xzdio";
1797 repo->av = poptGetArgs(repo->con);
1799 if (repo->av == NULL || repo->av[0] == NULL)
1802 if (repo->av != NULL)
1803 for (i = 0; repo->av[
i] != NULL; i++) {
1807 const char * lpath = NULL;
1808 int ut =
urlPath(repo->av[i], &lpath);
1809 size_t nb = (size_t)(lpath - repo->av[i]);
1810 int isdir = (lpath[strlen(lpath)-1] ==
'/');
1813 if (lpath[0] !=
'/') {
1817 rpath =
_free(rpath);
1830 assert(nb <
sizeof(fullpath));
1831 strncpy(fullpath, repo->av[i], nb);
1832 fullpath[nb] =
'\0';
1834 lpath =
_free(lpath);
1839 lpath = (isdir || (!
Stat(rpath, &sb) && S_ISDIR(sb.st_mode))
1841 if (lpath != NULL) {
1843 xx =
argvAdd(&repo->directories, lpath);
1844 lpath =
_free(lpath);
1846 xx =
argvAdd(&repo->pkglist, rpath);
1848 rpath =
_free(rpath);
1860 repo->primary.digest =
_free(repo->primary.digest);
1861 repo->primary.Zdigest =
_free(repo->primary.Zdigest);
1862 repo->filelists.digest =
_free(repo->filelists.digest);
1863 repo->filelists.Zdigest =
_free(repo->filelists.Zdigest);
1864 repo->other.digest =
_free(repo->other.digest);
1865 repo->other.Zdigest =
_free(repo->other.Zdigest);
1866 repo->repomd.digest =
_free(repo->repomd.digest);
1867 repo->repomd.Zdigest =
_free(repo->repomd.Zdigest);
1868 repo->outputdir =
_free(repo->outputdir);
1869 repo->pkglist =
argvFree(repo->pkglist);
1870 repo->directories =
argvFree(repo->directories);
1871 repo->manifests =
argvFree(repo->manifests);
1873 repo->excludeMire =
mireFreeAll(repo->excludeMire, repo->nexcludes);
1874 repo->includeMire =
mireFreeAll(repo->includeMire, repo->nincludes);
1876 repo->exclude_patterns =
argvFree(repo->exclude_patterns);
1877 repo->include_patterns =
argvFree(repo->include_patterns);
1879 repo->con = poptFreeContext(repo->con);
1892 if (_rpmrepoPool == NULL) {
1893 _rpmrepoPool =
rpmioNewPool(
"repo",
sizeof(*repo), -1, _rpmrepo_debug,
1898 memset(((
char *)repo)+
sizeof(repo->_item), 0,
sizeof(*repo)-
sizeof(repo->_item));
static const char * suffix[]
poptContext rpmioInit(int argc, char *const argv[], struct poptOption *optionsTable)
Initialize most everything needed by an rpmio executable context.
int rpmrepoDoFinalMove(rpmrepo repo)
Rename temporary repository to final paths.
struct poptOption rpmioDigestPoptTable[]
Digest options using popt.
static const char * primary_sql_init[]
rpmtime_t rpmswExit(rpmop op, ssize_t rc)
Exit timed operation.
FTS * Fts_open(char *const *argv, int options, int(*compar)(const FTSENT **, const FTSENT **))
Create a handle for file hierarchy traversal.
int mireApply(miRE mire, int nmire, const char *s, size_t slen, int rc)
Apply array of patterns to a string.
static struct rpmrepo_s __repo
static const char Packages_qfmt[]
const char bson_timestamp_t * ts
void rpmrepoError(int lvl, const char *fmt,...)
Print an error message and exit (if requested).
void * mireFreeAll(miRE mire, int nmire)
Destroy compiled patterns.
struct rpmrfile_s * rpmrfile
static const char filelists_xml_qfmt[]
const char const char * cmd
static const char primary_yaml_qfmt[]
size_t Fwrite(const void *buf, size_t size, size_t nmemb, FD_t fd)
fwrite(3) clone.
char * xstrdup(const char *str)
FD_t Fopen(const char *path, const char *_fmode)
fopen(3) clone.
rpmrepo rpmrepoLink(rpmrepo repo)
Reference a repo wrapper instance.
static int chkSuffix(const char *fn, const char *suffix)
Check file name for a suffix.
DIGEST_CTX rpmDigestInit(pgpHashAlgo hashalgo, rpmDigestFlags flags)
Initialize digest context.
char * rpmGetPath(const char *path,...)
Return (malloc'ed) expanded, canonicalized, file path.
static const char * fdGetOPath(FD_t fd)
int Access(const char *path, int amode)
access(2) clone.
static const char * filelists_sql_init[]
static const char * rpmrepoMDExpand(rpmrepo repo, rpmrfile rfile)
Return a repository metadata file item.
int rpmioMkpath(const char *path, mode_t mode, uid_t uid, gid_t gid)
Insure that directories in path exist, creating as needed.
struct poptOption rpmioFtsPoptTable[]
Popt option table for options to set Fts(3) options.
rpmtime_t rpmswAdd(rpmop to, rpmop from)
Sum statistic counters.
int Stat(const char *path, struct stat *st)
stat(2) clone.
const char ** rpmrepoGetFileList(rpmrepo repo, const char *roots[], const char *ext)
Walk file/directory trees, looking for files with an extension.
static int rpmrepoMkdir(rpmrepo repo, const char *dn)
Create directory path.
rpmop rpmtsOp(rpmts ts, rpmtsOpX opx)
Retrieve operation timestamp from a transaction set.
const char * rpmrepoRealpath(const char *lpath)
Return realpath(3) canonicalized absolute path.
pgpHashAlgo rpmioDigestHashAlgo
static struct poptOption repoCompressionPoptTable[]
static void fdInitDigest(FD_t fd, pgpHashAlgo hashalgo, int _flags)
Attach digest to fd.
char * headerSprintf(Header h, const char *fmt, headerTagTableEntry tags, headerSprintfExtension exts, errmsg_t *errmsg)
Return formatted output string from header tags.
static int rpmrepoRfileDigest(const rpmrepo repo, rpmrfile rfile, const char **digestp)
Compute digest of a file.
static rpmop fdstat_op(FD_t fd, fdOpX opx)
static const char * rfileHeaderSprintf(Header h, const char *qfmt)
Return header query.
static int rpmioExists(const char *fn, struct stat *st)
Return stat(2) for a file.
Yet Another syslog(3) API clone.
static const char filelists_xml_init[]
int rpmrepoDoRepoMetadata(rpmrepo repo)
Write repository manifest.
static const char * algo2tagname(uint32_t algo)
const char * Fstrerror(FD_t fd)
strerror(3) clone.
int rpmDigestUpdate(DIGEST_CTX ctx, const void *data, size_t len)
Update context with next plain text buffer.
static const char primary_sql_qfmt[]
int Rmdir(const char *path)
rmdir(2) clone.
int rpmrepoDoPkgMetadata(rpmrepo repo)
Write repository metadata files.
rpmioItem rpmioGetPool(rpmioPool pool, size_t size)
Get unused item from pool, or alloc a new item.
int Rename(const char *oldpath, const char *newpath)
rename(2) clone.
static const char other_xml_qfmt[]
int argvCount(const ARGV_t argv)
Return no.
int Mkdir(const char *path, mode_t mode)
mkdir(2) clone.
const char const bson * data
ARGV_t argvFree(ARGV_t argv)
Destroy an argv array.
static const char other_sql_qfmt[]
static struct poptOption rpmrepoOptionsTable[]
rpmRC rpmReadPackageFile(rpmts ts, FD_t fd, const char *fn, Header *hdrp)
Return package header from file handle, verifying digests/signatures.
The FD_t File Handle data structure.
static void fdFiniDigest(FD_t fd, pgpHashAlgo hashalgo, void *datap, size_t *lenp, int asAscii)
const char * rpmGenPath(const char *urlroot, const char *urlmdir, const char *urlfile)
Merge 3 args into path, any or all of which may be a url.
int argvAdd(ARGV_t *argvp, ARGstr_t val)
Add a string to an argv array.
static time_t rpmioCtime(const char *fn)
Return stat(2) creation time of a file.
Header headerFree(Header h)
Dereference a header instance.
FTSENT * Fts_read(FTS *sp)
Return next node in the file hierarchy traversal.
int rpmswEnter(rpmop op, ssize_t rc)
Enter timed operation.
const char const bson const bson * op
char * rpmExpand(const char *arg,...)
Return (malloc'ed) concatenated macro expansion(s).
size_t Fread(void *buf, size_t size, size_t nmemb, FD_t fd)
fread(3) clone.
int Fclose(FD_t fd)
fclose(3) clone.
Cumulative statistics for an operation.
const char const bson int mongo_write_concern int flags
void argvPrint(const char *msg, ARGV_t argv, FILE *fp)
Print argv array elements.
static const char repomd_xml_init[]
static const char * other_sql_init[]
static const char primary_xml_init[]
static const char other_yaml_qfmt[]
enum rpmRC_e rpmRC
RPM return codes.
static Header rpmrepoReadHeader(rpmrepo repo, const char *path)
Read a header from a repository package file, computing package file digest.
int Ferror(FD_t fd)
ferror(3) clone.
static int rpmrfileXMLWrite(rpmrfile rfile, const char *spew)
Write to a repository metadata file.
const char const bson * query
urltype urlPath(const char *url, const char **pathp)
Return path component of URL.
static int snprintf(char *buf, int nb, const char *fmt,...)
int rpmrepoCheckTimeStamps(rpmrepo repo)
Check that repository time stamp is newer than any contained package.
static int rpmrepoFclose(rpmrepo repo, FD_t fd)
Close an I/O stream, accumulating uncompress/digest statistics.
Methods to handle package elements.
rpmioPool rpmioNewPool(const char *name, size_t size, int limit, int flags, char *(*dbg)(void *item), void(*init)(void *item), void(*fini)(void *item))
Create a memory pool.
rpmrepo rpmrepoNew(char **av, int flags)
Create and load a repo wrapper.
char * stpcpy(char *dest, const char *src)
static int rpmrepoOpenMDFile(const rpmrepo repo, rpmrfile rfile)
Open a repository metadata file.
struct rpmts_s * rpmts
The RPM Transaction Set.
static void * _free(const void *p)
Wrapper to free(3), hides const compilation noise, permit NULL, return NULL.
Structures and prototypes used for an "rpmts" transaction set.
static const char filelists_yaml_qfmt[]
static const char * rpmrepoGetPath(rpmrepo repo, const char *dir, const char *type, int compress)
Return /repository/directory/component.markup.compression path.
static const char other_xml_fini[]
int Fts_close(FTS *sp)
Destroy a file hierarchy traversal handle.
struct rpmrepo_s * rpmrepo
int Fileno(FD_t fd)
fileno(3) clone.
static rpmrepo rpmrepoGetPool(rpmioPool pool)
int rpmDigestFinal(DIGEST_CTX ctx, void *datap, size_t *lenp, int asAscii)
Return digest and destroy context.
static void rpmrepoProgress(rpmrepo repo, const char *item, int current, int total)
Display progress.
static int repoWriteMetadataDocs(rpmrepo repo)
Export all package metadata to repository metadata file(s).
static const char filelists_sql_qfmt[]
static int rpmrepoCloseMDFile(const rpmrepo repo, rpmrfile rfile)
Close a repository metadata file.
static const char Sources_qfmt[]
static void rpmrepoFini(void *_repo)
struct poptOption rpmioAllPoptTable[]
Popt option table for options shared by all modes and executables.
static const char other_xml_init[]
static int rpmrepoInitPopt(rpmrepo repo, char **av)
static const char repomd_xml_fini[]
static const char primary_xml_fini[]
static const char primary_xml_qfmt[]
char * Realpath(const char *path, char *resolved_path)
realpath(3) clone.
int rpmrepoTestSetupDirs(rpmrepo repo)
Test for repository sanity.
static const char filelists_xml_fini[]
static int rpmrepoWriteMDFile(rpmrepo repo, rpmrfile rfile, Header h)
Export a single package's metadata to repository metadata file(s).
static struct poptOption _rpmrepoOptions[]
unsigned long int rpmtime_t
int Unlink(const char *path)
unlink(2) clone.