yum-metadata-parser-1.1.4/0000775000076400007640000000000011321445743015271 5ustar skvidalskvidalyum-metadata-parser-1.1.4/yum-metadata-parser.spec0000664000076400007640000000443511321445436022034 0ustar skvidalskvidal%{!?python_sitelib_platform: %define python_sitelib_platform %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} Summary: A fast metadata parser for yum Name: yum-metadata-parser Version: 1.1.4 Release: 1 Source0: %{name}-%{version}.tar.gz License: GPL Group: Development/Libraries URL: http://devel.linux.duke.edu/cgi-bin/viewcvs.cgi/yum-metadata-parser/ Requires: yum >= 2.6.2 BuildRequires: python-devel BuildRequires: glib2-devel BuildRequires: libxml2-devel BuildRequires: sqlite-devel BuildRequires: pkgconfig BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) %description Fast metadata parser for yum implemented in C. %prep %setup %build %{__python} setup.py build %install %{__python} setup.py install -O1 --root=%{buildroot} %clean %{__rm} -rf %{buildroot} %files %defattr(-,root,root) %doc README AUTHORS ChangeLog %{python_sitelib_platform}/_sqlitecache.so %{python_sitelib_platform}/sqlitecachec.py %{python_sitelib_platform}/sqlitecachec.pyc %{python_sitelib_platform}/sqlitecachec.pyo %{python_sitelib_platform}/*egg-info %changelog * Thu Jan 7 2010 Seth Vidal - 1.1.4 b/c I made a mistake * Thu Jan 7 2010 Seth Vidal - add the egginfo support for python 2.5 and above * Thu Jan 7 2010 Seth Vidal - bump the version number for an official release * Wed Aug 29 2007 Seth Vidal - remove the %{dist} which doesn't really belong in the upstream pkg * Fri Aug 24 2007 Seth Vidal - 1.1.2 * Wed May 16 2007 Paul Nasrat - Expose DBVERSION * Fri Apr 27 2007 Seth Vidal - split out 1.1.0 for dbversion 10 * Wed Apr 4 2007 Seth Vidal - 1.0.4 * Sun Jan 7 2007 Seth Vidal - 1.0.3 * Wed Jul 12 2006 Seth Vidal - 1.0.2 * Mon Jun 19 2006 Seth Vidal - 1.0.1 * Mon Jun 05 2006 Tambet Ingo - 1.0-3 - Require yum >= 2.6.2 * Sat Jun 04 2006 Terje Rosten - 1.0-2 - add buildrequires - doc files - url * Fri Jun 02 2006 Terje Rosten - 1.0-0.1 - initial package yum-metadata-parser-1.1.4/db.c0000664000076400007640000006767211321436455016044 0ustar skvidalskvidal/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ /* This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License, * version 2, as published by the Free Software Foundation * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA * 02111-1307, USA. */ #include #include #include "db.h" /* We have a lot of code so we can "quickly" update the .sqlite file using * the old .sqlite data and the new .xml data. However it seems to have weird * edge cases where it doesn't work, rhbz 465898 etc. ... so we turn it off. */ #define YMP_CONFIG_UPDATE_DB 0 GQuark yum_db_error_quark (void) { static GQuark quark; if (!quark) quark = g_quark_from_static_string ("yum_db_error"); return quark; } #define ENCODED_PACKAGE_FILE_FILES 2048 #define ENCODED_PACKAGE_FILE_TYPES 60 typedef struct { GString *files; GString *types; } EncodedPackageFile; static EncodedPackageFile * encoded_package_file_new (void) { EncodedPackageFile *enc; enc = g_new0 (EncodedPackageFile, 1); enc->files = g_string_sized_new (ENCODED_PACKAGE_FILE_FILES); enc->types = g_string_sized_new (ENCODED_PACKAGE_FILE_TYPES); return enc; } static void encoded_package_file_free (EncodedPackageFile *file) { g_string_free (file->files, TRUE); g_string_free (file->types, TRUE); g_free (file); } static GHashTable * package_files_to_hash (GSList *files) { GHashTable *hash; GSList *iter; PackageFile *file; EncodedPackageFile *enc; char *dir; char *name; hash = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify) g_free, (GDestroyNotify) encoded_package_file_free); for (iter = files; iter; iter = iter->next) { file = (PackageFile *) iter->data; dir = g_path_get_dirname (file->name); name = g_path_get_basename (file->name); enc = (EncodedPackageFile *) g_hash_table_lookup (hash, dir); if (!enc) { enc = encoded_package_file_new (); g_hash_table_insert (hash, dir, enc); } else g_free (dir); if (enc->files->len) g_string_append_c (enc->files, '/'); g_string_append (enc->files, name); g_free (name); if (!strcmp (file->type, "dir")) g_string_append_c (enc->types, 'd'); else if (!strcmp (file->type, "file")) g_string_append_c (enc->types, 'f'); else if (!strcmp (file->type, "ghost")) g_string_append_c (enc->types, 'g'); } return hash; } char * yum_db_filename (const char *prefix) { char *filename; filename = g_strconcat (prefix, ".sqlite", NULL); return filename; } typedef enum { DB_STATUS_OK, DB_STATUS_VERSION_MISMATCH, DB_STATUS_CHECKSUM_MISMATCH, DB_STATUS_ERROR } DBStatus; static DBStatus dbinfo_status (sqlite3 *db, const char *checksum) { const char *query; int rc; sqlite3_stmt *handle = NULL; DBStatus status = DB_STATUS_ERROR; query = "SELECT dbversion, checksum FROM db_info"; rc = sqlite3_prepare (db, query, -1, &handle, NULL); if (rc != SQLITE_OK) goto cleanup; while ((rc = sqlite3_step (handle)) == SQLITE_ROW) { int dbversion; const char *dbchecksum; dbversion = sqlite3_column_int (handle, 0); dbchecksum = (const char *) sqlite3_column_text (handle, 1); if (dbversion != YUM_SQLITE_CACHE_DBVERSION) { g_message ("Warning: cache file is version %d, we need %d, will regenerate", dbversion, YUM_SQLITE_CACHE_DBVERSION); status = DB_STATUS_VERSION_MISMATCH; } else if (strcmp (checksum, dbchecksum)) { g_message ("sqlite cache needs updating, reading in metadata"); status = DB_STATUS_CHECKSUM_MISMATCH; } else status = DB_STATUS_OK; break; } cleanup: if (handle) sqlite3_finalize (handle); return status; } static void yum_db_create_dbinfo_table (sqlite3 *db, GError **err) { int rc; const char *sql; sql = "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create db_info table: %s", sqlite3_errmsg (db)); } } sqlite3 * yum_db_open (const char *path, const char *checksum, CreateTablesFn create_tables, GError **err) { int rc; sqlite3 *db = NULL; gboolean db_existed; db_existed = g_file_test (path, G_FILE_TEST_EXISTS); rc = sqlite3_open (path, &db); if (rc == SQLITE_OK) { if (db_existed) { DBStatus status = dbinfo_status (db, checksum); switch (status) { case DB_STATUS_OK: /* Everything is up-to-date */ sqlite3_close (db); return NULL; break; case DB_STATUS_CHECKSUM_MISMATCH: if (YMP_CONFIG_UPDATE_DB) { sqlite3_exec (db, "PRAGMA synchronous = 0", NULL,NULL,NULL); sqlite3_exec (db, "DELETE FROM db_info", NULL, NULL, NULL); return db; break; } /* FALL THROUGH */ case DB_STATUS_VERSION_MISMATCH: case DB_STATUS_ERROR: sqlite3_close (db); db = NULL; unlink (path); break; } } } else { /* Let's try to delete it and try again, maybe it's a sqlite3 version mismatch. */ sqlite3_close (db); db = NULL; unlink (path); } if (!db) { rc = sqlite3_open (path, &db); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not open SQL database: %s", sqlite3_errmsg (db)); goto cleanup; } } yum_db_create_dbinfo_table (db, err); if (*err) goto cleanup; create_tables (db, err); if (*err) goto cleanup; sqlite3_exec (db, "PRAGMA synchronous = 0", NULL, NULL, NULL); cleanup: if (*err && db) { sqlite3_close (db); db = NULL; } return db; } void yum_db_dbinfo_update (sqlite3 *db, const char *checksum, GError **err) { int rc; char *sql; sql = g_strdup_printf ("INSERT INTO db_info (dbversion, checksum) VALUES (%d, '%s')", YUM_SQLITE_CACHE_DBVERSION, checksum); rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not update dbinfo table: %s", sqlite3_errmsg (db)); g_free (sql); } GHashTable * yum_db_read_package_ids (sqlite3 *db, GError **err) { const char *query; int rc; GHashTable *hash = NULL; sqlite3_stmt *handle = NULL; query = "SELECT pkgId, pkgKey FROM packages"; rc = sqlite3_prepare (db, query, -1, &handle, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not prepare SQL clause: %s", sqlite3_errmsg (db)); goto cleanup; } hash = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify) g_free, NULL); while ((rc = sqlite3_step (handle)) == SQLITE_ROW) { char *pkgId; gint pkgKey; pkgId = g_strdup ((char *) sqlite3_column_text (handle, 0)); pkgKey = sqlite3_column_int (handle, 1); g_hash_table_insert (hash, pkgId, GINT_TO_POINTER (pkgKey)); } if (rc != SQLITE_DONE) g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Error reading from SQL: %s", sqlite3_errmsg (db)); cleanup: if (handle) sqlite3_finalize (handle); return hash; } void yum_db_create_primary_tables (sqlite3 *db, GError **err) { int rc; const char *sql; sql = "CREATE TABLE packages (" " pkgKey INTEGER PRIMARY KEY," " pkgId TEXT," " name TEXT," " arch TEXT," " version TEXT," " epoch TEXT," " release TEXT," " summary TEXT," " description TEXT," " url TEXT," " time_file INTEGER," " time_build INTEGER," " rpm_license TEXT," " rpm_vendor TEXT," " rpm_group TEXT," " rpm_buildhost TEXT," " rpm_sourcerpm TEXT," " rpm_header_start INTEGER," " rpm_header_end INTEGER," " rpm_packager TEXT," " size_package INTEGER," " size_installed INTEGER," " size_archive INTEGER," " location_href TEXT," " location_base TEXT," " checksum_type TEXT)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create packages table: %s", sqlite3_errmsg (db)); return; } sql = "CREATE TABLE files (" " name TEXT," " type TEXT," " pkgKey INTEGER)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create files table: %s", sqlite3_errmsg (db)); return; } sql = "CREATE TABLE %s (" " name TEXT," " flags TEXT," " epoch TEXT," " version TEXT," " release TEXT," " pkgKey INTEGER %s)"; const char *deps[] = { "requires", "provides", "conflicts", "obsoletes", NULL }; int i; for (i = 0; deps[i]; i++) { const char *prereq; char *query; if (!strcmp(deps[i], "requires")) { prereq = ", pre BOOLEAN DEFAULT FALSE"; } else prereq = ""; query = g_strdup_printf (sql, deps[i], prereq); rc = sqlite3_exec (db, query, NULL, NULL, NULL); g_free (query); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create %s table: %s", deps[i], sqlite3_errmsg (db)); return; } } sql = "CREATE TRIGGER removals AFTER DELETE ON packages" " BEGIN" " DELETE FROM files WHERE pkgKey = old.pkgKey;" " DELETE FROM requires WHERE pkgKey = old.pkgKey;" " DELETE FROM provides WHERE pkgKey = old.pkgKey;" " DELETE FROM conflicts WHERE pkgKey = old.pkgKey;" " DELETE FROM obsoletes WHERE pkgKey = old.pkgKey;" " END;"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create removals trigger: %s", sqlite3_errmsg (db)); return; } } void yum_db_index_primary_tables (sqlite3 *db, GError **err) { int rc; const char *sql; sql = "CREATE INDEX IF NOT EXISTS packagename ON packages (name)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create packagename index: %s", sqlite3_errmsg (db)); return; } sql = "CREATE INDEX IF NOT EXISTS packageId ON packages (pkgId)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create packageId index: %s", sqlite3_errmsg (db)); return; } sql = "CREATE INDEX IF NOT EXISTS filenames ON files (name)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create filenames index: %s", sqlite3_errmsg (db)); return; } sql = "CREATE INDEX IF NOT EXISTS pkgfiles ON files (pkgKey)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create index on files table: %s", sqlite3_errmsg (db)); return; } const char *deps[] = { "requires", "provides", "conflicts", "obsoletes", NULL }; int i; const char *pkgindexsql = "CREATE INDEX IF NOT EXISTS pkg%s on %s (pkgKey)"; const char *nameindexsql = "CREATE INDEX IF NOT EXISTS %sname ON %s (name)"; for (i = 0; deps[i]; i++) { char *query; query = g_strdup_printf(pkgindexsql, deps[i], deps[i]); rc = sqlite3_exec (db, query, NULL, NULL, NULL); g_free (query); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create index on %s table: %s", deps[i], sqlite3_errmsg (db)); return; } if (i < 2) { query = g_strdup_printf(nameindexsql, deps[i], deps[i]); rc = sqlite3_exec (db, query, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create %sname index: %s", deps[i], sqlite3_errmsg (db)); return; } } } } sqlite3_stmt * yum_db_package_prepare (sqlite3 *db, GError **err) { int rc; sqlite3_stmt *handle = NULL; const char *query; query = "INSERT INTO packages (" " pkgId, name, arch, version, epoch, release, summary, description," " url, time_file, time_build, rpm_license, rpm_vendor, rpm_group," " rpm_buildhost, rpm_sourcerpm, rpm_header_start, rpm_header_end," " rpm_packager, size_package, size_installed, size_archive," " location_href, location_base, checksum_type) " "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?," " ?, ?, ?, ?, ?, ?, ?)"; rc = sqlite3_prepare (db, query, -1, &handle, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not prepare packages insertion: %s", sqlite3_errmsg (db)); sqlite3_finalize (handle); handle = NULL; } return handle; } void yum_db_package_write (sqlite3 *db, sqlite3_stmt *handle, Package *p) { int rc; sqlite3_bind_text (handle, 1, p->pkgId, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 2, p->name, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 3, p->arch, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 4, p->version, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 5, p->epoch, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 6, p->release, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 7, p->summary, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 8, p->description, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 9, p->url, -1, SQLITE_STATIC); sqlite3_bind_int (handle, 10, p->time_file); sqlite3_bind_int (handle, 11, p->time_build); sqlite3_bind_text (handle, 12, p->rpm_license, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 13, p->rpm_vendor, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 14, p->rpm_group, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 15, p->rpm_buildhost, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 16, p->rpm_sourcerpm, -1, SQLITE_STATIC); sqlite3_bind_int (handle, 17, p->rpm_header_start); sqlite3_bind_int (handle, 18, p->rpm_header_end); sqlite3_bind_text (handle, 19, p->rpm_packager, -1, SQLITE_STATIC); sqlite3_bind_int (handle, 20, p->size_package); sqlite3_bind_int (handle, 21, p->size_installed); sqlite3_bind_int (handle, 22, p->size_archive); sqlite3_bind_text (handle, 23, p->location_href, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 24, p->location_base, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 25, p->checksum_type, -1, SQLITE_STATIC); rc = sqlite3_step (handle); sqlite3_reset (handle); if (rc != SQLITE_DONE) { g_critical ("Error adding package to SQL: %s", sqlite3_errmsg (db)); } else p->pkgKey = sqlite3_last_insert_rowid (db); } sqlite3_stmt * yum_db_dependency_prepare (sqlite3 *db, const char *table, GError **err) { int rc; sqlite3_stmt *handle = NULL; char *query; const char *pre_name = ""; const char *pre_value = ""; if (!strcmp (table, "requires")) { pre_name = ", pre"; pre_value = ", ?"; } query = g_strdup_printf ("INSERT INTO %s (name, flags, epoch, version, release, pkgKey%s) " "VALUES (?, ?, ?, ?, ?, ?%s)", table, pre_name, pre_value); rc = sqlite3_prepare (db, query, -1, &handle, NULL); g_free (query); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not prepare dependency insertion: %s", sqlite3_errmsg (db)); sqlite3_finalize (handle); handle = NULL; } return handle; } void yum_db_dependency_write (sqlite3 *db, sqlite3_stmt *handle, gint64 pkgKey, Dependency *dep, gboolean isRequirement) { int rc; sqlite3_bind_text (handle, 1, dep->name, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 2, dep->flags, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 3, dep->epoch, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 4, dep->version, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 5, dep->release, -1, SQLITE_STATIC); sqlite3_bind_int (handle, 6, pkgKey); if (isRequirement) { if (dep->pre) sqlite3_bind_text (handle, 7, "TRUE", -1, SQLITE_TRANSIENT); else sqlite3_bind_text (handle, 7, "FALSE", -1, SQLITE_TRANSIENT); } rc = sqlite3_step (handle); sqlite3_reset (handle); if (rc != SQLITE_DONE) g_critical ("Error adding dependency to SQL: %s", sqlite3_errmsg (db)); } sqlite3_stmt * yum_db_file_prepare (sqlite3 *db, GError **err) { int rc; sqlite3_stmt *handle = NULL; const char *query; query = "INSERT INTO files (name, type, pkgKey) VALUES (?, ?, ?)"; rc = sqlite3_prepare (db, query, -1, &handle, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not prepare file insertion: %s", sqlite3_errmsg (db)); sqlite3_finalize (handle); handle = NULL; } return handle; } void yum_db_file_write (sqlite3 *db, sqlite3_stmt *handle, gint64 pkgKey, PackageFile *file) { int rc; sqlite3_bind_text (handle, 1, file->name, -1, SQLITE_STATIC); sqlite3_bind_text (handle, 2, file->type, -1, SQLITE_STATIC); sqlite3_bind_int (handle, 3, pkgKey); rc = sqlite3_step (handle); sqlite3_reset (handle); if (rc != SQLITE_DONE) g_critical ("Error adding package file to SQL: %s", sqlite3_errmsg (db)); } void yum_db_create_filelist_tables (sqlite3 *db, GError **err) { int rc; const char *sql; sql = "CREATE TABLE packages (" " pkgKey INTEGER PRIMARY KEY," " pkgId TEXT)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create packages table: %s", sqlite3_errmsg (db)); return; } sql = "CREATE TABLE filelist (" " pkgKey INTEGER," " dirname TEXT," " filenames TEXT," " filetypes TEXT)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create filelist table: %s", sqlite3_errmsg (db)); return; } sql = "CREATE TRIGGER remove_filelist AFTER DELETE ON packages" " BEGIN" " DELETE FROM filelist WHERE pkgKey = old.pkgKey;" " END;"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create remove_filelist trigger: %s", sqlite3_errmsg (db)); return; } } void yum_db_index_filelist_tables (sqlite3 *db, GError **err) { int rc; const char *sql; sql = "CREATE INDEX IF NOT EXISTS keyfile ON filelist (pkgKey)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create keyfile index: %s", sqlite3_errmsg (db)); return; } sql = "CREATE INDEX IF NOT EXISTS pkgId ON packages (pkgId)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create pkgId index: %s", sqlite3_errmsg (db)); return; } sql = "CREATE INDEX IF NOT EXISTS dirnames ON filelist (dirname)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create dirnames index: %s", sqlite3_errmsg (db)); return; } } sqlite3_stmt * yum_db_package_ids_prepare (sqlite3 *db, GError **err) { int rc; sqlite3_stmt *handle = NULL; const char *query; query = "INSERT INTO packages (pkgId) VALUES (?)"; rc = sqlite3_prepare (db, query, -1, &handle, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not prepare package ids insertion: %s", sqlite3_errmsg (db)); sqlite3_finalize (handle); handle = NULL; } return handle; } void yum_db_package_ids_write (sqlite3 *db, sqlite3_stmt *handle, Package *p) { int rc; sqlite3_bind_text (handle, 1, p->pkgId, -1, SQLITE_STATIC); rc = sqlite3_step (handle); sqlite3_reset (handle); if (rc != SQLITE_DONE) { g_critical ("Error adding package to SQL: %s", sqlite3_errmsg (db)); } else p->pkgKey = sqlite3_last_insert_rowid (db); } sqlite3_stmt * yum_db_filelists_prepare (sqlite3 *db, GError **err) { int rc; sqlite3_stmt *handle = NULL; const char *query; query = "INSERT INTO filelist (pkgKey, dirname, filenames, filetypes) " " VALUES (?, ?, ?, ?)"; rc = sqlite3_prepare (db, query, -1, &handle, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not prepare filelist insertion: %s", sqlite3_errmsg (db)); sqlite3_finalize (handle); handle = NULL; } return handle; } typedef struct { sqlite3 *db; sqlite3_stmt *handle; gint64 pkgKey; } FileWriteInfo; static void write_file (gpointer key, gpointer value, gpointer user_data) { EncodedPackageFile *file = (EncodedPackageFile *) value; FileWriteInfo *info = (FileWriteInfo *) user_data; int rc; sqlite3_bind_int (info->handle, 1, info->pkgKey); sqlite3_bind_text (info->handle, 2, (const char *) key, -1, SQLITE_STATIC); sqlite3_bind_text (info->handle, 3, file->files->str, -1, SQLITE_STATIC); sqlite3_bind_text (info->handle, 4, file->types->str, -1, SQLITE_STATIC); rc = sqlite3_step (info->handle); sqlite3_reset (info->handle); if (rc != SQLITE_DONE) { g_critical ("Error adding file to SQL: %s", sqlite3_errmsg (info->db)); } } void yum_db_filelists_write (sqlite3 *db, sqlite3_stmt *handle, Package *p) { GHashTable *hash; FileWriteInfo info; info.db = db; info.handle = handle; info.pkgKey = p->pkgKey; hash = package_files_to_hash (p->files); g_hash_table_foreach (hash, write_file, &info); g_hash_table_destroy (hash); } void yum_db_create_other_tables (sqlite3 *db, GError **err) { int rc; const char *sql; sql = "CREATE TABLE packages (" " pkgKey INTEGER PRIMARY KEY," " pkgId TEXT)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create packages table: %s", sqlite3_errmsg (db)); return; } sql = "CREATE TABLE changelog (" " pkgKey INTEGER," " author TEXT," " date INTEGER," " changelog TEXT)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create changelog table: %s", sqlite3_errmsg (db)); return; } sql = "CREATE TRIGGER remove_changelogs AFTER DELETE ON packages" " BEGIN" " DELETE FROM changelog WHERE pkgKey = old.pkgKey;" " END;"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create remove_changelogs trigger: %s", sqlite3_errmsg (db)); return; } } void yum_db_index_other_tables (sqlite3 *db, GError **err) { int rc; const char *sql; sql = "CREATE INDEX IF NOT EXISTS keychange ON changelog (pkgKey)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create keychange index: %s", sqlite3_errmsg (db)); return; } sql = "CREATE INDEX IF NOT EXISTS pkgId ON packages (pkgId)"; rc = sqlite3_exec (db, sql, NULL, NULL, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not create pkgId index: %s", sqlite3_errmsg (db)); return; } } sqlite3_stmt * yum_db_changelog_prepare (sqlite3 *db, GError **err) { int rc; sqlite3_stmt *handle = NULL; const char *query; query = "INSERT INTO changelog (pkgKey, author, date, changelog) " " VALUES (?, ?, ?, ?)"; rc = sqlite3_prepare (db, query, -1, &handle, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not prepare changelog insertion: %s", sqlite3_errmsg (db)); sqlite3_finalize (handle); handle = NULL; } return handle; } void yum_db_changelog_write (sqlite3 *db, sqlite3_stmt *handle, Package *p) { GSList *iter; ChangelogEntry *entry; int rc; for (iter = p->changelogs; iter; iter = iter->next) { entry = (ChangelogEntry *) iter->data; sqlite3_bind_int (handle, 1, p->pkgKey); sqlite3_bind_text (handle, 2, entry->author, -1, SQLITE_STATIC); sqlite3_bind_int (handle, 3, entry->date); sqlite3_bind_text (handle, 4, entry->changelog, -1, SQLITE_STATIC); rc = sqlite3_step (handle); sqlite3_reset (handle); if (rc != SQLITE_DONE) { g_critical ("Error adding changelog to SQL: %s", sqlite3_errmsg (db)); } } } yum-metadata-parser-1.1.4/db.h0000664000076400007640000000760211062025345016026 0ustar skvidalskvidal/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ /* This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License, * version 2, as published by the Free Software Foundation * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA * 02111-1307, USA. */ #ifndef __YUM_DB_H__ #define __YUM_DB_H__ #include #include #include "package.h" #define YUM_SQLITE_CACHE_DBVERSION 10 #define YUM_DB_ERROR yum_db_error_quark() GQuark yum_db_error_quark (void); typedef void (*CreateTablesFn) (sqlite3 *db, GError **err); char *yum_db_filename (const char *prefix); sqlite3 *yum_db_open (const char *path, const char *checksum, CreateTablesFn create_tables, GError **err); void yum_db_dbinfo_update (sqlite3 *db, const char *checksum, GError **err); GHashTable *yum_db_read_package_ids (sqlite3 *db, GError **err); /* Primary */ void yum_db_create_primary_tables (sqlite3 *db, GError **err); void yum_db_index_primary_tables (sqlite3 *db, GError **err); sqlite3_stmt *yum_db_package_prepare (sqlite3 *db, GError **err); void yum_db_package_write (sqlite3 *db, sqlite3_stmt *handle, Package *p); sqlite3_stmt *yum_db_dependency_prepare (sqlite3 *db, const char *table, GError **err); void yum_db_dependency_write (sqlite3 *db, sqlite3_stmt *handle, gint64 pkgKey, Dependency *dep, gboolean isRequirement); sqlite3_stmt *yum_db_file_prepare (sqlite3 *db, GError **err); void yum_db_file_write (sqlite3 *db, sqlite3_stmt *handle, gint64 pkgKey, PackageFile *file); /* Filelists */ void yum_db_create_filelist_tables (sqlite3 *db, GError **err); void yum_db_index_filelist_tables (sqlite3 *db, GError **err); sqlite3_stmt *yum_db_package_ids_prepare (sqlite3 *db, GError **err); void yum_db_package_ids_write (sqlite3 *db, sqlite3_stmt *handle, Package *p); sqlite3_stmt *yum_db_filelists_prepare (sqlite3 *db, GError **err); void yum_db_filelists_write (sqlite3 *db, sqlite3_stmt *handle, Package *p); /* Other */ void yum_db_create_other_tables (sqlite3 *db, GError **err); void yum_db_index_other_tables (sqlite3 *db, GError **err); sqlite3_stmt *yum_db_changelog_prepare (sqlite3 *db, GError **err); void yum_db_changelog_write (sqlite3 *db, sqlite3_stmt *handle, Package *p); #endif /* __YUM_DB_H__ */ yum-metadata-parser-1.1.4/setup.py0000664000076400007640000000177111321445411017001 0ustar skvidalskvidalimport os, string from distutils.core import setup, Extension pc = os.popen("pkg-config --cflags-only-I glib-2.0 libxml-2.0 sqlite3", "r") includes = map(lambda x:x[2:], string.split(pc.readline())) pc.close() pc = os.popen("pkg-config --libs-only-l glib-2.0 libxml-2.0 sqlite3", "r") libs = map(lambda x:x[2:], string.split(pc.readline())) pc.close() pc = os.popen("pkg-config --libs-only-L glib-2.0 libxml-2.0 sqlite3", "r") libdirs = map(lambda x:x[2:], string.split(pc.readline())) pc.close() module = Extension('_sqlitecache', include_dirs = includes, libraries = libs, library_dirs = libdirs, sources = ['package.c', 'xml-parser.c', 'db.c', 'sqlitecache.c']) setup (name = 'yum-metadata-parser', version = '1.1.4', description = 'A fast YUM meta-data parser', py_modules = ['sqlitecachec'], ext_modules = [module]) yum-metadata-parser-1.1.4/MANIFEST0000664000076400007640000000024611321445476016427 0ustar skvidalskvidalAUTHORS ChangeLog MANIFEST MANIFEST.in README db.c db.h package.c package.h setup.py sqlitecache.c sqlitecachec.py xml-parser.c xml-parser.h yum-metadata-parser.spec yum-metadata-parser-1.1.4/package.c0000664000076400007640000000441311055334426017031 0ustar skvidalskvidal/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ /* This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License, * version 2, as published by the Free Software Foundation * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA * 02111-1307, USA. */ #include "package.h" #define PACKAGE_CHUNK_SIZE 2048 Dependency * dependency_new (void) { Dependency *dep; dep = g_new0 (Dependency, 1); return dep; } PackageFile * package_file_new (void) { PackageFile *file; file = g_new0 (PackageFile, 1); return file; } ChangelogEntry * changelog_entry_new (void) { ChangelogEntry *entry; entry = g_new0 (ChangelogEntry, 1); return entry; } Package * package_new (void) { Package *package; package = g_new0 (Package, 1); package->chunk = g_string_chunk_new (PACKAGE_CHUNK_SIZE); return package; } void package_free (Package *package) { g_string_chunk_free (package->chunk); if (package->requires) { g_slist_foreach (package->requires, (GFunc) g_free, NULL); g_slist_free (package->requires); } if (package->provides) { g_slist_foreach (package->provides, (GFunc) g_free, NULL); g_slist_free (package->provides); } if (package->conflicts) { g_slist_foreach (package->conflicts, (GFunc) g_free, NULL); g_slist_free (package->conflicts); } if (package->obsoletes) { g_slist_foreach (package->obsoletes, (GFunc) g_free, NULL); g_slist_free (package->obsoletes); } if (package->files) { g_slist_foreach (package->files, (GFunc) g_free, NULL); g_slist_free (package->files); } if (package->changelogs) { g_slist_foreach (package->changelogs, (GFunc) g_free, NULL); g_slist_free (package->changelogs); } g_free (package); } yum-metadata-parser-1.1.4/README0000664000076400007640000000122211055334426016145 0ustar skvidalskvidalYUM metadata parser written in C. * Why? The biggest complaint people have with YUM is often the performance of parsing the metadata. This implementation should be ~10 times faster, parsing the primary.xml file under 1 second usually, filelists.xml under 3 seconds and other.xml under 4 seconds. It uses a lot less memory as well, some testings I have done show it uses ~4mb instead of 40mb standard YUM uses. * How? Should be really easy: python setup.py build sudo python setup.py install --prefix=/usr (Assuming you python prefix is /usr). The next time you use yum, it regenerates the sqlitecache because the database schema is slightly different. yum-metadata-parser-1.1.4/MANIFEST.in0000664000076400007640000000015211055334426017024 0ustar skvidalskvidalinclude README ChangeLog AUTHORS include MANIFEST.in MANIFEST include *.c *.h include *.py include *.spec yum-metadata-parser-1.1.4/xml-parser.h0000664000076400007640000000325311055334426017536 0ustar skvidalskvidal/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ /* This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License, * version 2, as published by the Free Software Foundation * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA * 02111-1307, USA. */ #ifndef __YUM_XML_PARSER_H__ #define __YUM_XML_PARSER_H__ #include "package.h" typedef void (*CountFn) (guint32 count, gpointer data); #define YUM_PARSER_ERROR yum_parser_error_quark() GQuark yum_parser_error_quark (void); void yum_xml_parse_primary (const char *filename, CountFn count_callback, PackageFn package_callback, gpointer user_data, GError **err); void yum_xml_parse_filelists (const char *filename, CountFn count_callback, PackageFn package_callback, gpointer user_data, GError **err); void yum_xml_parse_other (const char *filename, CountFn count_callback, PackageFn package_callback, gpointer user_data, GError **err); #endif /* __YUM_XML_PARSER_H__ */ yum-metadata-parser-1.1.4/ChangeLog0000664000076400007640000000556611321445402017047 0ustar skvidalskvidal2010-01-07 Seth Vidal * setup.py, yum-metadata-parser.spec: mark as 1.1.3 2009-11-02 James Antill * db.c: Add an index on files.pkgKey ... needed for .simpleFiles() to not suck 2009-08-18 Mike Bonnet * sqlitecachec.py: configure sqlite to return utf-8-encoded strs instead of unicode objects sqlite by default returns all text as unicode objects, and this causes a number of problems when merging repos which contain utf-8 characters in Provides or Requires (as the current F11/F12 repos do). For a testcase, try merging 2 F12 repos, and you should see it fail with a UnicodeDecodeError in packages.py:_dump_pco(). This patch instructs sqlite to return all text as utf-8-encoded strs, which avoids these encoding issues. 2008-10-14 James Antill * db.c: Turn off .sqlite updating from new .xml data, bug 465898 2008-09-10 Seth Vidal * db.c, db.h, sqlitecache.c: commit patches from Ville Skyttä to make indexes after the data has been inserted. Closed rh bug 461403 2008-09-10 Seth Vidal * sqlitecache.c: apply patch to improve error messages from Ville Skyttä from rh bug 461405 2008-01-25 Seth Vidal * sqlitecachec.py: apply patch from Panu Matilainen to setup the db for exclusive lock write access. closes rh bug 353171 2007-11-27 Paul Nasrat * sqlitecache.c: Fix segmentation fault experienced with a malformed primary.xml 2007-08-29 Seth Vidal * yum-metadata-parser.spec: remove %dist which doesn't really belong anyway 2007-08-24 Seth Vidal * ChangeLog: changelog merge 2007-08-24 Seth Vidal * setup.py, yum-metadata-parser.spec: bump version number 2007-08-24 Seth Vidal * db.c: commit Florian's patches to create more indexes in the sqlite files made by yum-metadata-parser 2007-07-03 James Bowes * xml-parser.c: Fix segfault in the xml parser 2007-06-03 James Bowes * xml-parser.c: Use a common sax_error function 2007-06-03 James Bowes * xml-parser.c: Use a common sax_characters function 2007-06-03 James Bowes * xml-parser.c: Use SAXContext for other 2007-06-03 James Bowes * xml-parser.c: Use SAXContext for filelists 2007-06-03 James Bowes * xml-parser.c: Extract out a common set of SAXContext members for the three file types, and use them with primary. 2007-06-03 James Bowes * xml-parser.c: Use a common sax warning callback for the three file types. 2007-05-30 James Bowes * Get SAX error callbacks for filelists and other to use the right context type. yum-metadata-parser-1.1.4/sqlitecachec.py0000664000076400007640000000471611242544023020274 0ustar skvidalskvidal# This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. try: import sqlite3 as sqlite except ImportError: import sqlite import _sqlitecache DBVERSION = _sqlitecache.DBVERSION class RepodataParserSqlite: def __init__(self, storedir, repoid, callback=None): self.callback = callback self.repoid = repoid def open_database(self, filename): if not filename: return None con = sqlite.connect(filename) con.text_factory = str if sqlite.version_info[0] > 1: con.row_factory = sqlite.Row cur = con.cursor() cur.execute("pragma locking_mode = EXCLUSIVE") del cur return con def getPrimary(self, location, checksum): """Load primary.xml.gz from an sqlite cache and update it if required""" return self.open_database(_sqlitecache.update_primary(location, checksum, self.callback, self.repoid)) def getFilelists(self, location, checksum): """Load filelist.xml.gz from an sqlite cache and update it if required""" return self.open_database(_sqlitecache.update_filelist(location, checksum, self.callback, self.repoid)) def getOtherdata(self, location, checksum): """Load other.xml.gz from an sqlite cache and update it if required""" return self.open_database(_sqlitecache.update_other(location, checksum, self.callback, self.repoid)) yum-metadata-parser-1.1.4/package.h0000664000076400007640000000421411055334426017035 0ustar skvidalskvidal/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ /* This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License, * version 2, as published by the Free Software Foundation * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA * 02111-1307, USA. */ #ifndef __YUM_PACKAGE_H__ #define __YUM_PACKAGE_H__ #include typedef struct { char *name; char *flags; char *epoch; char *version; char *release; gboolean pre; } Dependency; typedef struct { char *type; char *name; } PackageFile; typedef struct { char *author; gint64 date; char *changelog; } ChangelogEntry; typedef struct { gint64 pkgKey; char *pkgId; char *name; char *arch; char *version; char *epoch; char *release; char *summary; char *description; char *url; gint64 time_file; gint64 time_build; char *rpm_license; char *rpm_vendor; char *rpm_group; char *rpm_buildhost; char *rpm_sourcerpm; gint64 rpm_header_start; gint64 rpm_header_end; char *rpm_packager; gint64 size_package; gint64 size_installed; gint64 size_archive; char *location_href; char *location_base; char *checksum_type; GSList *requires; GSList *provides; GSList *conflicts; GSList *obsoletes; GSList *files; GSList *changelogs; GStringChunk *chunk; } Package; typedef void (*PackageFn) (Package *pkg, gpointer data); Dependency *dependency_new (void); PackageFile *package_file_new (void); ChangelogEntry *changelog_entry_new (void); Package *package_new (void); void package_free (Package *package); #endif /* __YUM_PACKAGE_H__ */ yum-metadata-parser-1.1.4/sqlitecache.c0000664000076400007640000004331111062025345017716 0ustar skvidalskvidal/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ /* This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License, * version 2, as published by the Free Software Foundation * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA * 02111-1307, USA. */ #include #include "xml-parser.h" #include "db.h" #include "package.h" /* Make room for 2500 package ids, 40 bytes + '\0' each */ #define PACKAGE_IDS_CHUNK 41 * 2500 typedef struct _UpdateInfo UpdateInfo; typedef void (*InfoInitFn) (UpdateInfo *update_info, sqlite3 *db, GError **err); typedef void (*InfoCleanFn) (UpdateInfo *update_info); typedef void (*XmlParseFn) (const char *filename, CountFn count_callback, PackageFn package_callback, gpointer user_data, GError **err); typedef void (*WriteDbPackageFn) (UpdateInfo *update_info, Package *package); typedef void (*IndexTablesFn) (sqlite3 *db, GError **err); struct _UpdateInfo { sqlite3 *db; sqlite3_stmt *remove_handle; guint32 count_from_md; guint32 packages_seen; guint32 add_count; guint32 del_count; GHashTable *current_packages; GHashTable *all_packages; GStringChunk *package_ids_chunk; GTimer *timer; gpointer python_callback; InfoInitFn info_init; InfoCleanFn info_clean; CreateTablesFn create_tables; WriteDbPackageFn write_package; XmlParseFn xml_parse; IndexTablesFn index_tables; gpointer user_data; }; static void update_info_init (UpdateInfo *info, GError **err) { const char *sql; int rc; sql = "DELETE FROM packages WHERE pkgKey = ?"; rc = sqlite3_prepare (info->db, sql, -1, &info->remove_handle, NULL); if (rc != SQLITE_OK) { g_set_error (err, YUM_DB_ERROR, YUM_DB_ERROR, "Can not prepare package removal: %s", sqlite3_errmsg (info->db)); sqlite3_finalize (info->remove_handle); return; } info->count_from_md = 0; info->packages_seen = 0; info->add_count = 0; info->del_count = 0; info->all_packages = g_hash_table_new (g_str_hash, g_str_equal); info->package_ids_chunk = g_string_chunk_new (PACKAGE_IDS_CHUNK); info->timer = g_timer_new (); g_timer_start (info->timer); info->current_packages = yum_db_read_package_ids (info->db, err); } static void remove_entry (gpointer key, gpointer value, gpointer user_data) { UpdateInfo *info = (UpdateInfo *) user_data; if (g_hash_table_lookup (info->all_packages, key) == NULL) { int rc; sqlite3_bind_int (info->remove_handle, 1, GPOINTER_TO_INT (value)); rc = sqlite3_step (info->remove_handle); sqlite3_reset (info->remove_handle); if (rc != SQLITE_DONE) g_warning ("Error removing package from SQL: %s", sqlite3_errmsg (info->db)); info->del_count++; } } static void update_info_remove_old_entries (UpdateInfo *info) { g_hash_table_foreach (info->current_packages, remove_entry, info); } static void count_cb (guint32 count, gpointer user_data) { UpdateInfo *info = (UpdateInfo *) user_data; info->count_from_md = count; } static void update_info_done (UpdateInfo *info, GError **err) { if (info->remove_handle) sqlite3_finalize (info->remove_handle); if (info->current_packages) g_hash_table_destroy (info->current_packages); if (info->all_packages) g_hash_table_destroy (info->all_packages); if (info->package_ids_chunk) g_string_chunk_free (info->package_ids_chunk); g_timer_stop (info->timer); if (!*err) { g_message ("Added %d new packages, deleted %d old in %.2f seconds", info->add_count, info->del_count, g_timer_elapsed (info->timer, NULL)); } g_timer_destroy (info->timer); } /* Primary */ typedef struct { UpdateInfo update_info; sqlite3_stmt *pkg_handle; sqlite3_stmt *requires_handle; sqlite3_stmt *provides_handle; sqlite3_stmt *conflicts_handle; sqlite3_stmt *obsoletes_handle; sqlite3_stmt *files_handle; } PackageWriterInfo; static void package_writer_info_init (UpdateInfo *update_info, sqlite3 *db, GError **err) { PackageWriterInfo *info = (PackageWriterInfo *) update_info; info->pkg_handle = yum_db_package_prepare (db, err); if (*err) return; info->requires_handle = yum_db_dependency_prepare (db, "requires", err); if (*err) return; info->provides_handle = yum_db_dependency_prepare (db, "provides", err); if (*err) return; info->conflicts_handle = yum_db_dependency_prepare (db, "conflicts", err); if (*err) return; info->obsoletes_handle = yum_db_dependency_prepare (db, "obsoletes", err); if (*err) return; info->files_handle = yum_db_file_prepare (db, err); } static void write_deps (sqlite3 *db, sqlite3_stmt *handle, gint64 pkgKey, GSList *deps) { GSList *iter; for (iter = deps; iter; iter = iter->next) yum_db_dependency_write (db, handle, pkgKey, (Dependency *) iter->data, FALSE); } static void write_requirements (sqlite3 *db, sqlite3_stmt *handle, gint64 pkgKey, GSList *deps) { GSList *iter; for (iter = deps; iter; iter = iter->next) yum_db_dependency_write (db, handle, pkgKey, (Dependency *) iter->data, TRUE); } static void write_files (sqlite3 *db, sqlite3_stmt *handle, Package *pkg) { GSList *iter; for (iter = pkg->files; iter; iter = iter->next) yum_db_file_write (db, handle, pkg->pkgKey, (PackageFile *) iter->data); } static void write_package_to_db (UpdateInfo *update_info, Package *package) { PackageWriterInfo *info = (PackageWriterInfo *) update_info; yum_db_package_write (update_info->db, info->pkg_handle, package); write_requirements (update_info->db, info->requires_handle, package->pkgKey, package->requires); write_deps (update_info->db, info->provides_handle, package->pkgKey, package->provides); write_deps (update_info->db, info->conflicts_handle, package->pkgKey, package->conflicts); write_deps (update_info->db, info->obsoletes_handle, package->pkgKey, package->obsoletes); write_files (update_info->db, info->files_handle, package); } static void package_writer_info_clean (UpdateInfo *update_info) { PackageWriterInfo *info = (PackageWriterInfo *) update_info; if (info->pkg_handle) sqlite3_finalize (info->pkg_handle); if (info->requires_handle) sqlite3_finalize (info->requires_handle); if (info->provides_handle) sqlite3_finalize (info->provides_handle); if (info->conflicts_handle) sqlite3_finalize (info->conflicts_handle); if (info->obsoletes_handle) sqlite3_finalize (info->obsoletes_handle); if (info->files_handle) sqlite3_finalize (info->files_handle); } /* Filelists */ typedef struct { UpdateInfo update_info; sqlite3_stmt *pkg_handle; sqlite3_stmt *file_handle; } FileListInfo; static void update_filelist_info_init (UpdateInfo *update_info, sqlite3 *db, GError **err) { FileListInfo *info = (FileListInfo *) update_info; info->pkg_handle = yum_db_package_ids_prepare (db, err); if (*err) return; info->file_handle = yum_db_filelists_prepare (db, err); } static void update_filelist_info_clean (UpdateInfo *update_info) { FileListInfo *info = (FileListInfo *) update_info; if (info->pkg_handle) sqlite3_finalize (info->pkg_handle); if (info->file_handle) sqlite3_finalize (info->file_handle); } static void write_filelist_package_to_db (UpdateInfo *update_info, Package *package) { FileListInfo *info = (FileListInfo *) update_info; yum_db_package_ids_write (update_info->db, info->pkg_handle, package); yum_db_filelists_write (update_info->db, info->file_handle, package); } /* Other */ typedef struct { UpdateInfo update_info; sqlite3_stmt *pkg_handle; sqlite3_stmt *changelog_handle; } UpdateOtherInfo; static void update_other_info_init (UpdateInfo *update_info, sqlite3 *db, GError **err) { UpdateOtherInfo *info = (UpdateOtherInfo *) update_info; info->pkg_handle = yum_db_package_ids_prepare (db, err); if (*err) return; info->changelog_handle = yum_db_changelog_prepare (db, err); } static void update_other_info_clean (UpdateInfo *update_info) { UpdateOtherInfo *info = (UpdateOtherInfo *) update_info; if (info->pkg_handle) sqlite3_finalize (info->pkg_handle); if (info->changelog_handle) sqlite3_finalize (info->changelog_handle); } static void write_other_package_to_db (UpdateInfo *update_info, Package *package) { UpdateOtherInfo *info = (UpdateOtherInfo *) update_info; yum_db_package_ids_write (update_info->db, info->pkg_handle, package); yum_db_changelog_write (update_info->db, info->changelog_handle, package); } /*****************************************************************************/ static void progress_cb (UpdateInfo *update_info) { PyObject *progress = (PyObject *) update_info->python_callback; PyObject *repoid = (PyObject *) update_info->user_data; PyObject *args; PyObject *result; Py_INCREF(repoid); args = PyTuple_New (3); PyTuple_SET_ITEM (args, 0, PyInt_FromLong (update_info->packages_seen)); PyTuple_SET_ITEM (args, 1, PyInt_FromLong (update_info->count_from_md)); PyTuple_SET_ITEM (args, 2, repoid); result = PyEval_CallObject (progress, args); Py_DECREF (args); Py_XDECREF (result); } static void update_package_cb (Package *p, gpointer user_data) { UpdateInfo *update_info = (UpdateInfo *) user_data; /* TODO: Wire in logging of skipped packages */ if (p->pkgId == NULL) { return; } g_hash_table_insert (update_info->all_packages, g_string_chunk_insert (update_info->package_ids_chunk, p->pkgId), GINT_TO_POINTER (1)); if (g_hash_table_lookup (update_info->current_packages, p->pkgId) == NULL) { update_info->write_package (update_info, p); update_info->add_count++; } if (update_info->count_from_md > 0 && update_info->python_callback) { update_info->packages_seen++; progress_cb (update_info); } } static char * update_packages (UpdateInfo *update_info, const char *md_filename, const char *checksum, gpointer python_callback, gpointer user_data, GError **err) { char *db_filename; db_filename = yum_db_filename (md_filename); update_info->db = yum_db_open (db_filename, checksum, update_info->create_tables, err); if (*err) goto cleanup; if (!update_info->db) return db_filename; update_info_init (update_info, err); if (*err) goto cleanup; update_info->python_callback = python_callback; update_info->user_data = user_data; update_info->info_init (update_info, update_info->db, err); if (*err) goto cleanup; sqlite3_exec (update_info->db, "BEGIN", NULL, NULL, NULL); update_info->xml_parse (md_filename, count_cb, update_package_cb, update_info, err); if (*err) goto cleanup; sqlite3_exec (update_info->db, "COMMIT", NULL, NULL, NULL); update_info->index_tables (update_info->db, err); if (*err) goto cleanup; update_info_remove_old_entries (update_info); yum_db_dbinfo_update (update_info->db, checksum, err); cleanup: update_info->info_clean (update_info); update_info_done (update_info, err); if (update_info->db) sqlite3_close (update_info->db); if (*err) { g_free (db_filename); db_filename = NULL; } return db_filename; } /*********************************************************************/ static gboolean py_parse_args (PyObject *args, const char **md_filename, const char **checksum, PyObject **log, PyObject **progress, PyObject **repoid) { PyObject *callback; if (!PyArg_ParseTuple (args, "ssOO", md_filename, checksum, &callback, repoid)) return FALSE; if (PyObject_HasAttrString (callback, "log")) { *log = PyObject_GetAttrString (callback, "log"); if (!PyCallable_Check (*log)) { PyErr_SetString (PyExc_TypeError, "parameter must be callable"); return FALSE; } } if (PyObject_HasAttrString (callback, "progressbar")) { *progress = PyObject_GetAttrString (callback, "progressbar"); if (!PyCallable_Check (*progress)) { PyErr_SetString (PyExc_TypeError, "parameter must be callable"); return FALSE; } } return TRUE; } static void log_cb (const gchar *log_domain, GLogLevelFlags log_level, const gchar *message, gpointer user_data) { PyObject *callback = (PyObject *) user_data; int level; PyObject *args; PyObject *result; if (!callback) return; args = PyTuple_New (2); switch (log_level) { case G_LOG_LEVEL_DEBUG: level = 2; break; case G_LOG_LEVEL_MESSAGE: level = 1; break; case G_LOG_LEVEL_WARNING: level = 0; break; case G_LOG_LEVEL_CRITICAL: default: level = -1; break; } PyTuple_SET_ITEM (args, 0, PyInt_FromLong (level)); PyTuple_SET_ITEM (args, 1, PyString_FromString (message)); result = PyEval_CallObject (callback, args); Py_DECREF (args); Py_XDECREF (result); } static PyObject * py_update (PyObject *self, PyObject *args, UpdateInfo *update_info) { const char *md_filename = NULL; const char *checksum = NULL; PyObject *log = NULL; PyObject *progress = NULL; PyObject *repoid = NULL; guint log_id = 0; char *db_filename; PyObject *ret = NULL; GError *err = NULL; if (!py_parse_args (args, &md_filename, &checksum, &log, &progress, &repoid)) return NULL; GLogLevelFlags level = G_LOG_LEVEL_MESSAGE | G_LOG_LEVEL_WARNING | G_LOG_LEVEL_CRITICAL | G_LOG_LEVEL_DEBUG; log_id = g_log_set_handler (NULL, level, log_cb, log); db_filename = update_packages (update_info, md_filename, checksum, progress, repoid, &err); g_log_remove_handler (NULL, log_id); if (db_filename) { ret = PyString_FromString (db_filename); g_free (db_filename); } else { PyErr_SetString (PyExc_TypeError, err->message); g_error_free (err); } return ret; } static PyObject * py_update_primary (PyObject *self, PyObject *args) { PackageWriterInfo info; memset (&info, 0, sizeof (PackageWriterInfo)); info.update_info.info_init = package_writer_info_init; info.update_info.info_clean = package_writer_info_clean; info.update_info.create_tables = yum_db_create_primary_tables; info.update_info.write_package = write_package_to_db; info.update_info.xml_parse = yum_xml_parse_primary; info.update_info.index_tables = yum_db_index_primary_tables; return py_update (self, args, (UpdateInfo *) &info); } static PyObject * py_update_filelist (PyObject *self, PyObject *args) { FileListInfo info; memset (&info, 0, sizeof (FileListInfo)); info.update_info.info_init = update_filelist_info_init; info.update_info.info_clean = update_filelist_info_clean; info.update_info.create_tables = yum_db_create_filelist_tables; info.update_info.write_package = write_filelist_package_to_db; info.update_info.xml_parse = yum_xml_parse_filelists; info.update_info.index_tables = yum_db_index_filelist_tables; return py_update (self, args, (UpdateInfo *) &info); } static PyObject * py_update_other (PyObject *self, PyObject *args) { UpdateOtherInfo info; memset (&info, 0, sizeof (UpdateOtherInfo)); info.update_info.info_init = update_other_info_init; info.update_info.info_clean = update_other_info_clean; info.update_info.create_tables = yum_db_create_other_tables; info.update_info.write_package = write_other_package_to_db; info.update_info.xml_parse = yum_xml_parse_other; info.update_info.index_tables = yum_db_index_other_tables; return py_update (self, args, (UpdateInfo *) &info); } static PyMethodDef SqliteMethods[] = { {"update_primary", py_update_primary, METH_VARARGS, "Parse YUM primary.xml metadata."}, {"update_filelist", py_update_filelist, METH_VARARGS, "Parse YUM filelists.xml metadata."}, {"update_other", py_update_other, METH_VARARGS, "Parse YUM other.xml metadata."}, {NULL, NULL, 0, NULL} }; PyMODINIT_FUNC init_sqlitecache (void) { PyObject * m, * d; m = Py_InitModule ("_sqlitecache", SqliteMethods); d = PyModule_GetDict(m); PyDict_SetItemString(d, "DBVERSION", PyInt_FromLong(YUM_SQLITE_CACHE_DBVERSION)); } yum-metadata-parser-1.1.4/xml-parser.c0000664000076400007640000007346611055334426017546 0ustar skvidalskvidal/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ /* This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License, * version 2, as published by the Free Software Foundation * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA * 02111-1307, USA. */ #include #include #include #include #include #include "xml-parser.h" #define PACKAGE_FIELD_SIZE 1024 GQuark yum_parser_error_quark (void) { static GQuark quark; if (!quark) quark = g_quark_from_static_string ("yum_parser_error"); return quark; } static guint32 string_to_guint32_with_default (const char *n, guint32 def) { char *ret; guint32 z; z = strtoul (n, &ret, 10); if (*ret != '\0') return def; else return z; } typedef struct { const char *md_type; xmlParserCtxt *xml_context; GError **error; CountFn count_fn; PackageFn package_fn; gpointer user_data; Package *current_package; gboolean want_text; GString *text_buffer; } SAXContext; typedef enum { PRIMARY_PARSER_TOPLEVEL = 0, PRIMARY_PARSER_PACKAGE, PRIMARY_PARSER_FORMAT, PRIMARY_PARSER_DEP, } PrimarySAXContextState; typedef struct { SAXContext sctx; PrimarySAXContextState state; GSList **current_dep_list; PackageFile *current_file; } PrimarySAXContext; static void primary_parser_toplevel_start (PrimarySAXContext *ctx, const char *name, const char **attrs) { SAXContext *sctx = &ctx->sctx; if (!strcmp (name, "package")) { g_assert (sctx->current_package == NULL); ctx->state = PRIMARY_PARSER_PACKAGE; sctx->current_package = package_new (); } else if (sctx->count_fn && !strcmp (name, "metadata")) { int i; const char *attr; const char *value; for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "packages")) { sctx->count_fn (string_to_guint32_with_default (value, 0), sctx->user_data); break; } } } } static void parse_version_info(const char **attrs, Package *p) { int i; const char *attr; const char *value; for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "epoch")) p->epoch = g_string_chunk_insert (p->chunk, value); else if (!strcmp (attr, "ver")) p->version = g_string_chunk_insert (p->chunk, value); else if (!strcmp (attr, "rel")) p->release = g_string_chunk_insert (p->chunk, value); } } static void primary_parser_package_start (PrimarySAXContext *ctx, const char *name, const char **attrs) { SAXContext *sctx = &ctx->sctx; Package *p = sctx->current_package; int i; const char *attr; const char *value; g_assert (p != NULL); sctx->want_text = TRUE; if (!strcmp (name, "format")) { ctx->state = PRIMARY_PARSER_FORMAT; } else if (!strcmp (name, "version")) { parse_version_info(attrs, p); } else if (!strcmp (name, "checksum")) { for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "type")) p->checksum_type = g_string_chunk_insert (p->chunk, value); } } else if (!strcmp (name, "time")) { for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "file")) p->time_file = strtol(value, NULL, 10); else if (!strcmp (attr, "build")) p->time_build = strtol(value, NULL, 10); } } else if (!strcmp (name, "size")) { for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "package")) p->size_package = strtol(value, NULL, 10); else if (!strcmp (attr, "installed")) p->size_installed = strtol(value, NULL, 10); else if (!strcmp (attr, "archive")) p->size_archive = strtol(value, NULL, 10); } } else if (!strcmp (name, "location")) { for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "href")) p->location_href = g_string_chunk_insert (p->chunk, value); else if (!strcmp (attr, "xml:base")) p->location_base = g_string_chunk_insert (p->chunk, value); } } } static void primary_parser_format_start (PrimarySAXContext *ctx, const char *name, const char **attrs) { SAXContext *sctx = &ctx->sctx; Package *p = sctx->current_package; int i; const char *attr; const char *value; g_assert (p != NULL); if (!strcmp (name, "rpm:header-range")) { for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "start")) p->rpm_header_start = strtol(value, NULL, 10); else if (!strcmp (attr, "end")) p->rpm_header_end = strtol(value, NULL, 10); } } else if (!strcmp (name, "rpm:provides")) { ctx->state = PRIMARY_PARSER_DEP; ctx->current_dep_list = &sctx->current_package->provides; } else if (!strcmp (name, "rpm:requires")) { ctx->state = PRIMARY_PARSER_DEP; ctx->current_dep_list = &sctx->current_package->requires; } else if (!strcmp (name, "rpm:obsoletes")) { ctx->state = PRIMARY_PARSER_DEP; ctx->current_dep_list = &sctx->current_package->obsoletes; } else if (!strcmp (name, "rpm:conflicts")) { ctx->state = PRIMARY_PARSER_DEP; ctx->current_dep_list = &sctx->current_package->conflicts; } else if (!strcmp (name, "file")) { for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "type")) { ctx->current_file = package_file_new (); ctx->current_file->type = g_string_chunk_insert_const (p->chunk, value); } } } } static void primary_parser_dep_start (PrimarySAXContext *ctx, const char *name, const char **attrs) { SAXContext *sctx = &ctx->sctx; const char *tmp_name = NULL; const char *tmp_version = NULL; const char *tmp_release = NULL; const char *tmp_epoch = NULL; const char *tmp_flags = NULL; gboolean tmp_pre = FALSE; Dependency *dep; int i; gboolean ignore = FALSE; const char *attr; const char *value; if (!strcmp (name, "rpm:entry")) { for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "name")) { if (!strncmp (value, "rpmlib(", strlen ("rpmlib("))) { ignore = TRUE; break; } tmp_name = value; } else if (!strcmp (attr, "flags")) tmp_flags = value; else if (!strcmp (attr, "epoch")) tmp_epoch = value; else if (!strcmp (attr, "ver")) tmp_version = value; else if (!strcmp (attr, "rel")) tmp_release = value; else if (!strcmp (attr, "pre")) tmp_pre = TRUE; } if (!ignore) { GStringChunk *chunk = sctx->current_package->chunk; dep = dependency_new (); dep->name = g_string_chunk_insert (chunk, tmp_name); if (tmp_flags) dep->flags = g_string_chunk_insert (chunk, tmp_flags); if (tmp_epoch) dep->epoch = g_string_chunk_insert (chunk, tmp_epoch); if (tmp_version) dep->version = g_string_chunk_insert (chunk, tmp_version); if (tmp_release) dep->release = g_string_chunk_insert (chunk, tmp_release); dep->pre = tmp_pre; *ctx->current_dep_list = g_slist_prepend (*ctx->current_dep_list, dep); } } } static void primary_sax_start_element (void *data, const char *name, const char **attrs) { PrimarySAXContext *ctx = (PrimarySAXContext *) data; SAXContext *sctx = &ctx->sctx; if (sctx->text_buffer->len) g_string_truncate (sctx->text_buffer, 0); switch (ctx->state) { case PRIMARY_PARSER_TOPLEVEL: primary_parser_toplevel_start (ctx, name, attrs); break; case PRIMARY_PARSER_PACKAGE: primary_parser_package_start (ctx, name, attrs); break; case PRIMARY_PARSER_FORMAT: primary_parser_format_start (ctx, name, attrs); break; case PRIMARY_PARSER_DEP: primary_parser_dep_start (ctx, name, attrs); break; default: break; } } static void primary_parser_package_end (PrimarySAXContext *ctx, const char *name) { SAXContext *sctx = &ctx->sctx; Package *p = sctx->current_package; g_assert (p != NULL); if (!strcmp (name, "package")) { if (sctx->package_fn && !*sctx->error) sctx->package_fn (p, sctx->user_data); package_free (p); sctx->current_package = NULL; sctx->want_text = FALSE; ctx->state = PRIMARY_PARSER_TOPLEVEL; } else if (sctx->text_buffer->len == 0) /* Nothing interesting to do here */ return; else if (!strcmp (name, "name")) p->name = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); else if (!strcmp (name, "arch")) p->arch = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); else if (!strcmp (name, "checksum")) p->pkgId = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); else if (!strcmp (name, "summary")) p->summary = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); else if (!strcmp (name, "description")) p->description = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); else if (!strcmp (name, "packager")) p->rpm_packager = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); else if (!strcmp (name, "url")) p->url = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); } static void primary_parser_format_end (PrimarySAXContext *ctx, const char *name) { SAXContext *sctx = &ctx->sctx; Package *p = sctx->current_package; g_assert (p != NULL); if (!strcmp (name, "rpm:license")) p->rpm_license = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); if (!strcmp (name, "rpm:vendor")) p->rpm_vendor = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); if (!strcmp (name, "rpm:group")) p->rpm_group = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); if (!strcmp (name, "rpm:buildhost")) p->rpm_buildhost = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); if (!strcmp (name, "rpm:sourcerpm")) p->rpm_sourcerpm = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); else if (!strcmp (name, "file")) { PackageFile *file = ctx->current_file != NULL ? ctx->current_file : package_file_new (); file->name = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); if (!file->type) file->type = g_string_chunk_insert_const (p->chunk, "file"); p->files = g_slist_prepend (p->files, file); ctx->current_file = NULL; } else if (!strcmp (name, "format")) ctx->state = PRIMARY_PARSER_PACKAGE; } static void primary_parser_dep_end (PrimarySAXContext *ctx, const char *name) { SAXContext *sctx = &ctx->sctx; g_assert (sctx->current_package != NULL); if (strcmp (name, "rpm:entry")) ctx->state = PRIMARY_PARSER_FORMAT; } static void primary_sax_end_element (void *data, const char *name) { PrimarySAXContext *ctx = (PrimarySAXContext *) data; SAXContext *sctx = &ctx->sctx; switch (ctx->state) { case PRIMARY_PARSER_PACKAGE: primary_parser_package_end (ctx, name); break; case PRIMARY_PARSER_FORMAT: primary_parser_format_end (ctx, name); break; case PRIMARY_PARSER_DEP: primary_parser_dep_end (ctx, name); break; default: break; } g_string_truncate (sctx->text_buffer, 0); } static void sax_characters (void *data, const char *ch, int len) { SAXContext *sctx = (SAXContext *) data; if (sctx->want_text) g_string_append_len (sctx->text_buffer, ch, len); } static void sax_warning (void *data, const char *msg, ...) { va_list args; char *tmp; va_start (args, msg); tmp = g_strdup_vprintf (msg, args); g_warning ("* SAX Warning: %s", tmp); g_free (tmp); va_end (args); } static void sax_error (void *data, const char *msg, ...) { SAXContext *sctx = (SAXContext *) data; va_list args; char *tmp; va_start (args, msg); tmp = g_strdup_vprintf (msg, args); g_set_error (sctx->error, YUM_PARSER_ERROR, YUM_PARSER_ERROR, "Parsing %s error: %s", sctx->md_type, tmp); g_free (tmp); va_end (args); } static xmlSAXHandler primary_sax_handler = { NULL, /* internalSubset */ NULL, /* isStandalone */ NULL, /* hasInternalSubset */ NULL, /* hasExternalSubset */ NULL, /* resolveEntity */ NULL, /* getEntity */ NULL, /* entityDecl */ NULL, /* notationDecl */ NULL, /* attributeDecl */ NULL, /* elementDecl */ NULL, /* unparsedEntityDecl */ NULL, /* setDocumentLocator */ NULL, /* startDocument */ NULL, /* endDocument */ (startElementSAXFunc) primary_sax_start_element, /* startElement */ (endElementSAXFunc) primary_sax_end_element, /* endElement */ NULL, /* reference */ (charactersSAXFunc) sax_characters, /* characters */ NULL, /* ignorableWhitespace */ NULL, /* processingInstruction */ NULL, /* comment */ sax_warning, /* warning */ sax_error, /* error */ sax_error, /* fatalError */ }; void sax_context_init (SAXContext *sctx, const char *md_type, CountFn count_callback, PackageFn package_callback, gpointer user_data, GError **err) { sctx->md_type = md_type; sctx->error = err; sctx->count_fn = count_callback; sctx->package_fn = package_callback; sctx->user_data = user_data; sctx->current_package = NULL; sctx->want_text = FALSE; sctx->text_buffer = g_string_sized_new (PACKAGE_FIELD_SIZE); } void yum_xml_parse_primary (const char *filename, CountFn count_callback, PackageFn package_callback, gpointer user_data, GError **err) { PrimarySAXContext ctx; SAXContext *sctx = &ctx.sctx; int rc; ctx.state = PRIMARY_PARSER_TOPLEVEL; ctx.current_dep_list = NULL; ctx.current_file = NULL; sax_context_init(sctx, "primary.xml", count_callback, package_callback, user_data, err); xmlSubstituteEntitiesDefault (1); rc = xmlSAXUserParseFile (&primary_sax_handler, &ctx, filename); if (sctx->current_package) { g_warning ("Incomplete package lost"); package_free (sctx->current_package); } g_string_free (sctx->text_buffer, TRUE); } /*****************************************************************************/ static void parse_package (const char **attrs, Package *p) { int i; const char *attr; const char *value; for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "pkgid")) p->pkgId = g_string_chunk_insert (p->chunk, value); if (!strcmp (attr, "name")) p->name = g_string_chunk_insert (p->chunk, value); else if (!strcmp (attr, "arch")) p->arch = g_string_chunk_insert (p->chunk, value); } } typedef enum { FILELIST_PARSER_TOPLEVEL = 0, FILELIST_PARSER_PACKAGE, } FilelistSAXContextState; typedef struct { SAXContext sctx; FilelistSAXContextState state; PackageFile *current_file; } FilelistSAXContext; static void filelist_parser_toplevel_start (FilelistSAXContext *ctx, const char *name, const char **attrs) { SAXContext *sctx = &ctx->sctx; if (!strcmp (name, "package")) { g_assert (sctx->current_package == NULL); ctx->state = FILELIST_PARSER_PACKAGE; sctx->current_package = package_new (); parse_package (attrs, sctx->current_package); } else if (sctx->count_fn && !strcmp (name, "filelists")) { int i; const char *attr; const char *value; for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "packages")) { sctx->count_fn (string_to_guint32_with_default (value, 0), sctx->user_data); break; } } } } static void filelist_parser_package_start (FilelistSAXContext *ctx, const char *name, const char **attrs) { SAXContext *sctx = &ctx->sctx; Package *p = sctx->current_package; int i; const char *attr; const char *value; g_assert (p != NULL); sctx->want_text = TRUE; if (!strcmp (name, "version")) { parse_version_info(attrs, p); } else if (!strcmp (name, "file")) { ctx->current_file = package_file_new (); for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "type")) ctx->current_file->type = g_string_chunk_insert_const (p->chunk, value); } } } static void filelist_sax_start_element (void *data, const char *name, const char **attrs) { FilelistSAXContext *ctx = (FilelistSAXContext *) data; SAXContext *sctx = &ctx->sctx; if (sctx->text_buffer->len) g_string_truncate (sctx->text_buffer, 0); switch (ctx->state) { case FILELIST_PARSER_TOPLEVEL: filelist_parser_toplevel_start (ctx, name, attrs); break; case FILELIST_PARSER_PACKAGE: filelist_parser_package_start (ctx, name, attrs); break; default: break; } } static void filelist_parser_package_end (FilelistSAXContext *ctx, const char *name) { SAXContext *sctx = &ctx->sctx; Package *p = sctx->current_package; g_assert (p != NULL); sctx->want_text = FALSE; if (!strcmp (name, "package")) { if (sctx->package_fn && !*sctx->error) sctx->package_fn (p, sctx->user_data); package_free (p); sctx->current_package = NULL; if (ctx->current_file) { g_free (ctx->current_file); ctx->current_file = NULL; } ctx->state = FILELIST_PARSER_TOPLEVEL; } else if (!strcmp (name, "file")) { PackageFile *file = ctx->current_file; file->name = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); if (!file->type) file->type = g_string_chunk_insert_const (p->chunk, "file"); p->files = g_slist_prepend (p->files, file); ctx->current_file = NULL; } } static void filelist_sax_end_element (void *data, const char *name) { FilelistSAXContext *ctx = (FilelistSAXContext *) data; SAXContext *sctx = &ctx->sctx; switch (ctx->state) { case FILELIST_PARSER_PACKAGE: filelist_parser_package_end (ctx, name); break; default: break; } g_string_truncate (sctx->text_buffer, 0); } static xmlSAXHandler filelist_sax_handler = { NULL, /* internalSubset */ NULL, /* isStandalone */ NULL, /* hasInternalSubset */ NULL, /* hasExternalSubset */ NULL, /* resolveEntity */ NULL, /* getEntity */ NULL, /* entityDecl */ NULL, /* notationDecl */ NULL, /* attributeDecl */ NULL, /* elementDecl */ NULL, /* unparsedEntityDecl */ NULL, /* setDocumentLocator */ NULL, /* startDocument */ NULL, /* endDocument */ (startElementSAXFunc) filelist_sax_start_element, /* startElement */ (endElementSAXFunc) filelist_sax_end_element, /* endElement */ NULL, /* reference */ (charactersSAXFunc) sax_characters, /* characters */ NULL, /* ignorableWhitespace */ NULL, /* processingInstruction */ NULL, /* comment */ sax_warning, /* warning */ sax_error, /* error */ sax_error, /* fatalError */ }; void yum_xml_parse_filelists (const char *filename, CountFn count_callback, PackageFn package_callback, gpointer user_data, GError **err) { FilelistSAXContext ctx; SAXContext *sctx = &ctx.sctx; int rc; ctx.state = FILELIST_PARSER_TOPLEVEL; ctx.current_file = NULL; sax_context_init(sctx, "filelists.xml", count_callback, package_callback, user_data, err); xmlSubstituteEntitiesDefault (1); rc = xmlSAXUserParseFile (&filelist_sax_handler, &ctx, filename); if (sctx->current_package) { g_warning ("Incomplete package lost"); package_free (sctx->current_package); } if (ctx.current_file) g_free (ctx.current_file); g_string_free (sctx->text_buffer, TRUE); } /*****************************************************************************/ typedef enum { OTHER_PARSER_TOPLEVEL = 0, OTHER_PARSER_PACKAGE, } OtherSAXContextState; typedef struct { SAXContext sctx; OtherSAXContextState state; ChangelogEntry *current_entry; } OtherSAXContext; static void other_parser_toplevel_start (OtherSAXContext *ctx, const char *name, const char **attrs) { SAXContext *sctx = &ctx->sctx; if (!strcmp (name, "package")) { g_assert (sctx->current_package == NULL); ctx->state = OTHER_PARSER_PACKAGE; sctx->current_package = package_new (); parse_package (attrs, sctx->current_package); } else if (sctx->count_fn && !strcmp (name, "otherdata")) { int i; const char *attr; const char *value; for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "packages")) { sctx->count_fn (string_to_guint32_with_default (value, 0), sctx->user_data); break; } } } } static void other_parser_package_start (OtherSAXContext *ctx, const char *name, const char **attrs) { SAXContext *sctx = &ctx->sctx; Package *p = sctx->current_package; int i; const char *attr; const char *value; g_assert (p != NULL); sctx->want_text = TRUE; if (!strcmp (name, "version")) { parse_version_info(attrs, p); } else if (!strcmp (name, "changelog")) { ctx->current_entry = changelog_entry_new (); for (i = 0; attrs && attrs[i]; i++) { attr = attrs[i]; value = attrs[++i]; if (!strcmp (attr, "author")) ctx->current_entry->author = g_string_chunk_insert_const (p->chunk, value); else if (!strcmp (attr, "date")) ctx->current_entry->date = strtol(value, NULL, 10); } } } static void other_sax_start_element (void *data, const char *name, const char **attrs) { OtherSAXContext *ctx = (OtherSAXContext *) data; SAXContext *sctx = &ctx->sctx; if (sctx->text_buffer->len) g_string_truncate (sctx->text_buffer, 0); switch (ctx->state) { case OTHER_PARSER_TOPLEVEL: other_parser_toplevel_start (ctx, name, attrs); break; case OTHER_PARSER_PACKAGE: other_parser_package_start (ctx, name, attrs); break; default: break; } } static void other_parser_package_end (OtherSAXContext *ctx, const char *name) { SAXContext *sctx = &ctx->sctx; Package *p = sctx->current_package; g_assert (p != NULL); sctx->want_text = FALSE; if (!strcmp (name, "package")) { if (p->changelogs) p->changelogs = g_slist_reverse (p->changelogs); if (sctx->package_fn && !*sctx->error) sctx->package_fn (p, sctx->user_data); package_free (p); sctx->current_package = NULL; if (ctx->current_entry) { g_free (ctx->current_entry); ctx->current_entry = NULL; } ctx->state = OTHER_PARSER_TOPLEVEL; } else if (!strcmp (name, "changelog")) { ctx->current_entry->changelog = g_string_chunk_insert_len (p->chunk, sctx->text_buffer->str, sctx->text_buffer->len); p->changelogs = g_slist_prepend (p->changelogs, ctx->current_entry); ctx->current_entry = NULL; } } static void other_sax_end_element (void *data, const char *name) { OtherSAXContext *ctx = (OtherSAXContext *) data; SAXContext *sctx = &ctx->sctx; switch (ctx->state) { case OTHER_PARSER_PACKAGE: other_parser_package_end (ctx, name); break; default: break; } g_string_truncate (sctx->text_buffer, 0); } static xmlSAXHandler other_sax_handler = { NULL, /* internalSubset */ NULL, /* isStandalone */ NULL, /* hasInternalSubset */ NULL, /* hasExternalSubset */ NULL, /* resolveEntity */ NULL, /* getEntity */ NULL, /* entityDecl */ NULL, /* notationDecl */ NULL, /* attributeDecl */ NULL, /* elementDecl */ NULL, /* unparsedEntityDecl */ NULL, /* setDocumentLocator */ NULL, /* startDocument */ NULL, /* endDocument */ (startElementSAXFunc) other_sax_start_element, /* startElement */ (endElementSAXFunc) other_sax_end_element, /* endElement */ NULL, /* reference */ (charactersSAXFunc) sax_characters, /* characters */ NULL, /* ignorableWhitespace */ NULL, /* processingInstruction */ NULL, /* comment */ sax_warning, /* warning */ sax_error, /* error */ sax_error, /* fatalError */ }; void yum_xml_parse_other (const char *filename, CountFn count_callback, PackageFn package_callback, gpointer user_data, GError **err) { OtherSAXContext ctx; SAXContext *sctx = &ctx.sctx; int rc; ctx.state = OTHER_PARSER_TOPLEVEL; ctx.current_entry = NULL; sax_context_init(sctx, "other.xml", count_callback, package_callback, user_data, err); xmlSubstituteEntitiesDefault (1); rc = xmlSAXUserParseFile (&other_sax_handler, &ctx, filename); if (sctx->current_package) { g_warning ("Incomplete package lost"); package_free (sctx->current_package); } if (ctx.current_entry) g_free (ctx.current_entry); g_string_free (sctx->text_buffer, TRUE); } yum-metadata-parser-1.1.4/PKG-INFO0000664000076400007640000000032511321445743016366 0ustar skvidalskvidalMetadata-Version: 1.0 Name: yum-metadata-parser Version: 1.1.4 Summary: A fast YUM meta-data parser Home-page: UNKNOWN Author: UNKNOWN Author-email: UNKNOWN License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN yum-metadata-parser-1.1.4/AUTHORS0000664000076400007640000000014011055334426016333 0ustar skvidalskvidalJames Bowes Florian Festi Tambet Ingo Jeremy Katz Paul Nasrat Seth Vidal