/*
 * Copyright (C) 2016-2025 Matthias Klumpp <matthias@tenstral.net>
 *
 * Licensed under the GNU Lesser General Public License Version 3
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the license, or
 * (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this software.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "defines.h"
#include "engine.h"

#include <algorithm>
#include <chrono>
#include <execution>
#include <filesystem>
#include <format>
#include <iostream>
#include <string>
#include <string_view>
#include <thread>
#include <unordered_set>

#include <appstream.h>
#include <glib.h>

#include <tbb/parallel_for.h>
#include <tbb/parallel_for_each.h>
#include <tbb/parallel_invoke.h>
#include <tbb/blocked_range.h>
#include <tbb/task_arena.h>
#include <inja/inja.hpp>

#include "datainjectpkg.h"
#include "extractor.h"
#include "hintregistry.h"
#include "logging.h"
#include "result.h"
#include "utils.h"
#include "yaml-utils.h"
#include "zarchive.h"
#include "backends/interfaces.h"

// Backends
#include "backends/dummy/pkgindex.h"
#include "backends/debian/debpkgindex.h"
#include "backends/ubuntu/ubupkgindex.h"
#include "backends/alpinelinux/apkpkgindex.h"
#include "backends/archlinux/alpkgindex.h"
#include "backends/rpmmd/rpmpkgindex.h"
#include "backends/freebsd/fbsdpkgindex.h"

namespace ASGenerator
{

Engine::Engine()
    : m_conf(&Config::get()),
      m_log(getLogger("engine")),
      m_forced(false)
{
    // Configure a TBB task arena to limit parallelism a little (use half the available CPU cores, or at least 6
    // threads) This avoids having too many parallel downloads on high-core-count machines, and also leaves some room
    // for additional parallelism of the used libraries, e.g. for image processing.
    const auto numCPU = std::thread::hardware_concurrency();
    const auto maxThreads = std::max((long)numCPU > 6 ? 6L : numCPU, std::lround(numCPU * 0.60));
    m_taskArena = std::make_unique<tbb::task_arena>(maxThreads);

    // Select backend
    switch (m_conf->backend) {
    case Backend::Dummy:
        m_pkgIndex = std::make_unique<DummyPackageIndex>(m_conf->archiveRoot);
        break;
    case Backend::Debian:
        m_pkgIndex = std::make_unique<DebianPackageIndex>(m_conf->archiveRoot);
        break;
    case Backend::Ubuntu:
        m_pkgIndex = std::make_unique<UbuntuPackageIndex>(m_conf->archiveRoot);
        break;
    case Backend::Archlinux:
        m_pkgIndex = std::make_unique<ArchPackageIndex>(m_conf->archiveRoot);
        break;
    case Backend::RpmMd:
        m_pkgIndex = std::make_unique<RPMPackageIndex>(m_conf->archiveRoot);
        break;
    case Backend::Alpinelinux:
        m_pkgIndex = std::make_unique<AlpinePackageIndex>(m_conf->archiveRoot);
        break;
    case Backend::FreeBSD:
        m_pkgIndex = std::make_unique<FreeBSDPackageIndex>(m_conf->archiveRoot);
        break;
    default:
        throw std::runtime_error("No backend specified, can not continue!");
    }

    // set preferred prefix for this backend
    m_backendPathPrefix = Utils::normalizePath(m_pkgIndex->dataPrefix());
    m_backendPrefixNotUsr = m_backendPathPrefix != "/usr";

    // Load global registry of issue hint templates
    loadHintsRegistry();

    // Create cache in cache directory on workspace
    m_dstore = std::make_shared<DataStore>();
    m_dstore->open(*m_conf);

    // Open package contents cache
    m_cstore = std::make_shared<ContentsStore>(m_backendPathPrefix);
    m_cstore->open(*m_conf);

    // Open statistics database
    m_sstore = std::make_shared<StatsStore>();
    m_sstore->open(*m_conf);

    // move statistics that older versions of the generator kept in the main database over
    if (const auto legacyStats = m_dstore->takeLegacyStatistics(); !legacyStats.empty()) {
        LOG_INFO(m_log, "Migrating {} statistics entries to the statistics database.", legacyStats.size());
        for (const auto &entry : legacyStats)
            m_sstore->addStatistics(entry);
    }
}

bool Engine::forced() const
{
    return m_forced;
}

void Engine::setForced(bool v)
{
    m_forced = v;
}

void Engine::logVersionInfo()
{
    std::string backendInfo = "";
    if (!m_conf->backendName.empty())
        backendInfo = std::format(" [{}]", m_conf->backendName);

    // Get AppStream version
    const char *asVersion = as_version_string();
    LOG_INFO(logRoot, "AppStream Generator {}, AS: {}{}", ASGEN_VERSION, asVersion, backendInfo);
}

void Engine::checkLibfyamlVersion()
{
    const auto version = Yaml::libfyamlVersion();
    if (version.empty()) {
        LOG_WARNING(
            m_log, "Unable to determine libfyaml version! Please ensure you have at least version 0.9 installed!");
        return;
    }

    if (as_vercmp_simple(version.c_str(), "0.9") < 0) {
        throw std::runtime_error(
            std::format(
                "libfyaml version {} is too old, at least version 0.9 is required! Please upgrade libfyaml, as "
                "versions <= "
                "0.8 have known bugs in unicode handling and may generate corrupted UTF-8 data.",
                version));
    }
}

void Engine::processPackages(
    const std::vector<std::shared_ptr<Package>> &pkgs,
    std::shared_ptr<IconHandler> iconh,
    std::shared_ptr<InjectedModifications> injMods,
    AscImageFormat imageFormat)
{
    g_autoptr(AsgLocaleUnit) localeUnit = asg_locale_unit_new(m_cstore, pkgs);

    {
        g_autoptr(GError) error = NULL;
        if (!asc_unit_open(ASC_UNIT(localeUnit), &error))
            throw std::runtime_error(
                std::format("Failed to open locale unit: {}", error ? error->message : "Unknown error"));
    }

    const auto numProcessors = std::thread::hardware_concurrency();
    std::size_t chunkSize = pkgs.size() / numProcessors / 10;
    if (chunkSize > 100)
        chunkSize = 100;
    if (chunkSize <= 10)
        chunkSize = 10;

    LOG_DEBUG(
        m_log,
        "Analyzing {} packages in batches of {} with {} parallel tasks",
        pkgs.size(),
        chunkSize,
        m_taskArena->max_concurrency());

    m_taskArena->execute([&] {
        tbb::parallel_for(
            tbb::blocked_range<std::size_t>(0, pkgs.size(), chunkSize),
            [&](const tbb::blocked_range<std::size_t> &range) {
                auto mde = std::make_unique<DataExtractor>(
                    m_dstore,
                    iconh,
                    localeUnit,
                    imageFormat,
                    injMods,
                    m_backendPrefixNotUsr ? m_backendPathPrefix : "");

                for (std::size_t i = range.begin(); i != range.end(); ++i) {
                    auto pkg = pkgs[i];
                    const auto &pkid = pkg->id();

                    if (m_dstore->packageExists(pkid))
                        continue;

                    auto res = mde->processPackage(pkg);
                    {
                        std::lock_guard<std::mutex> lock(m_mutex);
                        // Write resulting data into the database
                        m_dstore->addGeneratorResult(m_conf->metadataType, res);
                    }

                    LOG_INFO(
                        m_log,
                        "Processed {}, components: {}, hints: {}",
                        res.pkid(),
                        res.componentsCount(),
                        res.hintsCount());

                    // We don't need content data from this package anymore
                    pkg->finish();
                }
            });
    });

    // not strictly necessary, but let's close the unit explicitly
    asc_unit_close(ASC_UNIT(localeUnit));
}

// Helper to check if a package may contain interesting metadata
bool Engine::packageIsInteresting(std::shared_ptr<Package> pkg)
{
    // Common prefixes that we will always check
    constexpr std::string_view usr_share_apps = "/usr/share/applications/";
    constexpr std::string_view usr_share_meta = "/usr/share/metainfo/";
    constexpr std::string_view usr_share = "/usr/share/";

    // Additional data to check if a custom prefix has been set
    std::string extra_apps_path;
    std::string extra_meta_path;
    if (m_backendPrefixNotUsr) {
        extra_apps_path = m_backendPathPrefix + "/share/applications/";
        extra_meta_path = m_backendPathPrefix + "/share/metainfo/";
    }

    const auto &contents = pkg->contents();
    for (const auto &c : contents) {
        // Quick length check first - all interesting paths are at least 16 characters
        if (c.size() < 16)
            continue;

        std::string_view path_view{c};
        if (path_view.starts_with(usr_share)) {
            // Already know it starts with /usr/share/, check specific subdirs
            if (path_view.starts_with(usr_share_apps) || path_view.starts_with(usr_share_meta))
                return true;
        } else if (m_backendPrefixNotUsr && path_view.starts_with(m_backendPathPrefix)) {
            // Already know it starts with the backend prefix, check specific subdirs
            if (path_view.starts_with(extra_apps_path) || path_view.starts_with(extra_meta_path))
                return true;
        }
    }

    // Check for GStreamer codec info
    auto gst = pkg->gst();
    if (gst.has_value() && gst->isNotEmpty())
        return true;

    return false;
}

bool Engine::seedContentsData(
    const Suite &suite,
    const std::string &section,
    const std::string &arch,
    const std::vector<std::shared_ptr<Package>> &pkgs)
{
    const auto numProcessors = std::thread::hardware_concurrency();
    std::size_t workUnitSize = numProcessors * 2;
    if (workUnitSize >= pkgs.size())
        workUnitSize = 4;
    if (workUnitSize > 30)
        workUnitSize = 30;

    LOG_DEBUG(
        m_log,
        "Scanning {} packages, work unit size: {}, parallel tasks: {}",
        pkgs.size(),
        workUnitSize,
        m_taskArena->max_concurrency());

    // Check if the index has changed data, skip the update if there's nothing new
    if (pkgs.empty() && !m_pkgIndex->hasChanges(m_dstore, suite.name, section, arch) && !m_forced) {
        LOG_DEBUG(
            m_log, "Skipping contents cache update for {}/{} [{}], index has not changed.", suite.name, section, arch);
        return false;
    }

    LOG_INFO(m_log, "Scanning new packages for {}/{} [{}]", suite.name, section, arch);

    std::vector<std::shared_ptr<Package>> packagesToProcess = pkgs;
    if (packagesToProcess.empty())
        packagesToProcess = m_pkgIndex->packagesFor(suite.name, section, arch);

    // Get contents information for packages and add them to the database
    std::atomic_bool interestingFound = false;

    // First get the contents (only) of all packages in the base suite
    if (!suite.baseSuite.empty()) {
        LOG_INFO(m_log, "Scanning new packages for base suite {}/{} [{}]", suite.baseSuite, section, arch);
        auto baseSuitePkgs = m_pkgIndex->packagesFor(suite.baseSuite, section, arch);

        m_taskArena->execute([&] {
            tbb::parallel_for(
                tbb::blocked_range<std::size_t>(0, baseSuitePkgs.size(), workUnitSize),
                [&](const tbb::blocked_range<std::size_t> &range) {
                    for (size_t i = range.begin(); i != range.end(); ++i) {
                        auto pkg = baseSuitePkgs[i];
                        const auto &pkid = pkg->id();

                        if (!m_cstore->packageExists(pkid)) {
                            m_cstore->addContents(pkid, pkg->contents());
                            LOG_INFO(m_log, "Scanned {} for base suite.", pkid);
                        }

                        // Chances are that we might never want to extract data from these packages, so remove their
                        // temporary data for now - we can reopen the packages later if we actually need them.
                        pkg->cleanupTemp();
                    }
                });
        });
    }

    // And then scan the suite itself - here packages can be 'interesting'
    // in that they might end up in the output.
    m_taskArena->execute([&] {
        tbb::parallel_for(
            tbb::blocked_range<std::size_t>(0, packagesToProcess.size(), workUnitSize),
            [&](const tbb::blocked_range<std::size_t> &range) {
                for (std::size_t i = range.begin(); i != range.end(); ++i) {
                    auto pkg = packagesToProcess[i];
                    const auto &pkid = pkg->id();

                    std::vector<std::string> contents;
                    if (m_cstore->packageExists(pkid)) {
                        if (m_dstore->packageExists(pkid)) {
                            // TODO: Unfortunately, packages can move between suites without changing their ID.
                            // This means as soon as we have an interesting package, even if we already processed it,
                            // we need to regenerate the output metadata.
                            // For that to happen, we set interestingFound to true here. Later, a more elegant solution
                            // would be desirable here, ideally one which doesn't force us to track which package is
                            // in which suite as well.
                            if (!m_dstore->isIgnored(pkid))
                                interestingFound.store(true);
                            continue;
                        }
                        // We will complement the main database with ignore data, in case it
                        // went missing.
                        contents = m_cstore->getContents(pkid);
                    } else {
                        // Add contents to the index
                        contents = pkg->contents();
                        m_cstore->addContents(pkid, contents);
                    }

                    // Check if we can already mark this package as ignored, and print some log messages
                    if (!packageIsInteresting(pkg)) {
                        m_dstore->setPackageIgnore(pkid);
                        LOG_INFO(m_log, "Scanned {}, no interesting files found.", pkid);
                        // We won't use this anymore
                        pkg->finish();
                    } else {
                        LOG_INFO(m_log, "Scanned {}, could be interesting.", pkid);
                        interestingFound.store(true);
                    }
                }
            });
    });

    // Ensure the contents store is in a consistent state on disk,
    // since it might be accessed from other threads after this function
    // is run.
    m_cstore->sync();

    return interestingFound;
}

std::string Engine::getMetadataHead(const Suite &suite, const std::string &section)
{
    std::string head;

    // Determine origin as a lowercase string
    std::string origin;
    if (section.empty())
        origin = std::format("{}-{}", m_conf->projectName, suite.name);
    else
        origin = std::format("{}-{}-{}", m_conf->projectName, suite.name, section);
    std::transform(origin.begin(), origin.end(), origin.begin(), ::tolower);

    // Get current time in ISO8601 UTC
    auto now = std::chrono::floor<std::chrono::seconds>(std::chrono::system_clock::now());
    std::string timeNowIso8601 = std::format("{:%FT%TZ}", now);

    std::string mediaPoolUrl = fs::path(m_conf->mediaBaseUrl) / "pool";
    if (m_conf->feature.immutableSuites)
        mediaPoolUrl = std::format("{}/{}", m_conf->mediaBaseUrl, suite.name);

    const bool mediaBaseUrlAllowed = !m_conf->mediaBaseUrl.empty() && m_conf->feature.storeScreenshots;
    if (m_conf->metadataType == DataType::XML) {
        head = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";
        head += std::format("<components version=\"{}\" origin=\"{}\"", m_conf->formatVersionStr(), origin);
        if (suite.dataPriority != 0)
            head += std::format(" priority=\"{}\"", suite.dataPriority);
        if (mediaBaseUrlAllowed)
            head += std::format(" media_baseurl=\"{}\"", mediaPoolUrl);
        if (m_conf->feature.metadataTimestamps)
            head += std::format(" time=\"{}\"", timeNowIso8601);
        head += ">";
    } else {
        head = "%YAML 1.2\n---\n";
        head += std::format(
            "File: DEP-11\n"
            "Version: \"{}\"\n"
            "Origin: {}",
            m_conf->formatVersionStr(),
            origin);
        if (mediaBaseUrlAllowed)
            head += std::format("\nMediaBaseUrl: {}", mediaPoolUrl);
        if (suite.dataPriority != 0)
            head += std::format("\nPriority: {}", suite.dataPriority);
        if (m_conf->feature.metadataTimestamps)
            head += std::format("\nTime: \"{}\"", timeNowIso8601);
    }

    return head;
}

void Engine::exportMetadata(
    const Suite &suite,
    const std::string &section,
    const std::string &arch,
    const std::vector<std::shared_ptr<Package>> &pkgs)
{
    // the closing tag of an XML metadata document
    static constexpr std::string_view XmlDocumentEnd = "</components>\n";

    // we accumulate into plain strings rather than string streams: everything we append is
    // already a string, and this lets us reserve the final size once we know it
    std::string mdataFile;
    std::string hintsFile;

    // prepare hints file
    hintsFile += "[\n";

    LOG_INFO(m_log, "Exporting data for {} ({}/{})", suite.name, section, arch);

    // Add metadata document header
    mdataFile += getMetadataHead(suite, section);
    mdataFile += '\n';

    // Prepare destination
    const auto dataExportDir = m_conf->dataExportDir / suite.name / section;
    const auto hintsExportDir = m_conf->hintsExportDir / suite.name / section;

    fs::create_directories(dataExportDir);
    fs::create_directories(hintsExportDir);

    const bool useImmutableSuites = m_conf->feature.immutableSuites;
    // Select the media export target directory
    fs::path mediaExportDir;
    if (useImmutableSuites)
        mediaExportDir = m_dstore->mediaExportPoolDir().parent_path() / suite.name;
    else
        mediaExportDir = m_dstore->mediaExportPoolDir();

    // Collect metadata, icons and hints for the given packages.
    // Every package writes into its own slot instead of appending to the output directly.
    struct ComponentChunk {
        std::string cid;  // component-ID, empty if the GCID was malformed
        std::string gcid; // global component-ID
        std::string data; // the serialized metadata document, empty if we have none
    };

    std::vector<std::vector<ComponentChunk>> pkgComponents(pkgs.size());
    std::vector<std::string> pkgHints(pkgs.size());

    LOG_DEBUG(m_log, "Building final metadata and hints files.");

    tbb::parallel_for(static_cast<std::size_t>(0), pkgs.size(), [&](std::size_t i) {
        const auto &pkid = pkgs[i]->id();

        const auto gcids = m_dstore->getGCIDsForPackage(pkid);
        auto &components = pkgComponents[i];
        components.reserve(gcids.size());

        for (const auto &gcid : gcids) {
            const auto cid = Utils::getCidFromGlobalID(gcid);
            if (!cid.has_value())
                LOG_ERROR(m_log, "Could not extract component-ID from GCID: {}", gcid);

            components.emplace_back(
                cid.value_or(std::string{}), gcid, m_dstore->getMetadata(m_conf->metadataType, gcid));

            // Hardlink data from the pool to the suite-specific directories
            if (useImmutableSuites) {
                const auto gcidMediaPoolPath = m_dstore->mediaExportPoolDir() / gcid;
                const auto gcidMediaSuitePath = mediaExportDir / gcid;
                if (!fs::exists(gcidMediaSuitePath) && fs::exists(gcidMediaPoolPath))
                    Utils::copyDir(gcidMediaPoolPath.string(), gcidMediaSuitePath.string(), true);
            }
        }

        pkgHints[i] = m_dstore->getHints(pkid);
    });

    // merge the per-package results in a stable order: components by their component-ID, hints by package-ID
    std::unordered_map<std::string, std::string> cidGcidMap;

    std::vector<const ComponentChunk *> components;
    components.reserve(pkgs.size());
    std::size_t mdataSize = mdataFile.size();
    for (const auto &chunks : pkgComponents) {
        for (const auto &chunk : chunks) {
            components.push_back(&chunk);
            if (!chunk.data.empty())
                mdataSize += chunk.data.size() + 1; // the document plus its trailing newline
        }
    }
    if (m_conf->metadataType == DataType::XML)
        mdataSize += XmlDocumentEnd.size();
    mdataFile.reserve(mdataSize);

    // a malformed GCID has no component-ID to sort by, so fall back to the GCID itself
    const auto sortKey = [](const ComponentChunk *chunk) -> const std::string & {
        return chunk->cid.empty() ? chunk->gcid : chunk->cid;
    };
    std::ranges::stable_sort(components, [&](const auto *a, const auto *b) {
        const auto &keyA = sortKey(a);
        const auto &keyB = sortKey(b);
        if (keyA != keyB)
            return keyA < keyB;
        return a->gcid < b->gcid;
    });

    for (const auto chunk : components) {
        if (!chunk->data.empty()) {
            mdataFile += chunk->data;
            mdataFile += '\n';
        }
        if (!chunk->cid.empty())
            cidGcidMap[chunk->cid] = chunk->gcid;
    }

    std::vector<std::size_t> hintOrder;
    hintOrder.reserve(pkgs.size());
    for (std::size_t i = 0; i < pkgs.size(); ++i) {
        if (!pkgHints[i].empty())
            hintOrder.push_back(i);
    }
    std::sort(hintOrder.begin(), hintOrder.end(), [&](std::size_t a, std::size_t b) {
        return pkgs[a]->id() < pkgs[b]->id();
    });

    // an upper bound, as rtrimString() only ever shrinks its input
    std::size_t hintsSize = hintsFile.size() + sizeof("\n]\n");
    for (const auto i : hintOrder)
        hintsSize += pkgHints[i].size() + 2; // the entry plus its ",\n" separator
    hintsFile.reserve(hintsSize);

    bool firstHintEntry = true;
    for (const auto i : hintOrder) {
        if (firstHintEntry)
            firstHintEntry = false;
        else
            hintsFile += ",\n";
        hintsFile += Utils::rtrimString(pkgHints[i]);
    }

    fs::path dataBaseFname;
    if (m_conf->metadataType == DataType::XML)
        dataBaseFname = dataExportDir / std::format("Components-{}.xml", arch);
    else
        dataBaseFname = dataExportDir / std::format("Components-{}.yml", arch);

    const auto cidIndexFname = dataExportDir / std::format("CID-Index-{}.json", arch);
    const auto hintsBaseFname = hintsExportDir / std::format("Hints-{}.json", arch);

    // Write metadata
    LOG_INFO(m_log, "Writing metadata for {}/{} [{}]", suite.name, section, arch);

    // Add the closing XML tag for XML metadata
    if (m_conf->metadataType == DataType::XML)
        mdataFile += XmlDocumentEnd;

    // Compress metadata and save it to disk
    std::vector<std::uint8_t> mdataFileBytes(mdataFile.begin(), mdataFile.end());
    compressAndSave(mdataFileBytes, dataBaseFname.string() + ".gz", ArchiveType::GZIP);
    compressAndSave(mdataFileBytes, dataBaseFname.string() + ".xz", ArchiveType::XZ);

    // Component ID index
    inja::json cidIndexJson = inja::json::object();
    for (const auto &[cid, gcid] : cidGcidMap)
        cidIndexJson[cid] = gcid;

    auto cidIndexStr = cidIndexJson.dump(2); // Pretty print with 2-space indentation
    std::vector<std::uint8_t> cidIndexData(cidIndexStr.begin(), cidIndexStr.end());
    compressAndSave(cidIndexData, cidIndexFname.string() + ".gz", ArchiveType::GZIP);

    // Write hints
    LOG_INFO(m_log, "Writing hints for {}/{} [{}]", suite.name, section, arch);

    // Finalize the JSON hints document
    hintsFile += "\n]\n";

    // Compress hints
    std::vector<std::uint8_t> hintsFileBytes(hintsFile.begin(), hintsFile.end());
    compressAndSave(hintsFileBytes, hintsBaseFname.string() + ".gz", ArchiveType::GZIP);
    compressAndSave(hintsFileBytes, hintsBaseFname.string() + ".xz", ArchiveType::XZ);

    // Save a copy of the hints registry to be used by other tools
    // (this allows other apps to just resolve the hint tags to severities and explanations
    // without loading either AppStream or AppStream-Generator code)
    saveHintsRegistryToJsonFile((m_conf->hintsExportDir / suite.name / "hint-definitions.json").string());
}

void Engine::exportIconTarballs(
    const Suite &suite,
    const std::string &section,
    const std::vector<std::shared_ptr<Package>> &pkgs)
{
    // Determine data sources and destinations
    const auto dataExportDir = m_conf->dataExportDir / suite.name / section;
    fs::create_directories(dataExportDir);
    const bool useImmutableSuites = m_conf->feature.immutableSuites;
    const auto mediaExportDir = useImmutableSuites ? m_dstore->mediaExportPoolDir().parent_path() / suite.name
                                                   : m_dstore->mediaExportPoolDir();

    // Icons are always rendered in the image format the suite is configured for. If that
    // setting changed at some point, media directories may still hold files of the format we
    // used before, which no metadata we generate refers to any longer - so only the files in
    // the format we actually emit belong in the tarballs.
    const auto iconFileExt = std::format(".{}", asc_image_format_to_string(suite.imageFormat));

    // Prepare icon-tarball array
    std::unordered_map<std::string, std::vector<std::string>> iconTarFiles;

    // Initialize icon policy iterator
    AscIconPolicyIter policyIter;
    asc_icon_policy_iter_init(&policyIter, m_conf->iconPolicy());

    guint iconSizeInt;
    guint iconScale;
    AscIconState iconState;
    while (asc_icon_policy_iter_next(&policyIter, &iconSizeInt, &iconScale, &iconState)) {
        if (iconState == ASC_ICON_STATE_IGNORED || iconState == ASC_ICON_STATE_REMOTE_ONLY)
            continue; // We only want to create tarballs for cached icons

        const ImageSize iconSize(iconSizeInt, iconSizeInt, iconScale);
        iconTarFiles[iconSize.toString()] = std::vector<std::string>();
        iconTarFiles[iconSize.toString()].reserve(256);
    }

    LOG_INFO(m_log, "Creating icon tarballs for: {}/{}", suite.name, section);
    std::unordered_set<std::string> processedDirs;
    std::mutex dirMutex;
    std::mutex iconMutex;

    tbb::parallel_for_each(
        pkgs.begin(),
        pkgs.end(),
        [this, &mediaExportDir, &iconFileExt, &iconTarFiles, &processedDirs, &dirMutex, &iconMutex](
            std::shared_ptr<Package> pkg) {
            const auto &pkid = pkg->id();
            auto gcids = m_dstore->getGCIDsForPackage(pkid);
            if (gcids.empty())
                return;

            for (const auto &gcid : gcids) {
                // Compile list of icon-tarball files
                AscIconPolicyIter ipIter;
                asc_icon_policy_iter_init(&ipIter, m_conf->iconPolicy());

                guint taskIconSize;
                guint taskIconScale;
                AscIconState taskIconState;
                while (asc_icon_policy_iter_next(&ipIter, &taskIconSize, &taskIconScale, &taskIconState)) {
                    if (taskIconState == ASC_ICON_STATE_IGNORED || taskIconState == ASC_ICON_STATE_REMOTE_ONLY)
                        continue; // Only add icon to cache tarball if we want a cache for the particular size

                    const ImageSize iconSize(taskIconSize, taskIconSize, taskIconScale);
                    const auto iconDir = mediaExportDir / gcid / "icons" / iconSize.toString();

                    // Skip adding icon entries if we've already investigated this directory
                    {
                        std::lock_guard<std::mutex> lock(dirMutex);
                        if (processedDirs.contains(iconDir.string()))
                            continue;
                        else
                            processedDirs.insert(iconDir.string());
                    }

                    if (!fs::exists(iconDir))
                        continue;

                    for (const auto &entry : fs::directory_iterator(iconDir)) {
                        if (!entry.is_regular_file())
                            continue;
                        if (!entry.path().filename().string().ends_with(iconFileExt))
                            continue;

                        std::lock_guard<std::mutex> lock(iconMutex);
                        iconTarFiles[iconSize.toString()].push_back(entry.path().string());
                    }
                }
            }
        });

    // Create the icon tarballs
    asc_icon_policy_iter_init(&policyIter, m_conf->iconPolicy());
    while (asc_icon_policy_iter_next(&policyIter, &iconSizeInt, &iconScale, &iconState)) {
        if (iconState == ASC_ICON_STATE_IGNORED || iconState == ASC_ICON_STATE_REMOTE_ONLY)
            continue;

        const ImageSize iconSize(iconSizeInt, iconSizeInt, iconScale);
        auto iconTar = std::make_unique<ArchiveCompressor>(ArchiveType::GZIP);
        iconTar->open((dataExportDir / std::format("icons-{}.tar.gz", iconSize.toString())).string());

        // An icon tarball is flat, so its entries are addressed by their file name alone. One
        // package can render the same icon for several of its components, each into its own
        // media directory, which would put multiple files of that very same name in here - on
        // extraction they would only overwrite each other, so we just keep the first one.
        auto &iconFiles = iconTarFiles[iconSize.toString()];
        std::vector<std::pair<std::string, std::string>> uniqueIcons; // icon name -> file path
        uniqueIcons.reserve(iconFiles.size());
        for (const auto &fname : iconFiles)
            uniqueIcons.emplace_back(fs::path(fname).filename().string(), fname);

        std::ranges::sort(uniqueIcons);
        const auto duplicates = std::ranges::unique(uniqueIcons, {}, &std::pair<std::string, std::string>::first);
        uniqueIcons.erase(duplicates.begin(), duplicates.end());

        for (const auto &[iconName, fname] : uniqueIcons)
            iconTar->addFile(fname);

        iconTar->close();
    }
    LOG_INFO(m_log, "Icon tarballs built for: {}/{}", suite.name, section);
}

std::unordered_map<std::string, std::shared_ptr<Package>> Engine::getIconCandidatePackages(
    const Suite &suite,
    const std::string &section,
    const std::string &arch)
{
    // Always load the "main" and "universe" components, which contain most of the icon data
    // on Debian and Ubuntu. Load the "core" and "extra" components for Arch Linux.
    // FIXME: This is a hack, find a sane way to get rid of this, or at least get rid of the
    // distro-specific hardcoding.
    std::vector<std::shared_ptr<Package>> pkgs;

    for (const auto &newSection : std::vector<std::string>{"main", "universe", "core", "extra"}) {
        if (section != newSection && std::ranges::find(suite.sections, newSection) != suite.sections.end()) {
            auto sectionPkgs = m_pkgIndex->packagesFor(suite.name, newSection, arch);
            pkgs.insert(pkgs.end(), sectionPkgs.begin(), sectionPkgs.end());

            if (!suite.baseSuite.empty()) {
                auto basePkgs = m_pkgIndex->packagesFor(suite.baseSuite, newSection, arch);
                pkgs.insert(pkgs.end(), basePkgs.begin(), basePkgs.end());
            }
        }
    }

    if (!suite.baseSuite.empty()) {
        auto basePkgs = m_pkgIndex->packagesFor(suite.baseSuite, section, arch);
        pkgs.insert(pkgs.end(), basePkgs.begin(), basePkgs.end());
    }

    auto sectionPkgs = m_pkgIndex->packagesFor(suite.name, section, arch);
    pkgs.insert(pkgs.end(), sectionPkgs.begin(), sectionPkgs.end());

    std::unordered_map<std::string, std::shared_ptr<Package>> pkgMap;
    for (auto &pkg : pkgs) {
        const auto &pkid = pkg->id();
        pkgMap[pkid] = pkg;
    }

    return pkgMap;
}

std::shared_ptr<Package> Engine::processExtraMetainfoData(
    const Suite &suite,
    std::shared_ptr<IconHandler> iconh,
    const std::string &section,
    const std::string &arch,
    std::shared_ptr<InjectedModifications> injMods)
{
    if (suite.extraMetainfoDir.empty() && !injMods->hasRemovedComponents())
        return nullptr;

    const auto extraMIDir = suite.extraMetainfoDir / section;
    const auto archExtraMIDir = extraMIDir / arch;

    if (suite.extraMetainfoDir.empty())
        LOG_INFO(m_log, "Injecting component removal requests for {}/{}/{}", suite.name, section, arch);
    else
        LOG_INFO(m_log, "Loading additional metainfo from local directory for {}/{}/{}", suite.name, section, arch);

    // We create a dummy package to hold information for the injected components
    auto diPkg = std::make_shared<DataInjectPackage>(EXTRA_METAINFO_FAKE_PKGNAME, arch, m_backendPathPrefix);
    diPkg->setDataLocation(extraMIDir.string());
    diPkg->setArchDataLocation(archExtraMIDir.string());
    diPkg->setMaintainer("AppStream Generator Maintainer");

    // Ensure we have no leftover hints in the database.
    // Since this package never changes its version number, cruft data will not be automatically
    // removed for it.
    m_dstore->removePackage(diPkg->id());

    // Analyze our dummy package just like all other packages
    auto mde = std::make_unique<DataExtractor>(m_dstore, iconh, nullptr, suite.imageFormat, nullptr);
    auto gres = mde->processPackage(diPkg);

    // Add removal requests, as we can remove packages from frozen suites via overlays
    injMods->addRemovalRequestsToResult(&gres);

    // Write resulting data into the database
    m_dstore->addGeneratorResult(m_conf->metadataType, gres, true);

    return diPkg;
}

bool Engine::processSuiteSection(const Suite &suite, const std::string &section, std::shared_ptr<ReportGenerator> rgen)
{
    auto reportgen = std::move(rgen);
    if (!reportgen)
        reportgen = std::make_shared<ReportGenerator>(m_dstore.get(), m_sstore.get());

    // Load repo-level modifications
    auto injMods = std::make_shared<InjectedModifications>();
    try {
        injMods->loadForSuite(std::make_shared<Suite>(suite));
    } catch (const std::exception &e) {
        throw std::runtime_error(
            std::format("Unable to read modifications.json for suite {}: {}", suite.name, e.what()));
    }

    // Process packages by architecture
    std::vector<std::shared_ptr<Package>> sectionPkgs;
    bool suiteDataChanged = false;

    for (const auto &arch : suite.architectures) {
        // Update package contents information and flag boring packages as ignored
        const bool foundInteresting = seedContentsData(suite, section, arch) || m_forced;

        // Check if the suite/section/arch has actually changed
        if (!foundInteresting) {
            LOG_INFO(
                m_log,
                "Skipping {}/{} [{}], no interesting new packages since last update.",
                suite.name,
                section,
                arch);
            continue;
        }

        // Process new packages
        auto pkgs = m_pkgIndex->packagesFor(suite.name, section, arch);
        auto iconh = std::make_shared<IconHandler>(
            *m_cstore,
            getIconCandidatePackages(suite, section, arch),
            suite.imageFormat,
            suite.iconTheme,
            m_pkgIndex->dataPrefix());
        processPackages(pkgs, iconh, injMods, suite.imageFormat);

        // Read injected data and add it to the database as a fake package
        auto fakePkg = processExtraMetainfoData(suite, std::move(iconh), section, arch, injMods);
        if (fakePkg)
            pkgs.push_back(std::move(fakePkg));

        // Export package data
        exportMetadata(suite, section, arch, pkgs);
        suiteDataChanged = true;

        // We store the package info over all architectures to generate reports later
        sectionPkgs.reserve(sectionPkgs.capacity() + pkgs.size());
        sectionPkgs.insert(sectionPkgs.end(), pkgs.begin(), pkgs.end());

        // Log progress
        LOG_INFO(m_log, "Completed metadata processing of {}/{} [{}]", suite.name, section, arch);
    }

    // Finalize
    if (suiteDataChanged) {
        // Export icons for the found packages in this section
        exportIconTarballs(suite, section, sectionPkgs);

        // Write reports & statistics and render HTML, if that option is selected
        reportgen->processFor(suite.name, section, sectionPkgs);
    }

    // Release the index to free some memory
    m_pkgIndex->release();

    return suiteDataChanged;
}

SuiteUsabilityResult Engine::checkSuiteUsable(const std::string &suiteName)
{
    SuiteUsabilityResult res;
    res.suiteUsable = false;

    bool suiteFound = false;
    for (const auto &s : m_conf->suites) {
        if (s.name == suiteName) {
            res.suite = s;
            suiteFound = true;
            break;
        }
    }

    if (!suiteFound) {
        LOG_ERROR(m_log, "Suite '{}' was not found.", suiteName);
        return res;
    }

    if (res.suite.isImmutable) {
        // We also can't process anything if there are no architectures defined
        LOG_ERROR(m_log, "Suite '{}' is marked as immutable. No changes are allowed.", res.suite.name);
        return res;
    }

    if (res.suite.sections.empty()) {
        // If we have no sections, we can't do anything but exit...
        LOG_ERROR(m_log, "Suite '{}' has no sections. Can not continue.", res.suite.name);
        return res;
    }

    if (res.suite.architectures.empty()) {
        // We also can't process anything if there are no architectures defined
        LOG_ERROR(m_log, "Suite '{}' has no architectures defined. Can not continue.", res.suite.name);
        return res;
    }

    // If we are here, we can process this suite
    res.suiteUsable = true;
    return res;
}

bool Engine::processFile(
    const std::string &suiteName,
    const std::string &sectionName,
    const std::vector<std::string> &files)
{
    // Fetch suite and exit in case we can't write to it.
    auto suiteTuple = checkSuiteUsable(suiteName);
    if (!suiteTuple.suiteUsable)
        return false;
    auto suite = suiteTuple.suite;

    bool sectionValid = false;
    for (const auto &section : suite.sections) {
        if (section == sectionName) {
            sectionValid = true;
            break;
        }
    }
    if (!sectionValid) {
        LOG_ERROR(m_log, "Section '{}' does not exist in suite '{}'. Can not continue.", sectionName, suite.name);
        return false;
    }

    // ensure our libfyaml version is new enough
    checkLibfyamlVersion();

    std::unordered_map<std::string, std::vector<std::shared_ptr<Package>>> pkgByArch;
    for (const auto &fname : files) {
        auto pkg = m_pkgIndex->packageForFile(fname, suiteName, sectionName);
        if (!pkg) {
            LOG_ERROR(
                m_log,
                "Could not get package representation for file '{}' from backend '{}': The backend might not support "
                "this feature.",
                fname,
                m_conf->backendName);
            return false;
        }
        pkgByArch[pkg->arch()].push_back(pkg);
    }

    for (const auto &[arch, pkgs] : pkgByArch) {
        // Update package contents information and flag boring packages as ignored
        const bool foundInteresting = seedContentsData(suite, sectionName, arch, pkgs);

        // Skip if the new package files have no interesting data
        if (!foundInteresting) {
            LOG_INFO(m_log, "Skipping {}/{} [{}], no interesting new packages.", suite.name, sectionName, arch);
            continue;
        }

        // Process new packages
        auto iconh = std::make_shared<IconHandler>(
            *m_cstore,
            getIconCandidatePackages(suite, sectionName, arch),
            suite.imageFormat,
            suite.iconTheme,
            m_pkgIndex->dataPrefix());
        processPackages(pkgs, std::move(iconh), nullptr, suite.imageFormat);
    }

    return true;
}

void Engine::run()
{
    logVersionInfo();
    checkLibfyamlVersion();

    bool dataChanged = false;
    auto reportgen = std::make_shared<ReportGenerator>(m_dstore.get(), m_sstore.get());

    for (const auto &suite : m_conf->suites) {
        if (suite.isImmutable) {
            LOG_DEBUG(m_log, "Skipping immutable suite: {}", suite.name);
            continue;
        }

        // The `checkSuiteUsable` method will print an error
        // message in case the suite isn't usable.
        auto suiteCheck = checkSuiteUsable(suite.name);
        if (!suiteCheck.suiteUsable)
            continue;

        printHeaderBox(std::format("Processing: {}", suite.name));

        // process the suite
        for (const auto &section : suite.sections) {
            printSectionBox(std::format("{}: {}", suite.name, section));
            const bool ret = processSuiteSection(suite, section, reportgen);
            if (ret)
                dataChanged = true;
        }
    }

    // Render index pages & statistics
    printHeaderBox("Updating Global Data");
    const auto statsHistory = reportgen->collectStatistics();
    reportgen->updateIndexPages(statsHistory);
    if (dataChanged)
        reportgen->exportStatistics(statsHistory);
}

void Engine::run(const std::string &suiteName)
{
    // Fetch suite and exit in case we can't write to it.
    // The `checkSuiteUsable` method will print an error
    // message in case the suite isn't usable.
    auto suiteCheck = checkSuiteUsable(suiteName);
    if (!suiteCheck.suiteUsable)
        return;
    auto suite = suiteCheck.suite;

    logVersionInfo();
    checkLibfyamlVersion();

    auto reportgen = std::make_shared<ReportGenerator>(m_dstore.get(), m_sstore.get());

    bool dataChanged = false;
    for (const auto &section : suite.sections) {
        const bool ret = processSuiteSection(suite, section, reportgen);
        if (ret)
            dataChanged = true;
    }

    // Render index pages & statistics
    const auto statsHistory = reportgen->collectStatistics();
    reportgen->updateIndexPages(statsHistory);
    if (dataChanged)
        reportgen->exportStatistics(statsHistory);
}

void Engine::run(const std::string &suiteName, const std::string &sectionName)
{
    // Fetch suite and exit in case we can't write to it.
    // The `checkSuiteUsable` method will print an error
    // message in case the suite isn't usable.
    auto suiteCheck = checkSuiteUsable(suiteName);
    if (!suiteCheck.suiteUsable)
        return;
    auto suite = suiteCheck.suite;

    logVersionInfo();
    checkLibfyamlVersion();

    bool sectionValid = false;
    for (const auto &section : suite.sections) {
        if (section == sectionName) {
            sectionValid = true;
            break;
        }
    }
    if (!sectionValid) {
        LOG_ERROR(m_log, "Section '{}' does not exist in suite '{}'. Can not continue.", sectionName, suite.name);
        return;
    }

    auto reportgen = std::make_shared<ReportGenerator>(m_dstore.get(), m_sstore.get());
    auto dataChanged = processSuiteSection(suite, sectionName, reportgen);

    // Render index pages & statistics
    const auto statsHistory = reportgen->collectStatistics();
    reportgen->updateIndexPages(statsHistory);
    if (dataChanged)
        reportgen->exportStatistics(statsHistory);
}

void Engine::publishMetadataForSuiteSection(
    const Suite &suite,
    const std::string &section,
    std::shared_ptr<ReportGenerator> rgen)
{
    auto reportgen = std::move(rgen);
    if (!reportgen)
        reportgen = std::make_shared<ReportGenerator>(m_dstore.get(), m_sstore.get());

    std::vector<std::shared_ptr<Package>> sectionPkgs;
    for (const auto &arch : suite.architectures) {
        // Process new packages
        auto pkgs = m_pkgIndex->packagesFor(suite.name, section, arch);

        // Export package data
        exportMetadata(suite, section, arch, pkgs);

        // We store the package info over all architectures to generate reports later
        sectionPkgs.insert(sectionPkgs.end(), pkgs.begin(), pkgs.end());

        // Log progress
        LOG_INFO(m_log, "Completed publishing of data for {}/{} [{}]", suite.name, section, arch);
    }

    // Export icons for the found packages in this section
    exportIconTarballs(suite, section, sectionPkgs);

    // Write reports & statistics and render HTML, if that option is selected
    reportgen->processFor(suite.name, section, sectionPkgs);

    // Free some memory explicitly
    m_pkgIndex->release();
}

void Engine::publish(const std::string &suiteName)
{
    // Fetch suite and exit in case we can't write to it.
    auto scResult = checkSuiteUsable(suiteName);
    if (!scResult.suiteUsable)
        return;
    auto suite = scResult.suite;

    logVersionInfo();

    auto reportgen = std::make_shared<ReportGenerator>(m_dstore.get(), m_sstore.get());
    for (const auto &section : suite.sections)
        publishMetadataForSuiteSection(suite, section, reportgen);

    // Render index pages & statistics
    const auto statsHistory = reportgen->collectStatistics();
    reportgen->updateIndexPages(statsHistory);
    reportgen->exportStatistics(statsHistory);
}

void Engine::publish(const std::string &suiteName, const std::string &sectionName)
{
    // Fetch suite and exit in case we can't write to it.
    auto scResult = checkSuiteUsable(suiteName);
    if (!scResult.suiteUsable)
        return;
    auto suite = scResult.suite;

    logVersionInfo();

    bool sectionValid = false;
    for (const auto &section : suite.sections) {
        if (section == sectionName) {
            sectionValid = true;
            break;
        }
    }
    if (!sectionValid) {
        LOG_ERROR(m_log, "Section '{}' does not exist in suite '{}'. Can not continue.", sectionName, suite.name);
        return;
    }

    auto reportgen = std::make_shared<ReportGenerator>(m_dstore.get(), m_sstore.get());
    publishMetadataForSuiteSection(suite, sectionName, reportgen);

    // Render index pages & statistics
    const auto statsHistory = reportgen->collectStatistics();
    reportgen->updateIndexPages(statsHistory);
    reportgen->exportStatistics(statsHistory);
}

void Engine::cleanupStatistics()
{
    auto allStats = m_sstore->getStatistics();
    std::sort(allStats.begin(), allStats.end(), [](const auto &a, const auto &b) {
        return a.time < b.time;
    });

    std::unordered_map<std::string, std::unordered_map<std::string, MetaValue>> lastStatData;
    std::unordered_map<std::string, std::time_t> lastTime;

    for (const auto &entry : allStats) {
        // We don't clean up combined statistics entries, and therefore need to reset
        // the last-data hashmaps as soon as we encounter one to not lose data.
        auto suiteIt = entry.data.find("suite");
        auto sectionIt = entry.data.find("section");

        if (suiteIt == entry.data.end() || sectionIt == entry.data.end()) {
            lastStatData.clear();
            lastTime.clear();
            continue;
        }

        const auto ssid = std::format(
            "{}-{}", std::get<std::string>(suiteIt->second), std::get<std::string>(sectionIt->second));

        if (lastStatData.find(ssid) == lastStatData.end()) {
            lastStatData[ssid] = entry.data;
            lastTime[ssid] = entry.time;
            continue;
        }

        if (lastStatData[ssid] == entry.data) {
            LOG_INFO(m_log, "Removing superfluous statistics entry: {}", lastTime[ssid]);
            m_sstore->removeStatistics(lastTime[ssid]);
        }

        lastTime[ssid] = entry.time;
        lastStatData[ssid] = entry.data;
    }
}

void Engine::runCleanup()
{
    logVersionInfo();

    LOG_INFO(m_log, "Cleaning up left over temporary data.");
    const auto tmpDir = m_conf->cacheRootDir() / "tmp";
    if (fs::exists(tmpDir))
        fs::remove_all(tmpDir);

    LOG_INFO(m_log, "Collecting information.");

    // Get sets of all packages registered in the database
    std::unordered_set<std::string> pkidsContents;
    std::unordered_set<std::string> pkidsData;

    // Parallel collection of package IDs
    tbb::parallel_invoke(
        [&]() {
            pkidsContents = m_cstore->getPackageIdSet();
        },
        [&]() {
            pkidsData = m_dstore->getPackageIdSet();
        });
    LOG_INFO(
        m_log, "We have data on a total of {} packages (content lists on {})", pkidsData.size(), pkidsContents.size());

    // Build a set of all valid packages
    for (const auto &suite : m_conf->suites) {
        if (suite.isImmutable)
            continue; // Data from immutable suites is ignored

        for (const auto &section : suite.sections) {
            for (const auto &arch : suite.architectures) {
                // Fetch current packages without long descriptions, we really only are interested in the pkgid
                auto pkgs = m_pkgIndex->packagesFor(suite.name, section, arch, false);
                if (!suite.baseSuite.empty()) {
                    auto basePkgs = m_pkgIndex->packagesFor(suite.baseSuite, section, arch, false);
                    pkgs.insert(pkgs.end(), basePkgs.begin(), basePkgs.end());
                }

                {
                    std::lock_guard<std::mutex> lock(m_mutex);
                    for (const auto &pkg : pkgs) {
                        // Remove packages from the sets that are still active
                        pkidsContents.erase(pkg->id());
                        pkidsData.erase(pkg->id());
                    }

                    // Free some memory
                    m_pkgIndex->release();
                }
            }
        }
    }

    // Release index resources
    m_pkgIndex->release();

    LOG_INFO(
        m_log,
        "Cleaning up superseded data ({} hints/data, {} content lists).",
        pkidsData.size(),
        pkidsContents.size());

    // Remove packages from the caches which are no longer in the archive
    tbb::parallel_invoke(
        [&]() {
            m_cstore->removePackages(pkidsContents);
        },
        [&]() {
            m_dstore->removePackages(pkidsData);
        });

    // Remove orphaned data and media
    LOG_INFO(m_log, "Cleaning up obsolete media.");
    m_dstore->cleanupCruft();

    // Cleanup duplicate statistical entries
    LOG_INFO(m_log, "Cleaning up excess statistical data.");
    cleanupStatistics();
}

void Engine::removeHintsComponents(const std::string &suiteName)
{
    auto st = checkSuiteUsable(suiteName);
    if (!st.suiteUsable)
        return;
    auto suite = st.suite;

    logVersionInfo();

    for (const auto &section : suite.sections) {
        const auto &architectures = suite.architectures;

        tbb::parallel_for_each(architectures.begin(), architectures.end(), [&](const std::string &arch) {
            auto pkgs = m_pkgIndex->packagesFor(suite.name, section, arch, false);

            for (const auto &pkg : pkgs) {
                const auto &pkid = pkg->id();

                if (!m_dstore->packageExists(pkid))
                    continue;
                if (m_dstore->isIgnored(pkid))
                    continue;

                m_dstore->removePackage(pkid);
            }
        });

        m_pkgIndex->release();
    }

    m_dstore->cleanupCruft();
    m_pkgIndex->release();
}

void Engine::forgetPackage(const std::string &identifier)
{
    const auto slashCount = std::count(identifier.begin(), identifier.end(), '/');

    std::unordered_set<std::string> affectedGCIDs;
    if (slashCount == 2) {
        // We have a package-id, so we can do a targeted remove
        const auto pkid = identifier;
        LOG_DEBUG(m_log, "Considering {} to be a package-id.", pkid);

        const auto gcids = m_dstore->getGCIDsForPackage(pkid);
        affectedGCIDs.insert(gcids.begin(), gcids.end());

        if (m_cstore->packageExists(pkid))
            m_cstore->removePackage(pkid);
        if (m_dstore->packageExists(pkid))
            m_dstore->removePackage(pkid);
        LOG_INFO(m_log, "Removed package with ID: {}", pkid);
    } else {
        auto pkids = m_dstore->getPkidsMatching(identifier);
        for (const auto &pkid : pkids) {
            const auto gcids = m_dstore->getGCIDsForPackage(pkid);
            affectedGCIDs.insert(gcids.begin(), gcids.end());

            m_dstore->removePackage(pkid);
            if (m_cstore->packageExists(pkid))
                m_cstore->removePackage(pkid);
            LOG_INFO(m_log, "Removed package with ID: {}", pkid);
        }
    }

    // Remove orphaned data and media
    m_dstore->cleanupCruft();

    // Report if data is kept because packages keep the GCIDs around
    flushLogs();
    std::cout << "Packages still using the same data from the removed package(s):" << std::endl;
    auto pkgsForGCIDs = m_dstore->getPackagesForGCIDs(affectedGCIDs);
    if (pkgsForGCIDs.empty()) {
        std::cout << "  - None" << std::endl;
    } else {
        for (const auto &it : pkgsForGCIDs) {
            std::cout << "  - " << it.first << ": ";
            for (const auto &gcid : it.second)
                std::cout << gcid << " ";
            std::cout << std::endl;
        }
    }
}

bool Engine::printPackageInfo(const std::string &identifier)
{
    const auto slashCount = std::count(identifier.begin(), identifier.end(), '/');

    // everything below is written to stdout directly, so make sure no log
    // message is still pending and ends up in the middle of our output
    flushLogs();

    if (slashCount != 2) {
        std::cout << "Please enter a package-id in the format <name>/<version>/<arch>\n";
        return false;
    }
    const auto pkid = identifier;

    std::cout << "== " << pkid << " ==\n";
    std::cout << "Contents:\n";
    auto pkgContents = m_cstore->getContents(pkid);
    if (pkgContents.empty()) {
        std::cout << "~ No contents found.\n";
    } else {
        for (const auto &s : pkgContents)
            std::cout << " " << s << "\n";
    }
    std::cout << "\n";

    std::cout << "Icons:\n";
    auto pkgIcons = m_cstore->getIcons(pkid);
    if (pkgIcons.empty()) {
        std::cout << "~ No icons found.\n";
    } else {
        for (const auto &s : pkgIcons)
            std::cout << " " << s << "\n";
    }
    std::cout << "\n";

    if (m_dstore->isIgnored(pkid)) {
        std::cout << "Ignored: yes\n";
        std::cout << "\n";
    } else {
        std::cout << "Global Component IDs:\n";
        for (const auto &s : m_dstore->getGCIDsForPackage(pkid))
            std::cout << "- " << s << "\n";
        std::cout << "\n";

        std::cout << "Generated Data:\n";
        for (const auto &s : m_dstore->getMetadataForPackage(m_conf->metadataType, pkid))
            std::cout << s << "\n";
        std::cout << "\n";
    }

    if (m_dstore->hasHints(pkid)) {
        std::cout << "Hints:\n";
        std::cout << m_dstore->getHints(pkid) << "\n";
    } else {
        std::cout << "Hints: None\n";
    }

    std::cout << "\n";

    return true;
}

} // namespace ASGenerator
