use std::collections::{HashMap, HashSet};
use anyhow::{Context, Result};
use camino::{Utf8Path, Utf8PathBuf};
use cap_std_ext::cap_std::fs::Dir;
use indexmap::IndexMap;
use openssl::hash::{Hasher, MessageDigest};
use rpm_qa::FileInfo;
use crate::utils::{calculate_stability, canonicalize_parent_path};
use super::{ComponentId, ComponentInfo, ComponentsRepo, FileType};
const REPO_NAME: &str = "rpm";
const RPMDB_PATHS: &[&str] = &["usr/lib/sysimage/rpm", "usr/share/rpm", "var/lib/rpm"];
pub struct RpmRepo {
components: IndexMap<String, (u64, f64)>,
path_to_components: HashMap<Utf8PathBuf, Vec<(ComponentId, FileInfo)>>,
orphan_digest_index: HashMap<String, ComponentId>,
orphan_sizes: HashSet<u64>,
}
impl RpmRepo {
pub fn load(rootfs: &Dir, files: &super::FileMap, now: u64) -> Result<Option<Self>> {
if !has_rpmdb(rootfs)? {
return Ok(None);
}
let mut packages =
rpm_qa::load_from_rootfs_dir(rootfs).context("loading rpmdb from rootfs")?;
tracing::debug!(packages = packages.len(), "canonicalizing package paths");
canonicalize_package_paths(rootfs, files, &mut packages)
.context("canonicalizing package paths")?;
let mut repo = Self::load_from_packages(packages, now)?;
build_orphan_digest_index(&mut repo, files);
Ok(Some(repo))
}
pub fn load_from_packages(packages: rpm_qa::Packages, now: u64) -> Result<Self> {
let mut components: IndexMap<String, (u64, f64)> = IndexMap::new();
let mut path_to_components: HashMap<Utf8PathBuf, Vec<(ComponentId, FileInfo)>> =
HashMap::new();
let package_count = packages.len();
let mut non_sha256_count: usize = 0;
for pkg in packages.into_values() {
let component_name: &str = match pkg.sourcerpm.as_deref().map(parse_srpm_name) {
Some(name) => name,
None => {
tracing::warn!(package = %pkg.name, "missing sourcerpm, using package name");
&pkg.name
}
};
let entry = components.entry(component_name.to_string());
let stability = calculate_stability(&pkg.changelog_times, pkg.buildtime, now);
let component_id = ComponentId(entry.index());
match entry {
indexmap::map::Entry::Occupied(mut e) => {
let (existing_bt, existing_stability) = e.get_mut();
*existing_bt = (*existing_bt).max(pkg.buildtime);
if stability != *existing_stability && !pkg.changelog_times.is_empty() {
tracing::warn!(package = %pkg.name, "package has different changelog than sibling RPM");
}
*existing_stability = (*existing_stability).min(stability);
tracing::trace!(component = %component_name, buildtime = %existing_bt, stability = %existing_stability, "multiple rpm components from same srpm");
}
indexmap::map::Entry::Vacant(e) => {
tracing::trace!(component = %component_name, id = component_id.0, "rpm component created");
e.insert((pkg.buildtime, stability));
}
}
let is_sha256 = match pkg.digest_algo {
Some(rpm_qa::DigestAlgorithm::Sha256) => true,
Some(_) => {
non_sha256_count += 1;
false
}
None => false,
};
for (path, mut file_info) in pkg.files.into_iter() {
if !is_sha256 {
file_info.digest = None;
}
let entries = path_to_components.entry(path).or_default();
if !entries.iter().any(|(id, _)| *id == component_id) {
entries.push((component_id, file_info));
}
}
}
if non_sha256_count > 0 {
tracing::warn!(
non_sha256_count,
"packages with non-SHA-256 digest algorithm, move detection disabled for these"
);
}
tracing::debug!(
packages = package_count,
components = components.len(),
paths = path_to_components.len(),
"loaded rpm database"
);
Ok(Self {
components,
path_to_components,
orphan_digest_index: HashMap::new(),
orphan_sizes: HashSet::new(),
})
}
}
impl ComponentsRepo for RpmRepo {
fn name(&self) -> &'static str {
REPO_NAME
}
fn default_priority(&self) -> usize {
10
}
fn strong_claims_for_path(
&self,
path: &Utf8Path,
file_info: &super::FileInfo,
) -> Vec<ComponentId> {
if let Ok(rel_path) = path.strip_prefix("/")
&& RPMDB_PATHS.iter().any(|p| rel_path.starts_with(p))
{
return Vec::new();
}
self.path_to_components
.get(path)
.map(|entries| {
entries
.iter()
.filter(|(_, fi)| file_info_to_file_type(fi) == Some(file_info.file_type))
.map(|(id, _)| *id)
.collect()
})
.unwrap_or_default()
}
fn weak_claims_for_path(
&self,
rootfs: &Dir,
path: &Utf8Path,
file_info: &super::FileInfo,
) -> Result<Vec<ComponentId>> {
if self.orphan_digest_index.is_empty() {
return Ok(vec![]);
}
if file_info.file_type != super::FileType::File {
return Ok(vec![]);
}
if !self.orphan_sizes.contains(&file_info.size) {
return Ok(vec![]);
}
let digest = compute_sha256(rootfs, path)
.with_context(|| format!("hashing {path} for weak claim"))?;
if let Some(id) = self.orphan_digest_index.get(&digest) {
tracing::trace!(path = %path, component = self.components.get_index(id.0).unwrap().0, "reclaimed by digest");
return Ok(vec![*id]);
}
Ok(vec![])
}
fn component_info(&self, id: ComponentId) -> ComponentInfo<'_> {
let (name, (mtime, stability)) = self
.components
.get_index(id.0)
.expect("invalid ComponentId");
ComponentInfo {
name,
mtime_clamp: *mtime,
stability: *stability,
}
}
}
fn has_rpmdb(rootfs: &Dir) -> anyhow::Result<bool> {
for path in RPMDB_PATHS {
if rootfs
.try_exists(path)
.with_context(|| format!("checking for {path}"))?
{
return Ok(true);
}
}
Ok(false)
}
fn canonicalize_package_paths(
rootfs: &Dir,
files: &super::FileMap,
packages: &mut rpm_qa::Packages,
) -> Result<()> {
let mut cache = HashMap::new();
for package in packages.values_mut() {
let old_files = std::mem::take(&mut package.files);
for (path, info) in old_files {
let canonical = canonicalize_parent_path(rootfs, files, &path, &mut cache)
.with_context(|| format!("canonicalizing {}", path))?;
if canonical != path {
tracing::trace!(original = %path, canonical = %canonical, "path canonicalized");
}
package.files.insert(canonical, info);
}
}
Ok(())
}
fn build_orphan_digest_index(repo: &mut RpmRepo, files: &super::FileMap) {
let mut digest_to_ids: HashMap<String, Vec<ComponentId>> = HashMap::new();
let mut digest_to_size: HashMap<String, u64> = HashMap::new();
for (path, entries) in &repo.path_to_components {
if files.contains_key(path) {
continue;
}
for (component_id, fi) in entries {
if fi.flags.is_ghost() {
continue;
}
if fi.size == 0 {
continue;
}
let digest = match &fi.digest {
Some(d) if !d.is_empty() => d,
_ => continue,
};
digest_to_ids
.entry(digest.clone())
.or_default()
.push(*component_id);
digest_to_size.insert(digest.clone(), fi.size);
}
}
let mut orphan_digest_index: HashMap<String, ComponentId> = HashMap::new();
let mut orphan_sizes: HashSet<u64> = HashSet::new();
let mut ambiguous_count: usize = 0;
for (digest, ids) in digest_to_ids {
let unique_ids: HashSet<ComponentId> = ids.into_iter().collect();
if unique_ids.len() > 1 {
ambiguous_count += 1;
tracing::trace!(digest = %digest, components = unique_ids.len(), "excluding ambiguous orphan digest");
continue;
}
let id = unique_ids.into_iter().next().unwrap();
let size = digest_to_size.get(&digest).unwrap();
orphan_digest_index.insert(digest, id);
orphan_sizes.insert(*size);
}
tracing::debug!(
index_size = orphan_digest_index.len(),
unique_sizes = orphan_sizes.len(),
ambiguous_excluded = ambiguous_count,
"built orphan digest index"
);
repo.orphan_digest_index = orphan_digest_index;
repo.orphan_sizes = orphan_sizes;
}
fn compute_sha256(rootfs: &Dir, path: &Utf8Path) -> Result<String> {
let rel_path = path.strip_prefix("/").unwrap_or(path.as_ref());
let mut file = rootfs
.open(rel_path.as_str())
.with_context(|| format!("opening {path} for hashing"))?;
let mut hasher = Hasher::new(MessageDigest::sha256()).context("creating SHA-256 hasher")?;
std::io::copy(&mut file, &mut hasher).with_context(|| format!("hashing {path}"))?;
let digest = hasher.finish().context("finalizing SHA-256 hash")?;
Ok(hex::encode(digest))
}
fn parse_srpm_name(srpm: &str) -> &str {
let without_suffix = srpm.strip_suffix(".src.rpm").unwrap_or(srpm);
let parts: Vec<&str> = without_suffix.rsplitn(3, '-').collect();
if parts.len() >= 3 {
parts[2]
} else {
without_suffix
}
}
fn file_info_to_file_type(fi: &FileInfo) -> Option<FileType> {
let file_type = (fi.mode as libc::mode_t) & libc::S_IFMT;
match file_type {
libc::S_IFDIR => Some(FileType::Directory),
libc::S_IFREG => Some(FileType::File),
libc::S_IFLNK => Some(FileType::Symlink),
_ => None,
}
}
#[cfg(test)]
mod tests {
use camino::Utf8Path;
use cap_std_ext::cap_std::ambient_authority;
use rpm_qa::Package;
use super::*;
const FIXTURE: &str = include_str!("../../tests/fixtures/fedora.qf");
fn fi(file_type: FileType) -> crate::components::FileInfo {
crate::components::FileInfo::dummy(file_type)
}
#[test]
fn test_parse_srpm_name() {
assert_eq!(parse_srpm_name("bash-5.2.15-5.fc40.src.rpm"), "bash");
assert_eq!(parse_srpm_name("systemd-256.4-1.fc41.src.rpm"), "systemd");
assert_eq!(parse_srpm_name("python3-3.12.0-1.fc40.src.rpm"), "python3");
assert_eq!(parse_srpm_name("glibc-2.39-5.fc40.src.rpm"), "glibc");
assert_eq!(
parse_srpm_name("python-dateutil-2.8.2-1.fc40.src.rpm"),
"python-dateutil"
);
assert_eq!(
parse_srpm_name("cairo-dock-plugins-3.4.1-1.fc40.src.rpm"),
"cairo-dock-plugins"
);
assert_eq!(
parse_srpm_name("xorg-x11-server-1.20.14-1.fc40.src.rpm"),
"xorg-x11-server"
);
assert_eq!(parse_srpm_name("name-version"), "name-version");
assert_eq!(parse_srpm_name("bash-5.2.15-5.fc40"), "bash");
assert_eq!(parse_srpm_name("nodash"), "nodash");
}
#[test]
fn test_claims_for_path() {
let packages = rpm_qa::load_from_str(FIXTURE).unwrap();
let repo = RpmRepo::load_from_packages(packages, now_secs()).unwrap();
let claims =
repo.strong_claims_for_path(Utf8Path::new("/usr/bin/bash"), &fi(FileType::File));
assert_eq!(claims.len(), 1);
let info = repo.component_info(claims[0]);
assert_eq!(info.name, "bash");
assert_eq!(info.mtime_clamp, 1753299195);
let claims =
repo.strong_claims_for_path(Utf8Path::new("/usr/bin/sh"), &fi(FileType::Symlink));
assert_eq!(claims.len(), 1);
let info = repo.component_info(claims[0]);
assert_eq!(info.name, "bash");
let claims =
repo.strong_claims_for_path(Utf8Path::new("/usr/lib64/libc.so.6"), &fi(FileType::File));
assert_eq!(claims.len(), 1);
let info = repo.component_info(claims[0]);
assert_eq!(info.name, "glibc");
assert_eq!(info.mtime_clamp, 1771428496);
let claims =
repo.strong_claims_for_path(Utf8Path::new("/some/unowned/file"), &fi(FileType::File));
assert!(claims.is_empty());
for rpmdb_path in [
"/usr/lib/sysimage/rpm/rpmdb.sqlite",
"/usr/share/rpm/macros",
"/var/lib/rpm/Packages",
] {
let claims =
repo.strong_claims_for_path(Utf8Path::new(rpmdb_path), &fi(FileType::File));
assert!(
claims.is_empty(),
"RPMDB path {} should not be claimed",
rpmdb_path
);
}
}
#[test]
fn test_claims_for_path_wrong_type() {
let packages = rpm_qa::load_from_str(FIXTURE).unwrap();
let repo = RpmRepo::load_from_packages(packages, now_secs()).unwrap();
let claims =
repo.strong_claims_for_path(Utf8Path::new("/usr/bin/bash"), &fi(FileType::Symlink));
assert!(claims.is_empty());
let claims = repo.strong_claims_for_path(Utf8Path::new("/usr/bin/sh"), &fi(FileType::File));
assert!(claims.is_empty());
}
#[test]
fn test_shared_directories_claimed_by_multiple_components() {
let packages = rpm_qa::load_from_str(FIXTURE).unwrap();
let repo = RpmRepo::load_from_packages(packages, now_secs()).unwrap();
let claims = repo.strong_claims_for_path(
Utf8Path::new("/usr/lib/.build-id"),
&fi(FileType::Directory),
);
assert!(
claims.len() >= 2,
"shared dir should be claimed by multiple components"
);
let names: std::collections::HashSet<_> = claims
.iter()
.map(|id| repo.component_info(*id).name)
.collect();
for pkg in ["bash", "glibc", "coreutils"] {
assert!(names.contains(pkg), "{pkg} should claim /usr/lib/.build-id");
}
}
#[test]
fn test_load_from_rpmdb_sqlite() {
use std::process::Command;
let rpm_available = Command::new("rpm").arg("--version").output().is_ok();
if !rpm_available {
eprintln!("skipping test: rpm command not available");
return;
}
let tmp = tempfile::tempdir().unwrap();
let rpmdb_dir = tmp.path().join("usr/lib/sysimage/rpm");
std::fs::create_dir_all(&rpmdb_dir).unwrap();
let fixture_path =
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/rpmdb.sqlite");
std::fs::copy(&fixture_path, rpmdb_dir.join("rpmdb.sqlite")).unwrap();
let rootfs = Dir::open_ambient_dir(tmp.path(), ambient_authority()).unwrap();
let files = crate::scan::Scanner::new(&rootfs).scan().unwrap();
let repo = RpmRepo::load(&rootfs, &files, now_secs()).unwrap().unwrap();
let claims = repo.strong_claims_for_path(Utf8Path::new("/"), &fi(FileType::Directory));
assert!(!claims.is_empty(), "/ should be claimed");
assert_eq!(repo.component_info(claims[0]).name, "filesystem");
let claims = repo.strong_claims_for_path(Utf8Path::new("/etc"), &fi(FileType::Directory));
assert!(!claims.is_empty(), "/etc should be claimed");
assert_eq!(repo.component_info(claims[0]).name, "filesystem");
let claims = repo.strong_claims_for_path(Utf8Path::new("/etc/passwd"), &fi(FileType::File));
assert!(!claims.is_empty(), "/etc/passwd should be claimed");
assert_eq!(repo.component_info(claims[0]).name, "setup");
}
fn now_secs() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs()
}
fn assert_stability_in_range(stability: f64, min: f64, max: f64) {
assert!(
stability >= min && stability <= max,
"stability {stability} not in range [{min}, {max}]"
);
}
#[test]
fn test_calculate_stability_all_old_entries() {
use crate::components::SECS_PER_DAY;
let now = now_secs();
let old_time = now - (400 * SECS_PER_DAY);
let changelog_times = vec![old_time, old_time - SECS_PER_DAY];
let buildtime = old_time;
let stability = calculate_stability(&changelog_times, buildtime, now);
assert_eq!(stability, 0.99);
}
#[test]
fn test_calculate_stability_very_recent() {
let now = now_secs();
let recent_time = now - 3600;
let changelog_times = vec![recent_time];
let buildtime = recent_time;
let stability = calculate_stability(&changelog_times, buildtime, now);
assert_eq!(stability, 0.0);
}
#[test]
fn test_calculate_stability_no_changelog_uses_buildtime() {
use crate::components::SECS_PER_DAY;
let now = now_secs();
let buildtime = now - (30 * SECS_PER_DAY);
let changelog_times: Vec<u64> = vec![];
let stability = calculate_stability(&changelog_times, buildtime, now);
assert_stability_in_range(stability, 0.75, 0.85);
}
#[test]
fn test_calculate_stability_normal_case() {
use crate::components::SECS_PER_DAY;
let now = now_secs();
let changelog_times = vec![
now - (10 * SECS_PER_DAY),
now - (30 * SECS_PER_DAY),
now - (60 * SECS_PER_DAY),
now - (100 * SECS_PER_DAY),
];
let buildtime = now - (100 * SECS_PER_DAY);
let stability = calculate_stability(&changelog_times, buildtime, now);
assert_stability_in_range(stability, 0.70, 0.80);
}
#[test]
fn test_calculate_stability_high_frequency() {
use crate::components::SECS_PER_DAY;
let now = now_secs();
let changelog_times: Vec<u64> = (0..10)
.map(|i| now - ((2 + i * 2) * SECS_PER_DAY))
.collect();
let buildtime = now - (20 * SECS_PER_DAY);
let stability = calculate_stability(&changelog_times, buildtime, now);
assert_stability_in_range(stability, 0.0, 0.10);
}
#[test]
fn test_stability_min_across_subpackages() {
use std::collections::BTreeMap;
let now = 1_800_000_000;
let srpm = "foo-1.0-1.fc40.src.rpm";
let foo = rpm_qa::Package {
name: "foo".into(),
version: "1.0".into(),
release: "1.fc40".into(),
epoch: None,
arch: "x86_64".into(),
license: "MIT".into(),
size: 1000,
buildtime: now - 200000,
installtime: now,
sourcerpm: Some(srpm.into()),
digest_algo: None,
changelog_times: vec![now - 200000, now - 300000],
files: BTreeMap::new(),
};
let mut foo2 = foo.clone();
foo2.name = "foo2".into();
foo2.changelog_times = vec![now, now - 100000];
let stab_foo = calculate_stability(&foo.changelog_times, foo.buildtime, now);
let stab_foo2 = calculate_stability(&foo2.changelog_times, foo2.buildtime, now);
assert!(stab_foo > stab_foo2, "foo isn't more stable than foo2");
let assert_stability = |first: &Package, second: &Package| {
let mut packages: rpm_qa::Packages = HashMap::new();
packages.insert(first.name.to_string(), first.clone());
packages.insert(second.name.to_string(), second.clone());
let repo = RpmRepo::load_from_packages(packages, now).unwrap();
let info = repo.component_info(ComponentId(0));
assert_eq!(info.name, "foo");
assert_eq!(info.stability, stab_foo2);
};
assert_stability(&foo, &foo2);
assert_stability(&foo2, &foo);
}
#[test]
fn test_compute_sha256() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = Dir::open_ambient_dir(tmp.path(), ambient_authority()).unwrap();
rootfs.write("hello.txt", "hello world\n").unwrap();
let digest = compute_sha256(&rootfs, Utf8Path::new("/hello.txt")).unwrap();
assert_eq!(
digest,
"a948904f2f0f479b8f8197694b30184b0d2ed1c1cd2a1ec0fb85d299a192a447"
);
rootfs.write("empty", "").unwrap();
let digest = compute_sha256(&rootfs, Utf8Path::new("/empty")).unwrap();
assert_eq!(
digest,
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
);
}
#[test]
fn test_weak_claims_for_path() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = Dir::open_ambient_dir(tmp.path(), ambient_authority()).unwrap();
rootfs.create_dir("moved").unwrap();
let content = "x".repeat(1024);
rootfs.write("moved/bash", &content).unwrap();
let files = crate::scan::Scanner::new(&rootfs).scan().unwrap();
let digest = compute_sha256(&rootfs, Utf8Path::new("/moved/bash")).unwrap();
let size = files.get(Utf8Path::new("/moved/bash")).unwrap().size;
let mut packages = rpm_qa::load_from_str(FIXTURE).unwrap();
let bash = packages.get_mut("bash").unwrap();
let bash_fi = bash.files.get_mut(Utf8Path::new("/usr/bin/bash")).unwrap();
bash_fi.digest = Some(digest);
bash_fi.size = size;
let mut repo = RpmRepo::load_from_packages(packages, now_secs()).unwrap();
build_orphan_digest_index(&mut repo, &files);
let moved_fi = files.get(Utf8Path::new("/moved/bash")).unwrap();
let claims = repo
.weak_claims_for_path(&rootfs, Utf8Path::new("/moved/bash"), moved_fi)
.unwrap();
assert_eq!(claims.len(), 1);
assert_eq!(repo.component_info(claims[0]).name, "bash");
}
#[test]
fn test_weak_claims_for_path_ambiguous() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = Dir::open_ambient_dir(tmp.path(), ambient_authority()).unwrap();
let content = "x".repeat(1024);
rootfs.write("data.bin", &content).unwrap();
let files = crate::scan::Scanner::new(&rootfs).scan().unwrap();
let digest = compute_sha256(&rootfs, Utf8Path::new("/data.bin")).unwrap();
let size = files.get(Utf8Path::new("/data.bin")).unwrap().size;
let mut packages = rpm_qa::load_from_str(FIXTURE).unwrap();
let bash = packages.get_mut("bash").unwrap();
let bash_fi = bash.files.get_mut(Utf8Path::new("/usr/bin/bash")).unwrap();
bash_fi.digest = Some(digest.clone());
bash_fi.size = size;
let glibc = packages.get_mut("glibc").unwrap();
let glibc_fi = glibc
.files
.get_mut(Utf8Path::new("/usr/lib64/libc.so.6"))
.unwrap();
glibc_fi.digest = Some(digest);
glibc_fi.size = size;
let mut repo = RpmRepo::load_from_packages(packages, now_secs()).unwrap();
build_orphan_digest_index(&mut repo, &files);
let data_fi = files.get(Utf8Path::new("/data.bin")).unwrap();
let claims = repo
.weak_claims_for_path(&rootfs, Utf8Path::new("/data.bin"), data_fi)
.unwrap();
assert!(claims.is_empty(), "ambiguous digest should not claim");
}
}