// This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. //! License detection for the file view. //! //! A file that looks like a license (see [`is_license_path`]) is matched //! against the full SPDX license corpus by text similarity — the same //! approach as GitHub's `licensee` / `askalono`, not brittle substring //! matching. Each license text is normalized (lowercased, punctuation folded //! to spaces) and reduced to a bag of word-trigrams; the file is scored against //! every SPDX license by the Sørensen–Dice coefficient and the best match above //! a confidence threshold wins. This correctly handles licenses whose text //! quotes other licenses (e.g. the MPL names the GNU (A/L)GPL in its //! "Secondary License" clause) — the whole-document similarity dominates. //! //! For the ~40 licenses that [choosealicense.com](https://choosealicense.com) //! curates, we also show a permissions / conditions / limitations summary; any //! other SPDX license is still recognized and named, just without the columns. //! The corpus (`assets/licenses/corpus.jsonl.gz`) and rights data //! (`assets/licenses/rights.json`) are generated by `scripts/gen-licenses`. The //! rights summary is informational, never legal advice. use std::collections::HashMap; use std::io::Read; use std::sync::LazyLock; use flate2::read::GzDecoder; /// gzip of one `{"id","name","dep","text"}` JSON record per line; `text` is the /// license body already run through [`normalize`]. const CORPUS_GZ: &[u8] = include_bytes!("../../../assets/licenses/corpus.jsonl.gz"); /// choosealicense.com rights summaries, keyed by SPDX id. const RIGHTS_JSON: &[u8] = include_bytes!("../../../assets/licenses/rights.json"); /// Minimum Dice similarity to accept a match. Real license files fill in a /// copyright line and little else, so genuine matches score well above this; /// unrelated text scores far below it. const THRESHOLD: f64 = 0.9; /// Scores within this band of the best are treated as a tie and broken by /// [`pref_key`] (prefer a curated, non-deprecated, canonical id) — this keeps /// e.g. `GPL-3.0-only` from losing to an obscure GPL variant with equal text. const TIE_BAND: f64 = 0.02; /// The choosealicense rights summary for a license. struct Rights { title: &'static str, description: String, permissions: Vec, conditions: Vec, limitations: Vec, } /// One SPDX license in the searchable corpus. struct Entry { spdx: String, name: String, deprecated: bool, /// Sorted word-trigram hashes of the normalized text. fingerprint: Vec, /// Whether a [`Rights`] summary exists for this id. has_rights: bool, } /// The loaded, ready-to-query license database. struct Db { entries: Vec, rights: HashMap, } /// A detected license and (optionally) its rights summary. pub(crate) struct Detected { /// Human-readable name, e.g. "MIT License". pub name: String, /// SPDX identifier, e.g. "MIT". pub spdx: String, /// One-line plain description (only when a rights summary exists). pub description: Option, /// What the license permits (empty without a rights summary). pub permissions: Vec, /// What it requires. pub conditions: Vec, /// What it limits. pub limitations: Vec, /// Whether the permissions/conditions/limitations columns are populated. pub has_rights: bool, } /// Whether `path`'s filename looks like a license file (`LICENSE`, `LICENCE`, /// `COPYING`, optionally with an extension, case-insensitive). pub(crate) fn is_license_path(path: &str) -> bool { let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase(); let stem = name.split('.').next().unwrap_or(&name); matches!( stem, "license" | "licence" | "copying" | "copyright" | "unlicense" ) } /// Fold text to a canonical token stream: ASCII-lowercase, every run of /// non-alphanumeric characters becomes a single space, no leading/trailing /// space. Idempotent, so re-running it over already-normalized corpus text is a /// no-op. Must stay identical to the generator's `normalize`. fn normalize(s: &str) -> String { let mut out = String::with_capacity(s.len()); let mut prev_space = true; // trims leading separators for c in s.chars() { let c = c.to_ascii_lowercase(); if c.is_ascii_alphanumeric() { out.push(c); prev_space = false; } else if !prev_space { out.push(' '); prev_space = true; } } if out.ends_with(' ') { out.pop(); } out } /// FNV-1a over the bytes of three space-joined tokens. fn trigram_hash(a: &str, b: &str, c: &str) -> u64 { let mut h = 0xcbf2_9ce4_8422_2325u64; let mut feed = |bytes: &[u8]| { for &byte in bytes { h ^= u64::from(byte); h = h.wrapping_mul(0x0000_0100_0000_01b3); } }; feed(a.as_bytes()); feed(b" "); feed(b.as_bytes()); feed(b" "); feed(c.as_bytes()); h } /// Sorted multiset of word-trigram hashes for normalized `text`. Empty when the /// text has fewer than three tokens (too little to match on). fn fingerprint(text: &str) -> Vec { let toks: Vec<&str> = text.split(' ').filter(|t| !t.is_empty()).collect(); if toks.len() < 3 { return Vec::new(); } let mut fp: Vec = toks .windows(3) .map(|w| trigram_hash(w[0], w[1], w[2])) .collect(); fp.sort_unstable(); fp } /// Sørensen–Dice coefficient over two sorted multisets. The counts are /// trigram tallies (a few thousand at most), so the `usize`→`f64` casts cannot /// lose precision. #[allow(clippy::cast_precision_loss)] fn dice(a: &[u64], b: &[u64]) -> f64 { if a.is_empty() || b.is_empty() { return 0.0; } let (mut i, mut j, mut inter) = (0usize, 0usize, 0usize); while i < a.len() && j < b.len() { match a[i].cmp(&b[j]) { std::cmp::Ordering::Less => i += 1, std::cmp::Ordering::Greater => j += 1, std::cmp::Ordering::Equal => { inter += 1; i += 1; j += 1; } } } 2.0 * inter as f64 / (a.len() + b.len()) as f64 } /// Look up rights for an SPDX id, falling back to the base id when the id /// carries a GNU `-only` / `-or-later` / `+` disambiguator (choosealicense /// keys those under the plain id, e.g. `GPL-3.0`). fn rights_lookup<'a>(rights: &'a HashMap, spdx: &str) -> Option<&'a Rights> { if let Some(r) = rights.get(spdx) { return Some(r); } let base = spdx .strip_suffix("-or-later") .or_else(|| spdx.strip_suffix("-only")) .or_else(|| spdx.strip_suffix('+')) .unwrap_or(spdx); (base != spdx).then(|| rights.get(base)).flatten() } /// Ordering key for tie-breaking near-equal matches: prefer a curated /// (rights-bearing) id, then a non-deprecated one, then the shorter/earlier id. fn pref_key(e: &Entry) -> (bool, bool, usize, &str) { (!e.has_rights, e.deprecated, e.spdx.len(), e.spdx.as_str()) } /// Read one JSON string field out of a `serde_json::Value` object. fn str_field(v: &serde_json::Value, key: &str) -> String { v.get(key) .and_then(|x| x.as_str()) .unwrap_or_default() .to_string() } /// Read a JSON array-of-strings field. fn vec_field(v: &serde_json::Value, key: &str) -> Vec { v.get(key) .and_then(|x| x.as_array()) .map(|a| { a.iter() .filter_map(|s| s.as_str().map(str::to_string)) .collect() }) .unwrap_or_default() } /// The lazily-built, process-lifetime license database. static DB: LazyLock = LazyLock::new(build_db); fn build_db() -> Db { // Rights summaries (small, uncompressed JSON array). let mut rights: HashMap = HashMap::new(); if let Ok(serde_json::Value::Array(arr)) = serde_json::from_slice(RIGHTS_JSON) { for v in arr { let spdx = str_field(&v, "spdx"); if spdx.is_empty() { continue; } // `title` outlives the process via the leaked owned string; this runs // once, for ~40 entries, so the leak is bounded and intentional. let title: &'static str = Box::leak(str_field(&v, "title").into_boxed_str()); rights.insert( spdx, Rights { title, description: str_field(&v, "description"), permissions: vec_field(&v, "permissions"), conditions: vec_field(&v, "conditions"), limitations: vec_field(&v, "limitations"), }, ); } } // SPDX corpus (gzipped JSONL). let mut text = String::new(); let _ = GzDecoder::new(CORPUS_GZ).read_to_string(&mut text); let mut entries = Vec::new(); for line in text.lines() { if line.is_empty() { continue; } let Ok(v) = serde_json::from_str::(line) else { continue; }; let spdx = str_field(&v, "id"); let body = str_field(&v, "text"); if spdx.is_empty() || body.is_empty() { continue; } // `body` is already normalized; re-running is idempotent and guarantees // the corpus and query tokenize through the exact same code path. let fingerprint = fingerprint(&normalize(&body)); if fingerprint.is_empty() { continue; } let has_rights = rights_lookup(&rights, &spdx).is_some(); entries.push(Entry { name: str_field(&v, "name"), deprecated: v .get("dep") .and_then(serde_json::Value::as_bool) .unwrap_or(false), spdx, fingerprint, has_rights, }); } Db { entries, rights } } /// Detect the license from a file's text, or `None` if no SPDX license matches /// with enough confidence. pub(crate) fn detect(content: &str) -> Option { let db = &*DB; let fp = fingerprint(&normalize(content)); if fp.is_empty() { return None; } // Score against every corpus entry, skipping candidates whose length is too // far off to plausibly match. let mut best_score = 0.0f64; let mut best: Option<&Entry> = None; for e in &db.entries { // Skip candidates whose length is more than 2× off in either direction. let (la, lb) = (fp.len(), e.fingerprint.len()); if la * 2 < lb || lb * 2 < la { continue; } let score = dice(&fp, &e.fingerprint); let better = match best { None => true, Some(_) if score > best_score + TIE_BAND => true, Some(cur) if score >= best_score - TIE_BAND => pref_key(e) < pref_key(cur), _ => false, }; if better { best = Some(e); } best_score = best_score.max(score); } let e = best?; if best_score < THRESHOLD { return None; } let rights = rights_lookup(&db.rights, &e.spdx); Some(Detected { name: rights.map_or_else(|| e.name.clone(), |r| r.title.to_string()), spdx: e.spdx.clone(), description: rights.map(|r| r.description.clone()), permissions: rights.map(|r| r.permissions.clone()).unwrap_or_default(), conditions: rights.map(|r| r.conditions.clone()).unwrap_or_default(), limitations: rights.map(|r| r.limitations.clone()).unwrap_or_default(), has_rights: rights.is_some(), }) } #[cfg(test)] mod tests { #![allow(clippy::unwrap_used, clippy::expect_used)] use super::*; #[test] fn recognizes_license_filenames() { assert!(is_license_path("LICENSE")); assert!(is_license_path("license.md")); assert!(is_license_path("path/to/COPYING")); assert!(is_license_path("LICENCE.txt")); assert!(!is_license_path("src/main.rs")); assert!(!is_license_path("readme.md")); } const MIT: &str = r#"MIT License Copyright (c) 2024 Example Holder Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE."#; #[test] fn detects_mit_with_rights() { let d = detect(MIT).expect("MIT detected"); assert_eq!(d.spdx, "MIT"); assert_eq!(d.name, "MIT License"); assert!(d.has_rights); assert!(d.permissions.iter().any(|p| p == "Commercial use")); assert!(d.limitations.iter().any(|l| l == "Warranty")); } /// Regression: the MPL text names the GNU (A)GPL in its Secondary-License /// clause, which previously mis-detected it as AGPL-3.0. Whole-document /// similarity must pick MPL-2.0. Uses the crate's own MPL-2.0 LICENSE. #[test] fn detects_mpl_not_agpl() { let mpl = include_str!("../../../LICENSE"); let d = detect(mpl).expect("MPL detected"); assert_eq!(d.spdx, "MPL-2.0"); assert!(d.has_rights); assert!(d.conditions.iter().any(|c| c == "Disclose source")); } #[test] fn ignores_non_license_text() { assert!(detect("just some random text file with no legal content at all").is_none()); } }