blackmwk commented on code in PR #3414:
URL: https://github.com/apache/iceberg-rust/pull/3414#discussion_r4292374922
##########
dev/release/licenses.py:
##########
@@ -56,17 +75,296 @@ def require_cargo_deny() -> None:
)
-def command_check_dependencies(_args: argparse.Namespace) -> None:
- step("Check dependency licenses")
- require_cargo_deny()
+def parse_declared_clarifications() -> set[CrateName]:
+ """Crate names that deny.toml carries a [[licenses.clarify]] block for."""
+ with DENY_TOML.open("rb") as handle:
+ config = tomllib.load(handle)
+
+ blocks = config.get("licenses", {}).get("clarify", [])
+ if not blocks:
+ raise Failure(
+ f"Expected at least one [[licenses.clarify]] block in {DENY_TOML},
found none.\n"
+ "If the last one was deliberately removed, remove this check with
it."
+ )
+ pinned = sorted(block["crate"] for block in blocks if "@" in
block["crate"])
+ if pinned:
+ raise Failure(
+ textwrap.dedent("""
+ {deny_toml} pins a [[licenses.clarify]] block to a crate
version:
+ {specs}
+
+ Version pins are rejected by this script.
+ cargo-deny supports them, but will not report when they become
stale.
+ To keep things simple, this script requires that no pinning is
used.
+ Where newer versions change licensing, the script will reject
the clarification
+ and the deny.toml file should be updated.
+ """).format(
+ deny_toml=DENY_TOML,
+ specs="\n".join(f" {spec}" for spec in pinned),
+ )
+ )
+
+ return {block["crate"] for block in blocks}
+
+
+def run_cargo_deny() -> tuple[
+ dict[CrateName, set[CrateVersion]], dict[CrateName, set[CrateVersion]]
+]:
+ """
+ Run cargo-deny, returning mappings from crate name to crate version
+ for every crate version evaluated plus the subset which had a
clarification applied.
+ """
result = subprocess.run(
- ["cargo", "deny", "check", "license"],
+ [
+ "cargo",
+ "deny",
+ "--format=json",
+ "check",
+ "license",
+ # We emit the license accepted messages as warnings so we can
count them.
+ "--warn=accepted",
+ ],
cwd=REPO_ROOT,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ diagnostics = result.stderr
+
+ evaluated: dict[CrateName, set[CrateVersion]] = defaultdict(set)
+ clarified: dict[CrateName, set[CrateVersion]] = defaultdict(set)
+ errors: list[str] = []
+ for line in diagnostics.splitlines():
+ if not line.strip():
+ continue
+ line_truncated = line[:200]
+ try:
+ diagnostic = json.loads(line)
+ except json.JSONDecodeError:
+ raise Failure(
+ "cargo-deny emitted a line that is not JSON, so its output
cannot be "
+ f"parsed:\n {line_truncated}"
+ )
+
+ fields = diagnostic.get("fields", {})
+
+ if fields.get("severity") == "error" or fields.get("level") == "ERROR":
+ krate = (fields.get("graphs") or [{}])[0].get("Krate", {})
+ crate = f"{krate['name']} {krate['version']}: " if krate else ""
+ labels = fields.get("labels") or []
+
+ error_msg = f" {crate}{fields.get('message')}"
+ if fields.get("code") == "rejected":
+ # For rejections, include the expression. Other codes are too
noisy (such as TOML error).
+ span = labels[0].get("span")
+ error_msg = f"{error_msg} ({span})"
+ errors.append(error_msg)
+
+ if fields.get("code") != "accepted":
+ continue
+
+ graphs = fields.get("graphs") or []
+ if len(graphs) != 1:
+ raise Failure(
+ f"cargo-deny reported an 'accepted' license verdict covering "
+ f"{len(graphs)} crates, but one was expected:\n
{line_truncated}"
+ )
+ krate = graphs[0].get("Krate", {})
+
+ name, version = krate.get("name"), krate.get("version")
+ if name is None or version is None:
+ raise Failure(
+ "cargo-deny reported an 'accepted' license verdict without
naming both "
+ f"the crate and its version:\n {line_truncated}"
+ )
+
+ evaluated[name].add(version)
+ labels = fields.get("labels") or []
+
+ # There's no machine readable error code.
+ # If this error message changes, the script will fail closed anyway as
expected clarifications won't be found.
+ CLARIFICATION_APPLIED_MSG = "license expression retrieved via user
override"
+ if any(label.get("message") == CLARIFICATION_APPLIED_MSG for label in
labels):
+ clarified[name].add(version)
+
+ if result.returncode != 0:
+ # Emit list of errors, or simply diagnostics if error code is non-zero
but we failed to parse any errors.
+ detail = "\n".join(errors) if errors else diagnostics.rstrip("\n")
+ raise Failure(f"cargo-deny rejected the dependency
licenses:\n{detail}")
+
+ return dict(evaluated), dict(clarified)
+
+
+def cargo_metadata() -> dict:
+ """Resolved cargo metadata for this workspace."""
+ result = subprocess.run(
+ [
+ "cargo",
+ "metadata",
+ "--format-version",
+ "1",
+ "--locked",
+ "--all-features",
+ "--manifest-path",
+ str(REPO_ROOT / "Cargo.toml"),
+ ],
+ capture_output=True,
+ text=True,
check=False,
)
if result.returncode != 0:
- raise Failure("cargo-deny rejected the dependency licenses.")
+ raise Failure(f"cargo metadata failed:\n{result.stderr}")
+ return json.loads(result.stdout)
+
+
+def crate_source_dirs(crates: set[Crate]) -> dict[Crate, str]:
+ """Where cargo unpacked each crate's sources.
+
+ The directory is `<name>-<version>`, but names often contain hyphens (eg.
my-crate-5.0.3) and
+ versions can carry build metadata (eg. my-crate-5.0.3+mybuild).
+ """
+ packages = cargo_metadata()["packages"]
+
+ dict_crate_to_path = {}
+
+ for package in packages:
+ crate = (package["name"], package["version"])
+ if crate not in crates:
+ continue
+ dict_crate_to_path[crate] = str(Path(package["manifest_path"]).parent)
+
+ return dict_crate_to_path
+
+
+def require_clarifications_applied(
+ declared: set[CrateName],
+ evaluated: dict[CrateName, set[CrateVersion]],
+ actually_clarified: dict[CrateName, set[CrateVersion]],
+) -> None:
+ """
+ Fail if a clarification did not reach a crate version that was evaluated.
+
+ `declared` is all crates declared with a clarification in deny.toml,
+ `evaluated` is the list of all crate versions evaluated as part of the
dependency graph,
+ and `actually_clarified` is the subset of crate versions evaluated which
had a clarification applied.
+ """
+ crates_with_clarifications_missed: list[Crate] = []
+ for crate_name in declared:
+ if crate_name not in evaluated:
+ continue
+ versions_evaluated = evaluated[crate_name]
+ versions_clarified = actually_clarified.get(crate_name, set())
+ for crate_version in versions_evaluated - versions_clarified:
+ crates_with_clarifications_missed.append((crate_name,
crate_version))
+ crates_with_clarifications_missed.sort()
+ if not crates_with_clarifications_missed:
+ # Nothing to report
+ return
+
+ source_dirs = crate_source_dirs(set(crates_with_clarifications_missed))
+ error_line_output = []
+ for crate_name, crate_version in crates_with_clarifications_missed:
+ source_dir = source_dirs.get(
+ (crate_name, crate_version), "<source not unpacked>"
+ )
+ error_line_output.append(
+ f" {crate_name} {crate_version} ({source_dir}): clarification did
not apply"
+ )
+
+ raise Failure(
+ textwrap.dedent("""
+ {deny_toml} declares license clarifications that cargo-deny did
not apply:
+ {crates}
+
+ A clarification is discarded by cargo-deny when any of its
attested license
+ files are no longer valid (hash mismatch, etc.).
+
+ Review the clarification, checking all files still exist and the
hash matches for each.
+ You can compute the hash for a license file using the following
command:
+
+ uv run {script} dependencies clarification-hash \\
+ <crate directory above>/<license file path>
+
+ If a hash has changed, read the file and ensure the license
expression still matches.
+ If it does, an updated deny.toml should be committed.
+ """).format(
+ deny_toml=DENY_TOML,
+ crates="\n".join(error_line_output),
+ script=SCRIPT_PATH,
+ )
+ )
+
+
+def require_no_stale_clarifications(
+ declared: set[CrateName], evaluated: dict[CrateName, set[CrateVersion]]
+) -> None:
+ """Fail if a clarification names a crate that is not in the dependency
graph."""
+ stale: list[CrateName] = sorted(declared - evaluated.keys())
+ if not stale:
+ return
+
+ raise Failure(
+ textwrap.dedent(
+ """
+ {deny_toml} declares license clarifications for crates that
cargo-deny did
+ not find in the dependency graph:
+ {crates}
+
+ These are likely stale entries that should be removed.
+ """
+ ).format(
+ deny_toml=DENY_TOML,
+ crates="\n".join(f" {crate_name}" for crate_name in stale),
+ )
+ )
+
+
+def clarification_file_hash(path: Path) -> str:
+ """The hash cargo-deny expects in a [[licenses.clarify]] `license-files`
entry.
+
+ XxHash32 with seed 0, over the file with every line terminated by exactly
one LF.
+
+ cargo-deny appears to offer no way to compute this value,
+ so this command is added to make it easy (based on the cargo-deny source
code).
Review Comment:
**Clarify the hashing helper's provenance**
([licenses.py:323](https://github.com/apache/iceberg-rust/blob/683d60c194d243ccc3cf61946e877d2196212c62/dev/release/licenses.py#L323)).
The docstring says it is "based on the cargo-deny source code." Does that mean
independently implementing the algorithm, or copying/translating upstream code?
If code was copied or adapted, please document its origin and selected upstream
license and preserve applicable notices. This is a provenance question, not a
confirmed license violation. ASF's policy says: "Do not modify or remove any
copyright notices or licenses within third-party works."
##########
deny.toml:
##########
@@ -43,3 +47,33 @@ exceptions = [
{ allow = ["MPL-2.0"], crate = "colored" },
{ allow = ["MPL-2.0"], crate = "option-ext" },
]
+
+# Clarifications use a hash to verify if the license file bytes changed.
+# To recompute: uv run dev/release/licenses.py dependencies clarification-hash
<file>
+
+# TODO: zstd-sys 2.1 relicensed the wrapper and bindings to BSD-3-Clause.
+# Transient dependency, drop when we upgrade (likely when we upgrade to Arrow
60).
+# https://github.com/apache/iceberg-rust/issues/3307
+[[licenses.clarify]]
+crate = "zstd-sys"
+expression = "(MIT OR Apache-2.0) AND BSD-3-Clause AND (GPL-2.0-only OR
BSD-3-Clause)"
Review Comment:
**Include the embedded BSD-2-Clause code in the zstd clarification**
([deny.toml:59](https://github.com/apache/iceberg-rust/blob/683d60c194d243ccc3cf61946e877d2196212c62/deny.toml#L59)).
The packaged legacy decoders contain BSD-2-Clause licenses and Yann Collet
copyright notices, for example
[zstd_v05.c](https://github.com/facebook/zstd/blob/v1.5.7/lib/legacy/zstd_v05.c#L17).
I checked that the packaged source matches upstream v1.5.7 and that the
dependency graph enables `zstd-sys/legacy`. Please account for these embedded
licenses in the expression and supporting evidence, and preserve the
corresponding notices in the planned binary attribution output. ASF requires
that every third-party work include its associated license; checking the
crate's top-level license files alone misses these notices. See [ASF's
third-party source policy](https://www.apache.org/legal/src-headers.html) and
[binary distribution
guidance](https://infra.apache.org/licensing-howto.html#binary-distributions)
.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
