On Wed, Mar 25, 2026 at 11:39 AM Daniel Turull <[email protected]> wrote: > > Hi, > Should I send my conversion script to oe-core as alternative or we keep it > outside?
I would still like to look at it more closely. Maybe leave it in contrib for now, since that will make it a little easier for me to look at. > > Best regards, > > Daniel > ________________________________ > From: [email protected] > <[email protected]> on behalf of Joshua Watt via > lists.openembedded.org <[email protected]> > Sent: Wednesday, March 25, 2026 6:16 PM > To: [email protected] > <[email protected]> > Cc: Joshua Watt <[email protected]> > Subject: [OE-core][PATCH] Remove SPDX 2.2 support > > Removes SPDX 2.2 support in favor of SPDX 3 support being the only > option. The SPDX 3 data is far superior to SPDX 2.2 and thus more useful > for SBoM uses cases. > > Signed-off-by: Joshua Watt <[email protected]> > --- > meta/classes/create-spdx-2.2.bbclass | 990 --------------------------- > meta/lib/oe/sbom.py | 120 ---- > meta/lib/oe/spdx.py | 357 ---------- > meta/lib/oeqa/selftest/cases/spdx.py | 63 +- > 4 files changed, 1 insertion(+), 1529 deletions(-) > delete mode 100644 meta/classes/create-spdx-2.2.bbclass > delete mode 100644 meta/lib/oe/sbom.py > delete mode 100644 meta/lib/oe/spdx.py > > diff --git a/meta/classes/create-spdx-2.2.bbclass > b/meta/classes/create-spdx-2.2.bbclass > deleted file mode 100644 > index 1c43156559..0000000000 > --- a/meta/classes/create-spdx-2.2.bbclass > +++ /dev/null > @@ -1,990 +0,0 @@ > -# > -# Copyright OpenEmbedded Contributors > -# > -# SPDX-License-Identifier: GPL-2.0-only > -# > - > -inherit spdx-common > - > -SPDX_VERSION = "2.2" > - > -SPDX_ORG ??= "OpenEmbedded ()" > -SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" > -SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages > created from \ > - this recipe. For SPDX documents create using this class during the > build, this \ > - is the contact information for the person or organization who is doing > the \ > - build." > - > -SPDX_ARCHIVE_SOURCES ??= "0" > -SPDX_ARCHIVE_PACKAGED ??= "0" > - > -def get_namespace(d, name): > - import uuid > - namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, > d.getVar("SPDX_UUID_NAMESPACE")) > - return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, > str(uuid.uuid5(namespace_uuid, name))) > - > -SPDX_PACKAGE_VERSION ??= "${PV}" > -SPDX_PACKAGE_VERSION[doc] = "The version of a package, versionInfo in > recipe, package and image" > - > -def create_annotation(d, comment): > - from datetime import datetime, timezone > - > - creation_time = > datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") > - annotation = oe.spdx.SPDXAnnotation() > - annotation.annotationDate = creation_time > - annotation.annotationType = "OTHER" > - annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), > d.getVar("SPDX_TOOL_VERSION")) > - annotation.comment = comment > - return annotation > - > -def recipe_spdx_is_native(d, recipe): > - return any(a.annotationType == "OTHER" and > - a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), > d.getVar("SPDX_TOOL_VERSION")) and > - a.comment == "isNative" for a in recipe.annotations) > - > -def get_json_indent(d): > - if d.getVar("SPDX_PRETTY") == "1": > - return 2 > - return None > - > - > -def convert_license_to_spdx(lic, license_data, document, d, existing={}): > - from pathlib import Path > - import oe.spdx > - > - extracted = {} > - > - def add_extracted_license(ident, name): > - nonlocal document > - > - if name in extracted: > - return > - > - extracted_info = oe.spdx.SPDXExtractedLicensingInfo() > - extracted_info.name = name > - extracted_info.licenseId = ident > - extracted_info.extractedText = None > - > - if name == "PD": > - # Special-case this. > - extracted_info.extractedText = "Software released to the public > domain" > - else: > - # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH > - for directory in [d.getVar('COMMON_LICENSE_DIR')] + > (d.getVar('LICENSE_PATH') or '').split(): > - try: > - with (Path(directory) / name).open(errors="replace") as > f: > - extracted_info.extractedText = f.read() > - break > - except FileNotFoundError: > - pass > - if extracted_info.extractedText is None: > - # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set > - entry = d.getVarFlag('NO_GENERIC_LICENSE', name).split(';') > - filename = entry[0] > - params = {i.split('=')[0]: i.split('=')[1] for i in > entry[1:] if '=' in i} > - beginline = int(params.get('beginline', 1)) > - endline = params.get('endline', None) > - if endline: > - endline = int(endline) > - if filename: > - filename = d.expand("${S}/" + filename) > - with open(filename, errors="replace") as f: > - extracted_info.extractedText = "".join(line for idx, > line in enumerate(f, 1) if beginline <= idx and idx <= (endline or idx)) > - else: > - bb.fatal("Cannot find any text for license %s" % name) > - > - extracted[name] = extracted_info > - document.hasExtractedLicensingInfos.append(extracted_info) > - > - def convert(l): > - if l == "(" or l == ")": > - return l > - > - if l == "&": > - return "AND" > - > - if l == "|": > - return "OR" > - > - if l == "CLOSED": > - return "NONE" > - > - spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l > - if spdx_license in license_data["licenses"]: > - return spdx_license > - > - try: > - spdx_license = existing[l] > - except KeyError: > - spdx_license = "LicenseRef-" + l > - add_extracted_license(spdx_license, l) > - > - return spdx_license > - > - lic_split = lic.replace("(", " ( ").replace(")", " ) ").replace("|", " | > ").replace("&", " & ").split() > - > - return ' '.join(convert(l) for l in lic_split) > - > -def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, > archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): > - from pathlib import Path > - import oe.spdx > - import oe.spdx_common > - import hashlib > - > - source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") > - if source_date_epoch: > - source_date_epoch = int(source_date_epoch) > - > - sha1s = [] > - spdx_files = [] > - > - file_counter = 1 > - > - check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1" > - if check_compiled_sources: > - compiled_sources, types = oe.spdx_common.get_compiled_sources(d) > - bb.debug(1, f"Total compiled files: {len(compiled_sources)}") > - for subdir, dirs, files in os.walk(topdir): > - dirs[:] = [d for d in dirs if d not in ignore_dirs] > - if subdir == str(topdir): > - dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] > - > - for file in files: > - filepath = Path(subdir) / file > - filename = str(filepath.relative_to(topdir)) > - > - if not filepath.is_symlink() and filepath.is_file(): > - # Check if file is compiled > - if check_compiled_sources: > - if not oe.spdx_common.is_compiled_source(filename, > compiled_sources, types): > - continue > - spdx_file = oe.spdx.SPDXFile() > - spdx_file.SPDXID = get_spdxid(file_counter) > - for t in get_types(filepath): > - spdx_file.fileTypes.append(t) > - spdx_file.fileName = filename > - > - if archive is not None: > - with filepath.open("rb") as f: > - info = archive.gettarinfo(fileobj=f) > - info.name = filename > - info.uid = 0 > - info.gid = 0 > - info.uname = "root" > - info.gname = "root" > - > - if source_date_epoch is not None and info.mtime > > source_date_epoch: > - info.mtime = source_date_epoch > - > - archive.addfile(info, f) > - > - sha1 = bb.utils.sha1_file(filepath) > - sha1s.append(sha1) > - spdx_file.checksums.append(oe.spdx.SPDXChecksum( > - algorithm="SHA1", > - checksumValue=sha1, > - )) > - spdx_file.checksums.append(oe.spdx.SPDXChecksum( > - algorithm="SHA256", > - checksumValue=bb.utils.sha256_file(filepath), > - )) > - > - if "SOURCE" in spdx_file.fileTypes: > - extracted_lics = > oe.spdx_common.extract_licenses(filepath) > - if extracted_lics: > - spdx_file.licenseInfoInFiles = extracted_lics > - > - doc.files.append(spdx_file) > - doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) > - spdx_pkg.hasFiles.append(spdx_file.SPDXID) > - > - spdx_files.append(spdx_file) > - > - file_counter += 1 > - > - sha1s.sort() > - verifier = hashlib.sha1() > - for v in sha1s: > - verifier.update(v.encode("utf-8")) > - spdx_pkg.packageVerificationCode.packageVerificationCodeValue = > verifier.hexdigest() > - > - return spdx_files > - > - > -def add_package_sources_from_debug(d, package_doc, spdx_package, package, > package_files, sources): > - from pathlib import Path > - import hashlib > - import oe.packagedata > - import oe.spdx > - > - debug_search_paths = [ > - Path(d.getVar('PKGD')), > - Path(d.getVar('STAGING_DIR_TARGET')), > - Path(d.getVar('STAGING_DIR_NATIVE')), > - Path(d.getVar('STAGING_KERNEL_DIR')), > - ] > - > - pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) > - > - if pkg_data is None: > - return > - > - for file_path, file_data in pkg_data["files_info"].items(): > - if not "debugsrc" in file_data: > - continue > - > - for pkg_file in package_files: > - if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): > - break > - else: > - bb.fatal("No package file found for %s in %s; SPDX found: %s" % > (str(file_path), package, > - " ".join(p.fileName for p in package_files))) > - continue > - > - for debugsrc in file_data["debugsrc"]: > - ref_id = "NOASSERTION" > - for search in debug_search_paths: > - if debugsrc.startswith("/usr/src/kernel"): > - debugsrc_path = search / > debugsrc.replace('/usr/src/kernel/', '') > - else: > - debugsrc_path = search / debugsrc.lstrip("/") > - # We can only hash files below, skip directories, links, etc. > - if not os.path.isfile(debugsrc_path): > - continue > - > - file_sha256 = bb.utils.sha256_file(debugsrc_path) > - > - if file_sha256 in sources: > - source_file = sources[file_sha256] > - > - doc_ref = > package_doc.find_external_document_ref(source_file.doc.documentNamespace) > - if doc_ref is None: > - doc_ref = oe.spdx.SPDXExternalDocumentRef() > - doc_ref.externalDocumentId = > "DocumentRef-dependency-" + source_file.doc.name > - doc_ref.spdxDocument = > source_file.doc.documentNamespace > - doc_ref.checksum.algorithm = "SHA1" > - doc_ref.checksum.checksumValue = source_file.doc_sha1 > - package_doc.externalDocumentRefs.append(doc_ref) > - > - ref_id = "%s:%s" % (doc_ref.externalDocumentId, > source_file.file.SPDXID) > - else: > - bb.debug(1, "Debug source %s with SHA256 %s not found in > any dependency" % (str(debugsrc_path), file_sha256)) > - break > - else: > - bb.debug(1, "Debug source %s not found" % debugsrc) > - > - package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, > comment=debugsrc) > - > -add_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR" > - > -def collect_dep_recipes(d, doc, spdx_recipe, direct_deps): > - import json > - from pathlib import Path > - import oe.sbom > - import oe.spdx > - import oe.spdx_common > - > - deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) > - package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() > - package_archs.reverse() > - > - dep_recipes = [] > - > - for dep in direct_deps: > - # If this dependency is not calculated in the taskhash skip it. > - # Otherwise, it can result in broken links since this task won't > - # rebuild and see the new SPDX ID if the dependency changes > - if not dep.in_taskhash: > - continue > - > - dep_recipe_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, > package_archs, "recipe-" + dep.pn, dep.hashfn) > - if not dep_recipe_path: > - bb.fatal("Cannot find any SPDX file for recipe %s, %s" % > (dep.pn, dep.hashfn)) > - > - spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) > - > - for pkg in spdx_dep_doc.packages: > - if pkg.name == dep.pn: > - spdx_dep_recipe = pkg > - break > - else: > - continue > - > - dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, > spdx_dep_recipe)) > - > - dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() > - dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + > spdx_dep_doc.name > - dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace > - dep_recipe_ref.checksum.algorithm = "SHA1" > - dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 > - > - doc.externalDocumentRefs.append(dep_recipe_ref) > - > - doc.add_relationship( > - "%s:%s" % (dep_recipe_ref.externalDocumentId, > spdx_dep_recipe.SPDXID), > - "BUILD_DEPENDENCY_OF", > - spdx_recipe > - ) > - > - return dep_recipes > - > -collect_dep_recipes[vardepsexclude] = "SPDX_MULTILIB_SSTATE_ARCHS" > - > -def collect_dep_sources(d, dep_recipes): > - import oe.sbom > - > - sources = {} > - for dep in dep_recipes: > - # Don't collect sources from native recipes as they > - # match non-native sources also. > - if recipe_spdx_is_native(d, dep.recipe): > - continue > - recipe_files = set(dep.recipe.hasFiles) > - > - for spdx_file in dep.doc.files: > - if spdx_file.SPDXID not in recipe_files: > - continue > - > - if "SOURCE" in spdx_file.fileTypes: > - for checksum in spdx_file.checksums: > - if checksum.algorithm == "SHA256": > - sources[checksum.checksumValue] = > oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) > - break > - > - return sources > - > -def add_download_packages(d, doc, recipe): > - import os.path > - from bb.fetch2 import decodeurl, CHECKSUM_LIST > - import bb.process > - import oe.spdx > - import oe.sbom > - > - for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): > - f = bb.fetch2.FetchData(src_uri, d) > - > - package = oe.spdx.SPDXPackage() > - package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) > - package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) > - > - if f.type == "file": > - continue > - > - if f.method.supports_checksum(f): > - for checksum_id in CHECKSUM_LIST: > - if checksum_id.upper() not in > oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: > - continue > - > - expected_checksum = getattr(f, "%s_expected" % checksum_id) > - if expected_checksum is None: > - continue > - > - c = oe.spdx.SPDXChecksum() > - c.algorithm = checksum_id.upper() > - c.checksumValue = expected_checksum > - package.checksums.append(c) > - > - package.downloadLocation = oe.spdx_common.fetch_data_to_uri(f, > f.name) > - doc.packages.append(package) > - doc.add_relationship(doc, "DESCRIBES", package) > - # In the future, we might be able to do more fancy dependencies, > - # but this should be sufficient for now > - doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) > - > -def get_license_list_version(license_data, d): > - # Newer versions of the SPDX license list are SemVer > ("MAJOR.MINOR.MICRO"), > - # but SPDX 2 only uses "MAJOR.MINOR". > - return ".".join(license_data["licenseListVersion"].split(".")[:2]) > - > - > -# This task is added for compatibility with tasks shared with SPDX 3, but > -# doesn't do anything > -do_create_recipe_spdx() { > - : > -} > -do_create_recipe_spdx[noexec] = "1" > -addtask do_create_recipe_spdx > - > - > -python do_create_spdx() { > - from datetime import datetime, timezone > - import oe.sbom > - import oe.spdx > - import oe.spdx_common > - import uuid > - from pathlib import Path > - from contextlib import contextmanager > - import oe.cve_check > - > - license_data = oe.spdx_common.load_spdx_license_data(d) > - > - @contextmanager > - def optional_tarfile(name, guard, mode="w"): > - import tarfile > - import bb.compress.zstd > - > - num_threads = int(d.getVar("BB_NUMBER_THREADS")) > - > - if guard: > - name.parent.mkdir(parents=True, exist_ok=True) > - with bb.compress.zstd.open(name, mode=mode + "b", > num_threads=num_threads) as f: > - with tarfile.open(fileobj=f, mode=mode + "|") as tf: > - yield tf > - else: > - yield None > - > - > - deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) > - spdx_workdir = Path(d.getVar("SPDXWORK")) > - include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" > - archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" > - archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" > - pkg_arch = d.getVar("SSTATE_PKGARCH") > - > - creation_time = > datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") > - > - doc = oe.spdx.SPDXDocument() > - > - doc.name = "recipe-" + d.getVar("PN") > - doc.documentNamespace = get_namespace(d, doc.name) > - doc.creationInfo.created = creation_time > - doc.creationInfo.comment = "This document was created by analyzing > recipe files during the build." > - doc.creationInfo.licenseListVersion = > get_license_list_version(license_data, d) > - doc.creationInfo.creators.append("Tool: OpenEmbedded Core > create-spdx.bbclass") > - doc.creationInfo.creators.append("Organization: %s" % > d.getVar("SPDX_ORG")) > - doc.creationInfo.creators.append("Person: N/A ()") > - > - recipe = oe.spdx.SPDXPackage() > - recipe.name = d.getVar("PN") > - recipe.versionInfo = d.getVar("SPDX_PACKAGE_VERSION") > - recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) > - recipe.supplier = d.getVar("SPDX_SUPPLIER") > - if bb.data.inherits_class("native", d) or > bb.data.inherits_class("cross", d): > - recipe.annotations.append(create_annotation(d, "isNative")) > - > - homepage = d.getVar("HOMEPAGE") > - if homepage: > - recipe.homepage = homepage > - > - license = d.getVar("LICENSE") > - if license: > - recipe.licenseDeclared = convert_license_to_spdx(license, > license_data, doc, d) > - > - summary = d.getVar("SUMMARY") > - if summary: > - recipe.summary = summary > - > - description = d.getVar("DESCRIPTION") > - if description: > - recipe.description = description > - > - if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"): > - for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): > - recipe.annotations.append(create_annotation(d, var + "=" + > d.getVar(var))) > - > - # Some CVEs may be patched during the build process without incrementing > the version number, > - # so querying for CVEs based on the CPE id can lead to false positives. > To account for this, > - # save the CVEs fixed by patches to source information field in the SPDX. > - patched_cves = oe.cve_check.get_patched_cves(d) > - patched_cves = list(patched_cves) > - > - ignored_cves = d.getVar("CVE_CHECK_IGNORE") > - if ignored_cves: > - patched_cves.extend(ignored_cves.split()) > - > - patched_cves = ' '.join(patched_cves) > - if patched_cves: > - recipe.sourceInfo = "CVEs fixed: " + patched_cves > - > - cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), > d.getVar("CVE_VERSION")) > - if cpe_ids: > - for cpe_id in cpe_ids: > - cpe = oe.spdx.SPDXExternalReference() > - cpe.referenceCategory = "SECURITY" > - cpe.referenceType = "cpe23Type" > - cpe.referenceLocator = cpe_id > - recipe.externalRefs.append(cpe) > - > - doc.packages.append(recipe) > - doc.add_relationship(doc, "DESCRIBES", recipe) > - > - add_download_packages(d, doc, recipe) > - > - if oe.spdx_common.process_sources(d) and include_sources: > - recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + > ".tar.zst") > - with optional_tarfile(recipe_archive, archive_sources) as archive: > - oe.spdx_common.get_patched_src(d) > - > - add_package_files( > - d, > - doc, > - recipe, > - spdx_workdir, > - lambda file_counter: "SPDXRef-SourceFile-%s-%d" % > (d.getVar("PN"), file_counter), > - lambda filepath: ["SOURCE"], > - ignore_dirs=[".git"], > - ignore_top_level_dirs=["temp"], > - archive=archive, > - ) > - > - if archive is not None: > - recipe.packageFileName = str(recipe_archive.name) > - > - direct_deps = oe.spdx_common.collect_direct_deps(d, "do_create_spdx") > - dep_recipes = collect_dep_recipes(d, doc, recipe, direct_deps) > - > - doc_sha1 = oe.sbom.write_doc(d, doc, pkg_arch, "recipes", > indent=get_json_indent(d)) > - dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) > - > - recipe_ref = oe.spdx.SPDXExternalDocumentRef() > - recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name > - recipe_ref.spdxDocument = doc.documentNamespace > - recipe_ref.checksum.algorithm = "SHA1" > - recipe_ref.checksum.checksumValue = doc_sha1 > - > - sources = collect_dep_sources(d, dep_recipes) > - found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + > license.licenseId for license in doc.hasExtractedLicensingInfos} > - > - if not recipe_spdx_is_native(d, recipe): > - bb.build.exec_func("read_subpackage_metadata", d) > - > - pkgdest = Path(d.getVar("PKGDEST")) > - for package in d.getVar("PACKAGES").split(): > - if not oe.packagedata.packaged(package, d): > - continue > - > - package_doc = oe.spdx.SPDXDocument() > - pkg_name = d.getVar("PKG:%s" % package) or package > - package_doc.name = pkg_name > - package_doc.documentNamespace = get_namespace(d, > package_doc.name) > - package_doc.creationInfo.created = creation_time > - package_doc.creationInfo.comment = "This document was created by > analyzing packages created during the build." > - package_doc.creationInfo.licenseListVersion = > get_license_list_version(license_data, d) > - package_doc.creationInfo.creators.append("Tool: OpenEmbedded > Core create-spdx.bbclass") > - package_doc.creationInfo.creators.append("Organization: %s" % > d.getVar("SPDX_ORG")) > - package_doc.creationInfo.creators.append("Person: N/A ()") > - package_doc.externalDocumentRefs.append(recipe_ref) > - > - package_license = d.getVar("LICENSE:%s" % package) or > d.getVar("LICENSE") > - > - spdx_package = oe.spdx.SPDXPackage() > - > - spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) > - spdx_package.name = pkg_name > - spdx_package.versionInfo = d.getVar("SPDX_PACKAGE_VERSION") > - spdx_package.licenseDeclared = > convert_license_to_spdx(package_license, license_data, package_doc, d, > found_licenses) > - spdx_package.supplier = d.getVar("SPDX_SUPPLIER") > - > - package_doc.packages.append(spdx_package) > - > - package_doc.add_relationship(spdx_package, "GENERATED_FROM", > "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) > - package_doc.add_relationship(package_doc, "DESCRIBES", > spdx_package) > - > - package_archive = deploy_dir_spdx / "packages" / > (package_doc.name + ".tar.zst") > - with optional_tarfile(package_archive, archive_packaged) as > archive: > - package_files = add_package_files( > - d, > - package_doc, > - spdx_package, > - pkgdest / package, > - lambda file_counter: > oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), > - lambda filepath: ["BINARY"], > - ignore_top_level_dirs=['CONTROL', 'DEBIAN'], > - archive=archive, > - ) > - > - if archive is not None: > - spdx_package.packageFileName = str(package_archive.name) > - > - add_package_sources_from_debug(d, package_doc, spdx_package, > package, package_files, sources) > - > - oe.sbom.write_doc(d, package_doc, pkg_arch, "packages", > indent=get_json_indent(d)) > -} > -do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS" > -# NOTE: depending on do_unpack is a hack that is necessary to get it's > dependencies for archive the source > -addtask do_create_spdx after do_create_recipe_spdx do_package do_packagedata > do_unpack do_patch before do_populate_sdk do_build do_rm_work > - > -SSTATETASKS += "do_create_spdx" > -do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" > -do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" > - > -python do_create_spdx_setscene () { > - sstate_setscene(d) > -} > -addtask do_create_spdx_setscene > - > -do_create_spdx[deptask] += "do_create_spdx" > -do_create_spdx[dirs] = "${SPDXWORK}" > -do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" > -do_create_spdx[depends] += " \ > - ${PATCHDEPENDENCY} \ > - ${@create_spdx_source_deps(d)} \ > -" > - > -python do_create_runtime_spdx() { > - from datetime import datetime, timezone > - import oe.sbom > - import oe.spdx > - import oe.spdx_common > - import oe.packagedata > - from pathlib import Path > - > - deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) > - spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) > - is_native = bb.data.inherits_class("native", d) or > bb.data.inherits_class("cross", d) > - > - creation_time = > datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") > - > - license_data = oe.spdx_common.load_spdx_license_data(d) > - > - direct_deps = oe.spdx_common.collect_direct_deps(d, "do_create_spdx") > - > - providers = oe.spdx_common.collect_package_providers(d, direct_deps) > - pkg_arch = d.getVar("SSTATE_PKGARCH") > - package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() > - package_archs.reverse() > - > - if not is_native: > - bb.build.exec_func("read_subpackage_metadata", d) > - > - dep_package_cache = {} > - > - pkgdest = Path(d.getVar("PKGDEST")) > - for package in d.getVar("PACKAGES").split(): > - localdata = bb.data.createCopy(d) > - pkg_name = d.getVar("PKG:%s" % package) or package > - localdata.setVar("PKG", pkg_name) > - localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" > + package) > - > - if not oe.packagedata.packaged(package, localdata): > - continue > - > - pkg_spdx_path = oe.sbom.doc_path(deploy_dir_spdx, pkg_name, > pkg_arch, "packages") > - > - package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) > - > - for p in package_doc.packages: > - if p.name == pkg_name: > - spdx_package = p > - break > - else: > - bb.fatal("Package '%s' not found in %s" % (pkg_name, > pkg_spdx_path)) > - > - runtime_doc = oe.spdx.SPDXDocument() > - runtime_doc.name = "runtime-" + pkg_name > - runtime_doc.documentNamespace = get_namespace(localdata, > runtime_doc.name) > - runtime_doc.creationInfo.created = creation_time > - runtime_doc.creationInfo.comment = "This document was created by > analyzing package runtime dependencies." > - runtime_doc.creationInfo.licenseListVersion = > get_license_list_version(license_data, d) > - runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded > Core create-spdx.bbclass") > - runtime_doc.creationInfo.creators.append("Organization: %s" % > d.getVar("SPDX_ORG")) > - runtime_doc.creationInfo.creators.append("Person: N/A ()") > - > - package_ref = oe.spdx.SPDXExternalDocumentRef() > - package_ref.externalDocumentId = "DocumentRef-package-" + package > - package_ref.spdxDocument = package_doc.documentNamespace > - package_ref.checksum.algorithm = "SHA1" > - package_ref.checksum.checksumValue = package_doc_sha1 > - > - runtime_doc.externalDocumentRefs.append(package_ref) > - > - runtime_doc.add_relationship( > - runtime_doc.SPDXID, > - "AMENDS", > - "%s:%s" % (package_ref.externalDocumentId, > package_doc.SPDXID) > - ) > - > - deps = > bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") > - seen_deps = set() > - for dep, _ in deps.items(): > - if dep in seen_deps: > - continue > - > - if dep not in providers: > - continue > - > - (dep, dep_hashfn) = providers[dep] > - > - if not oe.packagedata.packaged(dep, localdata): > - continue > - > - dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) > - dep_pkg = dep_pkg_data["PKG"] > - > - if dep in dep_package_cache: > - (dep_spdx_package, dep_package_ref) = > dep_package_cache[dep] > - else: > - dep_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, > package_archs, dep_pkg, dep_hashfn) > - if not dep_path: > - bb.fatal("No SPDX file found for package %s, %s" % > (dep_pkg, dep_hashfn)) > - > - spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) > - > - for pkg in spdx_dep_doc.packages: > - if pkg.name == dep_pkg: > - dep_spdx_package = pkg > - break > - else: > - bb.fatal("Package '%s' not found in %s" % (dep_pkg, > dep_path)) > - > - dep_package_ref = oe.spdx.SPDXExternalDocumentRef() > - dep_package_ref.externalDocumentId = > "DocumentRef-runtime-dependency-" + spdx_dep_doc.name > - dep_package_ref.spdxDocument = > spdx_dep_doc.documentNamespace > - dep_package_ref.checksum.algorithm = "SHA1" > - dep_package_ref.checksum.checksumValue = spdx_dep_sha1 > - > - dep_package_cache[dep] = (dep_spdx_package, > dep_package_ref) > - > - runtime_doc.externalDocumentRefs.append(dep_package_ref) > - > - runtime_doc.add_relationship( > - "%s:%s" % (dep_package_ref.externalDocumentId, > dep_spdx_package.SPDXID), > - "RUNTIME_DEPENDENCY_OF", > - "%s:%s" % (package_ref.externalDocumentId, > spdx_package.SPDXID) > - ) > - seen_deps.add(dep) > - > - oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", > spdx_deploy, indent=get_json_indent(d)) > -} > - > -do_create_runtime_spdx[vardepsexclude] += "OVERRIDES > SPDX_MULTILIB_SSTATE_ARCHS" > - > -addtask do_create_runtime_spdx after do_create_spdx before do_build > do_rm_work > -SSTATETASKS += "do_create_runtime_spdx" > -do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" > -do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" > - > -python do_create_runtime_spdx_setscene () { > - sstate_setscene(d) > -} > -addtask do_create_runtime_spdx_setscene > - > -do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" > -do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" > -do_create_runtime_spdx[deptask] = "do_create_spdx" > -do_create_runtime_spdx[rdeptask] = "do_create_spdx" > - > -do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" > -do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" > - > -ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx" > - > -do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" > -do_populate_sdk[cleandirs] += "${SPDXSDKWORK}" > -POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " > sdk_host_combine_spdx" > -POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " > sdk_target_combine_spdx" > - > -python image_combine_spdx() { > - import os > - import oe.sbom > - from pathlib import Path > - from oe.rootfs import image_list_installed_packages > - > - image_name = d.getVar("IMAGE_NAME") > - image_link_name = d.getVar("IMAGE_LINK_NAME") > - imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) > - img_spdxid = oe.sbom.get_image_spdxid(image_name) > - packages = image_list_installed_packages(d) > - > - combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, > Path(d.getVar("SPDXIMAGEWORK"))) > - > - def make_image_link(target_path, suffix): > - if image_link_name: > - link = imgdeploydir / (image_link_name + suffix) > - if link != target_path: > - link.symlink_to(os.path.relpath(target_path, link.parent)) > - > - spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") > - make_image_link(spdx_tar_path, ".spdx.tar.zst") > -} > - > -python sdk_host_combine_spdx() { > - sdk_combine_spdx(d, "host") > -} > - > -python sdk_target_combine_spdx() { > - sdk_combine_spdx(d, "target") > -} > - > -def sdk_combine_spdx(d, sdk_type): > - import oe.sbom > - from pathlib import Path > - from oe.sdk import sdk_list_installed_packages > - > - sdk_name = d.getVar("TOOLCHAIN_OUTPUTNAME") + "-" + sdk_type > - sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) > - sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) > - sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") > - combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, > Path(d.getVar('SPDXSDKWORK'))) > - > -def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, > spdx_workdir): > - import os > - import oe.spdx > - import oe.sbom > - import oe.spdx_common > - import io > - import json > - from datetime import timezone, datetime > - from pathlib import Path > - import tarfile > - import bb.compress.zstd > - > - license_data = oe.spdx_common.load_spdx_license_data(d) > - > - direct_deps = oe.spdx_common.collect_direct_deps(d, "do_create_spdx") > - > - providers = oe.spdx_common.collect_package_providers(d, direct_deps) > - package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() > - package_archs.reverse() > - > - creation_time = > datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") > - deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) > - source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") > - > - doc = oe.spdx.SPDXDocument() > - doc.name = rootfs_name > - doc.documentNamespace = get_namespace(d, doc.name) > - doc.creationInfo.created = creation_time > - doc.creationInfo.comment = "This document was created by analyzing the > source of the Yocto recipe during the build." > - doc.creationInfo.licenseListVersion = > get_license_list_version(license_data, d) > - doc.creationInfo.creators.append("Tool: OpenEmbedded Core > create-spdx.bbclass") > - doc.creationInfo.creators.append("Organization: %s" % > d.getVar("SPDX_ORG")) > - doc.creationInfo.creators.append("Person: N/A ()") > - > - image = oe.spdx.SPDXPackage() > - image.name = d.getVar("PN") > - image.versionInfo = d.getVar("SPDX_PACKAGE_VERSION") > - image.SPDXID = rootfs_spdxid > - image.supplier = d.getVar("SPDX_SUPPLIER") > - > - doc.packages.append(image) > - > - if packages: > - for name in sorted(packages.keys()): > - if name not in providers: > - bb.note("Unable to find SPDX provider for '%s'" % name) > - continue > - > - pkg_name, pkg_hashfn = providers[name] > - > - pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, > package_archs, pkg_name, pkg_hashfn) > - if not pkg_spdx_path: > - bb.fatal("No SPDX file found for package %s, %s" % > (pkg_name, pkg_hashfn)) > - > - pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) > - > - for p in pkg_doc.packages: > - if p.name == name: > - pkg_ref = oe.spdx.SPDXExternalDocumentRef() > - pkg_ref.externalDocumentId = "DocumentRef-%s" % > pkg_doc.name > - pkg_ref.spdxDocument = pkg_doc.documentNamespace > - pkg_ref.checksum.algorithm = "SHA1" > - pkg_ref.checksum.checksumValue = pkg_doc_sha1 > - > - doc.externalDocumentRefs.append(pkg_ref) > - doc.add_relationship(image, "CONTAINS", "%s:%s" % > (pkg_ref.externalDocumentId, p.SPDXID)) > - break > - else: > - bb.fatal("Unable to find package with name '%s' in SPDX file > %s" % (name, pkg_spdx_path)) > - > - runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, > package_archs, "runtime-" + name, pkg_hashfn) > - if not runtime_spdx_path: > - bb.fatal("No runtime SPDX document found for %s, %s" % > (name, pkg_hashfn)) > - > - runtime_doc, runtime_doc_sha1 = > oe.sbom.read_doc(runtime_spdx_path) > - > - runtime_ref = oe.spdx.SPDXExternalDocumentRef() > - runtime_ref.externalDocumentId = "DocumentRef-%s" % > runtime_doc.name > - runtime_ref.spdxDocument = runtime_doc.documentNamespace > - runtime_ref.checksum.algorithm = "SHA1" > - runtime_ref.checksum.checksumValue = runtime_doc_sha1 > - > - # "OTHER" isn't ideal here, but I can't find a relationship that > makes sense > - doc.externalDocumentRefs.append(runtime_ref) > - doc.add_relationship( > - image, > - "OTHER", > - "%s:%s" % (runtime_ref.externalDocumentId, > runtime_doc.SPDXID), > - comment="Runtime dependencies for %s" % name > - ) > - bb.utils.mkdirhier(spdx_workdir) > - image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") > - > - with image_spdx_path.open("wb") as f: > - doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) > - > - num_threads = int(d.getVar("BB_NUMBER_THREADS")) > - > - visited_docs = set() > - > - index = {"documents": []} > - > - spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") > - with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) > as f: > - with tarfile.open(fileobj=f, mode="w|") as tar: > - def collect_spdx_document(path): > - nonlocal tar > - nonlocal deploy_dir_spdx > - nonlocal source_date_epoch > - nonlocal index > - > - if path in visited_docs: > - return > - > - visited_docs.add(path) > - > - with path.open("rb") as f: > - doc, sha1 = oe.sbom.read_doc(f) > - f.seek(0) > - > - if doc.documentNamespace in visited_docs: > - return > - > - bb.note("Adding SPDX document %s" % path) > - visited_docs.add(doc.documentNamespace) > - info = tar.gettarinfo(fileobj=f) > - > - info.name = doc.name + ".spdx.json" > - info.uid = 0 > - info.gid = 0 > - info.uname = "root" > - info.gname = "root" > - > - if source_date_epoch is not None and info.mtime > > int(source_date_epoch): > - info.mtime = int(source_date_epoch) > - > - tar.addfile(info, f) > - > - index["documents"].append({ > - "filename": info.name, > - "documentNamespace": doc.documentNamespace, > - "sha1": sha1, > - }) > - > - for ref in doc.externalDocumentRefs: > - ref_path = > oe.sbom.doc_find_by_namespace(deploy_dir_spdx, package_archs, > ref.spdxDocument) > - if not ref_path: > - bb.fatal("Cannot find any SPDX file for document %s" > % ref.spdxDocument) > - collect_spdx_document(ref_path) > - > - collect_spdx_document(image_spdx_path) > - > - index["documents"].sort(key=lambda x: x["filename"]) > - > - index_str = io.BytesIO(json.dumps( > - index, > - sort_keys=True, > - indent=get_json_indent(d), > - ).encode("utf-8")) > - > - info = tarfile.TarInfo() > - info.name = "index.json" > - info.size = len(index_str.getvalue()) > - info.uid = 0 > - info.gid = 0 > - info.uname = "root" > - info.gname = "root" > - > - tar.addfile(info, fileobj=index_str) > - > -combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS > SPDX_MULTILIB_SSTATE_ARCHS" > diff --git a/meta/lib/oe/sbom.py b/meta/lib/oe/sbom.py > deleted file mode 100644 > index fd4b6895d8..0000000000 > --- a/meta/lib/oe/sbom.py > +++ /dev/null > @@ -1,120 +0,0 @@ > -# > -# Copyright OpenEmbedded Contributors > -# > -# SPDX-License-Identifier: GPL-2.0-only > -# > - > -import collections > - > -DepRecipe = collections.namedtuple("DepRecipe", ("doc", "doc_sha1", > "recipe")) > -DepSource = collections.namedtuple("DepSource", ("doc", "doc_sha1", > "recipe", "file")) > - > - > -def get_recipe_spdxid(d): > - return "SPDXRef-%s-%s" % ("Recipe", d.getVar("PN")) > - > - > -def get_download_spdxid(d, idx): > - return "SPDXRef-Download-%s-%d" % (d.getVar("PN"), idx) > - > - > -def get_package_spdxid(pkg): > - return "SPDXRef-Package-%s" % pkg > - > - > -def get_source_file_spdxid(d, idx): > - return "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), idx) > - > - > -def get_packaged_file_spdxid(pkg, idx): > - return "SPDXRef-PackagedFile-%s-%d" % (pkg, idx) > - > - > -def get_image_spdxid(img): > - return "SPDXRef-Image-%s" % img > - > - > -def get_sdk_spdxid(sdk): > - return "SPDXRef-SDK-%s" % sdk > - > - > -def _doc_path_by_namespace(spdx_deploy, arch, doc_namespace): > - return spdx_deploy / "by-namespace" / arch / doc_namespace.replace("/", > "_") > - > - > -def doc_find_by_namespace(spdx_deploy, search_arches, doc_namespace): > - for pkgarch in search_arches: > - p = _doc_path_by_namespace(spdx_deploy, pkgarch, doc_namespace) > - if os.path.exists(p): > - return p > - return None > - > - > -def _doc_path_by_hashfn(spdx_deploy, arch, doc_name, hashfn): > - return ( > - spdx_deploy / "by-hash" / arch / hashfn.split()[1] / (doc_name + > ".spdx.json") > - ) > - > - > -def doc_find_by_hashfn(spdx_deploy, search_arches, doc_name, hashfn): > - for pkgarch in search_arches: > - p = _doc_path_by_hashfn(spdx_deploy, pkgarch, doc_name, hashfn) > - if os.path.exists(p): > - return p > - return None > - > - > -def doc_path(spdx_deploy, doc_name, arch, subdir): > - return spdx_deploy / arch / subdir / (doc_name + ".spdx.json") > - > - > -def write_doc(d, spdx_doc, arch, subdir, spdx_deploy=None, indent=None): > - from pathlib import Path > - > - if spdx_deploy is None: > - spdx_deploy = Path(d.getVar("SPDXDEPLOY")) > - > - dest = doc_path(spdx_deploy, spdx_doc.name, arch, subdir) > - dest.parent.mkdir(exist_ok=True, parents=True) > - with dest.open("wb") as f: > - doc_sha1 = spdx_doc.to_json(f, sort_keys=True, indent=indent) > - > - l = _doc_path_by_namespace(spdx_deploy, arch, spdx_doc.documentNamespace) > - l.parent.mkdir(exist_ok=True, parents=True) > - l.symlink_to(os.path.relpath(dest, l.parent)) > - > - l = _doc_path_by_hashfn( > - spdx_deploy, arch, spdx_doc.name, d.getVar("BB_HASHFILENAME") > - ) > - l.parent.mkdir(exist_ok=True, parents=True) > - l.symlink_to(os.path.relpath(dest, l.parent)) > - > - return doc_sha1 > - > - > -def read_doc(fn): > - import hashlib > - import oe.spdx > - import io > - import contextlib > - > - @contextlib.contextmanager > - def get_file(): > - if isinstance(fn, io.IOBase): > - yield fn > - else: > - with fn.open("rb") as f: > - yield f > - > - with get_file() as f: > - sha1 = hashlib.sha1() > - while True: > - chunk = f.read(4096) > - if not chunk: > - break > - sha1.update(chunk) > - > - f.seek(0) > - doc = oe.spdx.SPDXDocument.from_json(f) > - > - return (doc, sha1.hexdigest()) > diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py > deleted file mode 100644 > index 7aaf2af5ed..0000000000 > --- a/meta/lib/oe/spdx.py > +++ /dev/null > @@ -1,357 +0,0 @@ > -# > -# Copyright OpenEmbedded Contributors > -# > -# SPDX-License-Identifier: GPL-2.0-only > -# > - > -# > -# This library is intended to capture the JSON SPDX specification in a type > -# safe manner. It is not intended to encode any particular OE specific > -# behaviors, see the sbom.py for that. > -# > -# The documented SPDX spec document doesn't cover the JSON syntax for > -# particular configuration, which can make it hard to determine what the JSON > -# syntax should be. I've found it is actually much simpler to read the > official > -# SPDX JSON schema which can be found here: > https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2Fspdx%2Fspdx-spec&data=05%7C02%7Cdaniel.turull%40ericsson.com%7Cb55ec66fdc694391aa7208de8a924852%7C92e84cebfbfd47abbe52080c6b87953f%7C0%7C0%7C639100558059258824%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=%2BVln%2FHYRh1ohkoxiJBVPujrZaQqIW99UUvXjgAqAR8I%3D&reserved=0 > -# in schemas/spdx-schema.json > -# > - > -import hashlib > -import itertools > -import json > - > -SPDX_VERSION = "2.2" > - > - > -# > -# The following are the support classes that are used to implement SPDX > object > -# > - > -class _Property(object): > - """ > - A generic SPDX object property. The different types will derive from this > - class > - """ > - > - def __init__(self, *, default=None): > - self.default = default > - > - def setdefault(self, dest, name): > - if self.default is not None: > - dest.setdefault(name, self.default) > - > - > -class _String(_Property): > - """ > - A scalar string property for an SPDX object > - """ > - > - def __init__(self, **kwargs): > - super().__init__(**kwargs) > - > - def set_property(self, attrs, name): > - def get_helper(obj): > - return obj._spdx[name] > - > - def set_helper(obj, value): > - obj._spdx[name] = value > - > - def del_helper(obj): > - del obj._spdx[name] > - > - attrs[name] = property(get_helper, set_helper, del_helper) > - > - def init(self, source): > - return source > - > - > -class _Object(_Property): > - """ > - A scalar SPDX object property of a SPDX object > - """ > - > - def __init__(self, cls, **kwargs): > - super().__init__(**kwargs) > - self.cls = cls > - > - def set_property(self, attrs, name): > - def get_helper(obj): > - if not name in obj._spdx: > - obj._spdx[name] = self.cls() > - return obj._spdx[name] > - > - def set_helper(obj, value): > - obj._spdx[name] = value > - > - def del_helper(obj): > - del obj._spdx[name] > - > - attrs[name] = property(get_helper, set_helper) > - > - def init(self, source): > - return self.cls(**source) > - > - > -class _ListProperty(_Property): > - """ > - A list of SPDX properties > - """ > - > - def __init__(self, prop, **kwargs): > - super().__init__(**kwargs) > - self.prop = prop > - > - def set_property(self, attrs, name): > - def get_helper(obj): > - if not name in obj._spdx: > - obj._spdx[name] = [] > - return obj._spdx[name] > - > - def set_helper(obj, value): > - obj._spdx[name] = list(value) > - > - def del_helper(obj): > - del obj._spdx[name] > - > - attrs[name] = property(get_helper, set_helper, del_helper) > - > - def init(self, source): > - return [self.prop.init(o) for o in source] > - > - > -class _StringList(_ListProperty): > - """ > - A list of strings as a property for an SPDX object > - """ > - > - def __init__(self, **kwargs): > - super().__init__(_String(), **kwargs) > - > - > -class _ObjectList(_ListProperty): > - """ > - A list of SPDX objects as a property for an SPDX object > - """ > - > - def __init__(self, cls, **kwargs): > - super().__init__(_Object(cls), **kwargs) > - > - > -class MetaSPDXObject(type): > - """ > - A metaclass that allows properties (anything derived from a _Property > - class) to be defined for a SPDX object > - """ > - def __new__(mcls, name, bases, attrs): > - attrs["_properties"] = {} > - > - for key in attrs.keys(): > - if isinstance(attrs[key], _Property): > - prop = attrs[key] > - attrs["_properties"][key] = prop > - prop.set_property(attrs, key) > - > - return super().__new__(mcls, name, bases, attrs) > - > - > -class SPDXObject(metaclass=MetaSPDXObject): > - """ > - The base SPDX object; all SPDX spec classes must derive from this class > - """ > - def __init__(self, **d): > - self._spdx = {} > - > - for name, prop in self._properties.items(): > - prop.setdefault(self._spdx, name) > - if name in d: > - self._spdx[name] = prop.init(d[name]) > - > - def serializer(self): > - return self._spdx > - > - def __setattr__(self, name, value): > - if name in self._properties or name == "_spdx": > - super().__setattr__(name, value) > - return > - raise KeyError("%r is not a valid SPDX property" % name) > - > -# > -# These are the SPDX objects implemented from the spec. The *only* properties > -# that can be added to these objects are ones directly specified in the SPDX > -# spec, however you may add helper functions to make operations easier. > -# > -# Defaults should *only* be specified if the SPDX spec says there is a > certain > -# required value for a field (e.g. dataLicense), or if the field is mandatory > -# and has some sane "this field is unknown" (e.g. "NOASSERTION") > -# > - > -class SPDXAnnotation(SPDXObject): > - annotationDate = _String() > - annotationType = _String() > - annotator = _String() > - comment = _String() > - > -class SPDXChecksum(SPDXObject): > - algorithm = _String() > - checksumValue = _String() > - > - > -class SPDXRelationship(SPDXObject): > - spdxElementId = _String() > - relatedSpdxElement = _String() > - relationshipType = _String() > - comment = _String() > - annotations = _ObjectList(SPDXAnnotation) > - > - > -class SPDXExternalReference(SPDXObject): > - referenceCategory = _String() > - referenceType = _String() > - referenceLocator = _String() > - > - > -class SPDXPackageVerificationCode(SPDXObject): > - packageVerificationCodeValue = _String() > - packageVerificationCodeExcludedFiles = _StringList() > - > - > -class SPDXPackage(SPDXObject): > - ALLOWED_CHECKSUMS = [ > - "SHA1", > - "SHA224", > - "SHA256", > - "SHA384", > - "SHA512", > - "MD2", > - "MD4", > - "MD5", > - "MD6", > - ] > - > - name = _String() > - SPDXID = _String() > - versionInfo = _String() > - downloadLocation = _String(default="NOASSERTION") > - supplier = _String(default="NOASSERTION") > - homepage = _String() > - licenseConcluded = _String(default="NOASSERTION") > - licenseDeclared = _String(default="NOASSERTION") > - summary = _String() > - description = _String() > - sourceInfo = _String() > - copyrightText = _String(default="NOASSERTION") > - licenseInfoFromFiles = _StringList(default=["NOASSERTION"]) > - externalRefs = _ObjectList(SPDXExternalReference) > - packageVerificationCode = _Object(SPDXPackageVerificationCode) > - hasFiles = _StringList() > - packageFileName = _String() > - annotations = _ObjectList(SPDXAnnotation) > - checksums = _ObjectList(SPDXChecksum) > - > - > -class SPDXFile(SPDXObject): > - SPDXID = _String() > - fileName = _String() > - licenseConcluded = _String(default="NOASSERTION") > - copyrightText = _String(default="NOASSERTION") > - licenseInfoInFiles = _StringList(default=["NOASSERTION"]) > - checksums = _ObjectList(SPDXChecksum) > - fileTypes = _StringList() > - > - > -class SPDXCreationInfo(SPDXObject): > - created = _String() > - licenseListVersion = _String() > - comment = _String() > - creators = _StringList() > - > - > -class SPDXExternalDocumentRef(SPDXObject): > - externalDocumentId = _String() > - spdxDocument = _String() > - checksum = _Object(SPDXChecksum) > - > - > -class SPDXExtractedLicensingInfo(SPDXObject): > - name = _String() > - comment = _String() > - licenseId = _String() > - extractedText = _String() > - > - > -class SPDXDocument(SPDXObject): > - spdxVersion = _String(default="SPDX-" + SPDX_VERSION) > - dataLicense = _String(default="CC0-1.0") > - SPDXID = _String(default="SPDXRef-DOCUMENT") > - name = _String() > - documentNamespace = _String() > - creationInfo = _Object(SPDXCreationInfo) > - packages = _ObjectList(SPDXPackage) > - files = _ObjectList(SPDXFile) > - relationships = _ObjectList(SPDXRelationship) > - externalDocumentRefs = _ObjectList(SPDXExternalDocumentRef) > - hasExtractedLicensingInfos = _ObjectList(SPDXExtractedLicensingInfo) > - > - def __init__(self, **d): > - super().__init__(**d) > - > - def to_json(self, f, *, sort_keys=False, indent=None, separators=None): > - class Encoder(json.JSONEncoder): > - def default(self, o): > - if isinstance(o, SPDXObject): > - return o.serializer() > - > - return super().default(o) > - > - sha1 = hashlib.sha1() > - for chunk in Encoder( > - sort_keys=sort_keys, > - indent=indent, > - separators=separators, > - ).iterencode(self): > - chunk = chunk.encode("utf-8") > - f.write(chunk) > - sha1.update(chunk) > - > - return sha1.hexdigest() > - > - @classmethod > - def from_json(cls, f): > - return cls(**json.load(f)) > - > - def add_relationship(self, _from, relationship, _to, *, comment=None, > annotation=None): > - if isinstance(_from, SPDXObject): > - from_spdxid = _from.SPDXID > - else: > - from_spdxid = _from > - > - if isinstance(_to, SPDXObject): > - to_spdxid = _to.SPDXID > - else: > - to_spdxid = _to > - > - r = SPDXRelationship( > - spdxElementId=from_spdxid, > - relatedSpdxElement=to_spdxid, > - relationshipType=relationship, > - ) > - > - if comment is not None: > - r.comment = comment > - > - if annotation is not None: > - r.annotations.append(annotation) > - > - self.relationships.append(r) > - > - def find_by_spdxid(self, spdxid): > - for o in itertools.chain(self.packages, self.files): > - if o.SPDXID == spdxid: > - return o > - return None > - > - def find_external_document_ref(self, namespace): > - for r in self.externalDocumentRefs: > - if r.spdxDocument == namespace: > - return r > - return None > diff --git a/meta/lib/oeqa/selftest/cases/spdx.py > b/meta/lib/oeqa/selftest/cases/spdx.py > index 83e4890d44..c563fd1011 100644 > --- a/meta/lib/oeqa/selftest/cases/spdx.py > +++ b/meta/lib/oeqa/selftest/cases/spdx.py > @@ -4,74 +4,13 @@ > # SPDX-License-Identifier: MIT > # > > -import json > -import os > import textwrap > import hashlib > -from pathlib import Path > from oeqa.selftest.case import OESelftestTestCase > -from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars, runCmd > +from oeqa.utils.commands import bitbake, get_bb_var, get_bb_vars > import oe.spdx30 > > > -class SPDX22Check(OESelftestTestCase): > - @classmethod > - def setUpClass(cls): > - super().setUpClass() > - bitbake("python3-spdx-tools-native") > - bitbake("-c addto_recipe_sysroot python3-spdx-tools-native") > - > - def check_recipe_spdx(self, high_level_dir, spdx_file, target_name): > - config = textwrap.dedent( > - """\ > - INHERIT:remove = "create-spdx" > - INHERIT += "create-spdx-2.2" > - """ > - ) > - self.write_config(config) > - > - deploy_dir = get_bb_var("DEPLOY_DIR") > - arch_dir = get_bb_var("PACKAGE_ARCH", target_name) > - spdx_version = get_bb_var("SPDX_VERSION") > - # qemux86-64 creates the directory qemux86_64 > - # arch_dir = arch_var.replace("-", "_") > - > - full_file_path = os.path.join( > - deploy_dir, "spdx", spdx_version, arch_dir, high_level_dir, > spdx_file > - ) > - > - try: > - os.remove(full_file_path) > - except FileNotFoundError: > - pass > - > - bitbake("%s -c create_spdx" % target_name) > - > - def check_spdx_json(filename): > - with open(filename) as f: > - report = json.load(f) > - self.assertNotEqual(report, None) > - self.assertNotEqual(report["SPDXID"], None) > - > - python = os.path.join( > - get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), > - "nativepython3", > - ) > - validator = os.path.join( > - get_bb_var("STAGING_BINDIR", "python3-spdx-tools-native"), > "pyspdxtools" > - ) > - result = runCmd("{} {} -i {}".format(python, validator, > filename)) > - > - self.assertExists(full_file_path) > - result = check_spdx_json(full_file_path) > - > - def test_spdx_base_files(self): > - self.check_recipe_spdx("packages", "base-files.spdx.json", > "base-files") > - > - def test_spdx_tar(self): > - self.check_recipe_spdx("packages", "tar.spdx.json", "tar") > - > - > class SPDX3CheckBase(object): > """ > Base class for checking SPDX 3 based tests > -- > 2.53.0 >
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#233934): https://lists.openembedded.org/g/openembedded-core/message/233934 Mute This Topic: https://lists.openembedded.org/mt/118505113/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
