#!/usr/bin/env python3 """ @file sdl.py @brief Helpers for SDL process. @kaspersky_support Artiom N. @license Apache 2.0 License. @copyright © 2024 AO Kaspersky Lab @date 28.10.2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ import json import os from pathlib import Path import re import urllib.request from urllib.parse import urlparse, urlunparse, ParseResult from typing import Any, Collection ARTIFACTS_URL = os.getenv('ARTIFACTS_URL') SDL_ARTIFACTS_DIRECTORY = os.getenv('SDL_ARTIFACTS_DIRECTORY', 'build/SDL') HLA_FILENAME = 'hla.xml' THIRD_PARTY_FILENAME = 'third_party.xml' THIRD_PARTY_FILENAME_SERVICE = 'third_party_service.xml' THIRD_PARTY_LIST_FILENAMES = ['3d-party-requirements.xml', '3d-party-requirements-motiv.xml'] THIRD_PARTY_LIST_SERVICE = '3d-party-requirements-service.xml' SECURE_CODE_REVIEW_FILENAME = 'code_review.xml' DEVOPS_PIPELINE_FILENAME = 'devops_pipeline.xml' STATIC_ANALYSIS_FILENAME = 'static_analysis.xml' DYNAMIC_ANALYSIS_FILENAME = 'dynamic_analysis.xml' BUILD_NUMBER = os.getenv('BUILD_NUMBER') # TFS ID of the release card. RELEASE_NUMBER = os.getenv('TFS_RELEASE_NUMBER') SERVICE_NUMBER = os.getenv('TFS_SERVICE_NUMBER') COLLECTION_URI = os.getenv('COLLECTION_URI') KNP_ROOT = Path(__file__).resolve().parent.parent def artifact_url( file_name: str, build: str = BUILD_NUMBER, server: str = ARTIFACTS_URL, add_build_to_name: bool = True ) -> str: return f'{server}/{build}/SDL/{build}_{file_name}' if add_build_to_name else f'{server}/{build}/SDL/{file_name}' def generate_tfs_url(branch: str = 'develop', project: str = 'FT-SNN', repo: str = 'KNP', path: str = '/') -> str: url_comps = urlparse(f'{COLLECTION_URI}/{project}/_git/{repo}?path={path}&version=GB{branch}') return urlunparse( ParseResult( url_comps.scheme, url_comps.netloc, str(Path(url_comps.path)), url_comps.params, url_comps.query, url_comps.fragment, ) ) def generate_hla_artifact_xml() -> str: return ''' ''' def generate_third_party_xml(third_party_urls: list[str]) -> str: tp_list = [] for third_party_url in third_party_urls: tp_list.append(f'') tp_code = ('\n' + ' ' * 8).join(tp_list) return f''' {tp_code} ''' def generate_service_third_party_xml(third_party_url: str) -> str: return f''' ''' def generate_devops_pipeline_xml() -> str: return f''' ''' def generate_code_review_xml(link: str) -> str: return f''' ''' def get_config(filename: Path, regex: str) -> str: with open(filename, encoding='utf8') as config_file: file_content = config_file.read() m_res = re.findall(regex, file_content, re.MULTILINE | re.IGNORECASE) if not m_res: raise RuntimeError(f'"{filename}" doesn\'t contain configuration data!') return re.sub(r'\s\s+', '\n', '\n'.join(m_res)) def get_bandit_config() -> str: """ This is not real config, because real bandit configuration set on CI and may differ from this. """ return '''-r -lll -x ".*fixtures" $(Build.SourcesDirectory)/knp/python-framework $(Build.BinariesDirectory)/knp_python_framework/ -o $(Build.BinariesDirectory)/bandit.log''' def get_pvs_config(filename: Path = KNP_ROOT / 'knp' / 'CMakeLists.txt') -> str: return get_config(filename, r'pvs_studio_add_target\(([^\)]+)\)') def get_valgrind_config(filename: Path = KNP_ROOT / 'knp' / 'tests' / 'CMakeLists.txt') -> str: return get_config(filename, r'set\((CTEST_MEMORYCHECK_COMMAND[^\)]+)\)') def generate_static_analysis_xml() -> str: pvs_logs = [] # linux, windows for osname in ['linux']: pvs_logs_archive = f'{osname}_pvs_report.7z' pvs_config = f'{osname}_pvs_config.txt' with open(KNP_ROOT / SDL_ARTIFACTS_DIRECTORY / f'{BUILD_NUMBER}_{pvs_config}', 'w', encoding='utf8') as pc_f: pc_f.write(get_pvs_config()) pvs_logs.append( f'' f'' f'' '' ) with open(KNP_ROOT / SDL_ARTIFACTS_DIRECTORY / f'{BUILD_NUMBER}_bandit', 'w', encoding='utf8') as pc_f: pc_f.write(get_bandit_config()) # # # # return f''' {' '.join(pvs_logs)} ''' def generate_dynamic_analysis_xml() -> str: with open(KNP_ROOT / SDL_ARTIFACTS_DIRECTORY / f'{BUILD_NUMBER}_valgrind_config.txt', 'w', encoding='utf8') as pc_f: pc_f.write(get_valgrind_config()) return f''' ''' def generate_sdl_artifacts() -> None: sdl_path = KNP_ROOT / SDL_ARTIFACTS_DIRECTORY sdl_path.mkdir(parents=True, exist_ok=True) print(f'Generating SDL XMLs for build {BUILD_NUMBER} in path "{sdl_path}"...') with open(sdl_path / HLA_FILENAME, 'w', encoding='utf8') as f: f.write(generate_hla_artifact_xml()) with open(sdl_path / THIRD_PARTY_FILENAME, 'w', encoding='utf8') as f: f.write(generate_third_party_xml(THIRD_PARTY_LIST_FILENAMES)) with open(sdl_path / THIRD_PARTY_FILENAME_SERVICE, 'w', encoding='utf8') as f: f.write(generate_service_third_party_xml(THIRD_PARTY_LIST_SERVICE)) with open(sdl_path / DEVOPS_PIPELINE_FILENAME, 'w', encoding='utf8') as f: f.write(generate_devops_pipeline_xml()) with open(sdl_path / SECURE_CODE_REVIEW_FILENAME, 'w', encoding='utf8') as f: f.write(generate_code_review_xml(generate_tfs_url())) with open(sdl_path / STATIC_ANALYSIS_FILENAME, 'w', encoding='utf8') as f: f.write(generate_static_analysis_xml()) with open(sdl_path / DYNAMIC_ANALYSIS_FILENAME, 'w', encoding='utf8') as f: f.write(generate_dynamic_analysis_xml()) def generate_sdl_requests() -> ( tuple[dict[str, str | Collection[str] | dict[str, str]], dict[str, str | Collection[str] | dict[str, str]]] ): # major, minor, *_ = RELEASE_NUMBER.split('.') release_iteration = r'FT-SNN\Open Source Release' product_req = { 'release_tfs_id': RELEASE_NUMBER, 'build_number': BUILD_NUMBER, 'hla': artifact_url(HLA_FILENAME, add_build_to_name=False), 'third_party': {'type': 'uri', 'data': artifact_url(THIRD_PARTY_FILENAME, add_build_to_name=False)}, 'devops_pipeline': { 'type': 'uri', 'data': artifact_url(DEVOPS_PIPELINE_FILENAME, add_build_to_name=False), }, 'source_code': { 'type': 'uri', 'data': artifact_url(SECURE_CODE_REVIEW_FILENAME, add_build_to_name=False), }, 'static_analysis': { 'type': 'uri', 'data': artifact_url(STATIC_ANALYSIS_FILENAME, add_build_to_name=False), }, 'dynamic_analysis': { 'type': 'uri', 'data': artifact_url(DYNAMIC_ANALYSIS_FILENAME, add_build_to_name=False), }, 'iteration_path_for_bug': release_iteration, 'iteration_path_for_req': release_iteration, } service_req = { 'release_tfs_id': SERVICE_NUMBER, 'build_number': BUILD_NUMBER, 'third_party': {'type': 'uri', 'data': artifact_url(THIRD_PARTY_FILENAME_SERVICE, add_build_to_name=False)}, 'devops_pipeline': { 'type': 'uri', 'data': artifact_url(DEVOPS_PIPELINE_FILENAME, add_build_to_name=False), }, 'source_code': { 'type': 'uri', 'data': artifact_url(SECURE_CODE_REVIEW_FILENAME, add_build_to_name=False), }, 'iteration_path_for_bug': release_iteration, 'iteration_path_for_req': release_iteration, } return product_req, service_req def call_api( url: str, requests: tuple[ dict[str, str | Collection[str] | dict[str, str]], dict[str, str | Collection[str] | dict[str, str]] ], ) -> Any: for body in requests: req = urllib.request.Request( url=url, data=json.dumps(body).encode('utf8'), headers={'Content-Type': 'application/json'}, ) with urllib.request.urlopen(req) as resp: print(f'Returned code: {resp.getcode()}') def call_vulnbot_api(make_call: bool) -> None: sdl_requests = generate_sdl_requests() sdl_api_url = os.getenv('SDL_API_URL') print(f'Calling SDL API: {sdl_api_url}\n{sdl_requests}') if make_call: call_api(sdl_api_url, sdl_requests) if '__main__' == __name__: import argparse parser = argparse.ArgumentParser() parser.add_argument('--generate-artifacts', action='store_true', help='Generate SDL artifacts') parser.add_argument('--call-api', action='store_true', help='Call Vulnbot API') parser.add_argument('--imitate-call', action='store_true', help='Don\'t call Vulnbot API, only imitate call') args = parser.parse_args() # RELEASE_NUMBER = get_release_number() mode_set: bool = False if args.generate_artifacts: generate_sdl_artifacts() mode_set = True if args.call_api: call_vulnbot_api(not args.imitate_call) mode_set = True if not mode_set: raise ValueError('Set options!')