From 37a0f24d54a0a5ae9cab702b24c70692d235afae Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 24 Feb 2022 10:43:55 +0100 Subject: [PATCH 1/5] Script to watch the evolution of generated files Archive the evolution of generated files over a range of revisions. Signed-off-by: Gilles Peskine --- tools/bin/mbedtls-trace-files.py | 136 +++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100755 tools/bin/mbedtls-trace-files.py diff --git a/tools/bin/mbedtls-trace-files.py b/tools/bin/mbedtls-trace-files.py new file mode 100755 index 0000000..56c79bd --- /dev/null +++ b/tools/bin/mbedtls-trace-files.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +"""Archive the contents of the specified files for the specified Git revisions. + +Run this script from a clean Git worktree. +This script runs `make FILE` to generate the desired files. +The outputs are stored in a subdirectory named for each commit hash. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil +import subprocess +from typing import List, Optional + + +class UncommittedChangesException(Exception): + "You have uncommitted changes. Please stash or commit them." + pass + + +class Archiver: + """Archive the contents of some files for some Git revisions.""" + + def __init__( + self, + build_dir: Optional[str] = None, + output_dir: Optional[str] = None, + run_after: Optional[str] = None, + run_before: Optional[str] = None, + ) -> None: + """Configure an archiver for generated files. + + `build_dir`: directory where ``make`` will be run. + `output_dir`: parent directory for the per-revision directories. + `run_before`: shell command to run before ``make``. + `run_after`: shell command to run after ``make``. + """ + self.build_dir = build_dir if build_dir is not None else os.curdir + self.output_dir = output_dir if output_dir is not None else os.curdir + self.run_before = run_before + self.run_after = run_after + self.prepare() + + def prepare(self) -> None: + """Prepare the working directory.""" + try: + subprocess.check_call(['git', 'diff', '--quiet']) + except subprocess.CalledProcessError: + raise UncommittedChangesException() + self.initial_revision = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'] + ).decode('ascii').strip() + + def done(self) -> None: + """Restore the working directory.""" + subprocess.check_call(['git', 'checkout', self.initial_revision]) + + def archive_revision(self, revision: str, files: List[str]) -> None: + """Archive generated files for a given revision. + + `revision`: Git revision to check out. + `files`: list of files to archive. + """ + subprocess.check_call(['git', 'checkout', revision]) + if self.run_before: + subprocess.check_call(self.run_before, shell=True) + subprocess.check_call(['make'] + files, + cwd=self.build_dir) + for filename in files: + target_dir = os.path.join(self.output_dir, + revision, + os.path.dirname(filename)) + os.makedirs(target_dir, exist_ok=True) + shutil.copy2(filename, target_dir) + if self.run_after: + subprocess.check_call(self.run_after, shell=True) + + def archive_revisions(self, revision_range: str, files: List[str]) -> None: + """Archive generated files for a given revision range. + + `revision`: Git revision range to check out. + `files`: list of files to archive. + """ + self.prepare() + try: + revisions = subprocess.check_output( + ['git', 'log', '--format=%H', revision_range] + ).decode('ascii').split() + for revision in revisions: + self.archive_revision(revision, files) + finally: + self.done() + + +def main() -> None: + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--build-dir', '-b', metavar='DIR', + help='Run `make` in DIR') + parser.add_argument('--output-dir', '-o', metavar='DIR', + help='Put output directories under DIR') + parser.add_argument('--run-after', '-R', metavar='CMD', + help='Shell command to run after each build') + parser.add_argument('--run-before', '-r', metavar='CMD', + help='Shell command to run before each build') + parser.add_argument('revisions', metavar='REVISIONS', + help='Comma-separated of Git revisions (see gitrevisions(7))') + parser.add_argument('files', metavar='FILE', nargs='*', + help='File to archive') + options = parser.parse_args() + revision_ranges = options.revisions.split(',') + del options.revisions + files = options.files + del options.files + archiver = Archiver(**vars(options)) + for revision_range in revision_ranges: + archiver.archive_revisions(revision_range, files) + +if __name__ == '__main__': + main() From 7c55dcda0e969a686b9c12ff33b0050498473706 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 24 Feb 2022 18:30:41 +0100 Subject: [PATCH 2/5] Support a single revision, not just revision ranges Signed-off-by: Gilles Peskine --- tools/bin/mbedtls-trace-files.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tools/bin/mbedtls-trace-files.py b/tools/bin/mbedtls-trace-files.py index 56c79bd..27f7808 100755 --- a/tools/bin/mbedtls-trace-files.py +++ b/tools/bin/mbedtls-trace-files.py @@ -24,6 +24,7 @@ import argparse import os +import re import shutil import subprocess from typing import List, Optional @@ -91,6 +92,16 @@ def archive_revision(self, revision: str, files: List[str]) -> None: if self.run_after: subprocess.check_call(self.run_after, shell=True) + def list_revisions(self, revision_or_range: str) -> List[str]: + """Return the list of commits in revision_or_range. + + If revision_or_range is a single revision, return it in a one-element + list. Otherwise return the list of commits in that range. + """ + return subprocess.check_output( + ['git', 'rev-list', '--no-walk', revision_or_range] + ).decode('ascii').split() + def archive_revisions(self, revision_range: str, files: List[str]) -> None: """Archive generated files for a given revision range. @@ -99,14 +110,13 @@ def archive_revisions(self, revision_range: str, files: List[str]) -> None: """ self.prepare() try: - revisions = subprocess.check_output( - ['git', 'log', '--format=%H', revision_range] - ).decode('ascii').split() + revisions = self.list_revisions(revision_range) for revision in revisions: self.archive_revision(revision, files) finally: self.done() +REVISION_SEPARATOR = re.compile('[\t\n\f\r ,]') def main() -> None: """Command line entry point.""" @@ -120,11 +130,11 @@ def main() -> None: parser.add_argument('--run-before', '-r', metavar='CMD', help='Shell command to run before each build') parser.add_argument('revisions', metavar='REVISIONS', - help='Comma-separated of Git revisions (see gitrevisions(7))') + help='Comma/blank-separated list of Git revisions or ranges (see gitrevisions(7))') parser.add_argument('files', metavar='FILE', nargs='*', help='File to archive') options = parser.parse_args() - revision_ranges = options.revisions.split(',') + revision_ranges = REVISION_SEPARATOR.split(options.revisions) del options.revisions files = options.files del options.files From cc78a8764ba661c58ccab21d1ccb25b333cdb139 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 11 Dec 2024 11:22:10 +0100 Subject: [PATCH 3/5] Allow starting from a different revision number This is especially useful to continue from a previous series of commits that are already traced. Signed-off-by: Gilles Peskine --- tools/bin/mbedtls-trace-files.py | 45 ++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/tools/bin/mbedtls-trace-files.py b/tools/bin/mbedtls-trace-files.py index 27f7808..c367359 100755 --- a/tools/bin/mbedtls-trace-files.py +++ b/tools/bin/mbedtls-trace-files.py @@ -44,6 +44,7 @@ def __init__( output_dir: Optional[str] = None, run_after: Optional[str] = None, run_before: Optional[str] = None, + **kwargs ) -> None: """Configure an archiver for generated files. @@ -72,7 +73,10 @@ def done(self) -> None: """Restore the working directory.""" subprocess.check_call(['git', 'checkout', self.initial_revision]) - def archive_revision(self, revision: str, files: List[str]) -> None: + def archive_revision(self, + target_prefix: str, + revision: str, + files: List[str]) -> None: """Archive generated files for a given revision. `revision`: Git revision to check out. @@ -85,10 +89,10 @@ def archive_revision(self, revision: str, files: List[str]) -> None: cwd=self.build_dir) for filename in files: target_dir = os.path.join(self.output_dir, - revision, + target_prefix + revision, os.path.dirname(filename)) os.makedirs(target_dir, exist_ok=True) - shutil.copy2(filename, target_dir) + shutil.copy2(os.path.join(self.build_dir, filename), target_dir) if self.run_after: subprocess.check_call(self.run_after, shell=True) @@ -98,21 +102,37 @@ def list_revisions(self, revision_or_range: str) -> List[str]: If revision_or_range is a single revision, return it in a one-element list. Otherwise return the list of commits in that range. """ - return subprocess.check_output( - ['git', 'rev-list', '--no-walk', revision_or_range] - ).decode('ascii').split() - - def archive_revisions(self, revision_range: str, files: List[str]) -> None: + subsequent = [] + m = re.match(r'(.*)\.\.', revision_or_range) + if m: + subsequent = subprocess.check_output( + ['git', 'rev-list', '--no-walk', revision_or_range] + ).decode('ascii').split() + subsequent.reverse() + first_name = m.group(1) + else: + first_name = revision_or_range + first_sha = subprocess.check_output( + ['git', 'rev-parse', first_name] + ).decode('ascii').rstrip() + return [first_sha] + subsequent + + def archive_revisions(self, + starting_number: int, + revision_range: str, + files: List[str]) -> None: """Archive generated files for a given revision range. + `starting_number`: number used to name the directory for the first revision. `revision`: Git revision range to check out. `files`: list of files to archive. """ self.prepare() try: revisions = self.list_revisions(revision_range) - for revision in revisions: - self.archive_revision(revision, files) + prefix_format = '{:0' + str(len(str(len(revisions) - 1))) + '}-' + for n, revision in enumerate(revisions, starting_number): + self.archive_revision(prefix_format.format(n), revision, files) finally: self.done() @@ -123,6 +143,9 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--build-dir', '-b', metavar='DIR', help='Run `make` in DIR') + parser.add_argument('--number-from', '-f', metavar='NUM', + type=int, default=0, + help='Count revisions from NUM (default 0)') parser.add_argument('--output-dir', '-o', metavar='DIR', help='Put output directories under DIR') parser.add_argument('--run-after', '-R', metavar='CMD', @@ -140,7 +163,7 @@ def main() -> None: del options.files archiver = Archiver(**vars(options)) for revision_range in revision_ranges: - archiver.archive_revisions(revision_range, files) + archiver.archive_revisions(options.number_from, revision_range, files) if __name__ == '__main__': main() From 37905f7f34c08cf406c8a8b6c814338b8983d3de Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 11 Dec 2024 11:35:30 +0100 Subject: [PATCH 4/5] Allow skipping make Useful when the traced files are checked into Git, or when they're generated by some shell command (-r) rather than make. Signed-off-by: Gilles Peskine --- tools/bin/mbedtls-trace-files.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/bin/mbedtls-trace-files.py b/tools/bin/mbedtls-trace-files.py index c367359..a3d11cd 100755 --- a/tools/bin/mbedtls-trace-files.py +++ b/tools/bin/mbedtls-trace-files.py @@ -44,6 +44,7 @@ def __init__( output_dir: Optional[str] = None, run_after: Optional[str] = None, run_before: Optional[str] = None, + skip_make: bool = False, **kwargs ) -> None: """Configure an archiver for generated files. @@ -52,11 +53,13 @@ def __init__( `output_dir`: parent directory for the per-revision directories. `run_before`: shell command to run before ``make``. `run_after`: shell command to run after ``make``. + `skip_make`: if specified and true, don't run ``make``. """ self.build_dir = build_dir if build_dir is not None else os.curdir self.output_dir = output_dir if output_dir is not None else os.curdir self.run_before = run_before self.run_after = run_after + self.skip_make = skip_make self.prepare() def prepare(self) -> None: @@ -85,8 +88,9 @@ def archive_revision(self, subprocess.check_call(['git', 'checkout', revision]) if self.run_before: subprocess.check_call(self.run_before, shell=True) - subprocess.check_call(['make'] + files, - cwd=self.build_dir) + if not self.skip_make: + subprocess.check_call(['make'] + files, + cwd=self.build_dir) for filename in files: target_dir = os.path.join(self.output_dir, target_prefix + revision, @@ -142,7 +146,7 @@ def main() -> None: """Command line entry point.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--build-dir', '-b', metavar='DIR', - help='Run `make` in DIR') + help='Run `make` and collect files in DIR') parser.add_argument('--number-from', '-f', metavar='NUM', type=int, default=0, help='Count revisions from NUM (default 0)') @@ -152,6 +156,9 @@ def main() -> None: help='Shell command to run after each build') parser.add_argument('--run-before', '-r', metavar='CMD', help='Shell command to run before each build') + parser.add_argument('--skip-make', + action='store_true', + help='Do not run `make` (rely on -r to build the files)') parser.add_argument('revisions', metavar='REVISIONS', help='Comma/blank-separated list of Git revisions or ranges (see gitrevisions(7))') parser.add_argument('files', metavar='FILE', nargs='*', From 71b375ecb6514c0d44f0130696b16e18da177180 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 4 Jun 2025 14:06:06 +0200 Subject: [PATCH 5/5] Make sure to re-generate the targets In case the make command doesn't have correct dependencies, make sure that it at least regenerates the targets. This isn't enough if intermediate files need to be updated as well, but it's the best we can do here. Signed-off-by: Gilles Peskine --- tools/bin/mbedtls-trace-files.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/bin/mbedtls-trace-files.py b/tools/bin/mbedtls-trace-files.py index a3d11cd..18ecf3f 100755 --- a/tools/bin/mbedtls-trace-files.py +++ b/tools/bin/mbedtls-trace-files.py @@ -86,6 +86,15 @@ def archive_revision(self, `files`: list of files to archive. """ subprocess.check_call(['git', 'checkout', revision]) + # In case the make command doesn't have correct dependencies, + # make sure that it at least regenerates the targets. This isn't + # enough if intermediate files need to be updated as well, but + # it's the best we can do here. If make doesn't have correct + # dependencies, make sure to delete all relevant intermediate files + # in the run_before command. + for filename in files: + if os.path.exists(filename): + os.remove(filename) if self.run_before: subprocess.check_call(self.run_before, shell=True) if not self.skip_make: @@ -134,7 +143,8 @@ def archive_revisions(self, self.prepare() try: revisions = self.list_revisions(revision_range) - prefix_format = '{:0' + str(len(str(len(revisions) - 1))) + '}-' + max_number = starting_number + len(revisions) - 1 + prefix_format = '{:0' + str(len(str(max_number))) + '}-' for n, revision in enumerate(revisions, starting_number): self.archive_revision(prefix_format.format(n), revision, files) finally: