From c380e4c1167f2e5be9f8d59d65971d72cd536efb Mon Sep 17 00:00:00 2001 From: Scott Schneider Date: Thu, 31 Oct 2024 06:56:43 -0700 Subject: [PATCH] Generalize README benchmark --- .../decoders/benchmark_decoders_library.py | 41 +++++++++++-------- benchmarks/decoders/generate_readme_chart.py | 14 +++---- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/benchmarks/decoders/benchmark_decoders_library.py b/benchmarks/decoders/benchmark_decoders_library.py index bf961d6f..bc9618ae 100644 --- a/benchmarks/decoders/benchmark_decoders_library.py +++ b/benchmarks/decoders/benchmark_decoders_library.py @@ -3,7 +3,7 @@ import os import subprocess import timeit -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor, wait from itertools import product import matplotlib.pyplot as plt @@ -306,6 +306,7 @@ def generate_video(command): print(command) print(" ".join(command)) subprocess.check_call(command) + return True def generate_videos( @@ -321,6 +322,7 @@ def generate_videos( executor = ThreadPoolExecutor(max_workers=20) video_count = 0 + futures = [] for resolution, duration, fps, gop_size, encoding, pix_fmt in product( resolutions, durations, fpses, gop_sizes, encodings, pix_fmts ): @@ -342,9 +344,12 @@ def generate_videos( pix_fmt, outfile, ] - executor.submit(generate_video, command) + futures.append(executor.submit(generate_video, command)) video_count += 1 + wait(futures) + for f in futures: + assert f.result() executor.shutdown(wait=True) print(f"Generated {video_count} videos") @@ -442,21 +447,21 @@ def plot_data(df_data, plot_path): def run_benchmarks( decoder_dict, - video_paths, + video_files_paths, num_uniform_samples, min_runtime_seconds, benchmark_video_creation, ): results = [] df_data = [] - print(f"video_paths={video_paths}") + print(f"video_files_paths={video_files_paths}") verbose = False for decoder_name, decoder in decoder_dict.items(): - for video_path in video_paths: - print(f"video={video_path}, decoder={decoder_name}") + for video_file_path in video_files_paths: + print(f"video={video_file_path}, decoder={decoder_name}") # We only use the VideoDecoder to get the metadata and get # the list of PTS values to seek to. - simple_decoder = VideoDecoder(video_path) + simple_decoder = VideoDecoder(video_file_path) duration = simple_decoder.metadata.duration_seconds pts_list = [ i * duration / num_uniform_samples for i in range(num_uniform_samples) @@ -465,16 +470,16 @@ def run_benchmarks( metadata_string = f"{metadata.codec} {metadata.width}x{metadata.height}, {metadata.duration_seconds}s {metadata.average_fps}fps" if verbose: print( - f"video={video_path}, decoder={decoder_name}, pts_list={pts_list}" + f"video={video_file_path}, decoder={decoder_name}, pts_list={pts_list}" ) seeked_result = benchmark.Timer( stmt="decoder.get_frames_from_video(video_file, pts_list)", globals={ - "video_file": video_path, + "video_file": video_file_path, "pts_list": pts_list, "decoder": decoder, }, - label=f"video={video_path} {metadata_string}", + label=f"video={video_file_path} {metadata_string}", sub_label=decoder_name, description=f"{num_uniform_samples} seek()+next()", ) @@ -483,7 +488,7 @@ def run_benchmarks( ) df_item = {} df_item["decoder"] = decoder_name - df_item["video"] = video_path + df_item["video"] = video_file_path df_item["description"] = results[-1].description df_item["frame_count"] = num_uniform_samples df_item["median"] = results[-1].median @@ -498,11 +503,11 @@ def run_benchmarks( consecutive_frames_result = benchmark.Timer( stmt="decoder.get_consecutive_frames_from_video(video_file, consecutive_frames_to_extract)", globals={ - "video_file": video_path, + "video_file": video_file_path, "consecutive_frames_to_extract": num_consecutive_nexts, "decoder": decoder, }, - label=f"video={video_path} {metadata_string}", + label=f"video={video_file_path} {metadata_string}", sub_label=decoder_name, description=f"{num_consecutive_nexts} next()", ) @@ -513,7 +518,7 @@ def run_benchmarks( ) df_item = {} df_item["decoder"] = decoder_name - df_item["video"] = video_path + df_item["video"] = video_file_path df_item["description"] = results[-1].description df_item["frame_count"] = num_consecutive_nexts df_item["median"] = results[-1].median @@ -524,18 +529,18 @@ def run_benchmarks( df_item["fps_p25"] = 1.0 * num_consecutive_nexts / results[-1]._p25 df_data.append(df_item) - first_video_path = video_paths[0] + first_video_file_path = video_files_paths[0] if benchmark_video_creation: - simple_decoder = VideoDecoder(first_video_path) + simple_decoder = VideoDecoder(first_video_file_path) metadata = simple_decoder.metadata metadata_string = f"{metadata.codec} {metadata.width}x{metadata.height}, {metadata.duration_seconds}s {metadata.average_fps}fps" creation_result = benchmark.Timer( stmt="create_torchcodec_decoder_from_file(video_file)", globals={ - "video_file": first_video_path, + "video_file": first_video_file_path, "create_torchcodec_decoder_from_file": create_torchcodec_decoder_from_file, }, - label=f"video={first_video_path} {metadata_string}", + label=f"video={first_video_file_path} {metadata_string}", sub_label="TorchcodecNonCompiled", description="create()+next()", ) diff --git a/benchmarks/decoders/generate_readme_chart.py b/benchmarks/decoders/generate_readme_chart.py index 64d5c779..c1f040cc 100644 --- a/benchmarks/decoders/generate_readme_chart.py +++ b/benchmarks/decoders/generate_readme_chart.py @@ -42,10 +42,10 @@ def main() -> None: gop_sizes = [600] durations = [10] pix_fmts = ["yuv420p"] - ffmpeg_path = "/usr/local/bin/ffmpeg" - videos_path = "/tmp/videos" - shutil.rmtree(videos_path) - os.makedirs(videos_path) + ffmpeg_path = "ffmpeg" + videos_dir_path = "/tmp/torchcodec_benchmarking_videos" + shutil.rmtree(videos_dir_path, ignore_errors=True) + os.makedirs(videos_dir_path) generate_videos( resolutions, encodings, @@ -54,9 +54,9 @@ def main() -> None: durations, pix_fmts, ffmpeg_path, - videos_path, + videos_dir_path, ) - video_paths = glob.glob(f"{videos_path}/*.mp4") + video_files_paths = glob.glob(f"{videos_dir_path}/*.mp4") decoder_dict = {} decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions() @@ -73,7 +73,7 @@ def main() -> None: num_uniform_samples = 10 df_data = run_benchmarks( decoder_dict, - video_paths, + video_files_paths, num_uniform_samples, 10, False,