diff --git a/README.md b/README.md index d799962d..79aaf031 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ The instructions below assume you're on Linux. ## Benchmark Results -The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_chart.py) on a lightly loaded 22-core machine. +The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_data.py) on a lightly loaded 56-core machine. ![benchmark_results](./benchmarks/decoders/benchmark_readme_chart.png) diff --git a/benchmarks/decoders/benchmark_decoders_library.py b/benchmarks/decoders/benchmark_decoders_library.py index bc9618ae..d4762565 100644 --- a/benchmarks/decoders/benchmark_decoders_library.py +++ b/benchmarks/decoders/benchmark_decoders_library.py @@ -451,7 +451,7 @@ def run_benchmarks( num_uniform_samples, min_runtime_seconds, benchmark_video_creation, -): +) -> list[dict[str, str | float | int]]: results = [] df_data = [] print(f"video_files_paths={video_files_paths}") diff --git a/benchmarks/decoders/benchmark_readme_chart.png b/benchmarks/decoders/benchmark_readme_chart.png index d21aecb2..15908bfb 100644 Binary files a/benchmarks/decoders/benchmark_readme_chart.png and b/benchmarks/decoders/benchmark_readme_chart.png differ diff --git a/benchmarks/decoders/benchmark_readme_data.json b/benchmarks/decoders/benchmark_readme_data.json new file mode 100644 index 00000000..c3759ea9 --- /dev/null +++ b/benchmarks/decoders/benchmark_readme_data.json @@ -0,0 +1,153 @@ +[ + { + "decoder": "TorchCodec", + "description": "10 seek()+next()", + "fps": 296.7497929852154, + "fps_p25": 304.82592121401444, + "fps_p75": 286.39866868882336, + "frame_count": 10, + "iqr": 0.0021107543725520372, + "median": 0.03369842283427715, + "type": "seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchCodec", + "description": "1 next()", + "fps": 141.80053650468176, + "fps_p25": 144.12265279456386, + "fps_p75": 128.28507218641042, + "frame_count": 1, + "iqr": 0.0008566047297790648, + "median": 0.007052159495651722, + "type": "next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchCodec", + "description": "10 next()", + "fps": 638.7876251007046, + "fps_p25": 647.626546889273, + "fps_p75": 629.6538548699413, + "frame_count": 10, + "iqr": 0.00044074421748518944, + "median": 0.015654655173420906, + "type": "next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchCodec[num_threads=1]", + "description": "10 seek()+next()", + "fps": 126.6773946375974, + "fps_p25": 128.33021600580943, + "fps_p75": 124.2963412180317, + "frame_count": 10, + "iqr": 0.00252892030403018, + "median": 0.07894068257883191, + "type": "seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchCodec[num_threads=1]", + "description": "1 next()", + "fps": 410.70043288059617, + "fps_p25": 416.39979027639464, + "fps_p75": 405.8298526819803, + "frame_count": 1, + "iqr": 6.25486485660077e-05, + "median": 0.0024348647333681584, + "type": "next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchCodec[num_threads=1]", + "description": "10 next()", + "fps": 758.7565583035099, + "fps_p25": 766.9872077345758, + "fps_p75": 751.3583938952637, + "frame_count": 10, + "iqr": 0.00027120066806674004, + "median": 0.013179457746446133, + "type": "next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchVision[backend=VideoReader]", + "description": "10 seek()+next()", + "fps": 7.880664295730362, + "fps_p25": 7.924876540397429, + "fps_p75": 7.827668314297991, + "frame_count": 10, + "iqr": 0.01567032840102911, + "median": 1.2689285604283214, + "type": "seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchVision[backend=VideoReader]", + "description": "1 next()", + "fps": 209.3834723742803, + "fps_p25": 211.77546088016425, + "fps_p75": 199.25812670019334, + "frame_count": 1, + "iqr": 0.0002966334810480479, + "median": 0.00477592614479363, + "type": "next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchVision[backend=VideoReader]", + "description": "10 next()", + "fps": 531.7221665034224, + "fps_p25": 538.7259880033903, + "fps_p75": 522.8300241450709, + "frame_count": 10, + "iqr": 0.0005643628537654877, + "median": 0.018806814216077328, + "type": "next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchAudio", + "description": "10 seek()+next()", + "fps": 27.403418366590216, + "fps_p25": 27.57563496650987, + "fps_p75": 27.276339015442346, + "frame_count": 10, + "iqr": 0.003979140194132924, + "median": 0.36491797724738717, + "type": "seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchAudio", + "description": "1 next()", + "fps": 243.36030386646544, + "fps_p25": 246.1182470856226, + "fps_p75": 240.91573311529928, + "frame_count": 1, + "iqr": 8.774134330451558e-05, + "median": 0.004109133593738079, + "type": "next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "TorchAudio", + "description": "10 next()", + "fps": 564.5307153840273, + "fps_p25": 572.2932075367765, + "fps_p75": 558.1691247911658, + "frame_count": 10, + "iqr": 0.00044215633533895016, + "median": 0.01771382801234722, + "type": "next()", + "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4" + }, + { + "cpu_count": 56, + "machine": "x86_64", + "processor": "x86_64", + "python_version": "3.11.10", + "system": "Linux" + } +] diff --git a/benchmarks/decoders/generate_readme_chart.py b/benchmarks/decoders/generate_readme_chart.py index c1f040cc..3ec2b619 100644 --- a/benchmarks/decoders/generate_readme_chart.py +++ b/benchmarks/decoders/generate_readme_chart.py @@ -4,81 +4,20 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -import glob -import importlib.resources -import os -import shutil -from pathlib import Path - -from benchmark_decoders_library import ( - generate_videos, - plot_data, - run_benchmarks, - TorchAudioDecoder, - TorchcodecNonCompiledWithOptions, - TVNewAPIDecoderWithBackend, -) - - -def in_fbcode() -> bool: - return "FB_PAR_RUNTIME_FILES" in os.environ +import json +from pathlib import Path -def get_test_resource_path(filename: str) -> str: - if in_fbcode(): - resource = importlib.resources.files(__package__).joinpath(filename) - with importlib.resources.as_file(resource) as path: - return os.fspath(path) - - return str(Path(__file__).parent / f"../../test/resources/{filename}") +from benchmark_decoders_library import plot_data def main() -> None: - """Benchmarks the performance of a few video decoders on synthetic videos""" - - resolutions = ["640x480"] - encodings = ["libx264"] - fpses = [30] - gop_sizes = [600] - durations = [10] - pix_fmts = ["yuv420p"] - ffmpeg_path = "ffmpeg" - videos_dir_path = "/tmp/torchcodec_benchmarking_videos" - shutil.rmtree(videos_dir_path, ignore_errors=True) - os.makedirs(videos_dir_path) - generate_videos( - resolutions, - encodings, - fpses, - gop_sizes, - durations, - pix_fmts, - ffmpeg_path, - videos_dir_path, - ) - video_files_paths = glob.glob(f"{videos_dir_path}/*.mp4") - - decoder_dict = {} - decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions() - decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions( - num_threads=1 - ) - decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend( - "video_reader" - ) - decoder_dict["TorchAudio"] = TorchAudioDecoder() + data_json = Path(__file__).parent / "benchmark_readme_data.json" + with open(data_json, "r") as read_file: + data_from_file = json.load(read_file) output_png = Path(__file__).parent / "benchmark_readme_chart.png" - # These are the number of uniform seeks we do in the seek+decode benchmark. - num_uniform_samples = 10 - df_data = run_benchmarks( - decoder_dict, - video_files_paths, - num_uniform_samples, - 10, - False, - ) - plot_data(df_data, output_png) + plot_data(data_from_file, output_png) if __name__ == "__main__": diff --git a/benchmarks/decoders/generate_readme_data.py b/benchmarks/decoders/generate_readme_data.py new file mode 100644 index 00000000..6fb859c0 --- /dev/null +++ b/benchmarks/decoders/generate_readme_data.py @@ -0,0 +1,83 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import glob +import json +import os +import platform +import shutil +from pathlib import Path + +from benchmark_decoders_library import ( + generate_videos, + run_benchmarks, + TorchAudioDecoder, + TorchcodecNonCompiledWithOptions, + TVNewAPIDecoderWithBackend, +) + + +def main() -> None: + """Benchmarks the performance of a few video decoders on synthetic videos""" + + resolutions = ["640x480"] + encodings = ["libx264"] + fpses = [30] + gop_sizes = [600] + durations = [10] + pix_fmts = ["yuv420p"] + ffmpeg_path = "ffmpeg" + videos_dir_path = "/tmp/torchcodec_benchmarking_videos" + shutil.rmtree(videos_dir_path, ignore_errors=True) + os.makedirs(videos_dir_path) + generate_videos( + resolutions, + encodings, + fpses, + gop_sizes, + durations, + pix_fmts, + ffmpeg_path, + videos_dir_path, + ) + video_files_paths = glob.glob(f"{videos_dir_path}/*.mp4") + + decoder_dict = {} + decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions() + decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions( + num_threads=1 + ) + decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend( + "video_reader" + ) + decoder_dict["TorchAudio"] = TorchAudioDecoder() + + # These are the number of uniform seeks we do in the seek+decode benchmark. + num_uniform_samples = 10 + df_data = run_benchmarks( + decoder_dict, + video_files_paths, + num_uniform_samples, + 10, + False, + ) + df_data.append( + { + "cpu_count": os.cpu_count(), + "system": platform.system(), + "machine": platform.machine(), + "processor": platform.processor(), + "python_version": str(platform.python_version()), + } + ) + + data_json = Path(__file__).parent / "benchmark_readme_data.json" + with open(data_json, "w") as write_file: + json.dump(df_data, write_file, sort_keys=True, indent=4) + + +if __name__ == "__main__": + main()