Skip to content

Seperate out readme experiments from chart generation #328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 31, 2024
Merged

Seperate out readme experiments from chart generation #328

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -120,7 +120,7 @@ The instructions below assume you're on Linux.

## Benchmark Results

The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_chart.py) on a lightly loaded 22-core machine.
The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_data.py) on a lightly loaded 56-core machine.

![benchmark_results](./benchmarks/decoders/benchmark_readme_chart.png)

2 changes: 1 addition & 1 deletion benchmarks/decoders/benchmark_decoders_library.py
View file
Original file line number Diff line number Diff line change
@@ -451,7 +451,7 @@ def run_benchmarks(
num_uniform_samples,
min_runtime_seconds,
benchmark_video_creation,
):
) -> list[dict[str, str | float | int]]:
results = []
df_data = []
print(f"video_files_paths={video_files_paths}")
Binary file modified benchmarks/decoders/benchmark_readme_chart.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
153 changes: 153 additions & 0 deletions benchmarks/decoders/benchmark_readme_data.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
[
{
"decoder": "TorchCodec",
"description": "10 seek()+next()",
"fps": 296.7497929852154,
"fps_p25": 304.82592121401444,
"fps_p75": 286.39866868882336,
"frame_count": 10,
"iqr": 0.0021107543725520372,
"median": 0.03369842283427715,
"type": "seek()+next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchCodec",
"description": "1 next()",
"fps": 141.80053650468176,
"fps_p25": 144.12265279456386,
"fps_p75": 128.28507218641042,
"frame_count": 1,
"iqr": 0.0008566047297790648,
"median": 0.007052159495651722,
"type": "next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchCodec",
"description": "10 next()",
"fps": 638.7876251007046,
"fps_p25": 647.626546889273,
"fps_p75": 629.6538548699413,
"frame_count": 10,
"iqr": 0.00044074421748518944,
"median": 0.015654655173420906,
"type": "next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchCodec[num_threads=1]",
"description": "10 seek()+next()",
"fps": 126.6773946375974,
"fps_p25": 128.33021600580943,
"fps_p75": 124.2963412180317,
"frame_count": 10,
"iqr": 0.00252892030403018,
"median": 0.07894068257883191,
"type": "seek()+next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchCodec[num_threads=1]",
"description": "1 next()",
"fps": 410.70043288059617,
"fps_p25": 416.39979027639464,
"fps_p75": 405.8298526819803,
"frame_count": 1,
"iqr": 6.25486485660077e-05,
"median": 0.0024348647333681584,
"type": "next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchCodec[num_threads=1]",
"description": "10 next()",
"fps": 758.7565583035099,
"fps_p25": 766.9872077345758,
"fps_p75": 751.3583938952637,
"frame_count": 10,
"iqr": 0.00027120066806674004,
"median": 0.013179457746446133,
"type": "next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchVision[backend=VideoReader]",
"description": "10 seek()+next()",
"fps": 7.880664295730362,
"fps_p25": 7.924876540397429,
"fps_p75": 7.827668314297991,
"frame_count": 10,
"iqr": 0.01567032840102911,
"median": 1.2689285604283214,
"type": "seek()+next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchVision[backend=VideoReader]",
"description": "1 next()",
"fps": 209.3834723742803,
"fps_p25": 211.77546088016425,
"fps_p75": 199.25812670019334,
"frame_count": 1,
"iqr": 0.0002966334810480479,
"median": 0.00477592614479363,
"type": "next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchVision[backend=VideoReader]",
"description": "10 next()",
"fps": 531.7221665034224,
"fps_p25": 538.7259880033903,
"fps_p75": 522.8300241450709,
"frame_count": 10,
"iqr": 0.0005643628537654877,
"median": 0.018806814216077328,
"type": "next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchAudio",
"description": "10 seek()+next()",
"fps": 27.403418366590216,
"fps_p25": 27.57563496650987,
"fps_p75": 27.276339015442346,
"frame_count": 10,
"iqr": 0.003979140194132924,
"median": 0.36491797724738717,
"type": "seek()+next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchAudio",
"description": "1 next()",
"fps": 243.36030386646544,
"fps_p25": 246.1182470856226,
"fps_p75": 240.91573311529928,
"frame_count": 1,
"iqr": 8.774134330451558e-05,
"median": 0.004109133593738079,
"type": "next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"decoder": "TorchAudio",
"description": "10 next()",
"fps": 564.5307153840273,
"fps_p25": 572.2932075367765,
"fps_p75": 558.1691247911658,
"frame_count": 10,
"iqr": 0.00044215633533895016,
"median": 0.01771382801234722,
"type": "next()",
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
},
{
"cpu_count": 56,
"machine": "x86_64",
"processor": "x86_64",
"python_version": "3.11.10",
"system": "Linux"
}
]
75 changes: 7 additions & 68 deletions benchmarks/decoders/generate_readme_chart.py
View file
Original file line number Diff line number Diff line change
@@ -4,81 +4,20 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

import glob
import importlib.resources
import os
import shutil
from pathlib import Path

from benchmark_decoders_library import (
generate_videos,
plot_data,
run_benchmarks,
TorchAudioDecoder,
TorchcodecNonCompiledWithOptions,
TVNewAPIDecoderWithBackend,
)


def in_fbcode() -> bool:
return "FB_PAR_RUNTIME_FILES" in os.environ
import json

from pathlib import Path

def get_test_resource_path(filename: str) -> str:
if in_fbcode():
resource = importlib.resources.files(__package__).joinpath(filename)
with importlib.resources.as_file(resource) as path:
return os.fspath(path)

return str(Path(__file__).parent / f"../../test/resources/{filename}")
from benchmark_decoders_library import plot_data


def main() -> None:
"""Benchmarks the performance of a few video decoders on synthetic videos"""

resolutions = ["640x480"]
encodings = ["libx264"]
fpses = [30]
gop_sizes = [600]
durations = [10]
pix_fmts = ["yuv420p"]
ffmpeg_path = "ffmpeg"
videos_dir_path = "/tmp/torchcodec_benchmarking_videos"
shutil.rmtree(videos_dir_path, ignore_errors=True)
os.makedirs(videos_dir_path)
generate_videos(
resolutions,
encodings,
fpses,
gop_sizes,
durations,
pix_fmts,
ffmpeg_path,
videos_dir_path,
)
video_files_paths = glob.glob(f"{videos_dir_path}/*.mp4")

decoder_dict = {}
decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions()
decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions(
num_threads=1
)
decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend(
"video_reader"
)
decoder_dict["TorchAudio"] = TorchAudioDecoder()
data_json = Path(__file__).parent / "benchmark_readme_data.json"
with open(data_json, "r") as read_file:
data_from_file = json.load(read_file)

output_png = Path(__file__).parent / "benchmark_readme_chart.png"
# These are the number of uniform seeks we do in the seek+decode benchmark.
num_uniform_samples = 10
df_data = run_benchmarks(
decoder_dict,
video_files_paths,
num_uniform_samples,
10,
False,
)
plot_data(df_data, output_png)
plot_data(data_from_file, output_png)


if __name__ == "__main__":
83 changes: 83 additions & 0 deletions benchmarks/decoders/generate_readme_data.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

import glob
import json
import os
import platform
import shutil
from pathlib import Path

from benchmark_decoders_library import (
generate_videos,
run_benchmarks,
TorchAudioDecoder,
TorchcodecNonCompiledWithOptions,
TVNewAPIDecoderWithBackend,
)


def main() -> None:
"""Benchmarks the performance of a few video decoders on synthetic videos"""

resolutions = ["640x480"]
encodings = ["libx264"]
fpses = [30]
gop_sizes = [600]
durations = [10]
pix_fmts = ["yuv420p"]
ffmpeg_path = "ffmpeg"
videos_dir_path = "/tmp/torchcodec_benchmarking_videos"
shutil.rmtree(videos_dir_path, ignore_errors=True)
os.makedirs(videos_dir_path)
generate_videos(
resolutions,
encodings,
fpses,
gop_sizes,
durations,
pix_fmts,
ffmpeg_path,
videos_dir_path,
)
video_files_paths = glob.glob(f"{videos_dir_path}/*.mp4")

decoder_dict = {}
decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions()
decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions(
num_threads=1
)
decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend(
"video_reader"
)
decoder_dict["TorchAudio"] = TorchAudioDecoder()

# These are the number of uniform seeks we do in the seek+decode benchmark.
num_uniform_samples = 10
df_data = run_benchmarks(
decoder_dict,
video_files_paths,
num_uniform_samples,
10,
False,
)
df_data.append(
{
"cpu_count": os.cpu_count(),
"system": platform.system(),
"machine": platform.machine(),
"processor": platform.processor(),
"python_version": str(platform.python_version()),
}
)

data_json = Path(__file__).parent / "benchmark_readme_data.json"
with open(data_json, "w") as write_file:
json.dump(df_data, write_file, sort_keys=True, indent=4)


if __name__ == "__main__":
main()
Loading