diff --git a/.github/workflows/linux_wheel.yaml b/.github/workflows/linux_wheel.yaml index 9b862fbc..38f25733 100644 --- a/.github/workflows/linux_wheel.yaml +++ b/.github/workflows/linux_wheel.yaml @@ -1,4 +1,4 @@ -name: Build and test Linux wheels +name: Build and test Linux wheel on: pull_request: @@ -24,6 +24,7 @@ defaults: shell: bash -l -eo pipefail {0} jobs: + generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: @@ -34,11 +35,13 @@ jobs: with-xpu: disable with-rocm: disable with-cuda: disable + build-python-only: "disable" + build: needs: generate-matrix strategy: fail-fast: false - name: Build and Upload wheel + name: Build and Upload Linux wheel uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main with: repository: pytorch/torchcodec @@ -60,7 +63,6 @@ jobs: matrix: python-version: ['3.9'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - if: ${{ always() }} needs: build steps: - uses: actions/download-artifact@v3 @@ -121,7 +123,7 @@ jobs: ls - name: Smoke test run: | - python test/decoders/manual_smoke_test.py + python -X faulthandler test/decoders/manual_smoke_test.py - name: Run Python tests run: | pytest test -vvv diff --git a/.github/workflows/macos_wheel.yaml b/.github/workflows/macos_wheel.yaml index 9060b18c..ef637194 100644 --- a/.github/workflows/macos_wheel.yaml +++ b/.github/workflows/macos_wheel.yaml @@ -1,4 +1,4 @@ -name: Build and test MacOS +name: Build and test MacOS wheel on: pull_request: @@ -24,15 +24,54 @@ defaults: shell: bash -l -eo pipefail {0} jobs: + + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: macos-arm64 + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-xpu: disable + with-rocm: disable + with-cuda: disable + build-python-only: "disable" + + build: + needs: generate-matrix + strategy: + fail-fast: false + name: Build and Upload Mac wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main + with: + repository: pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/fake_smoke_test.py + runner-type: macos-m1-stable + package-name: torchcodec + trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build --wheel -vvv --no-isolation" + install-and-test: - runs-on: macos-m1-stable + runs-on: macos-14-xlarge strategy: fail-fast: false matrix: python-version: ['3.9'] ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - if: ${{ always() }} + needs: build steps: + - name: Download wheel + uses: actions/download-artifact@v3 + with: + name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_ + path: pytorch/torchcodec/dist/ + - name: Setup conda env uses: conda-incubator/setup-miniconda@v3 with: @@ -42,36 +81,49 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update pip run: python -m pip install --upgrade pip + - name: Install PyTorch run: | python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - - name: Check out repo + + - name: Install torchcodec from the wheel + run: | + wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv + + - name: Check out torchcodec repo uses: actions/checkout@v3 - - name: Install compile from source dependencies + + - name: Install ffmpeg run: | - conda install cmake pkg-config -c conda-forge + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version + - name: Install test dependencies run: | python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # Ideally we would find a way to get those dependencies from pyproject.toml python -m pip install numpy pytest pillow - - name: Install torchcodec from source, building against non-GPL FFmpeg - run: | - BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 pip install -e ".[dev]" --no-build-isolation - - name: Install ffmpeg, post build + + - name: Delete the src/ folder just for fun run: | - # Ideally we would have checked for that before installing the wheel, - # but we need to checkout the repo to access this file, and we don't - # want to checkout the repo before installing the wheel to avoid any - # side-effect. It's OK. - source packaging/helpers.sh - assert_ffmpeg_not_installed + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls -lh - conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - ffmpeg -version - name: Smoke test run: | - python test/decoders/manual_smoke_test.py + python -X faulthandler test/decoders/manual_smoke_test.py + - name: Run Python tests run: | pytest test -vvv diff --git a/packaging/check_glibcxx.py b/packaging/check_glibcxx.py index 0bf435ff..37ff654c 100644 --- a/packaging/check_glibcxx.py +++ b/packaging/check_glibcxx.py @@ -38,14 +38,15 @@ MAX_ALLOWED = (3, 4, 19) +symbol_matches = sys.argv[1].split("\n") all_symbols = set() -for line in sys.argv[1].split("\n"): +for line in symbol_matches: # We search for GLIBCXX_major.minor.micro if match := re.search(r"GLIBCXX_\d+\.\d+\.\d+", line): all_symbols.add(match.group(0)) if not all_symbols: - raise ValueError("No GLIBCXX symbols found. Something is wrong.") + raise ValueError(f"No GLIBCXX symbols found in {symbol_matches}. Something is wrong.") all_versions = (symbol.split("_")[1].split(".") for symbol in all_symbols) all_versions = (tuple(int(v) for v in version) for version in all_versions) diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index 721f0b61..ca8fd133 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,25 +1,35 @@ #!/bin/bash +set -ex + source packaging/helpers.sh wheel_path=$(pwd)/$(find dist -type f -name "*.whl") echo "Wheel content:" unzip -l $wheel_path -ffmpeg_versions=(4 5 6 7) +unamestr=$(uname) +if [[ "$unamestr" == 'Linux' ]]; then + ext="so" +elif [[ "$unamestr" == 'Darwin' ]]; then + ext="dylib" +else + echo "Unknown operating system: $unamestr" + exit 1 +fi # TODO: Make ffmpeg4 work with nvcc. -if [ "$ENABLE_CUDA" -eq 1 ]; then - ffmpeg_versions=(5 6 7) +if [[ "$ENABLE_CUDA" -eq 1 ]]; then + ffmpeg_versions=(5 6 7) fi -for ffmpeg_major_version in ${ffmepg_versions[@]}; do - assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so +for ffmpeg_major_version in ${ffmpeg_versions[@]}; do + assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.${ext} done -assert_not_in_wheel $wheel_path libtorchcodec.so +assert_not_in_wheel $wheel_path libtorchcodec.${ext} -for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do - assert_not_in_wheel $wheel_path $ffmpeg_so +for ffmpeg_ext in libavcodec.${ext} libavfilter.${ext} libavformat.${ext} libavutil.${ext} libavdevice.${ext} ; do + assert_not_in_wheel $wheel_path $ffmpeg_ext done assert_not_in_wheel $wheel_path "^test" @@ -27,19 +37,21 @@ assert_not_in_wheel $wheel_path "^doc" assert_not_in_wheel $wheel_path "^benchmarks" assert_not_in_wheel $wheel_path "^packaging" -# See invoked python script below for details about this check. -extracted_wheel_dir=$(mktemp -d) -unzip -q $wheel_path -d $extracted_wheel_dir -symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) -python packaging/check_glibcxx.py "$symbols_matches" +if [[ "$unamestr" == 'Linux' ]]; then + # See invoked python script below for details about this check. + extracted_wheel_dir=$(mktemp -d) + unzip -q $wheel_path -d $extracted_wheel_dir + symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) + python packaging/check_glibcxx.py "$symbols_matches" -echo "ls dist" -ls dist + echo "ls dist" + ls dist -old="linux_x86_64" -new="manylinux_2_17_x86_64.manylinux2014_x86_64" -echo "Replacing ${old} with ${new} in wheel name" -mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/") + old="linux_x86_64" + new="manylinux_2_17_x86_64.manylinux2014_x86_64" + echo "Replacing ${old} with ${new} in wheel name" + mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/") +fi echo "ls dist" ls dist diff --git a/src/torchcodec/decoders/_core/VideoDecoder.cpp b/src/torchcodec/decoders/_core/VideoDecoder.cpp index 8c9f4363..86b90469 100644 --- a/src/torchcodec/decoders/_core/VideoDecoder.cpp +++ b/src/torchcodec/decoders/_core/VideoDecoder.cpp @@ -676,9 +676,6 @@ void VideoDecoder::maybeSeekToBeforeDesiredPts() { int64_t desiredPtsForStream = *maybeDesiredPts_ * streamInfo.timeBase.den; if (!canWeAvoidSeekingForStream( streamInfo, streamInfo.currentPts, desiredPtsForStream)) { - VLOG(5) << "Seeking is needed for streamIndex=" << streamIndex - << " desiredPts=" << desiredPtsForStream - << " currentPts=" << streamInfo.currentPts; mustSeek = true; break; } @@ -727,13 +724,10 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( if (activeStreamIndices_.size() == 0) { throw std::runtime_error("No active streams configured."); } - VLOG(9) << "Starting getDecodedOutputWithFilter()"; resetDecodeStats(); if (maybeDesiredPts_.has_value()) { - VLOG(9) << "maybeDesiredPts_=" << *maybeDesiredPts_; maybeSeekToBeforeDesiredPts(); maybeDesiredPts_ = std::nullopt; - VLOG(9) << "seeking done"; } auto seekDone = std::chrono::high_resolution_clock::now(); // Need to get the next frame or error from PopFrame. @@ -748,13 +742,9 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( StreamInfo& streamInfo = streams_[streamIndex]; ffmpegStatus = avcodec_receive_frame(streamInfo.codecContext.get(), frame.get()); - VLOG(9) << "received frame" << " status=" << ffmpegStatus - << " streamIndex=" << streamInfo.stream->index; bool gotNonRetriableError = ffmpegStatus != AVSUCCESS && ffmpegStatus != AVERROR(EAGAIN); if (gotNonRetriableError) { - VLOG(9) << "Got non-retriable error from decoder: " - << getFFMPEGErrorStringFromErrorCode(ffmpegStatus); gotPermanentErrorOnAnyActiveStream = true; break; } @@ -784,7 +774,6 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( UniqueAVPacket packet(av_packet_alloc()); ffmpegStatus = av_read_frame(formatContext_.get(), packet.get()); decodeStats_.numPacketsRead++; - VLOG(9) << "av_read_frame returned status: " << ffmpegStatus; if (ffmpegStatus == AVERROR_EOF) { // End of file reached. We must drain all codecs by sending a nullptr // packet. @@ -807,8 +796,6 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( "Could not read frame from input file: " + getFFMPEGErrorStringFromErrorCode(ffmpegStatus)); } - VLOG(9) << "Got packet: stream_index=" << packet->stream_index - << " pts=" << packet->pts << " size=" << packet->size; if (activeStreamIndices_.count(packet->stream_index) == 0) { // This packet is not for any of the active streams. continue; @@ -846,10 +833,6 @@ VideoDecoder::RawDecodedOutput VideoDecoder::getDecodedOutputWithFilter( std::chrono::duration_cast(seekDone - start); auto seekToDecodeDone = std::chrono::duration_cast( decodeDone - seekDone); - VLOG(3) << "Got frame: stream_index=" << activeStream.stream->index - << " pts=" << frame->pts << " stats=" << decodeStats_ - << " startToSeekDone=" << startToSeekDone.count() << "ms" - << " seekToDecodeDone=" << seekToDecodeDone.count() << "ms"; RawDecodedOutput rawOutput; rawOutput.streamIndex = frameStreamIndex; rawOutput.frame = std::move(frame); diff --git a/src/torchcodec/decoders/_core/VideoDecoderOps.cpp b/src/torchcodec/decoders/_core/VideoDecoderOps.cpp index 6b91853c..0bf49621 100644 --- a/src/torchcodec/decoders/_core/VideoDecoderOps.cpp +++ b/src/torchcodec/decoders/_core/VideoDecoderOps.cpp @@ -292,12 +292,6 @@ bool _test_frame_pts_equality( int64_t frame_index, double pts_seconds_to_test) { auto videoDecoder = unwrapTensorToGetDecoder(decoder); - LOG(INFO) << "pts_seconds_to_test: " << std::setprecision(15) - << pts_seconds_to_test << std::endl; - LOG(INFO) << "frame pts : " << std::setprecision(15) - << videoDecoder->getPtsSecondsForFrame(stream_index, frame_index) - << std::endl - << std::endl; return pts_seconds_to_test == videoDecoder->getPtsSecondsForFrame(stream_index, frame_index); }