Ci/cmake to pytest only

CMakeLists.txt

@@ -616,6 +616,11 @@ if (BUILD_TESTING)
         target_compile_options(s2nd PRIVATE -flto)
     endif()
 
+    #1.	Checks if integration tests are enabled (S2N_INTEG_TESTS)
+    #2.	Discovers all test_*.py files under tests/integrationv2/
+    #3.	Generates a CTest entry per test file
+    #4.	Depending on the env (Nix vs CodeBuild), chooses how to run the test (pytest or tox)
+    #5.	Adds runtime config (paths, timeouts, logs) for each test
     if (S2N_INTEG_TESTS)
         find_package (Python3 COMPONENTS Interpreter Development)
         file(GLOB integv2_test_files "${PROJECT_SOURCE_DIR}/tests/integrationv2/test_*.py")
@@ -644,6 +649,7 @@ if (BUILD_TESTING)
                         ${test_file_path}
                         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integrationv2
                 )
+            # running in code build vs above is running in nix
             else()
                 # For use with libcryptos built into test-deps, and not in LD_LIBRARY_PATH.
                 # This is a duplication of tests/integrationv2/Makefile and

codebuild/bin/s2n_codebuild.sh

@@ -66,24 +66,36 @@ setup_apache_server() {
 }
 
 run_integration_v2_tests() {
+    # checks if the Apache-related integration test is being run
     setup_apache_server
+    # installs the current version of s2n (s2nc/s2nd) into a temporary directory
     "$CB_BIN_DIR/install_s2n_head.sh" "$(mktemp -d)"
+    # generates a CMake-based build system inside the build/ directory
     cmake . -Bbuild \
             -DCMAKE_PREFIX_PATH=$LIBCRYPTO_ROOT \
             -DBUILD_SHARED_LIBS=on \
-            -DS2N_INTEG_TESTS=on \
             -DPython3_EXECUTABLE=$(which python3)
+    # "Now build everything"
     cmake --build ./build --clean-first -- -j $(nproc)
+
+    # "doing a runtime check to verify which libcrypto library is actually linked to the s2nc and s2nd binaries"
     test_linked_libcrypto ./build/bin/s2nc
     test_linked_libcrypto ./build/bin/s2nd
+
+    # Ensure the s2nc and s2nd binaries are available
     cp -f ./build/bin/s2nc "$BASE_S2N_DIR"/bin/s2nc
     cp -f ./build/bin/s2nd "$BASE_S2N_DIR"/bin/s2nd
-    cd ./build/
-    for test_name in $TOX_TEST_NAME; do
-      test="${test_name//test_/}"
-      echo "Running... ctest --no-tests=error --output-on-failure --verbose -R ^integrationv2_${test}$"
-      ctest --no-tests=error --output-on-failure --verbose -R ^integrationv2_${test}$
-    done
+
+    export PATH="$BASE_S2N_DIR/bin:$PATH"
+
+    cd tests/integrationv2
+
+    # Run tests — THIS replaces the old ctest loop
+    uv run pytest . \
+        --provider-version=${S2N_LIBCRYPTO} \
+        -rpfs -n auto \
+        --timeout=100 \
+        -o log_cli=true --log-cli-level=DEBUG
 }
 
 run_unit_tests() {
@@ -109,4 +121,4 @@ if [[ "$TESTS" == "ALL" || "$TESTS" == "dynamicload" ]]; then ./codebuild/bin/te
 if [[ "$TESTS" == "sawHMAC" ]] && [[ "$OS_NAME" == "linux" ]]; then make -C tests/saw/ tmp/verify_HMAC.log ; fi
 if [[ "$TESTS" == "sawDRBG" ]]; then make -C tests/saw tmp/verify_drbg.log ; fi
 if [[ "$TESTS" == "ALL" || "$TESTS" == "tls" ]]; then make -C tests/saw tmp/verify_handshake.log ; fi
-if [[ "$TESTS" == "sawHMACFailure" ]]; then make -C tests/saw failure-tests ; fi
+if [[ "$TESTS" == "sawHMACFailure" ]]; then make -C tests/saw failure-tests ; fi

codebuild/spec/buildspec_ubuntu_integrationv2.yml

@@ -53,13 +53,26 @@ env:
 phases:
   pre_build:
     commands:
+      - apt-get update
+      - apt-get install -y gnutls-bin
+
+      - echo "Installing uv..."
+      - curl -LsSf https://astral.sh/uv/install.sh | sh
+      - export PATH="/root/.local/bin:$PATH"
+      - echo "Verifying uv install"
+      - uv --version
+      - uv venv
+      - uv pip install pytest>=8.3.4 pytest-rerunfailures>=15.0 pytest-xdist>=3.6.1 ruff>=0.9.7 sslyze>=6.1.0 pytest-timeout>=2.3.1
+
       - |
         if [ -d "third-party-src" ]; then
           cd third-party-src;
           ln -s /usr/local $CODEBUILD_SRC_DIR/third-party-src/test-deps;
         fi
   build:
     commands:
+      - export PATH="$HOME/.cargo/bin:$PATH"
+      - uv --version   
       # e.g. awslc-fips-2022_gcc6
       - echo "CI Configuration - s2n build preset is $S2N_BUILD_PRESET"
       # e.g. "test_cross_compatibility test_client_authentication"

tests/integrationv2/configuration.py

@@ -65,8 +65,24 @@
     Certificates.ECDSA_521,
 ]
 
-
+# 1 of each authentication type + encryp
+# 1 of each cipher h mac (AES128_SHA), AES128_GCM_SHA256)
+# no every combination
+# test buffered send, we don't want to reduce for happy path (other way???)
 # List of all ciphers that will be tested.
+
+
+"""
+OVERALL SPEEEED
+make tests run in parallel?
+when tests run on nix, we don't have an issue of the parallel stuff ()
+- n auto in pytest is running stuff in parallel, also works in nix, doesn't work in codebuild (that's why it's slow), 
+nix and codebuild overlapping test cases because each can't run all of them
+
+QUESTIONS
+Should I use minimal test ciphers for every testcase except for happy_path
+Questions on notes
+"""
 ALL_TEST_CIPHERS = [
     Ciphers.DHE_RSA_AES128_SHA,
     Ciphers.DHE_RSA_AES256_SHA,
@@ -75,26 +91,32 @@
     Ciphers.DHE_RSA_AES128_GCM_SHA256,
     Ciphers.DHE_RSA_AES256_GCM_SHA384,
     Ciphers.DHE_RSA_CHACHA20_POLY1305,
+
+    # what authentication is this???
     Ciphers.AES128_SHA,
     Ciphers.AES256_SHA,
     Ciphers.AES128_SHA256,
     Ciphers.AES256_SHA256,
-    Ciphers.AES128_GCM_SHA256,
-    Ciphers.AES256_GCM_SHA384,
-    Ciphers.ECDHE_ECDSA_AES128_GCM_SHA256,
-    Ciphers.ECDHE_ECDSA_AES256_GCM_SHA384,
-    Ciphers.ECDHE_ECDSA_AES128_SHA256,
-    Ciphers.ECDHE_ECDSA_AES256_SHA384,
+
     Ciphers.ECDHE_ECDSA_AES128_SHA,
     Ciphers.ECDHE_ECDSA_AES256_SHA,
+    Ciphers.ECDHE_ECDSA_AES128_SHA256,
+    Ciphers.ECDHE_ECDSA_AES256_SHA384,
+    Ciphers.ECDHE_ECDSA_AES128_GCM_SHA256,
+    Ciphers.ECDHE_ECDSA_AES256_GCM_SHA384,
     Ciphers.ECDHE_ECDSA_CHACHA20_POLY1305,
+
     Ciphers.ECDHE_RSA_AES128_SHA,
     Ciphers.ECDHE_RSA_AES256_SHA,
     Ciphers.ECDHE_RSA_AES128_SHA256,
     Ciphers.ECDHE_RSA_AES256_SHA384,
     Ciphers.ECDHE_RSA_AES128_GCM_SHA256,
     Ciphers.ECDHE_RSA_AES256_GCM_SHA384,
     Ciphers.ECDHE_RSA_CHACHA20_POLY1305,
+
+    # List of TLS13 Ciphers
+    Ciphers.AES128_GCM_SHA256,
+    Ciphers.AES256_GCM_SHA384,
     Ciphers.CHACHA20_POLY1305_SHA256,
 ]
 

tests/integrationv2/pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "pytest-xdist>=3.6.1",
     "ruff>=0.9.7",
     "sslyze>=6.1.0",
+    "pytest-timeout>=2.3.1",
 ]
 
 # setuptools made a breaking change in setup file validation: https://github.com/pypa/setuptools/blob/main/NEWS.rst#deprecations-and-removals