Testing: Refactor software tests into dedicated directory tests

git mv src/crate/client/test* tests/client/
git mv src/crate/testing/test* tests/testing/

Author: amotl

CHANGES.txt

@@ -12,6 +12,8 @@ Unreleased
   "Threads may share the module, but not connections."
 - Added ``error_trace`` to string representation of an Error to relay
   server stacktraces into exception messages.
+- Refactoring: The module namespace ``crate.client.test_util`` has been
+  renamed to ``crate.testing.util``.
 
 .. _Migrate from crate.client to sqlalchemy-cratedb: https://cratedb.com/docs/sqlalchemy-cratedb/migrate-from-crate-client.html
 .. _sqlalchemy-cratedb: https://pypi.org/project/sqlalchemy-cratedb/

DEVELOP.rst

@@ -32,34 +32,40 @@ see, for example, `useful command-line options for zope-testrunner`_.
 
 Run all tests::
 
-    ./bin/test -vvvv
+    bin/test
 
 Run specific tests::
 
-    ./bin/test -vvvv -t test_score
+    # Select modules.
+    bin/test -t test_cursor
+    bin/test -t client
+    bin/test -t testing
+
+    # Select doctests.
+    bin/test -t http.rst
 
 Ignore specific test directories::
 
-    ./bin/test -vvvv --ignore_dir=testing
+    bin/test --ignore_dir=testing
 
 The ``LayerTest`` test cases have quite some overhead. Omitting them will save
 a few cycles (~70 seconds runtime)::
 
-    ./bin/test -t '!LayerTest'
+    bin/test -t '!LayerTest'
 
-Invoke all tests without integration tests (~15 seconds runtime)::
+Invoke all tests without integration tests (~10 seconds runtime)::
 
-    ./bin/test --layer '!crate.testing.layer.crate' --test '!LayerTest'
+    bin/test --layer '!crate.testing.layer.crate' --test '!LayerTest'
 
-Yet ~130 test cases, but only ~5 seconds runtime::
+Yet ~60 test cases, but only ~1 second runtime::
 
-    ./bin/test --layer '!crate.testing.layer.crate' --test '!LayerTest' \
+    bin/test --layer '!crate.testing.layer.crate' --test '!LayerTest' \
         -t '!test_client_threaded' -t '!test_no_retry_on_read_timeout' \
         -t '!test_wait_for_http' -t '!test_table_clustered_by'
 
 To inspect the whole list of test cases, run::
 
-    ./bin/test --list-tests
+    bin/test --list-tests
 
 You can run the tests against multiple Python interpreters with `tox`_::
 

bin/test

@@ -12,6 +12,6 @@ sys.argv[0] = os.path.abspath(sys.argv[0])
 
 if __name__ == '__main__':
     zope.testrunner.run([
-        '-vvv', '--auto-color',
-        '--test-path', join(base, 'src')],
-    )
+        '-vvvv', '--auto-color',
+        '--path', join(base, 'tests'),
+    ])

docs/by-example/connection.rst

@@ -21,7 +21,7 @@ connect()
 This section sets up a connection object, and inspects some of its attributes.
 
     >>> from crate.client import connect
-    >>> from crate.client.test_util import ClientMocked
+    >>> from crate.testing.util import ClientMocked
 
     >>> connection = connect(client=ClientMocked())
     >>> connection.lowest_server_version.version

docs/by-example/cursor.rst

@@ -23,7 +23,7 @@ up the response for subsequent cursor operations.
     >>> from crate.client import connect
     >>> from crate.client.converter import DefaultTypeConverter
     >>> from crate.client.cursor import Cursor
-    >>> from crate.client.test_util import ClientMocked
+    >>> from crate.testing.util import ClientMocked
 
     >>> connection = connect(client=ClientMocked())
     >>> cursor = connection.cursor()

src/crate/client/test_util.py

@@ -1,3 +1,74 @@
+# -*- coding: utf-8; -*-
+#
+# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
+# license agreements.  See the NOTICE file distributed with this work for
+# additional information regarding copyright ownership.  Crate licenses
+# this file to you under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# However, if you have executed another commercial license agreement
+# with Crate these terms will supersede the license and you may use the
+# software solely pursuant to the terms of the relevant commercial agreement.
+import unittest
+
+
+class ClientMocked(object):
+
+    active_servers = ["http://localhost:4200"]
+
+    def __init__(self):
+        self.response = {}
+        self._server_infos = ("http://localhost:4200", "my server", "2.0.0")
+
+    def sql(self, stmt=None, parameters=None, bulk_parameters=None):
+        return self.response
+
+    def server_infos(self, server):
+        return self._server_infos
+
+    def set_next_response(self, response):
+        self.response = response
+
+    def set_next_server_infos(self, server, server_name, version):
+        self._server_infos = (server, server_name, version)
+
+    def close(self):
+        pass
+
+
+class ParametrizedTestCase(unittest.TestCase):
+    """
+    TestCase classes that want to be parametrized should
+    inherit from this class.
+
+    https://eli.thegreenplace.net/2011/08/02/python-unit-testing-parametrized-test-cases
+    """
+    def __init__(self, methodName="runTest", param=None):
+        super(ParametrizedTestCase, self).__init__(methodName)
+        self.param = param
+
+    @staticmethod
+    def parametrize(testcase_klass, param=None):
+        """ Create a suite containing all tests taken from the given
+            subclass, passing them the parameter 'param'.
+        """
+        testloader = unittest.TestLoader()
+        testnames = testloader.getTestCaseNames(testcase_klass)
+        suite = unittest.TestSuite()
+        for name in testnames:
+            suite.addTest(testcase_klass(name, param=param))
+        return suite
+
+
 class ExtraAssertions:
     """
     Additional assert methods for unittest.

src/crate/testing/util.py

@@ -34,11 +34,11 @@
 
 import stopit
 
-from crate.testing.layer import CrateLayer
-from crate.testing.settings import \
-    crate_host, crate_path, crate_port, \
-    crate_transport_port, docs_path, localhost
 from crate.client import connect
+from crate.testing.layer import CrateLayer
+from .settings import \
+    assets_path, crate_host, crate_path, crate_port, \
+    crate_transport_port, localhost
 
 
 makeSuite = unittest.TestLoader().loadTestsFromTestCase
@@ -104,15 +104,15 @@ def setUpCrateLayerBaseline(test):
     with connect(crate_host) as conn:
         cursor = conn.cursor()
 
-        with open(docs_path('testing/testdata/mappings/locations.sql')) as s:
+        with open(assets_path('mappings/locations.sql')) as s:
             stmt = s.read()
             cursor.execute(stmt)
             stmt = ("select count(*) from information_schema.tables "
                     "where table_name = 'locations'")
             cursor.execute(stmt)
             assert cursor.fetchall()[0][0] == 1
 
-        data_path = docs_path('testing/testdata/data/test_a.json')
+        data_path = assets_path('import/test_a.json')
         # load testing data into crate
         cursor.execute("copy locations from ?", (data_path,))
         # refresh location table so imported data is visible immediately
@@ -145,10 +145,8 @@ def tearDownDropEntitiesBaseline(test):
 class HttpsTestServerLayer:
     PORT = 65534
     HOST = "localhost"
-    CERT_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                "pki/server_valid.pem"))
-    CACERT_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                  "pki/cacert_valid.pem"))
+    CERT_FILE = assets_path("pki/server_valid.pem")
+    CACERT_FILE = assets_path("pki/cacert_valid.pem")
 
     __name__ = "httpsserver"
     __bases__ = tuple()
@@ -237,18 +235,10 @@ def setUpWithHttps(test):
     test.globs['pprint'] = pprint
     test.globs['print'] = cprint
 
-    test.globs['cacert_valid'] = os.path.abspath(
-        os.path.join(os.path.dirname(__file__), "pki/cacert_valid.pem")
-    )
-    test.globs['cacert_invalid'] = os.path.abspath(
-        os.path.join(os.path.dirname(__file__), "pki/cacert_invalid.pem")
-    )
-    test.globs['clientcert_valid'] = os.path.abspath(
-        os.path.join(os.path.dirname(__file__), "pki/client_valid.pem")
-    )
-    test.globs['clientcert_invalid'] = os.path.abspath(
-        os.path.join(os.path.dirname(__file__), "pki/client_invalid.pem")
-    )
+    test.globs['cacert_valid'] = assets_path("pki/cacert_valid.pem")
+    test.globs['cacert_invalid'] = assets_path("pki/cacert_invalid.pem")
+    test.globs['clientcert_valid'] = assets_path("pki/client_valid.pem")
+    test.globs['clientcert_invalid'] = assets_path("pki/client_invalid.pem")
 
 
 def _execute_statements(statements, on_error="ignore"):

tests/__init__.py

@@ -21,27 +21,20 @@
 # software solely pursuant to the terms of the relevant commercial agreement.
 from __future__ import absolute_import
 
-import os
+from pathlib import Path
 
 
-def docs_path(*parts):
-    return os.path.abspath(
-        os.path.join(
-            os.path.dirname(os.path.dirname(__file__)), *parts
-        )
-    )
+def assets_path(*parts) -> str:
+    return str((project_root() / "tests" / "assets").joinpath(*parts).absolute())
 
 
-def project_root(*parts):
-    return os.path.abspath(
-        os.path.join(docs_path("..", ".."), *parts)
-    )
+def crate_path() -> str:
+    return str(project_root() / "parts" / "crate")
 
 
-def crate_path(*parts):
-    return os.path.abspath(
-        project_root("parts", "crate", *parts)
-    )
+def project_root() -> Path:
+    return Path(__file__).parent.parent.parent
+
 
 
 crate_port = 44209

src/crate/testing/testdata/data/test_a.json renamed to tests/assets/import/test_a.json

@@ -2,12 +2,12 @@
 
 from urllib3 import Timeout
 
-from .connection import Connection
-from .http import Client
+from crate.client.connection import Connection
+from crate.client.http import Client
 from crate.client import connect
 from unittest import TestCase
 
-from ..testing.settings import crate_host
+from .settings import crate_host
 
 
 class ConnectionTest(TestCase):

src/crate/testing/testdata/mappings/locations.sql renamed to tests/assets/mappings/locations.sql

@@ -33,7 +33,7 @@
 from crate.client import connect
 from crate.client.converter import DataType, DefaultTypeConverter
 from crate.client.http import Client
-from crate.client.test_util import ClientMocked
+from crate.testing.util import ClientMocked
 
 
 class CursorTest(TestCase):

src/crate/client/pki/cacert_invalid.pem renamed to tests/assets/pki/cacert_invalid.pem

@@ -43,8 +43,8 @@
 import uuid
 import certifi
 
-from .http import Client, CrateJsonEncoder, _get_socket_opts, _remove_certs_for_non_https
-from .exceptions import ConnectionError, ProgrammingError, IntegrityError
+from crate.client.http import Client, CrateJsonEncoder, _get_socket_opts, _remove_certs_for_non_https
+from crate.client.exceptions import ConnectionError, ProgrammingError, IntegrityError
 
 REQUEST = 'crate.client.http.Server.request'
 CA_CERT_PATH = certifi.where()

src/crate/client/pki/cacert_valid.pem renamed to tests/assets/pki/cacert_valid.pem

@@ -1,12 +1,12 @@
 import doctest
 import unittest
 
-from crate.client.test_connection import ConnectionTest
-from crate.client.test_cursor import CursorTest
-from crate.client.test_http import HttpClientTest, KeepAliveClientTest, ThreadSafeHttpClientTest, ParamsTest, \
+from .test_connection import ConnectionTest
+from .test_cursor import CursorTest
+from .test_http import HttpClientTest, KeepAliveClientTest, ThreadSafeHttpClientTest, ParamsTest, \
     RetryOnTimeoutServerTest, RequestsCaBundleTest, TestUsernameSentAsHeader, TestCrateJsonEncoder, \
     TestDefaultSchemaHeader
-from crate.client.test_support import makeSuite, setUpWithHttps, HttpsTestServerLayer, setUpCrateLayerBaseline, \
+from .layer import makeSuite, setUpWithHttps, HttpsTestServerLayer, setUpCrateLayerBaseline, \
     tearDownDropEntitiesBaseline, ensure_cratedb_layer