Intermediate changes

contrib/libs/googleapis-common-protos/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [1.63.2](https://github.com/googleapis/python-api-common-protos/compare/v1.63.1...v1.63.2) (2024-06-19)
+
+
+### Bug Fixes
+
+* **deps:** Require protobuf>=3.20.2 ([c77c0dc](https://github.com/googleapis/python-api-common-protos/commit/c77c0dc5d29ef780d781a3c5d757736a9ed09674))
+* Regenerate pb2 files for compatibility with protobuf 5.x ([c77c0dc](https://github.com/googleapis/python-api-common-protos/commit/c77c0dc5d29ef780d781a3c5d757736a9ed09674))
+
 ## [1.63.1](https://github.com/googleapis/python-api-common-protos/compare/v1.63.0...v1.63.1) (2024-05-30)
 
 

contrib/libs/googleapis-common-protos/ya.make

@@ -6,9 +6,9 @@ LICENSE(Apache-2.0)
 
 LICENSE_TEXTS(.yandex_meta/licenses.list.txt)
 
-VERSION(1.63.1)
+VERSION(1.63.2)
 
-ORIGINAL_SOURCE(https://github.com/googleapis/python-api-common-protos/archive/v1.63.1.tar.gz)
+ORIGINAL_SOURCE(https://github.com/googleapis/python-api-common-protos/archive/v1.63.2.tar.gz)
 
 PY_NAMESPACE(.)
 

contrib/python/clickhouse-connect/.dist-info/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clickhouse-connect
-Version: 0.7.12
+Version: 0.7.14
 Summary: ClickHouse Database Core Driver for Python, Pandas, and Superset
 Home-page: https://github.com/ClickHouse/clickhouse-connect
 Author: ClickHouse Inc.
@@ -34,7 +34,7 @@ Requires-Dist: pandas ; extra == 'pandas'
 Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy <2.0,>1.3.21 ; extra == 'sqlalchemy'
 Provides-Extra: tzlocal
-Requires-Dist: tzlocal ; extra == 'tzlocal'
+Requires-Dist: tzlocal >=4.0 ; extra == 'tzlocal'
 
 ## ClickHouse Connect
 

contrib/python/clickhouse-connect/clickhouse_connect/__version__.py

@@ -1 +1 @@
-version = '0.7.12'
+version = '0.7.14'

contrib/python/clickhouse-connect/clickhouse_connect/datatypes/special.py

@@ -42,7 +42,7 @@ class UUID(ClickHouseType):
         if isinstance(first, str) or self.write_format(ctx) == 'string':
             for v in column:
                 if v:
-                    x = int(v, 16)
+                    x = int(v.replace('-', ''), 16)
                     dest += (x >> 64).to_bytes(8, 'little') + (x & 0xffffffffffffffff).to_bytes(8, 'little')
                 else:
                     dest += empty

contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py

@@ -133,7 +133,10 @@ class Client(ABC):
 
     def _prep_query(self, context: QueryContext):
         if context.is_select and not context.has_limit and self.query_limit:
-            return f'{context.final_query}\n LIMIT {self.query_limit}'
+            limit = f'\n LIMIT {self.query_limit}'
+            if isinstance(context.query, bytes):
+                return context.final_query + limit.encode()
+            return context.final_query + limit
         return context.final_query
 
     def _check_tz_change(self, new_tz) -> Optional[tzinfo]:
@@ -386,7 +389,7 @@ class Client(ABC):
                                    streaming=True).df_stream
 
     def create_query_context(self,
-                             query: Optional[str] = None,
+                             query: Optional[Union[str, bytes]] = None,
                              parameters: Optional[Union[Sequence, Dict[str, Any]]] = None,
                              settings: Optional[Dict[str, Any]] = None,
                              query_formats: Optional[Dict[str, str]] = None,

contrib/python/clickhouse-connect/clickhouse_connect/driver/external.py

@@ -43,16 +43,16 @@ class ExternalFile:
             self.file_name = file_name
         else:
             raise ProgrammingError('Either data or file_path must be specified for external data')
+        self.structure = None
+        self.types = None
         if types:
             if structure:
                 raise ProgrammingError('Only types or structure should be specified for external data, not both')
-            self.structure = None
             if isinstance(types, str):
                 self.types = types
             else:
                 self.types = ','.join(types)
         elif structure:
-            self.types = None
             if isinstance(structure, str):
                 self.structure = structure
             else:

contrib/python/clickhouse-connect/clickhouse_connect/driver/httpclient.py

@@ -15,17 +15,17 @@ from urllib3.response import HTTPResponse
 from clickhouse_connect import common
 from clickhouse_connect.datatypes import registry
 from clickhouse_connect.datatypes.base import ClickHouseType
-from clickhouse_connect.driver.ctypes import RespBuffCls
 from clickhouse_connect.driver.client import Client
 from clickhouse_connect.driver.common import dict_copy, coerce_bool, coerce_int
 from clickhouse_connect.driver.compression import available_compression
+from clickhouse_connect.driver.ctypes import RespBuffCls
 from clickhouse_connect.driver.exceptions import DatabaseError, OperationalError, ProgrammingError
 from clickhouse_connect.driver.external import ExternalData
 from clickhouse_connect.driver.httputil import ResponseSource, get_pool_manager, get_response_data, \
     default_pool_manager, get_proxy_manager, all_managers, check_env_proxy, check_conn_expiration
 from clickhouse_connect.driver.insert import InsertContext
-from clickhouse_connect.driver.summary import QuerySummary
 from clickhouse_connect.driver.query import QueryResult, QueryContext, quote_identifier, bind_query
+from clickhouse_connect.driver.summary import QuerySummary
 from clickhouse_connect.driver.transform import NativeTransform
 
 logger = logging.getLogger(__name__)
@@ -173,7 +173,10 @@ class HttpClient(Client):
         final_query = super()._prep_query(context)
         if context.is_insert:
             return final_query
-        return f'{final_query}\n FORMAT {self._read_format}'
+        fmt = f'\n FORMAT {self._read_format}'
+        if isinstance(final_query, bytes):
+            return final_query + fmt.encode()
+        return final_query + fmt
 
     def _query_with_context(self, context: QueryContext) -> QueryResult:
         headers = {}
@@ -349,20 +352,21 @@ class HttpClient(Client):
         return QuerySummary(self._summary(response))
 
     def _error_handler(self, response: HTTPResponse, retried: bool = False) -> None:
-        err_str = f'HTTPDriver for {self.url} returned response code {response.status})'
-        try:
-            err_content = get_response_data(response)
-        except Exception: # pylint: disable=broad-except
-            err_content = None
-        finally:
-            response.close()
-
-        if err_content:
-            if self.show_clickhouse_errors:
+        if self.show_clickhouse_errors:
+            try:
+                err_content = get_response_data(response)
+            except Exception:  # pylint: disable=broad-except
+                err_content = None
+            finally:
+                response.close()
+
+            err_str = f'HTTPDriver for {self.url} returned response code {response.status})'
+            if err_content:
                 err_msg = common.format_error(err_content.decode(errors='backslashreplace'))
-                err_str = f':{err_str}\n {err_msg}'
-            else:
-                err_str = 'The ClickHouse server returned an error.'
+                err_str = f'{err_str}\n {err_msg}'
+        else:
+            err_str = 'The ClickHouse server returned an error.'
+
         raise OperationalError(err_str) if retried else DatabaseError(err_str) from None
 
     def _raw_request(self,
@@ -426,7 +430,8 @@ class HttpClient(Client):
                         logger.debug('Retrying remotely closed connection')
                         continue
                 logger.warning('Unexpected Http Driver Exception')
-                raise OperationalError(f'Error {ex} executing HTTP request attempt {attempts} {self.url}') from ex
+                err_url = f' ({self.url})' if self.show_clickhouse_errors else ''
+                raise OperationalError(f'Error {ex} executing HTTP request attempt {attempts}{err_url}') from ex
             finally:
                 if query_session:
                     self._active_session = None  # Make sure we always clear this
@@ -471,14 +476,16 @@ class HttpClient(Client):
                         fmt: str,
                         use_database: bool,
                         external_data: Optional[ExternalData]):
-        final_query, bind_params = bind_query(query, parameters, self.server_tz)
         if fmt:
-            final_query += f'\n FORMAT {fmt}'
+            query += f'\n FORMAT {fmt}'
+        final_query, bind_params = bind_query(query, parameters, self.server_tz)
         params = self._validate_settings(settings or {})
         if use_database and self.database:
             params['database'] = self.database
         params.update(bind_params)
         if external_data:
+            if isinstance(final_query, bytes):
+                raise ProgrammingError('Cannot combine binary query data with `External Data`')
             body = bytes()
             params['query'] = final_query
             params.update(external_data.query_params)

contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py

@@ -40,7 +40,7 @@ class QueryContext(BaseQueryContext):
 
     # pylint: disable=duplicate-code,too-many-arguments,too-many-locals
     def __init__(self,
-                 query: str = '',
+                 query: Union[str, bytes] = '',
                  parameters: Optional[Dict[str, Any]] = None,
                  settings: Optional[Dict[str, Any]] = None,
                  query_formats: Optional[Dict[str, str]] = None,
@@ -177,7 +177,7 @@ class QueryContext(BaseQueryContext):
         return active_tz
 
     def updated_copy(self,
-                     query: Optional[str] = None,
+                     query: Optional[Union[str, bytes]] = None,
                      parameters: Optional[Dict[str, Any]] = None,
                      settings: Optional[Dict[str, Any]] = None,
                      query_formats: Optional[Dict[str, str]] = None,
@@ -218,7 +218,11 @@ class QueryContext(BaseQueryContext):
 
     def _update_query(self):
         self.final_query, self.bind_params = bind_query(self.query, self.parameters, self.server_tz)
-        self.uncommented_query = remove_sql_comments(self.final_query)
+        if isinstance(self.final_query, bytes):
+            # If we've embedded binary data in the query, all bets are off, and we check the original query for comments
+            self.uncommented_query = remove_sql_comments(self.query)
+        else:
+            self.uncommented_query = remove_sql_comments(self.final_query)
 
 
 class QueryResult(Closable):
@@ -368,9 +372,36 @@ def bind_query(query: str, parameters: Optional[Union[Sequence, Dict[str, Any]]]
         query = query[:-1]
     if not parameters:
         return query, {}
+
+    binary_binds = None
+    if isinstance(parameters, dict):
+        binary_binds = {k: v for k, v in parameters.items() if k.startswith('$') and k.endswith('$') and len(k) > 1}
+        for key in binary_binds.keys():
+            del parameters[key]
     if external_bind_re.search(query) is None:
-        return finalize_query(query, parameters, server_tz), {}
-    return query, {f'param_{k}': format_bind_value(v, server_tz) for k, v in parameters.items()}
+        query, bound_params = finalize_query(query, parameters, server_tz), {}
+    else:
+        bound_params = {f'param_{k}': format_bind_value(v, server_tz) for k, v in parameters.items()}
+    if binary_binds:
+        binary_query = query.encode()
+        binary_indexes = {}
+        for k, v in binary_binds.items():
+            key = k.encode()
+            item_index = 0
+            while True:
+                item_index = binary_query.find(key, item_index)
+                if item_index == -1:
+                    break
+                binary_indexes[item_index + len(key)] = key, v
+                item_index += len(key)
+        query = b''
+        start = 0
+        for loc in sorted(binary_indexes.keys()):
+            key, value = binary_indexes[loc]
+            query += binary_query[start:loc] + value + key
+            start = loc
+        query += binary_query[start:]
+    return query, bound_params
 
 
 def format_str(value: str):

contrib/python/clickhouse-connect/ya.make

@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(0.7.12)
+VERSION(0.7.14)
 
 LICENSE(Apache-2.0)