Squashing against "dev/dataframe-interchange-api"

Author: Nikita Kulikov

onedal/common/dtype_dispatcher.cpp

@@ -13,10 +13,11 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *******************************************************************************/
-
+#include <iostream>
 #include <pybind11/pybind11.h>
 
 #include "oneapi/dal/common.hpp"
+#include "oneapi/dal/table/common.hpp"
 
 #include "onedal/common.hpp"
 #include "onedal/common/dtype_dispatcher.hpp"
@@ -25,8 +26,8 @@ namespace py = pybind11;
 
 namespace oneapi::dal::python {
 
-ONEDAL_PY_INIT_MODULE(dtype_dispatcher) {
-    py::enum_<dal::data_type> py_dtype(m, "dtype");
+inline void instantiate_data_type(py::module& pm) {
+    py::enum_<dal::data_type> py_dtype(pm, "dtype");
     py_dtype.value("int8", dal::data_type::int8);
     py_dtype.value("int16", dal::data_type::int16);
     py_dtype.value("int32", dal::data_type::int32);
@@ -38,6 +39,20 @@ ONEDAL_PY_INIT_MODULE(dtype_dispatcher) {
     py_dtype.value("float32", dal::data_type::float32);
     py_dtype.value("float64", dal::data_type::float64);
     py_dtype.export_values();
+}
+
+inline void instantiate_feature_type(py::module& pm) {
+    py::enum_<dal::feature_type> py_ftype(pm, "ftype");
+    py_ftype.value("ratio", dal::feature_type::ratio);
+    py_ftype.value("nominal", dal::feature_type::nominal);
+    py_ftype.value("ordinal", dal::feature_type::ordinal);
+    py_ftype.value("interval", dal::feature_type::interval);
+    py_ftype.export_values();
+}
+
+ONEDAL_PY_INIT_MODULE(dtype_dispatcher) {
+    (void)instantiate_feature_type(m);
+    (void)instantiate_data_type(m);
 } // ONEDAL_PY_INIT_MODULE(dtype_dispatcher)
 
 } // namespace oneapi::dal::python

onedal/data_management/chunked_array/chunked_array.cpp

@@ -43,6 +43,7 @@ void instantiate_chunked_array_by_type(py::module& m) {
 
     py::class_<chunked_array_t> py_array(m, c_name);
     py_array.def(py::init<>());
+    py_array.def(py::init<chunked_array_t>());
     py_array.def(py::pickle(
         [](const chunked_array_t& m) -> py::bytes {
             return serialize(m);
@@ -86,19 +87,31 @@ void instantiate_chunked_array_by_type(py::module& m) {
     });
 }
 
+inline void instantiate_make_chunked_array(py::module& pm) {
+    constexpr const char name[] = "make_chunked_array";
+    pm.def(name, [](data_type dtype, std::int64_t chunk_count) -> py::object {
+        return detail::dispatch_by_data_type(dtype, [&](auto type_tag) -> py::object {
+            using type_t = std::decay_t<decltype(type_tag)>;
+            auto result = chunked_array<type_t>(chunk_count);
+            return py::cast(std::move(result));
+        });
+    });
+}
+
 template <typename... Types>
 inline void instantiate_chunked_array_impl(py::module& pm,
                                            const std::tuple<Types...>* const = nullptr) {
     auto instantiate = [&](auto type_tag) -> void {
         using type_t = std::decay_t<decltype(type_tag)>;
-        return instantiate_chunked_array_by_type<type_t>(pm);
+        instantiate_chunked_array_by_type<type_t>(pm);
     };
     return detail::apply(instantiate, Types{}...);
 }
 
 void instantiate_chunked_array(py::module& pm) {
     constexpr const supported_types_t* types = nullptr;
-    return instantiate_chunked_array_impl(pm, types);
+    (void)instantiate_chunked_array_impl(pm, types);
+    (void)instantiate_make_chunked_array(pm);
 }
 
 } // namespace oneapi::dal::python::data_management

onedal/data_management/heterogen_table/heterogen_table.cpp

@@ -30,6 +30,9 @@
 #include "oneapi/dal/table/heterogen.hpp"
 #include "oneapi/dal/table/common.hpp"
 
+#include "oneapi/dal/table/detail/table_iface.hpp"
+#include "oneapi/dal/table/detail/table_utils.hpp"
+
 #include "oneapi/dal/detail/common.hpp"
 
 namespace py = pybind11;
@@ -53,6 +56,23 @@ inline void make_chunked_array_setter(py::class_<dal::heterogen_table>& table) {
     });
 }
 
+inline void instantiate_getter(py::class_<dal::heterogen_table>& table) {
+    constexpr const char name[] = "get_column";
+    table.def(name, [](const dal::heterogen_table& table, std::int64_t col) -> py::object {
+        const auto dtype = table.get_metadata().get_data_type(col);
+        return detail::dispatch_by_data_type(dtype, [&](auto type_tag) -> py::object {
+            using dtype_t = std::decay_t<decltype(type_tag)>;
+            using column_t = dal::chunked_array<dtype_t>;
+            const detail::pimpl_accessor acc{};
+
+            auto iface = detail::get_heterogen_table_iface(table);
+            auto raw_column = iface->get_column(col);
+            auto column = acc.make<column_t>(raw_column);
+            return py::cast(std::move(column));
+        });
+    });
+}
+
 template <typename Table, typename... Types>
 void instantiate_setters(py::class_<Table>& table, const std::tuple<Types...>* const = nullptr) {
     return detail::apply(
@@ -75,8 +95,9 @@ void instantiate_heterogen_table(py::module& pm) {
 
     instantiate_table_iface(py_heterogen_table);
 
+    (void)instantiate_getter(py_heterogen_table);
     constexpr const supported_types_t* types = nullptr;
-    instantiate_setters(py_heterogen_table, types);
+    (void)instantiate_setters(py_heterogen_table, types);
 }
 
 } // namespace oneapi::dal::python::data_management

onedal/data_management/table_metadata/table_metadata.cpp

@@ -25,7 +25,7 @@ namespace py = pybind11;
 
 namespace oneapi::dal::python::data_management {
 
-void instantiate_table_metadata(py::module& pm) {
+inline void instantiate_table_metadata_impl(py::module& pm) {
     constexpr const char name[] = "table_metadata";
     using dtype_arr_t = dal::array<dal::data_type>;
     using ftype_arr_t = dal::array<dal::feature_type>;
@@ -34,6 +34,21 @@ void instantiate_table_metadata(py::module& pm) {
     py_metadata.def(py::init());
     py_metadata.def(py::init<dal::table_metadata>());
     py_metadata.def(py::init<dtype_arr_t, ftype_arr_t>());
+    // TODO: Remove this tight conversions
+    using dtype_t = typename py::enum_<dal::data_type>::Scalar;
+    using ftype_t = typename py::enum_<dal::feature_type>::Scalar;
+    py_metadata.def(py::init([](const dal::array<dtype_t>& dtypes,
+                                const dal::array<ftype_t>& ftypes) -> dal::table_metadata {
+        dal::array<dal::data_type> casted_dtypes =
+            dal::array<dal::data_type>(dtypes, //
+                                       reinterpret_cast<const dal::data_type*>(dtypes.get_data()),
+                                       dtypes.get_count());
+        dal::array<dal::feature_type> casted_ftypes = dal::array<dal::feature_type>(
+            ftypes, //
+            reinterpret_cast<const dal::feature_type*>(ftypes.get_data()),
+            ftypes.get_count());
+        return dal::table_metadata(casted_dtypes, casted_ftypes);
+    }));
     py_metadata.def(py::pickle(
         [](const table_metadata& m) -> py::bytes {
             return serialize(m);
@@ -48,4 +63,8 @@ void instantiate_table_metadata(py::module& pm) {
     py_metadata.def("get_feature_count", &table_metadata::get_feature_count);
 }
 
+void instantiate_table_metadata(py::module& pm) {
+    (void)instantiate_table_metadata_impl(pm);
+}
+
 } // namespace oneapi::dal::python::data_management

onedal/interop/dataframes/__init__.py

@@ -0,0 +1,19 @@
+# ==============================================================================
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from .table_interop import is_dataframe_entity, to_table
+
+__all__ = ["is_dataframe_entity", "to_table"]

onedal/interop/dataframes/column_builder.py

@@ -0,0 +1,71 @@
+# ==============================================================================
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import onedal
+
+from .dataframe_protocol import Column
+from .dtype_conversion import get_data_type
+from ..array import to_array, is_array_entity
+
+data_type = onedal._backend.dtype
+make_chunked_array = onedal._backend.data_management.make_chunked_array
+
+class ChunkedArrayBuilder:
+    def __init__(self, dtype: data_type):
+        self.__dtype = dtype
+        self.chunks = list()
+
+    def append(self, chunk):
+        assert is_array_entity(chunk)
+        self.chunks.append(chunk)
+        return self
+    
+    @property
+    def chunk_count(self) -> int:
+        return len(self.chunks)
+
+    @property
+    def dtype(self):
+        return self.__dtype
+    
+    def __validate(self, result):
+        count = result.get_chunk_count()
+        assert count == self.chunk_count
+        assert result.validate()
+
+    def build(self):
+        dtype, count = self.dtype, self.chunk_count
+        result = make_chunked_array(dtype, count)
+
+        for index in range(count):
+            chunk = self.chunks[index]
+            array = to_array(chunk)
+            result.set_chunk(index, array)
+
+        self.__validate(result)
+        return result
+
+def build_from_column(column: Column):
+    dtype = get_data_type(column.dtype)
+    builder = ChunkedArrayBuilder(dtype)
+
+    for chunk in column.get_chunks():
+        assert chunk.num_chunks() == 1
+        buffers = chunk.get_buffers()
+        raw_chunk, _ = buffers["data"] 
+        builder.append(raw_chunk)
+
+    return builder.build()

onedal/interop/dataframes/dataframe_builder.py

@@ -0,0 +1,99 @@
+# ==============================================================================
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import numpy as np
+
+import onedal
+from ..array import to_array
+
+from .dtype_conversion import get_data_type
+from .column_builder import build_from_column
+from .dataframe_protocol import Column, DataFrame
+
+feature_type = onedal._backend.ftype
+type_array = onedal._backend.data_management.array_q
+metadata = onedal._backend.data_management.table_metadata
+heterogen_table = onedal._backend.data_management.heterogen_table
+
+class DataFrameBuilder:
+    def __init__(self):
+        self.dtypes = []
+        self.columns = []
+
+    @property
+    def column_count(self) -> int:
+        result = len(self.columns)
+        control = len(self.dtypes)
+        assert result == control
+        return result
+
+    def append(self, column: Column):
+        self.dtypes.append(column.dtype)
+        self.columns.append(column)
+        return self
+
+    def build_dtype_array(self) -> type_array:
+        result = list()
+        for index in range(self.column_count):
+            dtype = self.dtypes[index]
+            dal_dtype = get_data_type(dtype)
+            result.append(dal_dtype)
+        result = np.asarray(result)
+        result = result.astype(np.int32)
+        return to_array(result)
+    
+    # TODO: implement logic supporting 
+    def build_ftype_array(self) -> type_array:
+        ratio = feature_type.ratio
+        col_count = self.column_count
+        result = np.full(col_count, ratio)
+        result = result.astype(np.int32)
+        return to_array(result)
+    
+    def build_metadata(self) -> metadata:
+        dtypes = self.build_dtype_array()
+        ftypes = self.build_ftype_array()
+        return metadata(dtypes, ftypes)
+    
+    def __validate(self, result: heterogen_table):
+        column_count = result.get_column_count()
+        assert column_count == self.column_count
+    
+    def build(self) -> heterogen_table:
+        meta = self.build_metadata()
+        result = heterogen_table(meta)
+
+
+        for index in range(self.column_count):
+            column = self.columns[index]
+            array = build_from_column(column)
+            result.set_column(index, array)
+
+        self.__validate(result)
+        return result
+    
+def build_from_dataframe(df: DataFrame) -> heterogen_table:
+    column_count = df.num_columns()
+    builder = DataFrameBuilder()
+
+    for index in range(column_count):
+        column = df.get_column(index)
+        builder.append(column)
+
+    result = builder.build()
+    row_count = result.get_row_count()
+    assert row_count == df.num_rows()
+    return result