Support vector index

Author: annbaek

README.md

@@ -21,8 +21,9 @@ AliSQL is Alibaba's MySQL branch, forked from official MySQL and used extensivel
 
 - **[DuckDB Storage Engine](./wiki/duckdb/duckdb.md)**:AliSQL integrates DuckDB as a native storage engine, allowing users to operate DuckDB with the same experience as MySQL. By leveraging AliSQL for rapid deployment of DuckDB service nodes, users can easily achieve lightweight analytical capabilities.
 
+- **[Vector Storage](https://www.alibabacloud.com/help/en/rds/apsaradb-rds-for-mysql/vector-storage-1?spm=a2c63.p38356.help-menu-26090.d_3_3_0.6bb8d111D06xOW)**:AliSQL natively supports enterprise-grade vector processing for up to 16,383 dimensions. By integrating a highly optimized HNSW algorithm for high-performance Approximate Nearest Neighbor (ANN) search, AliSQL empowers users to build AI-driven applications—such as semantic search and recommendation systems—seamlessly using standard SQL interfaces.
+
 ## Roadmap
-- **[Vector Storage](https://www.alibabacloud.com/help/en/rds/apsaradb-rds-for-mysql/vector-storage-1?spm=a2c63.p38356.help-menu-26090.d_3_3_0.6bb8d111D06xOW)** *(planned)*:AliSQL natively supports enterprise-grade vector processing for up to 16,383 dimensions. By integrating a highly optimized HNSW algorithm for high-performance Approximate Nearest Neighbor (ANN) search, AliSQL empowers users to build AI-driven applications—such as semantic search and recommendation systems—seamlessly using standard SQL interfaces.
 
 - **[DDL Optimization](https://www.alibabacloud.com/help/en/rds/apsaradb-rds-for-mysql/alisql-ddl-best-practices?spm=a2c63.p38356.help-menu-26090.d_2_8_0.1f7a28a5F1ZVeK)** *(planned)*:AliSQL delivers a faster, safer, and lighter DDL experience through innovations such as enhanced Instant DDL, parallel B+tree construction, a non-blocking lock mechanism, and real-time DDL apply—significantly improving schema change efficiency and virtually eliminating replication lag.
 
@@ -88,6 +89,7 @@ AliSQL is based on MySQL, which is licensed under GPL-2.0. The DuckDB integratio
 ## See Also
 - [AliSQL Release Notes](./wiki/changes-in-alisql-8.0.44.md)
 - [DuckDB Storage Engine in AliSQL](./wiki/duckdb/duckdb.md)
+- [Vector Index in AliSQL](./wiki/vidx/vidx_readme.md)
 - [MySQL 8.0 Documentation](https://dev.mysql.com/doc/refman/8.0/en/)
 - [MySQL 8.0 Github Repository](https://github.com/mysql/mysql-server)
 - [DuckDB Official Documentation](https://duckdb.org/docs/stable/)

config.h.cmake

@@ -44,6 +44,7 @@
 #cmakedefine HAVE_EXECINFO_H 1
 #cmakedefine HAVE_FPU_CONTROL_H 1
 #cmakedefine HAVE_GRP_H 1
+#cmakedefine HAVE_IMMINTRIN_H 1
 #cmakedefine HAVE_LANGINFO_H 1
 #cmakedefine HAVE_MALLOC_H 1
 #cmakedefine HAVE_NETINET_IN_H 1

configure.cmake

@@ -195,6 +195,7 @@ CHECK_INCLUDE_FILES (endian.h HAVE_ENDIAN_H)
 CHECK_INCLUDE_FILES (execinfo.h HAVE_EXECINFO_H)
 CHECK_INCLUDE_FILES (fpu_control.h HAVE_FPU_CONTROL_H)
 CHECK_INCLUDE_FILES (grp.h HAVE_GRP_H)
+CHECK_INCLUDE_FILES (immintrin.h HAVE_IMMINTRIN_H) # vidx
 CHECK_INCLUDE_FILES (langinfo.h HAVE_LANGINFO_H)
 CHECK_INCLUDE_FILES (malloc.h HAVE_MALLOC_H)
 CHECK_INCLUDE_FILES (netinet/in.h HAVE_NETINET_IN_H)

include/m_ctype.h

@@ -782,4 +782,14 @@ static inline bool is_supported_parser_charset(const CHARSET_INFO *cs) {
   return (cs->mbminlen == 1);
 }
 
+static inline void my_ci_hash_sort(CHARSET_INFO *ci, const uchar *key,
+                                   size_t len, ulong *nr1, ulong *nr2) {
+  (ci->coll->hash_sort)(ci, key, len, nr1, nr2);
+}
+
+extern "C" {
+void my_hash_sort_bin(const CHARSET_INFO *cs [[maybe_unused]], const uchar *key,
+                      size_t len, uint64 *nr1, uint64 *nr2);
+}  // extern "C"
+
 #endif  // M_CTYPE_INCLUDED

include/my_base.h

@@ -520,7 +520,8 @@ enum ha_base_keytype {
 /* The combination of the above can be used for key type comparison. */
 #define HA_KEYFLAG_MASK                                                       \
   (HA_NOSAME | HA_PACK_KEY | HA_AUTO_KEY | HA_BINARY_PACK_KEY | HA_FULLTEXT | \
-   HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY)
+   HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY |      \
+   HA_VECTOR)
 
 /** Fulltext index uses [pre]parser */
 #define HA_USES_PARSER (1 << 14)
@@ -555,6 +556,7 @@ enum ha_base_keytype {
 
 constexpr const ulong HA_INDEX_USES_ENGINE_ATTRIBUTE{1UL << 20};
 constexpr const ulong HA_INDEX_USES_SECONDARY_ENGINE_ATTRIBUTE{1UL << 21};
+constexpr const ulong HA_VECTOR{1UL << 22};
 
 /* These flags can be added to key-seg-flag */
 

include/my_bit.h

@@ -51,6 +51,45 @@ static inline uint my_bit_log2(ulong value) {
   return bit;
 }
 
+/*
+  my_bit_log2_xxx()
+
+  In the given value, find the highest bit set,
+  which is the smallest X that satisfies the condition: (2^X >= value).
+  Can be used as a reverse operation for (1<<X), to find X.
+
+  Examples:
+  - returns 0 for (1<<0)
+  - returns 1 for (1<<1)
+  - returns 2 for (1<<2)
+  - returns 1 for 3, which has (1<<1) as the highest bit set.
+
+  Note, the behaviour of log2(0) is not defined.
+  Let's return 0 for the input 0, for the code simplicity.
+  See the 000x branch. It covers both (1<<0) and 0.
+*/
+static inline constexpr uint my_bit_log2_hex_digit(uint8 value) {
+  return value & 0x0C ? /*1100*/ (value & 0x08 ? /*1000*/ 3 : /*0100*/ 2) :
+                      /*0010*/ (value & 0x02 ? /*0010*/ 1 : /*000x*/ 0);
+}
+static inline constexpr uint my_bit_log2_uint8(uint8 value) {
+  return value & 0xF0 ? my_bit_log2_hex_digit((uint8)(value >> 4)) + 4
+                      : my_bit_log2_hex_digit(value);
+}
+static inline constexpr uint my_bit_log2_uint16(uint16 value) {
+  return value & 0xFF00 ? my_bit_log2_uint8((uint8)(value >> 8)) + 8
+                        : my_bit_log2_uint8((uint8)value);
+}
+static inline constexpr uint my_bit_log2_uint32(uint32 value) {
+  return value & 0xFFFF0000UL ? my_bit_log2_uint16((uint16)(value >> 16)) + 16
+                              : my_bit_log2_uint16((uint16)value);
+}
+static inline constexpr uint my_bit_log2_uint64(ulonglong value) {
+  return value & 0xFFFFFFFF00000000ULL
+             ? my_bit_log2_uint32((uint32)(value >> 32)) + 32
+             : my_bit_log2_uint32((uint32)value);
+}
+
 static inline uint my_count_bits(ulonglong v) {
 #if SIZEOF_LONG_LONG > 4
   /* The following code is a bit faster on 16 bit machines than if we would

include/my_byteorder.h

@@ -322,6 +322,11 @@ inline uchar *store32be(uchar *ptr, uint32 val) {
   return pointer_cast<uchar *>(store32be(pointer_cast<char *>(ptr), val));
 }
 
+/* convenience helpers */
+static inline float get_float(const void *from) {
+  return float4get(((const uchar *)from));
+}
+
 #endif /* __cplusplus */
 
 #endif /* MY_BYTEORDER_INCLUDED */

include/my_sys.h

@@ -789,11 +789,13 @@ extern bool my_init_dynamic_array(DYNAMIC_ARRAY *array, PSI_memory_key key,
 
 #define dynamic_element(array, array_index, type) \
   ((type)((array)->buffer) + (array_index))
+#define reset_dynamic(array) ((array)->elements = 0)
 
 /* Some functions are still in use in C++, because HASH uses DYNAMIC_ARRAY */
 extern bool insert_dynamic(DYNAMIC_ARRAY *array, const void *element);
 extern void *alloc_dynamic(DYNAMIC_ARRAY *array);
 extern void delete_dynamic(DYNAMIC_ARRAY *array);
+extern void *pop_dynamic(DYNAMIC_ARRAY *);
 
 extern bool init_dynamic_string(DYNAMIC_STRING *str, const char *init_str,
                                 size_t init_alloc);

include/vidx/SIMD.h

@@ -0,0 +1,55 @@
+#ifndef SIMD_INCLUDED
+#define SIMD_INCLUDED
+
+/*
+MIT License
+
+Copyright (c) 2023 Sasha Krassovsky
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// https://save-buffer.github.io/bloom_filter.html
+
+/*
+  Use gcc function multiversioning to optimize for a specific CPU with run-time
+  detection. Works only for x86, for other architectures we provide only one
+  implementation for now.
+*/
+#define DEFAULT_IMPLEMENTATION
+#if __GNUC__ > 7
+#ifdef __x86_64__
+#ifdef HAVE_IMMINTRIN_H
+#include <immintrin.h>
+#undef DEFAULT_IMPLEMENTATION
+#define DEFAULT_IMPLEMENTATION __attribute__((target("default")))
+#define AVX2_IMPLEMENTATION __attribute__((target("avx2,avx,fma")))
+#if __GNUC__ > 9
+#define AVX512_IMPLEMENTATION __attribute__((target("avx512f,avx512bw")))
+#endif
+#endif
+#endif
+#ifdef __aarch64__
+#include <arm_neon.h>
+#undef DEFAULT_IMPLEMENTATION
+#define NEON_IMPLEMENTATION
+#endif
+#endif
+
+#endif /* SIMD_INCLUDED */