Skip to content

Commit e9641a9

Browse files
rojiroji
authored
.Net: Fix column nullability in SQL MEVD providers (#13622)
Closes #12560
1 parent b712ffc commit e9641a9

File tree

12 files changed

+390
-22
lines changed

12 files changed

+390
-22
lines changed

‎dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,27 +102,25 @@ static bool TryGetBaseType(Type type, [NotNullWhen(true)] out string? typeName)
102102

103103
var propertyType = property.Type;
104104

105-
// TODO: Handle NRTs properly via NullabilityInfoContext
106-
107105
(string PgType, bool IsNullable) result;
108106

109107
if (TryGetBaseType(propertyType, out string? pgType))
110108
{
111-
result = (pgType, !propertyType.IsValueType);
109+
result = (pgType, property.IsNullable);
112110
}
113111
// Handle nullable types (e.g. Nullable<int>)
114112
else if (Nullable.GetUnderlyingType(propertyType) is Type unwrappedType
115113
&& TryGetBaseType(unwrappedType, out string? underlyingPgType))
116114
{
117-
result = (underlyingPgType, true);
115+
result = (underlyingPgType, property.IsNullable);
118116
}
119117
// Handle collections
120118
else if ((propertyType.IsArray && TryGetBaseType(propertyType.GetElementType()!, out string? elementPgType))
121119
|| (propertyType.IsGenericType
122120
&& propertyType.GetGenericTypeDefinition() == typeof(List<>)
123121
&& TryGetBaseType(propertyType.GetGenericArguments()[0], out elementPgType)))
124122
{
125-
result = (elementPgType + "[]", true);
123+
result = (elementPgType + "[]", property.IsNullable);
126124
}
127125
else
128126
{
@@ -169,7 +167,7 @@ public static (string PgType, bool IsNullable) GetPgVectorTypeName(VectorPropert
169167
_ => throw new NotSupportedException($"Type {vectorProperty.EmbeddingType.Name} is not supported by this store.")
170168
};
171169

172-
return ($"{pgType}({vectorProperty.Dimensions})", unwrappedEmbeddingType != vectorProperty.EmbeddingType);
170+
return ($"{pgType}({vectorProperty.Dimensions})", vectorProperty.IsNullable);
173171
}
174172

175173
public static NpgsqlParameter GetNpgsqlParameter(object? value)

‎dotnet/src/VectorData/SqlServer/SqlServerCommandBuilder.cs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,22 @@ internal static List<SqlCommand> CreateTable(
6262

6363
foreach (var property in model.DataProperties)
6464
{
65-
sb.AppendIdentifier(property.StorageName).Append(' ').Append(Map(property)).AppendLine(",");
65+
sb.AppendIdentifier(property.StorageName).Append(' ').Append(Map(property));
66+
if (!property.IsNullable)
67+
{
68+
sb.Append(" NOT NULL");
69+
}
70+
sb.AppendLine(",");
6671
}
6772

6873
foreach (var property in model.VectorProperties)
6974
{
70-
sb.AppendIdentifier(property.StorageName).Append(" VECTOR(").Append(property.Dimensions).AppendLine("),");
75+
sb.AppendIdentifier(property.StorageName).Append(" VECTOR(").Append(property.Dimensions).Append(')');
76+
if (!property.IsNullable)
77+
{
78+
sb.Append(" NOT NULL");
79+
}
80+
sb.AppendLine(",");
7181
}
7282

7383
sb.Append("PRIMARY KEY (").AppendIdentifier(model.KeyProperty.StorageName).AppendLine(")");

‎dotnet/src/VectorData/SqliteVec/SqliteColumn.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ internal sealed class SqliteColumn(
1818

1919
public bool IsPrimary { get; set; } = isPrimary;
2020

21+
public bool IsNullable { get; set; }
22+
2123
public bool HasIndex { get; set; }
2224

2325
public Dictionary<string, object>? Configuration { get; set; }

‎dotnet/src/VectorData/SqliteVec/SqliteCommandBuilder.cs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public static DbCommand BuildCreateTableCommand(SqliteConnection connection, str
5151
}
5252
builder.AppendIdentifier(tableName).AppendLine(" (");
5353

54-
builder.AppendLine(string.Join(",\n", columns.Select(column => GetColumnDefinition(column, quote: true))));
54+
builder.AppendLine(string.Join(",\n", columns.Select(column => GetColumnDefinition(column, quote: true, includeNullability: true))));
5555
builder.AppendLine(");");
5656

5757
foreach (var column in columns)
@@ -91,7 +91,7 @@ public static DbCommand BuildCreateVirtualTableCommand(
9191
builder.AppendIdentifier(tableName).AppendLine(" USING vec0(");
9292

9393
// The vector extension is currently uncapable of handling quoted identifiers.
94-
builder.AppendLine(string.Join(",\n", columns.Select(column => GetColumnDefinition(column, quote: false))));
94+
builder.AppendLine(string.Join(",\n", columns.Select(column => GetColumnDefinition(column, quote: false, includeNullability: false))));
9595
builder.Append(");");
9696

9797
var command = connection.CreateCommand();
@@ -479,7 +479,7 @@ private static StringBuilder AppendWhereClause(this StringBuilder builder, strin
479479
return builder;
480480
}
481481

482-
private static string GetColumnDefinition(SqliteColumn column, bool quote)
482+
private static string GetColumnDefinition(SqliteColumn column, bool quote, bool includeNullability)
483483
{
484484
const string PrimaryKeyIdentifier = "PRIMARY KEY";
485485

@@ -490,6 +490,11 @@ private static string GetColumnDefinition(SqliteColumn column, bool quote)
490490
columnDefinitionParts.Add(PrimaryKeyIdentifier);
491491
}
492492

493+
if (includeNullability && !column.IsPrimary && !column.IsNullable)
494+
{
495+
columnDefinitionParts.Add("NOT NULL");
496+
}
497+
493498
if (column.Configuration is { Count: > 0 })
494499
{
495500
columnDefinitionParts.AddRange(column.Configuration

‎dotnet/src/VectorData/SqliteVec/SqlitePropertyMapping.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ public static List<SqliteColumn> GetColumns(IReadOnlyList<PropertyModel> propert
7070

7171
var column = new SqliteColumn(property.StorageName, propertyType, isPrimary)
7272
{
73+
IsNullable = property.IsNullable,
7374
Configuration = configuration,
7475
HasIndex = property is DataPropertyModel { IsIndexed: true }
7576
};

‎dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/PropertyModel.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,36 @@ public string StorageName
6262
/// </remarks>
6363
public Dictionary<string, object?>? ProviderAnnotations { get; set; }
6464

65+
/// <summary>
66+
/// Gets whether the property type is nullable. For value types, this is <see langword="true"/> when the type is
67+
/// <see cref="Nullable{T}"/>. For reference types on .NET 6+, this uses NRT annotations via
68+
/// <c>NullabilityInfoContext</c> when a <see cref="PropertyInfo"/> is available
69+
/// (i.e., POCO mapping); otherwise, reference types are assumed nullable.
70+
/// </summary>
71+
public bool IsNullable
72+
{
73+
get
74+
{
75+
// Value types: nullable only if Nullable<T>
76+
if (this.Type.IsValueType)
77+
{
78+
return Nullable.GetUnderlyingType(this.Type) is not null;
79+
}
80+
81+
// Reference types: check NRT annotation via NullabilityInfoContext when available
82+
#if NET
83+
if (this.PropertyInfo is { } propertyInfo)
84+
{
85+
var nullabilityInfo = new NullabilityInfoContext().Create(propertyInfo);
86+
return nullabilityInfo.ReadState != NullabilityState.NotNull;
87+
}
88+
#endif
89+
90+
// Dynamic mapping or old framework: assume nullable for reference types
91+
return true;
92+
}
93+
}
94+
6595
/// <summary>
6696
/// Reads the property from the given <paramref name="record"/>, returning the value as an <see cref="object"/>.
6797
/// </summary>

‎dotnet/test/VectorData/PgVector.UnitTests/PostgresSqlBuilderTests.cs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -594,4 +594,76 @@ public void TestBuildDeleteBatchCommand_WithSchema()
594594
}
595595

596596
#endregion
597+
598+
#region NRT (Nullable Reference Type) detection
599+
600+
#if NET // NRT detection via NullabilityInfoContext is only available on .NET 6+
601+
[Fact]
602+
public void TestBuildCreateTableCommand_WithNrtAnnotations()
603+
{
604+
var model = new PostgresModelBuilder().Build(
605+
typeof(NrtTestRecord),
606+
typeof(long),
607+
definition: null,
608+
defaultEmbeddingGenerator: null);
609+
610+
var sql = PostgresSqlBuilder.BuildCreateTableSql(schema: null, "testcollection", model, pgVersion: new Version(18, 0));
611+
612+
// Non-nullable reference types should be NOT NULL
613+
Assert.Contains("\"NonNullableString\" TEXT NOT NULL", sql);
614+
Assert.Contains("\"NonNullableByteArray\" BYTEA NOT NULL", sql);
615+
616+
// Nullable reference types should not have NOT NULL
617+
Assert.Contains("\"NullableString\" TEXT", sql);
618+
Assert.DoesNotContain("\"NullableString\" TEXT NOT NULL", sql);
619+
Assert.Contains("\"NullableByteArray\" BYTEA", sql);
620+
Assert.DoesNotContain("\"NullableByteArray\" BYTEA NOT NULL", sql);
621+
622+
// Non-nullable value types should be NOT NULL (unchanged from before)
623+
Assert.Contains("\"NonNullableInt\" INTEGER NOT NULL", sql);
624+
Assert.Contains("\"NonNullableBool\" BOOLEAN NOT NULL", sql);
625+
626+
// Nullable value types should not have NOT NULL (unchanged from before)
627+
Assert.Contains("\"NullableInt\" INTEGER", sql);
628+
Assert.DoesNotContain("\"NullableInt\" INTEGER NOT NULL", sql);
629+
630+
this._output.WriteLine(sql);
631+
}
632+
633+
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor
634+
#pragma warning disable CA1812 // Class is used via reflection
635+
private sealed class NrtTestRecord
636+
{
637+
[VectorStoreKey]
638+
public long Id { get; set; }
639+
640+
[VectorStoreData]
641+
public string NonNullableString { get; set; }
642+
643+
[VectorStoreData]
644+
public string? NullableString { get; set; }
645+
646+
[VectorStoreData]
647+
public byte[] NonNullableByteArray { get; set; }
648+
649+
[VectorStoreData]
650+
public byte[]? NullableByteArray { get; set; }
651+
652+
[VectorStoreData]
653+
public int NonNullableInt { get; set; }
654+
655+
[VectorStoreData]
656+
public int? NullableInt { get; set; }
657+
658+
[VectorStoreData]
659+
public bool NonNullableBool { get; set; }
660+
661+
[VectorStoreVector(10)]
662+
public ReadOnlyMemory<float> Embedding { get; set; }
663+
}
664+
#pragma warning restore CA1812
665+
#pragma warning restore CS8618
666+
#endif
667+
668+
#endregion
597669
}

‎dotnet/test/VectorData/SqlServer.ConformanceTests/SqlServerCommandBuilderTests.cs‎

Lines changed: 73 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,10 @@ public void CreateTable(bool ifNotExists)
114114
new VectorStoreKeyProperty("id", typeof(long)),
115115
new VectorStoreDataProperty("simpleName", typeof(string)),
116116
new VectorStoreDataProperty("with space", typeof(int)) { IsIndexed = true },
117-
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10)
117+
new VectorStoreDataProperty("nullableInt", typeof(int?)),
118+
new VectorStoreDataProperty("flag", typeof(bool)),
119+
new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory<float>), 10),
120+
new VectorStoreVectorProperty("nullableEmbedding", typeof(ReadOnlyMemory<float>?), 10)
118121
]);
119122

120123
using SqlConnection connection = CreateConnection();
@@ -128,8 +131,11 @@ public void CreateTable(bool ifNotExists)
128131
CREATE TABLE [schema].[table] (
129132
[id] BIGINT IDENTITY,
130133
[simpleName] NVARCHAR(MAX),
131-
[with space] INT,
132-
[embedding] VECTOR(10),
134+
[with space] INT NOT NULL,
135+
[nullableInt] INT,
136+
[flag] BIT NOT NULL,
137+
[embedding] VECTOR(10) NOT NULL,
138+
[nullableEmbedding] VECTOR(10),
133139
PRIMARY KEY ([id])
134140
);
135141
CREATE INDEX index_table_withspace ON [schema].[table]([with space]);
@@ -168,7 +174,7 @@ public void CreateTable_WithDiskAnnIndex()
168174
CREATE TABLE [schema].[table] (
169175
[id] BIGINT IDENTITY,
170176
[name] NVARCHAR(MAX),
171-
[embedding] VECTOR(10),
177+
[embedding] VECTOR(10) NOT NULL,
172178
PRIMARY KEY ([id])
173179
);
174180
END;
@@ -204,7 +210,7 @@ public void CreateTable_WithDiskAnnIndex_EuclideanDistance()
204210
BEGIN
205211
CREATE TABLE [schema].[table] (
206212
[id] BIGINT IDENTITY,
207-
[embedding] VECTOR(10),
213+
[embedding] VECTOR(10) NOT NULL,
208214
PRIMARY KEY ([id])
209215
);
210216
END;
@@ -534,4 +540,66 @@ private static SqlConnection CreateConnection()
534540
private static CollectionModel BuildModel(List<VectorStoreProperty> properties)
535541
=> new SqlServerModelBuilder()
536542
.BuildDynamic(new() { Properties = properties }, defaultEmbeddingGenerator: null);
543+
544+
#if NET // NRT detection via NullabilityInfoContext is only available on .NET 6+
545+
[Fact]
546+
public void CreateTable_WithNrtAnnotations()
547+
{
548+
var model = new SqlServerModelBuilder().Build(
549+
typeof(NrtTestRecord),
550+
typeof(long),
551+
definition: null,
552+
defaultEmbeddingGenerator: null);
553+
554+
using SqlConnection connection = CreateConnection();
555+
556+
var commands = SqlServerCommandBuilder.CreateTable(connection, "schema", "table", ifNotExists: false, model);
557+
558+
var command = Assert.Single(commands);
559+
560+
Assert.Equal(
561+
"""
562+
BEGIN
563+
CREATE TABLE [schema].[table] (
564+
[Id] BIGINT IDENTITY,
565+
[NonNullableString] NVARCHAR(MAX) NOT NULL,
566+
[NullableString] NVARCHAR(MAX),
567+
[NonNullableInt] INT NOT NULL,
568+
[NullableInt] INT,
569+
[NonNullableBool] BIT NOT NULL,
570+
[Embedding] VECTOR(10) NOT NULL,
571+
PRIMARY KEY ([Id])
572+
);
573+
END;
574+
""", command.CommandText, ignoreLineEndingDifferences: true);
575+
}
576+
577+
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor
578+
#pragma warning disable CA1812 // Class is used via reflection
579+
private sealed class NrtTestRecord
580+
{
581+
[VectorStoreKey]
582+
public long Id { get; set; }
583+
584+
[VectorStoreData]
585+
public string NonNullableString { get; set; }
586+
587+
[VectorStoreData]
588+
public string? NullableString { get; set; }
589+
590+
[VectorStoreData]
591+
public int NonNullableInt { get; set; }
592+
593+
[VectorStoreData]
594+
public int? NullableInt { get; set; }
595+
596+
[VectorStoreData]
597+
public bool NonNullableBool { get; set; }
598+
599+
[VectorStoreVector(10)]
600+
public ReadOnlyMemory<float> Embedding { get; set; }
601+
}
602+
#pragma warning restore CA1812
603+
#pragma warning restore CS8618
604+
#endif
537605
}

‎dotnet/test/VectorData/SqliteVec.UnitTests/SqliteCommandBuilderTests.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,9 @@ public void ItBuildsCreateTableCommand(bool ifNotExists)
5050
var columns = new List<SqliteColumn>
5151
{
5252
new("Column1", "Type1", isPrimary: true),
53-
new("Column2", "Type2", isPrimary: false) { Configuration = new() { ["distance_metric"] = "l2" } },
53+
new("Column2", "Type2", isPrimary: false) { IsNullable = true, Configuration = new() { ["distance_metric"] = "l2" } },
54+
new("Column3", "Type3", isPrimary: false) { IsNullable = false },
55+
new("Column4", "Type4", isPrimary: false) { IsNullable = true },
5456
};
5557

5658
// Act
@@ -64,6 +66,9 @@ public void ItBuildsCreateTableCommand(bool ifNotExists)
6466

6567
Assert.Contains("\"Column1\" Type1 PRIMARY KEY", command.CommandText);
6668
Assert.Contains("\"Column2\" Type2 distance_metric=l2", command.CommandText);
69+
Assert.Contains("\"Column3\" Type3 NOT NULL", command.CommandText);
70+
Assert.Contains("\"Column4\" Type4", command.CommandText);
71+
Assert.DoesNotContain("\"Column4\" Type4 NOT NULL", command.CommandText);
6772
}
6873

6974
[Theory]
@@ -77,7 +82,7 @@ public void ItBuildsCreateVirtualTableCommand(bool ifNotExists)
7782
var columns = new List<SqliteColumn>
7883
{
7984
new("Column1", "Type1", isPrimary: true),
80-
new("Column2", "Type2", isPrimary: false) { Configuration = new() { ["distance_metric"] = "l2" } },
85+
new("Column2", "Type2", isPrimary: false) { IsNullable = true, Configuration = new() { ["distance_metric"] = "l2" } },
8186
};
8287

8388
// Act

0 commit comments

Comments
 (0)