Files
Rafał Tyński 25ce901021 feat(backend): add semantic matchmaking with embedding pipeline
Add a pluggable embedding provider with an offline deterministic default, vector indexing for problems and innovations, cosine semantic ranking, similar cases and solution suggestions, area filtering and an admin reindex endpoint. Refresh the index when innovations or submissions change.
2026-10-04 02:44:03 +02:00

104 lines
2.5 KiB
C#

using System.Text;
namespace TarninaHub.Application.Matchmaking;
/// <summary>
/// Deterministic, offline embedding provider (feature hashing). Produces
/// L2-normalized vectors so cosine similarity reflects shared vocabulary.
/// Interchangeable with a real LLM provider behind <see cref="IEmbeddingProvider"/>
/// (see ADR-0005). Used as the default until an external provider is wired.
/// </summary>
public sealed class HashingEmbeddingProvider : IEmbeddingProvider
{
public int Dimension => EmbeddingConstants.Dimension;
public Task<float[]> EmbedAsync(string text, CancellationToken cancellationToken)
{
return Task.FromResult(Embed(text));
}
public Task<IReadOnlyList<float[]>> EmbedBatchAsync(
IReadOnlyList<string> texts,
CancellationToken cancellationToken)
{
IReadOnlyList<float[]> result = texts.Select(Embed).ToList();
return Task.FromResult(result);
}
public float[] Embed(string text)
{
var vector = new float[Dimension];
if (string.IsNullOrWhiteSpace(text))
{
return vector;
}
foreach (var token in Tokenize(text))
{
var index = (int)(Fnv1a(token) % Dimension);
vector[index] += 1f;
}
Normalize(vector);
return vector;
}
private static IEnumerable<string> Tokenize(string text)
{
var buffer = new StringBuilder();
foreach (var character in text)
{
if (char.IsLetterOrDigit(character))
{
buffer.Append(char.ToLowerInvariant(character));
}
else if (buffer.Length > 0)
{
yield return buffer.ToString();
buffer.Clear();
}
}
if (buffer.Length > 0)
{
yield return buffer.ToString();
}
}
private static void Normalize(float[] vector)
{
double sumOfSquares = vector.Sum(value => (double)value * value);
var magnitude = Math.Sqrt(sumOfSquares);
if (magnitude <= 0)
{
return;
}
for (var i = 0; i < vector.Length; i++)
{
vector[i] = (float)(vector[i] / magnitude);
}
}
private static uint Fnv1a(string token)
{
const uint offset = 2166136261;
const uint prime = 16777619;
var hash = offset;
foreach (var character in token)
{
hash ^= character;
hash *= prime;
}
return hash;
}
}