Generate vectors in query scripting pipeline

The following sample script generates vector queries using OpenAI:

Referenced assemblies:

  • Newtonsoft.Json.dll

  • System.Net.Http.dll

Imported namespaces

  • System.Net.Http;

  • Newtonsoft.Json;

  • Newtonsoft.Json.Linq;

Query Scripting stage sample code:

Copy
string apiKey = ""; // your OpenAI API Key here
string apiUrl = "https://api.openai.com/v1/embeddings";
string model = "text-embedding-ada-002";
// this line is optional, it is just an example that you can decide when you want to append vector search and not always applying it
if(!Query.IsComplexQuery){ 
     double[] embeddings;
    //preparing the request body for the OpenAI embeddings generation request 
    var request = new Dictionary<string, object>
        {
            { "model", model },
            { "input", Query.QueryText }
        };
    //setting up the Http call using the API key, model, endpoint and input text as part of request body
    var httpClient = new HttpClient();
    httpClient.DefaultRequestHeaders.Add("Authorization", "Bearer " + apiKey);
    var requestJson = JsonConvert.SerializeObject(request);
    var requestContent = new StringContent(requestJson, Encoding.UTF8, "application/json");
    var httpResponseMessage = httpClient.PostAsync(apiUrl, requestContent).ConfigureAwait(false).GetAwaiter().GetResult();
    //reading the response from OpenAI API as JSON format string 
    var jsonString = httpResponseMessage.Content.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult();
    try
    {
         //collecting the actual embeddings and transforming the into an array of doubles
        JObject jsonObject = JObject.Parse(jsonString);
        JArray embeddingArray = jsonObject["data"][0]["embedding"] as JArray;
        embeddings = embeddingArray.Select(e => (double)e).ToArray();
    }
    catch (Exception ex)
    {
        Logger.Error("Error parsing Embedding data: " + ex.Message);
        embeddings = new double[0];
    }
    // instantiate a new VectorQuery object
    var vectorQuery = new VectorQuery();
    // specify the field(s) for which the vector query should be used
    vectorQuery.Fields = new List<string> { "vector_field" };
    // set the actual vector for the vector query
    vectorQuery.Embeddings = embeddings;
    //setting the embeddings as a vector query component 
    Query.VectorQueries = new List<VectorQuery> {vectorQuery};
}