Allow embeddings requests to be tokens or strings (#417)

* Allow raw tokens to be used as embedding input

* fix linting issues (lines too long)

* add endpoint test for embedding from tokens

* remove redundant comments

* fix comment to match new param name

* change interface to any

* Rename methods and implement convert for base req

* add comments to CreateEmbeddings

* update tests

* shorten line length

* rename parameter

embeddings_test.go

@@ -32,6 +32,7 @@ func TestEmbedding(t *testing.T) {
 		BabbageCodeSearchText,
 	}
 	for _, model := range embeddedModels {
+		// test embedding request with strings (simple embedding request)
 		embeddingReq := EmbeddingRequest{
 			Input: []string{
 				"The food was delicious and the waiter",
@@ -46,6 +47,34 @@ func TestEmbedding(t *testing.T) {
 		if !bytes.Contains(marshaled, []byte(`"model":"`+model.String()+`"`)) {
 			t.Fatalf("Expected embedding request to contain model field")
 		}
+
+		// test embedding request with strings
+		embeddingReqStrings := EmbeddingRequestStrings{
+			Input: []string{
+				"The food was delicious and the waiter",
+				"Other examples of embedding request",
+			},
+			Model: model,
+		}
+		marshaled, err = json.Marshal(embeddingReqStrings)
+		checks.NoError(t, err, "Could not marshal embedding request")
+		if !bytes.Contains(marshaled, []byte(`"model":"`+model.String()+`"`)) {
+			t.Fatalf("Expected embedding request to contain model field")
+		}
+
+		// test embedding request with tokens
+		embeddingReqTokens := EmbeddingRequestTokens{
+			Input: [][]int{
+				{464, 2057, 373, 12625, 290, 262, 46612},
+				{6395, 6096, 286, 11525, 12083, 2581},
+			},
+			Model: model,
+		}
+		marshaled, err = json.Marshal(embeddingReqTokens)
+		checks.NoError(t, err, "Could not marshal embedding request")
+		if !bytes.Contains(marshaled, []byte(`"model":"`+model.String()+`"`)) {
+			t.Fatalf("Expected embedding request to contain model field")
+		}
 	}
 }
 
@@ -75,6 +104,15 @@ func TestEmbeddingEndpoint(t *testing.T) {
 			fmt.Fprintln(w, string(resBytes))
 		},
 	)
+	// test create embeddings with strings (simple embedding request)
 	_, err := client.CreateEmbeddings(context.Background(), EmbeddingRequest{})
 	checks.NoError(t, err, "CreateEmbeddings error")
+
+	// test create embeddings with strings
+	_, err = client.CreateEmbeddings(context.Background(), EmbeddingRequestStrings{})
+	checks.NoError(t, err, "CreateEmbeddings strings error")
+
+	// test create embeddings with tokens
+	_, err = client.CreateEmbeddings(context.Background(), EmbeddingRequestTokens{})
+	checks.NoError(t, err, "CreateEmbeddings tokens error")
 }