const data = await req.json();
const { url, prompt } = data;
const model = new ChatOpenAI({ modelName: "gpt-3.5-turbo" });
const loader = new CheerioWebBaseLoader(url);
const docs = await loader.load();
const splitter = RecursiveCharacterTextSplitter.fromLanguage("html");
const transformer = new HtmlToTextTransformer({
removeLineBreaks: true,
removeMultipleSpaces: true,
});
const sequence = splitter.pipe(transformer);
const newDocuments = await sequence.invoke(docs);
const client = createClient(
process.env.SUPABASE_URL,
process.env.SUPABASE_API_KEY
);
const vectorStore = await SupabaseVectorStore.fromDocuments(
newDocuments,
new OpenAIEmbeddings(),
{
client,
tableName: "documents",
queryName: "match_documents",
}
);
I create a document from a URL and save it to the vector database, but it is saving it to the database even if the same URL is sent each time. How can I control this?