-
Notifications
You must be signed in to change notification settings - Fork 3.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test: add new tests based on new basic examples + vector store tests (#…
…2879) * 🔧 (.github/workflows/typescript_test.yml): add BRAVE_SEARCH_API_KEY secret to workflow environment variables ✨ (frontend/src/modals/IOModal/components/chatView/chatMessage/index.tsx): add data-testid attribute to div element for testing purposes ✨ (frontend/tests/end-to-end/Hierarchical Tasks Agent.spec.ts): add end-to-end test for Hierarchical Tasks Agent ✨ (frontend/tests/end-to-end/Sequential Tasks Agent.spec.ts): add end-to-end test for Sequential Tasks Agent * updating hierarchical task json * 🔧 (.github/workflows/typescript_test.yml): add secrets for ASTRA_DB_APPLICATION_TOKEN and ASTRA_DB_API_ENDPOINT ♻️ (playwright.config.ts): refactor timeout value to 3 minutes 📝 (Complex Agent.spec.ts): update test to use BRAVE_SEARCH_API_KEY instead of OPENAI_API_KEY 📝 (Complex Agent.spec.ts): update test to use "apple" instead of "agile" 📝 (Hierarchical Tasks Agent.spec.ts): update test to use "langflow" instead of "agile" 📝 (Sequential Tasks Agent.spec.ts): update test to use waitForTimeout instead of isVisible ✨ (Vector Store.spec.ts): Add checks to skip tests if required environment variables are not set 🔧 (Vector Store.spec.ts): Update test logic to fill in required environment variables before running tests to ensure proper test execution
- Loading branch information
1 parent
9ecf101
commit ae096a3
Showing
8 changed files
with
776 additions
and
388 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
667 changes: 356 additions & 311 deletions
667
src/backend/base/langflow/initial_setup/starter_projects/Hierarchical Agent.json
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
import { expect, test } from "@playwright/test"; | ||
import * as dotenv from "dotenv"; | ||
import path from "path"; | ||
|
||
test("Complex Agent", async ({ page }) => { | ||
test.skip( | ||
!process?.env?.OPENAI_API_KEY, | ||
"OPENAI_API_KEY required to run this test", | ||
); | ||
|
||
test.skip( | ||
!process?.env?.BRAVE_SEARCH_API_KEY, | ||
"BRAVE_SEARCH_API_KEY required to run this test", | ||
); | ||
|
||
if (!process.env.CI) { | ||
dotenv.config({ path: path.resolve(__dirname, "../../.env") }); | ||
} | ||
|
||
await page.goto("/"); | ||
await page.waitForTimeout(2000); | ||
|
||
let modalCount = 0; | ||
try { | ||
const modalTitleElement = await page?.getByTestId("modal-title"); | ||
if (modalTitleElement) { | ||
modalCount = await modalTitleElement.count(); | ||
} | ||
} catch (error) { | ||
modalCount = 0; | ||
} | ||
|
||
while (modalCount === 0) { | ||
await page.getByText("New Project", { exact: true }).click(); | ||
await page.waitForTimeout(5000); | ||
modalCount = await page.getByTestId("modal-title")?.count(); | ||
} | ||
|
||
await page.getByRole("heading", { name: "Complex Agent" }).click(); | ||
|
||
await page.waitForSelector('[title="fit view"]', { | ||
timeout: 100000, | ||
}); | ||
|
||
await page.getByTitle("fit view").click(); | ||
await page.getByTitle("zoom out").click(); | ||
await page.getByTitle("zoom out").click(); | ||
await page.getByTitle("zoom out").click(); | ||
|
||
let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); | ||
|
||
while (outdatedComponents > 0) { | ||
await page.getByTestId("icon-AlertTriangle").first().click(); | ||
await page.waitForTimeout(1000); | ||
outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); | ||
} | ||
|
||
await page | ||
.getByTestId("popover-anchor-input-api_key") | ||
.last() | ||
.fill(process.env.BRAVE_SEARCH_API_KEY ?? ""); | ||
|
||
await page.waitForTimeout(2000); | ||
|
||
let openAiLlms = await page.getByText("OpenAI", { exact: true }).count(); | ||
|
||
for (let i = 0; i < openAiLlms; i++) { | ||
await page | ||
.getByTestId("popover-anchor-input-api_key") | ||
.nth(i) | ||
.fill(process.env.OPENAI_API_KEY ?? ""); | ||
|
||
await page.getByTestId("dropdown-model_name").nth(i).click(); | ||
await page.getByTestId("gpt-4o-1-option").last().click(); | ||
|
||
await page.waitForTimeout(1000); | ||
} | ||
|
||
await page.getByTestId("button_run_chat output").click(); | ||
await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 }); | ||
|
||
await page.getByText("built successfully").last().click({ | ||
timeout: 15000, | ||
}); | ||
|
||
await page.getByText("Playground", { exact: true }).click(); | ||
|
||
await page.waitForTimeout(2000); | ||
|
||
expect( | ||
page.getByText("Could you search info about AAPL?", { exact: true }).last(), | ||
).toBeVisible(); | ||
|
||
const textContents = await page | ||
.getByTestId("div-chat-message") | ||
.allTextContents(); | ||
|
||
const concatAllText = textContents.join(" "); | ||
expect(concatAllText.toLocaleLowerCase()).toContain("apple"); | ||
const allTextLength = concatAllText.length; | ||
expect(allTextLength).toBeGreaterThan(500); | ||
}); |
106 changes: 106 additions & 0 deletions
106
src/frontend/tests/end-to-end/Hierarchical Tasks Agent.spec.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
import { expect, test } from "@playwright/test"; | ||
import * as dotenv from "dotenv"; | ||
import path from "path"; | ||
|
||
test("Hierarchical Tasks Agent", async ({ page }) => { | ||
test.skip( | ||
!process?.env?.OPENAI_API_KEY, | ||
"OPENAI_API_KEY required to run this test", | ||
); | ||
|
||
test.skip( | ||
!process?.env?.BRAVE_SEARCH_API_KEY, | ||
"BRAVE_SEARCH_API_KEY required to run this test", | ||
); | ||
|
||
if (!process.env.CI) { | ||
dotenv.config({ path: path.resolve(__dirname, "../../.env") }); | ||
} | ||
|
||
await page.goto("/"); | ||
await page.waitForTimeout(2000); | ||
|
||
let modalCount = 0; | ||
try { | ||
const modalTitleElement = await page?.getByTestId("modal-title"); | ||
if (modalTitleElement) { | ||
modalCount = await modalTitleElement.count(); | ||
} | ||
} catch (error) { | ||
modalCount = 0; | ||
} | ||
|
||
while (modalCount === 0) { | ||
await page.getByText("New Project", { exact: true }).click(); | ||
await page.waitForTimeout(5000); | ||
modalCount = await page.getByTestId("modal-title")?.count(); | ||
} | ||
|
||
await page.getByRole("heading", { name: "Hierarchical Tasks Agent" }).click(); | ||
|
||
await page.waitForSelector('[title="fit view"]', { | ||
timeout: 100000, | ||
}); | ||
|
||
await page.getByTitle("fit view").click(); | ||
await page.getByTitle("zoom out").click(); | ||
await page.getByTitle("zoom out").click(); | ||
await page.getByTitle("zoom out").click(); | ||
|
||
let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); | ||
|
||
while (outdatedComponents > 0) { | ||
await page.getByTestId("icon-AlertTriangle").first().click(); | ||
await page.waitForTimeout(1000); | ||
outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); | ||
} | ||
|
||
await page | ||
.getByTestId("popover-anchor-input-api_key") | ||
.first() | ||
.fill(process.env.OPENAI_API_KEY ?? ""); | ||
|
||
await page | ||
.getByTestId("popover-anchor-input-api_key") | ||
.nth(1) | ||
.fill(process.env.OPENAI_API_KEY ?? ""); | ||
|
||
await page.getByTestId("dropdown-model_name").first().click(); | ||
await page.getByTestId("gpt-4o-1-option").first().click(); | ||
|
||
await page.waitForTimeout(2000); | ||
|
||
await page.getByTestId("dropdown-model_name").last().click(); | ||
await page.getByTestId("gpt-4o-1-option").last().click(); | ||
|
||
await page.waitForTimeout(2000); | ||
|
||
await page | ||
.getByTestId("popover-anchor-input-api_key") | ||
.last() | ||
.fill(process.env.BRAVE_SEARCH_API_KEY ?? ""); | ||
|
||
await page.waitForTimeout(2000); | ||
|
||
await page.getByTestId("button_run_chat output").click(); | ||
await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 }); | ||
|
||
await page.getByText("built successfully").last().click({ | ||
timeout: 15000, | ||
}); | ||
|
||
await page.getByText("Playground", { exact: true }).click(); | ||
|
||
await page.waitForTimeout(2000); | ||
|
||
expect(page.getByText("What is Langflow?", { exact: true })).toBeVisible(); | ||
|
||
const textContents = await page | ||
.getByTestId("div-chat-message") | ||
.allTextContents(); | ||
|
||
const concatAllText = textContents.join(" "); | ||
expect(concatAllText.toLocaleLowerCase()).toContain("langflow"); | ||
const allTextLength = concatAllText.length; | ||
expect(allTextLength).toBeGreaterThan(500); | ||
}); |
106 changes: 106 additions & 0 deletions
106
src/frontend/tests/end-to-end/Sequential Tasks Agent.spec.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
import { expect, test } from "@playwright/test"; | ||
import * as dotenv from "dotenv"; | ||
import path from "path"; | ||
|
||
test("Sequential Tasks Agent", async ({ page }) => { | ||
test.skip( | ||
!process?.env?.OPENAI_API_KEY, | ||
"OPENAI_API_KEY required to run this test", | ||
); | ||
|
||
test.skip( | ||
!process?.env?.BRAVE_SEARCH_API_KEY, | ||
"BRAVE_SEARCH_API_KEY required to run this test", | ||
); | ||
|
||
if (!process.env.CI) { | ||
dotenv.config({ path: path.resolve(__dirname, "../../.env") }); | ||
} | ||
|
||
await page.goto("/"); | ||
await page.waitForTimeout(2000); | ||
|
||
let modalCount = 0; | ||
try { | ||
const modalTitleElement = await page?.getByTestId("modal-title"); | ||
if (modalTitleElement) { | ||
modalCount = await modalTitleElement.count(); | ||
} | ||
} catch (error) { | ||
modalCount = 0; | ||
} | ||
|
||
while (modalCount === 0) { | ||
await page.getByText("New Project", { exact: true }).click(); | ||
await page.waitForTimeout(5000); | ||
modalCount = await page.getByTestId("modal-title")?.count(); | ||
} | ||
|
||
await page.getByRole("heading", { name: "Sequential Tasks Agent" }).click(); | ||
|
||
await page.waitForSelector('[title="fit view"]', { | ||
timeout: 100000, | ||
}); | ||
|
||
await page.getByTitle("fit view").click(); | ||
await page.getByTitle("zoom out").click(); | ||
await page.getByTitle("zoom out").click(); | ||
await page.getByTitle("zoom out").click(); | ||
|
||
let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); | ||
|
||
while (outdatedComponents > 0) { | ||
await page.getByTestId("icon-AlertTriangle").first().click(); | ||
await page.waitForTimeout(1000); | ||
outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); | ||
} | ||
|
||
await page | ||
.getByTestId("popover-anchor-input-api_key") | ||
.first() | ||
.fill(process.env.OPENAI_API_KEY ?? ""); | ||
|
||
await page.getByTestId("dropdown-model_name").click(); | ||
await page.getByTestId("gpt-4o-1-option").click(); | ||
|
||
await page.waitForTimeout(2000); | ||
|
||
await page | ||
.getByTestId("popover-anchor-input-api_key") | ||
.last() | ||
.fill(process.env.BRAVE_SEARCH_API_KEY ?? ""); | ||
|
||
await page.waitForTimeout(2000); | ||
|
||
await page.getByTestId("button_run_chat output").click(); | ||
await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 }); | ||
|
||
await page.getByText("built successfully").last().click({ | ||
timeout: 15000, | ||
}); | ||
|
||
await page.getByText("Playground", { exact: true }).click(); | ||
|
||
await page.waitForTimeout(2000); | ||
|
||
expect( | ||
page | ||
.getByPlaceholder("No chat input variables found. Click to run your flow") | ||
.last(), | ||
).toBeVisible(); | ||
|
||
await page.getByText("Topic", { exact: true }).nth(1).isVisible(); | ||
await page.getByText("Topic", { exact: true }).nth(1).click(); | ||
expect(await page.getByPlaceholder("Enter text...").inputValue()).toBe( | ||
"Agile", | ||
); | ||
|
||
const textContents = await page | ||
.getByTestId("div-chat-message") | ||
.allTextContents(); | ||
|
||
const concatAllText = textContents.join(" "); | ||
expect(concatAllText.toLocaleLowerCase()).toContain("agile"); | ||
const allTextLength = concatAllText.length; | ||
expect(allTextLength).toBeGreaterThan(500); | ||
}); |
Oops, something went wrong.