diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 00000000..f06235c4 --- /dev/null +++ b/.eslintignore @@ -0,0 +1,2 @@ +node_modules +dist diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 13531280..00a3c52d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -27,7 +27,8 @@ If applicable, add screenshots to help explain your problem. If applicable, add exported flow in order to help replicating the problem. **Setup** -- Installation [e.g. docker, `npx flowise start`, `yarn start`] + +- Installation [e.g. docker, `npx flowise start`, `pnpm start`] - Flowise Version [e.g. 1.2.11] - OS: [e.g. macOS, Windows, Linux] - Browser [e.g. chrome, safari] diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 759f195f..200202b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,10 +28,10 @@ jobs: with: node-version: ${{ matrix.node-version }} - - run: npm i -g yarn + - run: npm i -g pnpm - - run: yarn install --ignore-engines + - run: pnpm install - - run: yarn lint + - run: pnpm lint - - run: yarn build + - run: pnpm build diff --git a/.husky/pre-commit b/.husky/pre-commit index 7aba0476..8e47731f 100644 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,5 +1,5 @@ #!/bin/sh . "$(dirname "$0")/_/husky.sh" -yarn quick # prettify -yarn lint-staged # eslint lint(also include prettify but prettify support more file extensions than eslint, so run prettify first) \ No newline at end of file +pnpm quick # prettify +pnpm lint-staged # eslint lint(also include prettify but prettify support more file extensions than eslint, so run prettify first) diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000..96784e59 --- /dev/null +++ b/.npmrc @@ -0,0 +1,6 @@ +auto-install-peers = true +strict-peer-dependencies = false +prefer-workspace-packages = true +link-workspace-packages = deep +hoist = true +shamefully-hoist = true diff --git a/CONTRIBUTING-ZH.md b/CONTRIBUTING-ZH.md index 1211391d..13c36ca9 100644 --- a/CONTRIBUTING-ZH.md +++ b/CONTRIBUTING-ZH.md @@ -44,9 +44,9 @@ Flowise 在一个单一的单体存储库中有 3 个不同的模块。 #### 先决条件 -- 安装 [Yarn v1](https://classic.yarnpkg.com/en/docs/install) +- 安装 [PNPM](https://pnpm.io/installation) ```bash - npm i -g yarn + npm i -g pnpm ``` #### 逐步指南 @@ -71,45 +71,45 @@ Flowise 在一个单一的单体存储库中有 3 个不同的模块。 6. 安装所有模块的依赖项: ```bash - yarn install + pnpm install ``` 7. 构建所有代码: ```bash - yarn build + pnpm build ``` 8. 在[http://localhost:3000](http://localhost:3000)上启动应用程序 ```bash - yarn start + pnpm start ``` 9. 开发时: - - 在`packages/ui`中创建`.env`文件并指定`PORT`(参考`.env.example`) + - 在`packages/ui`中创建`.env`文件并指定`VITE_PORT`(参考`.env.example`) - 在`packages/server`中创建`.env`文件并指定`PORT`(参考`.env.example`) - 运行 ```bash - yarn dev + pnpm dev ``` 对`packages/ui`或`packages/server`进行的任何更改都将反映在[http://localhost:8080](http://localhost:8080)上 - 对于`packages/components`中进行的更改,再次运行`yarn build`以应用更改。 + 对于`packages/components`中进行的更改,再次运行`pnpm build`以应用更改。 10. 做完所有的更改后,运行以下命令来确保在生产环境中一切正常: ```bash - yarn build + pnpm build ``` 和 ```bash - yarn start + pnpm start ``` 11. 提交代码并从指向 [Flowise 主分支](https://github.com/FlowiseAI/Flowise/tree/master) 的分叉分支上提交 Pull Request。 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c09206e..ccf1379a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,9 +44,9 @@ Flowise has 3 different modules in a single mono repository. #### Prerequisite -- Install [Yarn v1](https://classic.yarnpkg.com/en/docs/install) +- Install [PNPM](https://pnpm.io/installation) ```bash - npm i -g yarn + npm i -g pnpm ``` #### Step by step @@ -71,45 +71,45 @@ Flowise has 3 different modules in a single mono repository. 6. Install all dependencies of all modules: ```bash - yarn install + pnpm install ``` 7. Build all the code: ```bash - yarn build + pnpm build ``` 8. Start the app on [http://localhost:3000](http://localhost:3000) ```bash - yarn start + pnpm start ``` 9. For development: - - Create `.env` file and specify the `PORT` (refer to `.env.example`) in `packages/ui` + - Create `.env` file and specify the `VITE_PORT` (refer to `.env.example`) in `packages/ui` - Create `.env` file and specify the `PORT` (refer to `.env.example`) in `packages/server` - Run ```bash - yarn dev + pnpm dev ``` Any changes made in `packages/ui` or `packages/server` will be reflected on [http://localhost:8080](http://localhost:8080) - For changes made in `packages/components`, run `yarn build` again to pickup the changes. + For changes made in `packages/components`, run `pnpm build` again to pickup the changes. 10. After making all the changes, run ```bash - yarn build + pnpm build ``` and ```bash - yarn start + pnpm start ``` to make sure everything works fine in production. diff --git a/Dockerfile b/Dockerfile index e485cd3e..60f2400b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,30 +12,21 @@ RUN apk add --no-cache build-base cairo-dev pango-dev # Install Chromium RUN apk add --no-cache chromium +#install PNPM globaly +RUN npm install -g pnpm + ENV PUPPETEER_SKIP_DOWNLOAD=true ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser -WORKDIR /usr/src/packages - -# Copy root package.json and lockfile -COPY package.json yarn.loc[k] ./ - -# Copy components package.json -COPY packages/components/package.json ./packages/components/package.json - -# Copy ui package.json -COPY packages/ui/package.json ./packages/ui/package.json - -# Copy server package.json -COPY packages/server/package.json ./packages/server/package.json - -RUN yarn install +WORKDIR /usr/src # Copy app source COPY . . -RUN yarn build +RUN pnpm install + +RUN pnpm build EXPOSE 3000 -CMD [ "yarn", "start" ] +CMD [ "pnpm", "start" ] diff --git a/README-ZH.md b/README-ZH.md index b6f389ed..d2fe6223 100644 --- a/README-ZH.md +++ b/README-ZH.md @@ -75,9 +75,9 @@ Flowise 在一个单一的代码库中有 3 个不同的模块。 ### 先决条件 -- 安装 [Yarn v1](https://classic.yarnpkg.com/en/docs/install) +- 安装 [PNPM](https://pnpm.io/installation) ```bash - npm i -g yarn + npm i -g pnpm ``` ### 设置 @@ -97,31 +97,31 @@ Flowise 在一个单一的代码库中有 3 个不同的模块。 3. 安装所有模块的依赖: ```bash - yarn install + pnpm install ``` 4. 构建所有代码: ```bash - yarn build + pnpm build ``` 5. 启动应用: ```bash - yarn start + pnpm start ``` 现在可以在 [http://localhost:3000](http://localhost:3000) 访问应用 6. 用于开发构建: - - 在 `packages/ui` 中创建 `.env` 文件并指定 `PORT`(参考 `.env.example`) + - 在 `packages/ui` 中创建 `.env` 文件并指定 `VITE_PORT`(参考 `.env.example`) - 在 `packages/server` 中创建 `.env` 文件并指定 `PORT`(参考 `.env.example`) - 运行 ```bash - yarn dev + pnpm dev ``` 任何代码更改都会自动重新加载应用程序,访问 [http://localhost:8080](http://localhost:8080) diff --git a/README.md b/README.md index 438647e8..6ada9a2f 100644 --- a/README.md +++ b/README.md @@ -75,9 +75,9 @@ Flowise has 3 different modules in a single mono repository. ### Prerequisite -- Install [Yarn v1](https://classic.yarnpkg.com/en/docs/install) +- Install [Yarn v1](https://pnpm.io/installation) ```bash - npm i -g yarn + npm i -g pnpm ``` ### Setup @@ -97,31 +97,31 @@ Flowise has 3 different modules in a single mono repository. 3. Install all dependencies of all modules: ```bash - yarn install + pnpm install ``` 4. Build all the code: ```bash - yarn build + pnpm build ``` 5. Start the app: ```bash - yarn start + pnpm start ``` You can now access the app on [http://localhost:3000](http://localhost:3000) 6. For development build: - - Create `.env` file and specify the `PORT` (refer to `.env.example`) in `packages/ui` + - Create `.env` file and specify the `VITE_PORT` (refer to `.env.example`) in `packages/ui` - Create `.env` file and specify the `PORT` (refer to `.env.example`) in `packages/server` - Run ```bash - yarn dev + pnpm dev ``` Any code changes will reload the app automatically on [http://localhost:8080](http://localhost:8080) diff --git a/package.json b/package.json index cdacf38e..2aa30b2a 100644 --- a/package.json +++ b/package.json @@ -11,19 +11,19 @@ ], "scripts": { "build": "turbo run build", - "build-force": "turbo run build --force", + "build-force": "pnpm clean && turbo run build --force", "dev": "turbo run dev --parallel", "start": "run-script-os", "start:windows": "cd packages/server/bin && run start", "start:default": "cd packages/server/bin && ./run start", - "clean": "npm exec -ws -- rimraf dist build", + "clean": "pnpm --filter \"./packages/**\" clean", + "nuke": "pnpm --filter \"./packages/**\" nuke && rimraf node_modules .turbo", "format": "prettier --write \"**/*.{ts,tsx,md}\"", - "test": "turbo run test", "lint": "eslint \"**/*.{js,jsx,ts,tsx,json,md}\"", - "lint-fix": "yarn lint --fix", + "lint-fix": "pnpm lint --fix", "quick": "pretty-quick --staged", "postinstall": "husky install", - "migration:create": "yarn typeorm migration:create" + "migration:create": "pnpm typeorm migration:create" }, "lint-staged": { "*.{js,jsx,ts,tsx,json,md}": "eslint --fix" @@ -43,16 +43,23 @@ "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-unused-imports": "^2.0.0", "husky": "^8.0.1", + "kill-port": "^2.0.1", "lint-staged": "^13.0.3", "prettier": "^2.7.1", "pretty-quick": "^3.1.3", "rimraf": "^3.0.2", "run-script-os": "^1.1.6", - "turbo": "^1.7.4", + "turbo": "1.10.16", "typescript": "^4.8.4" }, + "packageManager": "pnpm@8.14.0", + "pnpm": { + "onlyBuiltDependencies": [ + "sqlite3" + ] + }, "engines": { - "node": ">=18.15.0" + "node": ">=18.15.0 <19.0.0 || ^20" }, "resolutions": { "@qdrant/openapi-typescript-fetch": "1.2.1" diff --git a/packages/components/gulpfile.ts b/packages/components/gulpfile.ts index c4d2d353..cacfb076 100644 --- a/packages/components/gulpfile.ts +++ b/packages/components/gulpfile.ts @@ -1,6 +1,4 @@ -import gulp from 'gulp' - -const { src, dest } = gulp +const { src, dest } = require('gulp') function copyIcons() { return src(['nodes/**/*.{jpg,png,svg}']).pipe(dest('dist/nodes')) diff --git a/packages/components/nodes/chatmodels/ChatHuggingFace/core.ts b/packages/components/nodes/chatmodels/ChatHuggingFace/core.ts index 0d74bbe7..7d26c145 100644 --- a/packages/components/nodes/chatmodels/ChatHuggingFace/core.ts +++ b/packages/components/nodes/chatmodels/ChatHuggingFace/core.ts @@ -107,7 +107,7 @@ export class HuggingFaceInference extends LLM implements HFInput { const { HfInference } = await import('@huggingface/inference') return { HfInference } } catch (e) { - throw new Error('Please install huggingface as a dependency with, e.g. `yarn add @huggingface/inference`') + throw new Error('Please install huggingface as a dependency with, e.g. `pnpm install @huggingface/inference`') } } } diff --git a/packages/components/nodes/llms/Cohere/core.ts b/packages/components/nodes/llms/Cohere/core.ts index 30c5e240..bdf66ab1 100644 --- a/packages/components/nodes/llms/Cohere/core.ts +++ b/packages/components/nodes/llms/Cohere/core.ts @@ -72,7 +72,7 @@ export class Cohere extends LLM implements CohereInput { const { default: cohere } = await import('cohere-ai') return { cohere } } catch (e) { - throw new Error('Please install cohere-ai as a dependency with, e.g. `yarn add cohere-ai`') + throw new Error('Please install cohere-ai as a dependency with, e.g. `pnpm install cohere-ai`') } } } diff --git a/packages/components/package.json b/packages/components/package.json index 40a0f16e..417b1e4a 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -6,7 +6,11 @@ "types": "dist/src/index.d.ts", "scripts": { "build": "tsc && gulp", - "dev": "tsc --watch" + "dev:gulp": "gulp", + "dev": "tsc-watch --noClear -p ./tsconfig.json --onSuccess \"pnpm dev:gulp\"", + "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "clean": "rimraf dist", + "nuke": "rimraf dist node_modules .turbo" }, "keywords": [], "homepage": "https://flowiseai.com", @@ -52,6 +56,8 @@ "cheerio": "^1.0.0-rc.12", "chromadb": "^1.5.11", "cohere-ai": "^6.2.0", + "crypto-js": "^4.1.1", + "css-what": "^6.1.0", "d3-dsv": "2", "dotenv": "^16.0.0", "express": "^4.17.3", @@ -61,8 +67,8 @@ "google-auth-library": "^9.4.0", "graphql": "^16.6.0", "html-to-text": "^9.0.5", - "husky": "^8.0.3", "ioredis": "^5.3.2", + "jsdom": "^22.1.0", "jsonpointer": "^5.0.1", "langchain": "^0.1.20", "langfuse": "3.3.1", @@ -70,6 +76,7 @@ "langsmith": "0.1.6", "linkifyjs": "^4.1.1", "llamaindex": "^0.0.48", + "lodash": "^4.17.21", "lunary": "^0.6.16", "mammoth": "^1.5.1", "moment": "^2.29.3", @@ -88,22 +95,33 @@ "pyodide": ">=0.21.0-alpha.2", "redis": "^4.6.7", "replicate": "^0.12.3", + "socket.io": "^4.6.1", "srt-parser-2": "^1.2.3", + "typeorm": "^0.3.6", "vm2": "^3.9.19", "weaviate-ts-client": "^1.1.0", + "winston": "^3.9.0", "ws": "^8.9.0", + "zod": "^3.22.4", "zod-to-json-schema": "^3.21.4" }, "devDependencies": { + "@swc/core": "^1.3.99", + "@types/crypto-js": "^4.1.1", "@types/gulp": "4.0.9", + "@types/lodash": "^4.14.202", "@types/node-fetch": "2.6.2", "@types/object-hash": "^3.0.2", "@types/pg": "^8.10.2", "@types/ws": "^8.5.3", + "babel-register": "^6.26.0", "eslint-plugin-markdown": "^3.0.1", "eslint-plugin-react": "^7.33.2", "eslint-plugin-react-hooks": "^4.6.0", "gulp": "^4.0.2", + "rimraf": "^5.0.5", + "tsc-watch": "^6.0.4", + "tslib": "^2.6.2", "typescript": "^4.8.4" } } diff --git a/packages/components/tsconfig.json b/packages/components/tsconfig.json index d213dabc..bda815f9 100644 --- a/packages/components/tsconfig.json +++ b/packages/components/tsconfig.json @@ -16,5 +16,6 @@ "declaration": true, "module": "commonjs" }, - "include": ["src", "nodes", "credentials"] + "include": ["src", "nodes", "credentials"], + "exclude": ["gulpfile.ts", "node_modules", "dist"] } diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json index bf25eb00..d4688a54 100644 --- a/packages/server/nodemon.json +++ b/packages/server/nodemon.json @@ -1,6 +1,6 @@ { "ignore": ["**/*.spec.ts", ".git", "node_modules"], "watch": ["commands", "index.ts", "src", "../components/nodes", "../components/src"], - "exec": "yarn oclif-dev", + "exec": "pnpm start", "ext": "ts" } diff --git a/packages/server/package.json b/packages/server/package.json index 63a80cdc..d57c09a5 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -22,14 +22,16 @@ "scripts": { "build": "tsc", "start": "run-script-os", + "clean": "rimraf dist", + "nuke": "rimraf dist node_modules .turbo", "start:windows": "cd bin && run start", "start:default": "cd bin && ./run start", - "dev": "concurrently \"yarn watch\" \"nodemon\"", + "dev": "tsc-watch --noClear -p ./tsconfig.json --onSuccess \"pnpm start\"", "oclif-dev": "run-script-os", "oclif-dev:windows": "cd bin && dev start", "oclif-dev:default": "cd bin && ./dev start", "postpack": "shx rm -f oclif.manifest.json", - "prepack": "yarn build && oclif manifest && oclif readme", + "prepack": "pnpm build && oclif manifest && oclif readme", "typeorm": "typeorm-ts-node-commonjs", "watch": "tsc --watch", "version": "oclif readme && git add README.md" @@ -41,11 +43,13 @@ "email": "henryheng@flowiseai.com" }, "engines": { - "node": ">=18.15.0" + "node": ">=18.15.0 <19.0.0 || ^20" }, "license": "SEE LICENSE IN LICENSE.md", "dependencies": { "@oclif/core": "^1.13.10", + "@types/lodash": "^4.14.202", + "@types/uuid": "^9.0.7", "async-mutex": "^0.4.0", "axios": "1.6.2", "content-disposition": "0.5.4", @@ -55,11 +59,14 @@ "express": "^4.17.3", "express-basic-auth": "^1.2.1", "express-rate-limit": "^6.9.0", - "flowise-components": "*", - "flowise-ui": "*", + "flowise-components": "workspace:^", + "flowise-ui": "workspace:^", + "lodash": "^4.17.21", + "moment": "^2.29.3", "moment-timezone": "^0.5.34", "multer": "^1.4.5-lts.1", "mysql": "^2.18.1", + "openai": "^4.20.0", "pg": "^8.11.1", "posthog-node": "^3.5.0", "reflect-metadata": "^0.1.13", @@ -77,11 +84,13 @@ "@types/multer": "^1.4.7", "@types/sanitize-html": "^2.9.5", "concurrently": "^7.1.0", - "nodemon": "^2.0.15", + "nodemon": "^2.0.22", "oclif": "^3", + "rimraf": "^5.0.5", "run-script-os": "^1.1.6", "shx": "^0.3.3", "ts-node": "^10.7.0", + "tsc-watch": "^6.0.4", "typescript": "^4.8.4" } } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index ea7fe532..c1e68d09 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -11,7 +11,7 @@ import logger from './utils/logger' import { expressRequestLogger } from './utils/logger' import { v4 as uuidv4 } from 'uuid' import OpenAI from 'openai' -import { FindOptionsWhere, MoreThanOrEqual, LessThanOrEqual } from 'typeorm' +import { DataSource, FindOptionsWhere, MoreThanOrEqual, LessThanOrEqual } from 'typeorm' import { IChatFlow, IncomingInput, @@ -89,7 +89,7 @@ export class App { chatflowPool: ChatflowPool cachePool: CachePool telemetry: Telemetry - AppDataSource = getDataSource() + AppDataSource: DataSource = getDataSource() constructor() { this.app = express() diff --git a/packages/ui/.env.example b/packages/ui/.env.example index 25241b73..db18b4cd 100644 --- a/packages/ui/.env.example +++ b/packages/ui/.env.example @@ -1 +1 @@ -PORT=8080 +VITE_PORT=8080 diff --git a/packages/ui/index.html b/packages/ui/index.html new file mode 100644 index 00000000..64875b7b --- /dev/null +++ b/packages/ui/index.html @@ -0,0 +1,46 @@ + + +
+