diff --git a/.gitignore b/.gitignore index 64ccbfd..2347b61 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ .nuxt .output .temp +.gradle +.qodana +build *.local *.log *.vsix diff --git a/packages/gpt-runner-shared/build.config.ts b/packages/gpt-runner-shared/build.config.ts index 336600c..a98b6af 100644 --- a/packages/gpt-runner-shared/build.config.ts +++ b/packages/gpt-runner-shared/build.config.ts @@ -21,6 +21,7 @@ export default defineBuildConfig({ clean: true, declaration: true, externals: [ + '@kvs/node-localstorage', 'unconfig', 'express', 'debug', @@ -29,5 +30,14 @@ export default defineBuildConfig({ rollup: { emitCJS: true, inlineDependencies: true, + dts: { + compilerOptions: { + baseUrl: '.', + paths: { + // fix types error + '@kvs/storage': ['./node_modules/@kvs/storage/'], + }, + }, + }, }, }) diff --git a/packages/gpt-runner-shared/package.json b/packages/gpt-runner-shared/package.json index f0960fc..156c96d 100644 --- a/packages/gpt-runner-shared/package.json +++ b/packages/gpt-runner-shared/package.json @@ -56,6 +56,8 @@ "stub": "unbuild --stub" }, "peerDependencies": { + "@kvs/node-localstorage": "*", + "cachedir": "*", "debug": "*", "find-free-ports": "*", "ip": "*", @@ -63,6 +65,9 @@ "zod": "*" }, "dependencies": { + "@kvs/storage": "^2.1.3", + "@kvs/node-localstorage": "^2.1.3", + "cachedir": "^2.3.0", "debug": "^4.3.4", "find-free-ports": "^3.1.1", "ip": "^1.1.8", diff --git a/packages/gpt-runner-shared/src/common/types/enum.ts b/packages/gpt-runner-shared/src/common/types/enum.ts index 22951cb..9ec5908 100644 --- a/packages/gpt-runner-shared/src/common/types/enum.ts +++ b/packages/gpt-runner-shared/src/common/types/enum.ts @@ -21,3 +21,8 @@ export enum GptFileTreeItemType { File = 'file', Chat = 'chat', } + +export enum ServerStorageName { + FrontendState = 'frontend-state', + WebPreset = 'web-preset', +} diff --git a/packages/gpt-runner-shared/src/common/types/server.ts b/packages/gpt-runner-shared/src/common/types/server.ts index f0ffb86..d12030a 100644 --- a/packages/gpt-runner-shared/src/common/types/server.ts +++ b/packages/gpt-runner-shared/src/common/types/server.ts @@ -1,4 +1,5 @@ import type { GptFileInfo, GptFileInfoTree, SingleChatMessage, SingleFileConfig, UserConfig } from './config' +import type { ServerStorageName } from './enum' export interface BaseResponse { type: 'Success' | 'Fail' @@ -35,20 +36,22 @@ export interface GetUserConfigResData { userConfig: UserConfig } -export interface GetStateReqParams { +export interface GetStorageReqParams { + storageName: ServerStorageName key: string } -export type FrontendState = Record | null | undefined +export type ServerStorageValue = Record | null | undefined -export interface GetStateResData { - state: FrontendState +export interface GetStorageResData { + value: ServerStorageValue cacheDir: string } -export interface SaveStateReqParams { +export interface SaveStorageReqParams { + storageName: ServerStorageName key: string - state: FrontendState + value?: ServerStorageValue } -export type SaveStateResData = null +export type SaveStorageResData = null diff --git a/packages/gpt-runner-shared/src/common/zod/enum.zod.ts b/packages/gpt-runner-shared/src/common/zod/enum.zod.ts index 58bad23..9284239 100644 --- a/packages/gpt-runner-shared/src/common/zod/enum.zod.ts +++ b/packages/gpt-runner-shared/src/common/zod/enum.zod.ts @@ -1,5 +1,5 @@ import { z } from 'zod' -import { ChatMessageStatus, ChatRole, ClientEventName, GptFileTreeItemType } from '../types' +import { ChatMessageStatus, ChatRole, ClientEventName, GptFileTreeItemType, ServerStorageName } from '../types' export const ChatRoleSchema = z.nativeEnum(ChatRole) @@ -8,3 +8,5 @@ export const ChatMessageStatusSchema = z.nativeEnum(ChatMessageStatus) export const ClientEventNameSchema = z.nativeEnum(ClientEventName) export const GptFileTreeItemTypeSchema = z.nativeEnum(GptFileTreeItemType) + +export const ServerStorageNameSchema = z.nativeEnum(ServerStorageName) diff --git a/packages/gpt-runner-shared/src/common/zod/server.zod.ts b/packages/gpt-runner-shared/src/common/zod/server.zod.ts index 8131653..ab70936 100644 --- a/packages/gpt-runner-shared/src/common/zod/server.zod.ts +++ b/packages/gpt-runner-shared/src/common/zod/server.zod.ts @@ -1,6 +1,7 @@ import { z } from 'zod' -import type { ChatStreamReqParams, GetGptFilesReqParams, GetUserConfigReqParams } from '../types' +import type { ChatStreamReqParams, GetGptFilesReqParams, GetStorageReqParams, GetUserConfigReqParams, SaveStorageReqParams } from '../types' import { SingleChatMessageSchema, SingleFileConfigSchema } from './config.zod' +import { ServerStorageNameSchema } from './enum.zod' export const ChatStreamReqParamsSchema = z.object({ messages: z.array(SingleChatMessageSchema), @@ -18,11 +19,13 @@ export const GetUserConfigReqParamsSchema = z.object({ rootPath: z.string(), }) satisfies z.ZodType -export const GetStateReqParamsSchema = z.object({ +export const GetStorageReqParamsSchema = z.object({ + storageName: ServerStorageNameSchema, key: z.string(), -}) +}) satisfies z.ZodType -export const SaveStateReqParamsSchema = z.object({ +export const SaveStorageReqParamsSchema = z.object({ + storageName: ServerStorageNameSchema, key: z.string(), - state: z.record(z.any()).nullable().optional(), -}) + value: z.record(z.any()).nullable().optional(), +}) satisfies z.ZodType diff --git a/packages/gpt-runner-shared/src/node/helpers/get-cache-dir.ts b/packages/gpt-runner-shared/src/node/helpers/get-cache-dir.ts new file mode 100644 index 0000000..e84e712 --- /dev/null +++ b/packages/gpt-runner-shared/src/node/helpers/get-cache-dir.ts @@ -0,0 +1,19 @@ +import fs from 'node:fs' + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error +// @ts-ignore +import getCacheDir from 'cachedir' +import { PathUtils } from './path-utils' + +export async function getGlobalCacheDir(name: string) { + const cacheDir = getCacheDir(name) + await createCacheDir(cacheDir) + return cacheDir +} + +async function createCacheDir(cacheDir: string) { + if (await PathUtils.isAccessible(cacheDir, 'W')) + return + + await fs.promises.mkdir(cacheDir, { recursive: true }) +} diff --git a/packages/gpt-runner-shared/src/node/helpers/get-storage.ts b/packages/gpt-runner-shared/src/node/helpers/get-storage.ts new file mode 100644 index 0000000..7d48d39 --- /dev/null +++ b/packages/gpt-runner-shared/src/node/helpers/get-storage.ts @@ -0,0 +1,18 @@ +import { kvsLocalStorage } from '@kvs/node-localstorage' +import type { ServerStorageName } from '../../common/types' +import { getGlobalCacheDir } from './get-cache-dir' + +export async function getStorage(storageName: ServerStorageName) { + const cacheFolder = await getGlobalCacheDir('gpt-runner-server') + + const storage = await kvsLocalStorage | null>>({ + name: storageName, + storeFilePath: cacheFolder, + version: 1, + }) + + return { + cacheDir: cacheFolder, + storage, + } +} diff --git a/packages/gpt-runner-shared/src/node/helpers/index.ts b/packages/gpt-runner-shared/src/node/helpers/index.ts index 9a3a5f1..2a0cd32 100644 --- a/packages/gpt-runner-shared/src/node/helpers/index.ts +++ b/packages/gpt-runner-shared/src/node/helpers/index.ts @@ -1,4 +1,6 @@ export * from './file-utils' +export * from './get-cache-dir' +export * from './get-storage' +export * from './network' export * from './path-utils' export * from './request' -export * from './server' diff --git a/packages/gpt-runner-shared/src/node/helpers/server.ts b/packages/gpt-runner-shared/src/node/helpers/network.ts similarity index 100% rename from packages/gpt-runner-shared/src/node/helpers/server.ts rename to packages/gpt-runner-shared/src/node/helpers/network.ts diff --git a/packages/gpt-runner-web/client/src/networks/server-storage.ts b/packages/gpt-runner-web/client/src/networks/server-storage.ts new file mode 100644 index 0000000..c1219c5 --- /dev/null +++ b/packages/gpt-runner-web/client/src/networks/server-storage.ts @@ -0,0 +1,39 @@ +import type { BaseResponse, GetStorageReqParams, GetStorageResData, SaveStorageReqParams, SaveStorageResData } from '@nicepkg/gpt-runner-shared/common' +import { getGlobalConfig } from '../helpers/global-config' + +export interface GetServerStorageParams extends GetStorageReqParams { +} + +export type GetServerStorageRes = BaseResponse + +export async function getServerStorage(params: GetServerStorageParams): Promise { + const { storageName, key } = params + + const res = await fetch(`${getGlobalConfig().serverBaseUrl}/api/storage?storageName=${storageName}&key=${key}`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + }, + }) + const data = await res.json() + return data +} + +export interface SaveServerStorageParams extends SaveStorageReqParams { +} + +export type SaveServerStorageRes = BaseResponse + +export async function saveServerStorage(params: SaveServerStorageParams): Promise { + const { storageName, key, value } = params + + const res = await fetch(`${getGlobalConfig().serverBaseUrl}/api/storage`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ storageName, key, value }), + }) + const data = await res.json() + return data +} diff --git a/packages/gpt-runner-web/client/src/networks/state.ts b/packages/gpt-runner-web/client/src/networks/state.ts deleted file mode 100644 index 9d3f481..0000000 --- a/packages/gpt-runner-web/client/src/networks/state.ts +++ /dev/null @@ -1,39 +0,0 @@ -import type { BaseResponse, GetStateReqParams, GetStateResData, SaveStateReqParams, SaveStateResData } from '@nicepkg/gpt-runner-shared/common' -import { getGlobalConfig } from '../helpers/global-config' - -export interface FetchStateParams extends GetStateReqParams { -} - -export type FetchStateRes = BaseResponse - -export async function fetchState(params: FetchStateParams): Promise { - const { key } = params - - const res = await fetch(`${getGlobalConfig().serverBaseUrl}/api/state?key=${key}`, { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - }, - }) - const data = await res.json() - return data -} - -export interface SaveStateParams extends SaveStateReqParams { -} - -export type SaveStateRes = BaseResponse - -export async function saveState(params: SaveStateParams): Promise { - const { key, state } = params - - const res = await fetch(`${getGlobalConfig().serverBaseUrl}/api/state`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ key, state }), - }) - const data = await res.json() - return data -} diff --git a/packages/gpt-runner-web/client/src/store/zustand/storage.ts b/packages/gpt-runner-web/client/src/store/zustand/storage.ts index d1ffe50..fed3c57 100644 --- a/packages/gpt-runner-web/client/src/store/zustand/storage.ts +++ b/packages/gpt-runner-web/client/src/store/zustand/storage.ts @@ -1,27 +1,36 @@ import type { StateStorage } from 'zustand/middleware' -import type { FrontendState } from '@nicepkg/gpt-runner-shared/common' -import { debounce, runOnceByKey, tryParseJson } from '@nicepkg/gpt-runner-shared/common' +import type { ServerStorageValue } from '@nicepkg/gpt-runner-shared/common' +import { ServerStorageName, debounce, tryParseJson } from '@nicepkg/gpt-runner-shared/common' import { getGlobalConfig } from '../../helpers/global-config' -import { fetchState, saveState } from '../../networks/state' +import { getServerStorage, saveServerStorage } from '../../networks/server-storage' -let hasUpdateStateFromRemote = false - -const debounceSaveState = debounce(async (key: string, state: FrontendState) => { - if (!hasUpdateStateFromRemote) +// just only request once onload +let hasUpdateStateFromRemote: 'pending' | 'finish' | false = false +async function getStateFromServerOnce(key: string) { + if (hasUpdateStateFromRemote !== false) return - return await saveState({ key, state }) -}, 1000) - -function updateStateFromRemoteOnce(key: string) { - return runOnceByKey(async (key: string) => { - const res = await fetchState({ key }) - hasUpdateStateFromRemote = true - - return res - }, key)(key) + hasUpdateStateFromRemote = 'pending' + const res = await getServerStorage({ + storageName: ServerStorageName.FrontendState, + key, + }) + hasUpdateStateFromRemote = 'finish' + return res?.data?.value } +// will save each action state to server +const debounceSaveStateToServer = debounce(async (key: string, value: ServerStorageValue) => { + if (hasUpdateStateFromRemote !== 'finish') + return + + return await saveServerStorage({ + storageName: ServerStorageName.FrontendState, + key, + value, + }) +}, 1000) + export class CustomStorage implements StateStorage { #storage: Storage @@ -36,7 +45,7 @@ export class CustomStorage implements StateStorage { getItem = async (key: string) => { const finalKey = CustomStorage.prefixKey + key - const remoteSourceValue = (await updateStateFromRemoteOnce(finalKey))?.data?.state + const remoteSourceValue = await getStateFromServerOnce(finalKey) if (remoteSourceValue !== undefined) { const remoteString = JSON.stringify(remoteSourceValue) @@ -52,7 +61,7 @@ export class CustomStorage implements StateStorage { const finalKey = CustomStorage.prefixKey + key // save to server - debounceSaveState(finalKey, tryParseJson(value)) + debounceSaveStateToServer(finalKey, tryParseJson(value)) return this.#storage.setItem(finalKey, value) } @@ -61,7 +70,7 @@ export class CustomStorage implements StateStorage { const finalKey = CustomStorage.prefixKey + key // save to server - debounceSaveState(finalKey, null) + debounceSaveStateToServer(finalKey, null) return this.#storage.removeItem(finalKey) } diff --git a/packages/gpt-runner-web/package.json b/packages/gpt-runner-web/package.json index 6f32c3f..efb4e0d 100644 --- a/packages/gpt-runner-web/package.json +++ b/packages/gpt-runner-web/package.json @@ -62,13 +62,11 @@ "stub": "unbuild --stub" }, "dependencies": { - "@kvs/node-localstorage": "^2.1.3", "@microsoft/fetch-event-source": "^2.0.1", "@nicepkg/gpt-runner-core": "workspace:*", "@nicepkg/gpt-runner-shared": "workspace:*", "@tanstack/react-query": "^4.29.11", "@vscode/webview-ui-toolkit": "^1.2.2", - "cachedir": "^2.3.0", "clsx": "^1.2.1", "commander": "^10.0.1", "connect-history-api-fallback": "^2.0.0", diff --git a/packages/gpt-runner-web/server/src/controllers/index.ts b/packages/gpt-runner-web/server/src/controllers/index.ts index ad6e4b1..7e7ab4a 100644 --- a/packages/gpt-runner-web/server/src/controllers/index.ts +++ b/packages/gpt-runner-web/server/src/controllers/index.ts @@ -3,14 +3,14 @@ import type { Controller, ControllerConfig } from '../types' import { chatgptControllers } from './chatgpt.controller' import { configControllers } from './config.controller' import { gptFilesControllers } from './gpt-files.controller' -import { stateControllers } from './state.controller' +import { storageControllers } from './storage.controller' export function processControllers(router: Router) { const allControllersConfig: ControllerConfig[] = [ chatgptControllers, configControllers, gptFilesControllers, - stateControllers, + storageControllers, ] allControllersConfig.forEach((controllerConfig) => { diff --git a/packages/gpt-runner-web/server/src/controllers/state.controller.ts b/packages/gpt-runner-web/server/src/controllers/state.controller.ts deleted file mode 100644 index 4876661..0000000 --- a/packages/gpt-runner-web/server/src/controllers/state.controller.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { sendSuccessResponse, verifyParamsByZod } from '@nicepkg/gpt-runner-shared/node' -import type { GetStateReqParams, GetStateResData, SaveStateReqParams, SaveStateResData } from '@nicepkg/gpt-runner-shared/common' -import { Debug, GetStateReqParamsSchema, SaveStateReqParamsSchema } from '@nicepkg/gpt-runner-shared/common' -import { kvsLocalStorage } from '@kvs/node-localstorage' -import { getGlobalCacheDir } from '../helpers/get-cache-dir' -import type { ControllerConfig } from '../types' - -const debug = new Debug('state.controller') - -async function getStorage() { - const cacheFolder = await getGlobalCacheDir('gpt-runner-server') - debug.log('cacheFolder', cacheFolder) - - const storage = await kvsLocalStorage | null>>({ - name: 'frontend-state', - storeFilePath: cacheFolder, - version: 1, - }) - - return { - cacheDir: cacheFolder, - storage, - } -} - -export const stateControllers: ControllerConfig = { - namespacePath: '/state', - controllers: [ - { - url: '/', - method: 'get', - handler: async (req, res) => { - const query = req.query as GetStateReqParams - - verifyParamsByZod(query, GetStateReqParamsSchema) - - const { key } = query - - const { storage, cacheDir } = await getStorage() - const state = await storage.get(key) - - sendSuccessResponse(res, { - data: { - state, - cacheDir, - } satisfies GetStateResData, - }) - }, - }, - { - url: '/', - method: 'post', - handler: async (req, res) => { - const body = req.body as SaveStateReqParams - - verifyParamsByZod(body, SaveStateReqParamsSchema) - - const { key, state } = body - - const { storage } = await getStorage() - - switch (state) { - case undefined: - // remove - await storage.delete(key) - break - default: - // set - await storage.set(key, state) - break - } - - sendSuccessResponse(res, { - data: null satisfies SaveStateResData, - }) - }, - }, - ], -} diff --git a/packages/gpt-runner-web/server/src/controllers/storage.controller.ts b/packages/gpt-runner-web/server/src/controllers/storage.controller.ts new file mode 100644 index 0000000..0207541 --- /dev/null +++ b/packages/gpt-runner-web/server/src/controllers/storage.controller.ts @@ -0,0 +1,59 @@ +import { getStorage, sendSuccessResponse, verifyParamsByZod } from '@nicepkg/gpt-runner-shared/node' +import type { GetStorageReqParams, GetStorageResData, SaveStorageReqParams, SaveStorageResData } from '@nicepkg/gpt-runner-shared/common' +import { GetStorageReqParamsSchema, SaveStorageReqParamsSchema } from '@nicepkg/gpt-runner-shared/common' +import type { ControllerConfig } from '../types' + +export const storageControllers: ControllerConfig = { + namespacePath: '/storage', + controllers: [ + { + url: '/', + method: 'get', + handler: async (req, res) => { + const query = req.query as GetStorageReqParams + + verifyParamsByZod(query, GetStorageReqParamsSchema) + + const { key, storageName } = query + + const { storage, cacheDir } = await getStorage(storageName) + const value = await storage.get(key) + + sendSuccessResponse(res, { + data: { + value, + cacheDir, + } satisfies GetStorageResData, + }) + }, + }, + { + url: '/', + method: 'post', + handler: async (req, res) => { + const body = req.body as SaveStorageReqParams + + verifyParamsByZod(body, SaveStorageReqParamsSchema) + + const { storageName, key, value } = body + + const { storage } = await getStorage(storageName) + + switch (value) { + case undefined: + // remove + await storage.delete(key) + break + default: + // set + await storage.set(key, value) + break + } + + sendSuccessResponse(res, { + data: null satisfies SaveStorageResData, + }) + }, + }, + ], +} diff --git a/packages/gpt-runner-web/server/src/helpers/get-storage.ts b/packages/gpt-runner-web/server/src/helpers/get-storage.ts new file mode 100644 index 0000000..b9a9e3b --- /dev/null +++ b/packages/gpt-runner-web/server/src/helpers/get-storage.ts @@ -0,0 +1,22 @@ +import { kvsLocalStorage } from '@kvs/node-localstorage' +import { getGlobalCacheDir } from './get-cache-dir' + +export enum StorageName { + FrontendState = 'frontend-state', + WebPreset = 'web-preset', +} + +export async function getStorage(storageName: StorageName) { + const cacheFolder = await getGlobalCacheDir('gpt-runner-server') + + const storage = await kvsLocalStorage | null>>({ + name: storageName, + storeFilePath: cacheFolder, + version: 1, + }) + + return { + cacheDir: cacheFolder, + storage, + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eb158c2..ae20346 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -199,6 +199,15 @@ importers: packages/gpt-runner-shared: dependencies: + '@kvs/node-localstorage': + specifier: ^2.1.3 + version: 2.1.3 + '@kvs/storage': + specifier: ^2.1.3 + version: 2.1.3 + cachedir: + specifier: ^2.3.0 + version: 2.3.0 debug: specifier: ^4.3.4 version: 4.3.4 @@ -252,9 +261,6 @@ importers: packages/gpt-runner-web: dependencies: - '@kvs/node-localstorage': - specifier: ^2.1.3 - version: 2.1.3 '@microsoft/fetch-event-source': specifier: ^2.0.1 version: 2.0.1 @@ -270,9 +276,6 @@ importers: '@vscode/webview-ui-toolkit': specifier: ^1.2.2 version: 1.2.2(react@18.2.0) - cachedir: - specifier: ^2.3.0 - version: 2.3.0 clsx: specifier: ^1.2.1 version: 1.2.1 diff --git a/tsconfig.json b/tsconfig.json index 16687bb..196131b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,6 +6,7 @@ "moduleResolution": "node", "useDefineForClassFields": true, "esModuleInterop": true, + "noImplicitAny": true, "allowSyntheticDefaultImports": true, "forceConsistentCasingInFileNames": true, "strict": true,