From 2f53d993597d93a76d620bf9aacce9347e6ad73b Mon Sep 17 00:00:00 2001 From: roitium Date: Mon, 22 Jul 2024 12:06:50 +0800 Subject: [PATCH 1/5] =?UTF-8?q?fix:=20=E5=A2=9E=E5=8A=A0=E8=AE=B0=E5=BF=86?= =?UTF-8?q?=E7=94=9F=E6=88=90=E5=A4=B1=E8=B4=A5=E6=97=B6=E7=9A=84=E6=97=A5?= =?UTF-8?q?=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/services/bot/memory/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/services/bot/memory/index.ts b/src/services/bot/memory/index.ts index 86d5700..83a3470 100644 --- a/src/services/bot/memory/index.ts +++ b/src/services/bot/memory/index.ts @@ -7,7 +7,9 @@ import { openai } from "../../openai"; import { MessageContext } from "../conversation"; import { LongTermMemoryAgent } from "./long-term"; import { ShortTermMemoryAgent } from "./short-term"; +import {Logger} from "../../../utils/log"; +export const memoryLogger = Logger.create({ tag: "Memory" }); export class MemoryManager { private room: Room; @@ -118,6 +120,7 @@ export class MemoryManager { lastMemory, }); if (!newMemory) { + memoryLogger.error("💀 生成短期记忆失败"); return false; } const res = await ShortTermMemoryCRUD.addOrUpdate({ @@ -151,6 +154,7 @@ export class MemoryManager { lastMemory, }); if (!newMemory) { + memoryLogger.error("💀 生成长期记忆失败"); return false; } const res = await LongTermMemoryCRUD.addOrUpdate({ From c985e5b08fd0f92a376678220884a944d949d65e Mon Sep 17 00:00:00 2001 From: roitium Date: Mon, 22 Jul 2024 12:13:02 +0800 Subject: [PATCH 2/5] =?UTF-8?q?fix:=20=E5=8E=BB=E9=99=A4=E6=8E=89ai?= =?UTF-8?q?=E8=BF=94=E5=9B=9E=E5=86=85=E5=AE=B9=E4=B8=AD=E7=9A=84=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=9D=97=E6=A0=B7=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/services/bot/memory/long-term.ts | 4 ++++ src/services/bot/memory/short-term.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/services/bot/memory/long-term.ts b/src/services/bot/memory/long-term.ts index fe62553..751ece8 100644 --- a/src/services/bot/memory/long-term.ts +++ b/src/services/bot/memory/long-term.ts @@ -67,6 +67,10 @@ export class LongTermMemoryAgent { shortTermMemory: lastOf(newMemories)!.text, }), }); + // 如果返回内容是个markdown代码块,就让他变回普通json + res?.content?.trim(); + if (res?.content?.startsWith("```json")) {res.content = res?.content?.replace("```json", "");} + if (res?.content?.endsWith("```")) {res.content = res?.content?.replace("```", "");} return jsonDecode(res?.content)?.longTermMemories?.toString(); } } diff --git a/src/services/bot/memory/short-term.ts b/src/services/bot/memory/short-term.ts index 6e039f5..85767ef 100644 --- a/src/services/bot/memory/short-term.ts +++ b/src/services/bot/memory/short-term.ts @@ -78,6 +78,10 @@ export class ShortTermMemoryAgent { .join("\n"), }), }); + // 如果返回内容是个markdown代码块,就让他变回普通json + res?.content?.trim(); + if (res?.content?.startsWith("```json")) {res.content = res?.content?.replace("```json", "");} + if (res?.content?.endsWith("```")) {res.content = res?.content?.replace("```", "");} return jsonDecode(res?.content)?.shortTermMemories?.toString(); } } From 208b2db60b0af84702e6ea8479ddfc940b7e5ce5 Mon Sep 17 00:00:00 2001 From: roitium Date: Wed, 24 Jul 2024 10:18:24 +0800 Subject: [PATCH 3/5] =?UTF-8?q?fix:=20=E5=B0=86json=E6=B8=85=E6=B4=97?= =?UTF-8?q?=E6=93=8D=E4=BD=9C=E5=B0=81=E8=A3=85=E4=B8=BA=E5=87=BD=E6=95=B0?= =?UTF-8?q?=20refactor:=20=E5=B0=86json=E7=BC=96=E8=A7=A3=E7=A0=81?= =?UTF-8?q?=E6=93=8D=E4=BD=9C=E6=94=BE=E5=85=A5/utils/parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/services/speaker/base.ts | 3 ++- src/utils/base.ts | 19 +------------------ src/utils/io.ts | 2 +- src/utils/parse.ts | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 src/utils/parse.ts diff --git a/src/services/speaker/base.ts b/src/services/speaker/base.ts index 08ac031..5875faf 100644 --- a/src/services/speaker/base.ts +++ b/src/services/speaker/base.ts @@ -5,7 +5,8 @@ import { getMiIOT, getMiNA, } from "mi-service-lite"; -import { clamp, jsonEncode, sleep } from "../../utils/base"; +import { clamp, sleep } from "../../utils/base"; +import { jsonEncode } from "../../utils/parse"; import { Logger } from "../../utils/log"; import { StreamResponse } from "./stream"; import { kAreYouOK } from "../../utils/string"; diff --git a/src/utils/base.ts b/src/utils/base.ts index e98b41a..e80fd84 100644 --- a/src/utils/base.ts +++ b/src/utils/base.ts @@ -1,4 +1,5 @@ import { isEmpty } from "./is"; +import { jsonEncode } from "./parse" export function timestamp() { return new Date().getTime(); @@ -87,24 +88,6 @@ export function toSet(datas: T[], byKey?: (e: T) => any) { return Array.from(new Set(datas)); } -export function jsonEncode(obj: any, options?: { prettier?: boolean }) { - const { prettier } = options ?? {}; - try { - return prettier ? JSON.stringify(obj, undefined, 4) : JSON.stringify(obj); - } catch (error) { - return undefined; - } -} - -export function jsonDecode(json: string | null | undefined) { - if (json == undefined) return undefined; - try { - return JSON.parse(json!); - } catch (error) { - return undefined; - } -} - export function withDefault(e: any, defaultValue: T): T { return isEmpty(e) ? defaultValue : e; } diff --git a/src/utils/io.ts b/src/utils/io.ts index 24275c0..5e07e5a 100644 --- a/src/utils/io.ts +++ b/src/utils/io.ts @@ -1,7 +1,7 @@ import fs from "fs-extra"; import path from "path"; -import { jsonDecode, jsonEncode } from "./base"; +import { jsonDecode, jsonEncode } from "./parse"; export const kRoot = process.cwd(); diff --git a/src/utils/parse.ts b/src/utils/parse.ts new file mode 100644 index 0000000..6dbdc94 --- /dev/null +++ b/src/utils/parse.ts @@ -0,0 +1,32 @@ +/** + * 清理掉json字符串前后的其他字符,并解码 + */ +export function cleanJsonAndDecode(input: string | undefined | null) { + if (input == undefined) return undefined; + const pattern = /(\{[\s\S]*?"\s*:\s*[\s\S]*?})/; + const match = input.match(pattern); + + if (!match) { + return undefined; + } + + return jsonDecode(match[0]); +} + +export function jsonEncode(obj: any, options?: { prettier?: boolean }) { + const { prettier } = options ?? {}; + try { + return prettier ? JSON.stringify(obj, undefined, 4) : JSON.stringify(obj); + } catch (error) { + return undefined; + } +} + +export function jsonDecode(json: string | null | undefined) { + if (json == undefined) return undefined; + try { + return JSON.parse(json!); + } catch (error) { + return undefined; + } +} From 11dc25aa60f513c300f76d8ff7912752131263c7 Mon Sep 17 00:00:00 2001 From: roitium Date: Wed, 24 Jul 2024 10:18:57 +0800 Subject: [PATCH 4/5] =?UTF-8?q?refactor:=20=E8=B0=83=E7=94=A8=E5=B0=81?= =?UTF-8?q?=E8=A3=85=E5=90=8E=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/services/bot/memory/index.ts | 10 +++++----- src/services/bot/memory/long-term.ts | 9 +++------ src/services/bot/memory/short-term.ts | 8 ++------ 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/services/bot/memory/index.ts b/src/services/bot/memory/index.ts index 83a3470..c3538fe 100644 --- a/src/services/bot/memory/index.ts +++ b/src/services/bot/memory/index.ts @@ -1,5 +1,6 @@ import { Memory, Message, Room, User } from "@prisma/client"; import { firstOf, lastOf } from "../../../utils/base"; +import { Logger } from "../../../utils/log"; import { MemoryCRUD } from "../../db/memory"; import { LongTermMemoryCRUD } from "../../db/memory-long-term"; import { ShortTermMemoryCRUD } from "../../db/memory-short-term"; @@ -7,9 +8,7 @@ import { openai } from "../../openai"; import { MessageContext } from "../conversation"; import { LongTermMemoryAgent } from "./long-term"; import { ShortTermMemoryAgent } from "./short-term"; -import {Logger} from "../../../utils/log"; -export const memoryLogger = Logger.create({ tag: "Memory" }); export class MemoryManager { private room: Room; @@ -17,6 +16,7 @@ export class MemoryManager { * owner 为空时，即房间自己的公共记忆 */ private owner?: User; + private _logger = Logger.create({ tag: "Memory" }); constructor(room: Room, owner?: User) { this.room = room; @@ -100,7 +100,7 @@ export class MemoryManager { threshold?: number; } ) { - const { threshold = 10 } = options; + const { threshold = 1 } = options; const lastMemory = firstOf(await this.getShortTermMemories({ take: 1 })); const newMemories: (Memory & { msg: Message & { @@ -120,7 +120,7 @@ export class MemoryManager { lastMemory, }); if (!newMemory) { - memoryLogger.error("💀 生成短期记忆失败"); + this._logger.error("💀 生成短期记忆失败"); return false; } const res = await ShortTermMemoryCRUD.addOrUpdate({ @@ -154,7 +154,7 @@ export class MemoryManager { lastMemory, }); if (!newMemory) { - memoryLogger.error("💀 生成长期记忆失败"); + this._logger.error("💀 生成长期记忆失败"); return false; } const res = await LongTermMemoryCRUD.addOrUpdate({ diff --git a/src/services/bot/memory/long-term.ts b/src/services/bot/memory/long-term.ts index 751ece8..f51fe81 100644 --- a/src/services/bot/memory/long-term.ts +++ b/src/services/bot/memory/long-term.ts @@ -1,6 +1,7 @@ import { LongTermMemory, ShortTermMemory } from "@prisma/client"; -import { jsonDecode, lastOf } from "../../../utils/base"; +import { lastOf } from "../../../utils/base"; import { buildPrompt } from "../../../utils/string"; +import { cleanJsonAndDecode } from "../../../utils/parse"; import { openai } from "../../openai"; import { MessageContext } from "../conversation"; @@ -67,10 +68,6 @@ export class LongTermMemoryAgent { shortTermMemory: lastOf(newMemories)!.text, }), }); - // 如果返回内容是个markdown代码块,就让他变回普通json - res?.content?.trim(); - if (res?.content?.startsWith("```json")) {res.content = res?.content?.replace("```json", "");} - if (res?.content?.endsWith("```")) {res.content = res?.content?.replace("```", "");} - return jsonDecode(res?.content)?.longTermMemories?.toString(); + return cleanJsonAndDecode(res?.content)?.longTermMemories?.toString(); } } diff --git a/src/services/bot/memory/short-term.ts b/src/services/bot/memory/short-term.ts index 85767ef..a804ab2 100644 --- a/src/services/bot/memory/short-term.ts +++ b/src/services/bot/memory/short-term.ts @@ -1,5 +1,5 @@ import { Memory, Message, ShortTermMemory, User } from "@prisma/client"; -import { jsonDecode } from "../../../utils/base"; +import { cleanJsonAndDecode } from "../../../utils/parse"; import { buildPrompt, formatMsg } from "../../../utils/string"; import { openai } from "../../openai"; import { MessageContext } from "../conversation"; @@ -78,10 +78,6 @@ export class ShortTermMemoryAgent { .join("\n"), }), }); - // 如果返回内容是个markdown代码块,就让他变回普通json - res?.content?.trim(); - if (res?.content?.startsWith("```json")) {res.content = res?.content?.replace("```json", "");} - if (res?.content?.endsWith("```")) {res.content = res?.content?.replace("```", "");} - return jsonDecode(res?.content)?.shortTermMemories?.toString(); + return cleanJsonAndDecode(res?.content)?.shortTermMemories?.toString(); } } From bba886e2752dfc991571870dc0e9d9ad7e1565e7 Mon Sep 17 00:00:00 2001 From: roitium Date: Wed, 24 Jul 2024 10:21:44 +0800 Subject: [PATCH 5/5] =?UTF-8?q?fix:=20=E6=81=A2=E5=A4=8D=E5=8E=9Fthreshold?= =?UTF-8?q?=E6=95=B0=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/services/bot/memory/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/bot/memory/index.ts b/src/services/bot/memory/index.ts index c3538fe..6307783 100644 --- a/src/services/bot/memory/index.ts +++ b/src/services/bot/memory/index.ts @@ -100,7 +100,7 @@ export class MemoryManager { threshold?: number; } ) { - const { threshold = 1 } = options; + const { threshold = 10 } = options; const lastMemory = firstOf(await this.getShortTermMemories({ take: 1 })); const newMemories: (Memory & { msg: Message & {