feat: add ip region for chat logs

packages/global/core/app/logs/constants.ts

@@ -13,7 +13,8 @@ export enum AppLogKeysEnum {
   ANNOTATED_COUNT = 'annotatedCount',
   POINTS = 'points',
   RESPONSE_TIME = 'responseTime',
-  ERROR_COUNT = 'errorCount'
+  ERROR_COUNT = 'errorCount',
+  REGION = 'region'
 }
 
 export const AppLogKeysEnumMap = {
@@ -29,7 +30,8 @@ export const AppLogKeysEnumMap = {
   [AppLogKeysEnum.ANNOTATED_COUNT]: i18nT('app:logs_keys_annotatedCount'),
   [AppLogKeysEnum.POINTS]: i18nT('app:logs_keys_points'),
   [AppLogKeysEnum.RESPONSE_TIME]: i18nT('app:logs_keys_responseTime'),
-  [AppLogKeysEnum.ERROR_COUNT]: i18nT('app:logs_keys_errorCount')
+  [AppLogKeysEnum.ERROR_COUNT]: i18nT('app:logs_keys_errorCount'),
+  [AppLogKeysEnum.REGION]: i18nT('app:logs_keys_region')
 };
 
 export const DefaultAppLogKeys = [
@@ -45,7 +47,8 @@ export const DefaultAppLogKeys = [
   { key: AppLogKeysEnum.ANNOTATED_COUNT, enable: false },
   { key: AppLogKeysEnum.POINTS, enable: false },
   { key: AppLogKeysEnum.RESPONSE_TIME, enable: false },
-  { key: AppLogKeysEnum.ERROR_COUNT, enable: false }
+  { key: AppLogKeysEnum.ERROR_COUNT, enable: false },
+  { key: AppLogKeysEnum.REGION, enable: false }
 ];
 
 export enum AppLogTimespanEnum {

packages/service/common/file/image/controller.ts

@@ -78,9 +78,13 @@ export const copyAvatarImage = async ({
   const avatarSource = getS3AvatarSource();
   if (isS3ObjectKey(imageUrl?.slice(avatarSource.prefix.length), 'avatar')) {
     const filename = (() => {
-      const last = imageUrl.split('/').pop()?.split('-')[1];
-      if (!last) return getNanoid(6).concat(path.extname(imageUrl));
-      return `${getNanoid(6)}-${last}`;
+      const last = imageUrl.split('/').pop();
+      if (!last) {
+        return getNanoid(6).concat(path.extname(imageUrl));
+      }
+      const firstDashIndex = last.indexOf('-');
+      const filename = last.slice(firstDashIndex === -1 ? 0 : firstDashIndex + 1);
+      return `${getNanoid(6)}-${filename}`;
     })();
     const key = await getS3AvatarSource().copyAvatar({
       key: imageUrl,

packages/service/common/geo/constants.ts

@@ -0,0 +1,15 @@
+import path from 'node:path';
+import type { LocationName } from './type';
+
+export const dbPath = path.join(process.cwd(), 'data/GeoLite2-City.mmdb');
+
+export const privateOrOtherLocationName: LocationName = {
+  city: undefined,
+  country: {
+    en: 'Other',
+    zh: '其他'
+  },
+  province: undefined
+};
+
+export const cleanupIntervalMs = 6 * 60 * 60 * 1000; // Run cleanup every 6 hours

packages/service/common/geo/index.ts

@@ -0,0 +1,91 @@
+import fs from 'node:fs';
+import type { ReaderModel } from '@maxmind/geoip2-node';
+import { Reader } from '@maxmind/geoip2-node';
+import { cleanupIntervalMs, dbPath, privateOrOtherLocationName } from './constants';
+import type { LocationName } from './type';
+import { extractLocationData } from './utils';
+import type { NextApiRequest } from 'next';
+import { getClientIp } from 'request-ip';
+import { addLog } from '../system/log';
+
+let reader: ReaderModel | null = null;
+
+const locationIpMap = new Map<string, LocationName>();
+
+function loadGeoDB() {
+  const dbBuffer = fs.readFileSync(dbPath);
+  reader = Reader.openBuffer(dbBuffer);
+  return reader;
+}
+
+export function getGeoReader() {
+  if (!reader) {
+    return loadGeoDB();
+  }
+  return reader;
+}
+
+export function getLocationFromIp(ip: string): LocationName {
+  const reader = getGeoReader();
+
+  let locationName = locationIpMap.get(ip);
+  if (locationName) {
+    return locationName;
+  }
+
+  try {
+    const response = reader.city(ip);
+    const data = extractLocationData(response);
+    locationName = {
+      city: {
+        en: data.city.en,
+        zh: data.city.zh
+      },
+      country: {
+        en: data.country.en,
+        zh: data.country.zh
+      },
+      province: {
+        en: data.province.en,
+        zh: data.province.zh
+      }
+    };
+    locationIpMap.set(ip, locationName);
+    return locationName;
+  } catch (error) {
+    locationIpMap.set(ip, privateOrOtherLocationName);
+    return privateOrOtherLocationName;
+  }
+}
+
+let cleanupInterval: NodeJS.Timeout | null = null;
+function cleanupIpMap() {
+  locationIpMap.clear();
+}
+
+export function clearCleanupInterval() {
+  if (cleanupInterval) {
+    clearInterval(cleanupInterval);
+    cleanupInterval = null;
+  }
+}
+
+export function initGeo() {
+  cleanupInterval = setInterval(cleanupIpMap, cleanupIntervalMs);
+
+  try {
+    loadGeoDB();
+  } catch (error) {
+    clearCleanupInterval();
+    addLog.error(`Failed to load geo db`, error);
+    throw error;
+  }
+}
+
+export function getIpFromRequest(request: NextApiRequest): string {
+  const ip = getClientIp(request);
+  if (!ip || ip === '::1') {
+    return '127.0.0.1';
+  }
+  return ip;
+}

packages/service/common/geo/type.ts

@@ -0,0 +1,10 @@
+export type I18nName = {
+  zh?: string;
+  en?: string;
+};
+
+export type LocationName = {
+  country?: I18nName;
+  province?: I18nName;
+  city?: I18nName;
+};

packages/service/common/geo/utils.ts

@@ -0,0 +1,21 @@
+import type { City } from '@maxmind/geoip2-node';
+
+export function extractLocationData(response: City) {
+  return {
+    city: {
+      id: response.city?.geonameId,
+      en: response.city?.names.en,
+      zh: response.city?.names['zh-CN']
+    },
+    country: {
+      id: response.country?.geonameId,
+      en: response.country?.names.en,
+      zh: response.country?.names['zh-CN']
+    },
+    province: {
+      id: response.subdivisions?.[0]?.geonameId,
+      en: response.subdivisions?.[0]?.names.en,
+      zh: response.subdivisions?.[0]?.names['zh-CN']
+    }
+  };
+}

packages/service/package.json

@@ -31,6 +31,7 @@
     "https-proxy-agent": "^7.0.6",
     "iconv-lite": "^0.6.3",
     "ioredis": "^5.6.0",
+    "ip2region.js": "^3.1.6",
     "joplin-turndown-plugin-gfm": "^1.0.12",
     "json5": "^2.2.3",
     "jsonpath-plus": "^10.3.0",

packages/web/i18n/en/app.json

@@ -209,6 +209,7 @@
   "logs_keys_lastConversationTime": "last conversation time",
   "logs_keys_messageCount": "Message Count",
   "logs_keys_points": "Points Consumed",
+  "logs_keys_region": "User IP",
   "logs_keys_responseTime": "Average Response Time",
   "logs_keys_sessionId": "Session ID",
   "logs_keys_source": "Source",

packages/web/i18n/zh-CN/app.json

@@ -214,6 +214,7 @@
   "logs_keys_lastConversationTime": "最后对话时间",
   "logs_keys_messageCount": "消息总数",
   "logs_keys_points": "积分消耗",
+  "logs_keys_region": "使用者IP",
   "logs_keys_responseTime": "平均响应时长",
   "logs_keys_sessionId": "会话 ID",
   "logs_keys_source": "来源",

projects/app/Dockerfile

@@ -73,9 +73,11 @@ COPY --from=maindeps /app/node_modules/tiktoken ./node_modules/tiktoken
 RUN rm -rf ./node_modules/tiktoken/encoders
 COPY --from=maindeps /app/node_modules/@zilliz/milvus2-sdk-node ./node_modules/@zilliz/milvus2-sdk-node
 # copy package.json to version file
-COPY --from=builder /app/projects/app/package.json ./package.json 
+COPY --from=builder /app/projects/app/package.json ./package.json
 # copy config
 COPY ./projects/app/data/config.json /app/data/config.json
+# copy GeoLite2-City.mmdb
+COPY ./projects/app/data/GeoLite2-City.mmdb /app/data/GeoLite2-City.mmdb
 
 RUN chown -R nextjs:nodejs /app/data
 
@@ -92,4 +94,4 @@ USER nextjs
 
 ENV serverPath=./projects/app/server.js
 
-ENTRYPOINT ["sh","-c","node --max-old-space-size=4096 ${serverPath}"]
+ENTRYPOINT ["sh","-c","node --max-old-space-size=4096 ${serverPath}"]

projects/app/package.json

@@ -39,6 +39,7 @@
     "hyperdown": "^2.4.29",
     "i18next": "23.16.8",
     "immer": "^9.0.19",
+    "ip2region.js": "^3.1.6",
     "js-yaml": "^4.1.1",
     "json5": "^2.2.3",
     "jsondiffpatch": "^0.7.2",

projects/app/src/components/core/chat/ChatContainer/ChatBox/hooks/useFileUpload.tsx

@@ -188,9 +188,6 @@ export const useFileUpload = (props: UseFileUploadOptions) => {
           Object.entries(fields).forEach(([k, v]) => formData.set(k, v));
           formData.set('file', copyFile.rawFile);
           await POST(url, formData, {
-            headers: {
-              'Content-Type': 'multipart/form-data; charset=utf-8'
-            },
             onUploadProgress: (e) => {
               if (!e.total) return;
               const percent = Math.round((e.loaded / e.total) * 100);