commit fdcf5016eedb3dd8ed6f67b4f054a9a0ef2af593 Author: abearxiong Date: Tue Nov 11 10:53:35 2025 +0800 update diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c170c64 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +node_modules + +storage + +.env +!.env*example \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..6eaf561 --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "test-lancedb", + "version": "0.0.1", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "abearxiong (https://www.xiongxiao.me)", + "license": "MIT", + "packageManager": "pnpm@10.19.0", + "type": "module", + "dependencies": { + "@kevisual/ai": "^0.0.11", + "@lancedb/lancedb": "^0.22.3", + "dotenv": "^17.2.3" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..fd7b47d --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,331 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@kevisual/ai': + specifier: ^0.0.11 + version: 0.0.11 + '@lancedb/lancedb': + specifier: ^0.22.3 + version: 0.22.3(apache-arrow@18.1.0) + dotenv: + specifier: ^17.2.3 + version: 17.2.3 + +packages: + + '@kevisual/ai@0.0.11': + resolution: {integrity: sha512-SceIxodtjttLurZuXImZmJbimSt/NJW7N8vQArr0AaxqUam/WwcOTPYSVCleGhUW196Vew/FKC5k0vjyBMZZ5Q==} + + '@kevisual/logger@0.0.4': + resolution: {integrity: sha512-+fpr92eokSxoGOW1SIRl/27lPuO+zyY+feR5o2Q4YCNlAdt2x64NwC/w8r/3NEC5QenLgd4K0azyKTI2mHbARw==} + + '@lancedb/lancedb-darwin-arm64@0.22.3': + resolution: {integrity: sha512-oP2Kic51nLqs27Xo+AzSVlcMgmmfZbU/PseQ3KBtU92rczO5DYU2St1Y7qDUWcjw+RF3H2v/DKzYed16h1wCBQ==} + engines: {node: '>= 18'} + cpu: [arm64] + os: [darwin] + + '@lancedb/lancedb-darwin-x64@0.22.3': + resolution: {integrity: sha512-wOwgZkvBgQM8asjolz4NeyPa8W/AjZv4fwyQxJhTqKTGlB3ntrpdn1m84K5qncTmFFDcDfGgZ4DkNVkVK+ydoQ==} + engines: {node: '>= 18'} + cpu: [x64] + os: [darwin] + + '@lancedb/lancedb-linux-arm64-gnu@0.22.3': + resolution: {integrity: sha512-YUbFuBKQniTZOR9h2/es1f7lDzdHNt8qXs5GaqFmLQv2GNWpnvKXVA/vVffhCNpFB5nV132o1VhXW3KoMubPsw==} + engines: {node: '>= 18'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@lancedb/lancedb-linux-arm64-musl@0.22.3': + resolution: {integrity: sha512-jVRMtXxxYaDlZSaclCIHB2N+NJvQ1Fj9EaPeBx+HxG2VqUg0vXKef+yiaD2aGo9sAH6mMmkKJsrPhwABpUC4rQ==} + engines: {node: '>= 18'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@lancedb/lancedb-linux-x64-gnu@0.22.3': + resolution: {integrity: sha512-co7idTwvNAtbFoFHojhHlTpKsydOm5sZfbtAsQRdoa7g6a61yIrqrMm8D7Ngh756JfzZLFQBMkDUZEW3X4vP/g==} + engines: {node: '>= 18'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@lancedb/lancedb-linux-x64-musl@0.22.3': + resolution: {integrity: sha512-+ipFsn5PCODK7mOMq1gZ5OAZWks5YlgmjAlnYMmU8XxvaK0b6lZdA3s1hTmBaBO9+wv+31ulO55oBN4/U8Yldg==} + engines: {node: '>= 18'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@lancedb/lancedb-win32-arm64-msvc@0.22.3': + resolution: {integrity: sha512-E0XywJYnelIe4pzOlvog+aMHKt5ChW27tgmT2V80Z6PXcX6eN9I69Fj0Q6DK6z1YCTPIPu6Na1Hd6d4GqUNKPw==} + engines: {node: '>= 18'} + cpu: [arm64] + os: [win32] + + '@lancedb/lancedb-win32-x64-msvc@0.22.3': + resolution: {integrity: sha512-/1feFnjz5MIhzXOEU4+1OeGwpAFYczGfefuOGZRsmGWDdt4V6/fza7Hkkxyb2OnTzqpBfy6BdW2+iBguE1JMyQ==} + engines: {node: '>= 18'} + cpu: [x64] + os: [win32] + + '@lancedb/lancedb@0.22.3': + resolution: {integrity: sha512-nRC0fkg+d7dzCtudKHT+VH7znk6KUXRZyuS6HJYNnIrbvXBxaT6wAPjEbf70KTuqvP2znj48Zg+kiwRqkRnAJw==} + engines: {node: '>= 18'} + cpu: [x64, arm64] + os: [darwin, linux, win32] + peerDependencies: + apache-arrow: '>=15.0.0 <=18.1.0' + + '@swc/helpers@0.5.17': + resolution: {integrity: sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==} + + '@types/command-line-args@5.2.3': + resolution: {integrity: sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==} + + '@types/command-line-usage@5.0.4': + resolution: {integrity: sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==} + + '@types/node@20.19.24': + resolution: {integrity: sha512-FE5u0ezmi6y9OZEzlJfg37mqqf6ZDSF2V/NLjUyGrR9uTZ7Sb9F7bLNZ03S4XVUNRWGA7Ck4c1kK+YnuWjl+DA==} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + apache-arrow@18.1.0: + resolution: {integrity: sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg==} + hasBin: true + + array-back@3.1.0: + resolution: {integrity: sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==} + engines: {node: '>=6'} + + array-back@6.2.2: + resolution: {integrity: sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==} + engines: {node: '>=12.17'} + + chalk-template@0.4.0: + resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} + engines: {node: '>=12'} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + command-line-args@5.2.1: + resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} + engines: {node: '>=4.0.0'} + + command-line-usage@7.0.3: + resolution: {integrity: sha512-PqMLy5+YGwhMh1wS04mVG44oqDsgyLRSKJBdOo1bnYhMKBW65gZF1dRp2OZRhiTjgUHljy99qkO7bsctLaw35Q==} + engines: {node: '>=12.20.0'} + + dotenv@17.2.3: + resolution: {integrity: sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==} + engines: {node: '>=12'} + + find-replace@3.0.0: + resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} + engines: {node: '>=4.0.0'} + + flatbuffers@24.12.23: + resolution: {integrity: sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + json-bignum@0.0.3: + resolution: {integrity: sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==} + engines: {node: '>=0.8'} + + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + + reflect-metadata@0.2.2: + resolution: {integrity: sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + table-layout@4.1.1: + resolution: {integrity: sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==} + engines: {node: '>=12.17'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + typical@4.0.0: + resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} + engines: {node: '>=8'} + + typical@7.3.0: + resolution: {integrity: sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==} + engines: {node: '>=12.17'} + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + wordwrapjs@5.1.1: + resolution: {integrity: sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==} + engines: {node: '>=12.17'} + +snapshots: + + '@kevisual/ai@0.0.11': + dependencies: + '@kevisual/logger': 0.0.4 + + '@kevisual/logger@0.0.4': {} + + '@lancedb/lancedb-darwin-arm64@0.22.3': + optional: true + + '@lancedb/lancedb-darwin-x64@0.22.3': + optional: true + + '@lancedb/lancedb-linux-arm64-gnu@0.22.3': + optional: true + + '@lancedb/lancedb-linux-arm64-musl@0.22.3': + optional: true + + '@lancedb/lancedb-linux-x64-gnu@0.22.3': + optional: true + + '@lancedb/lancedb-linux-x64-musl@0.22.3': + optional: true + + '@lancedb/lancedb-win32-arm64-msvc@0.22.3': + optional: true + + '@lancedb/lancedb-win32-x64-msvc@0.22.3': + optional: true + + '@lancedb/lancedb@0.22.3(apache-arrow@18.1.0)': + dependencies: + apache-arrow: 18.1.0 + reflect-metadata: 0.2.2 + optionalDependencies: + '@lancedb/lancedb-darwin-arm64': 0.22.3 + '@lancedb/lancedb-darwin-x64': 0.22.3 + '@lancedb/lancedb-linux-arm64-gnu': 0.22.3 + '@lancedb/lancedb-linux-arm64-musl': 0.22.3 + '@lancedb/lancedb-linux-x64-gnu': 0.22.3 + '@lancedb/lancedb-linux-x64-musl': 0.22.3 + '@lancedb/lancedb-win32-arm64-msvc': 0.22.3 + '@lancedb/lancedb-win32-x64-msvc': 0.22.3 + + '@swc/helpers@0.5.17': + dependencies: + tslib: 2.8.1 + + '@types/command-line-args@5.2.3': {} + + '@types/command-line-usage@5.0.4': {} + + '@types/node@20.19.24': + dependencies: + undici-types: 6.21.0 + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + apache-arrow@18.1.0: + dependencies: + '@swc/helpers': 0.5.17 + '@types/command-line-args': 5.2.3 + '@types/command-line-usage': 5.0.4 + '@types/node': 20.19.24 + command-line-args: 5.2.1 + command-line-usage: 7.0.3 + flatbuffers: 24.12.23 + json-bignum: 0.0.3 + tslib: 2.8.1 + + array-back@3.1.0: {} + + array-back@6.2.2: {} + + chalk-template@0.4.0: + dependencies: + chalk: 4.1.2 + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + command-line-args@5.2.1: + dependencies: + array-back: 3.1.0 + find-replace: 3.0.0 + lodash.camelcase: 4.3.0 + typical: 4.0.0 + + command-line-usage@7.0.3: + dependencies: + array-back: 6.2.2 + chalk-template: 0.4.0 + table-layout: 4.1.1 + typical: 7.3.0 + + dotenv@17.2.3: {} + + find-replace@3.0.0: + dependencies: + array-back: 3.1.0 + + flatbuffers@24.12.23: {} + + has-flag@4.0.0: {} + + json-bignum@0.0.3: {} + + lodash.camelcase@4.3.0: {} + + reflect-metadata@0.2.2: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + table-layout@4.1.1: + dependencies: + array-back: 6.2.2 + wordwrapjs: 5.1.1 + + tslib@2.8.1: {} + + typical@4.0.0: {} + + typical@7.3.0: {} + + undici-types@6.21.0: {} + + wordwrapjs@5.1.1: {} diff --git a/src/ai.ts b/src/ai.ts new file mode 100644 index 0000000..1c62eb2 --- /dev/null +++ b/src/ai.ts @@ -0,0 +1,25 @@ +import { BailianProvider } from "@kevisual/ai"; +import './config' +export const textV4 = 'text-embedding-v4' +export const provider = new BailianProvider({ + model: 'qwen-turbo-latest', + apiKey: process.env.BAILIAN_API_KEY, +}); + + + +const chat = async () => { + const response = await provider.chat([ + { role: "user", content: "1+1=?" }, + ]); + console.log(response.choices[0].message); +} +// await chat(); + +const embedding = async () => { + const response = await provider.generateEmbeddingCore( + ["苹果", "香蕉", "橘子", '黄瓜'], + { model: "text-embedding-v4" }); + console.log(response.data); +} +// await embedding(); \ No newline at end of file diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..93efa4b --- /dev/null +++ b/src/config.ts @@ -0,0 +1,3 @@ +import dotenv from "dotenv"; + +dotenv.config(); diff --git a/src/demo.ts b/src/demo.ts new file mode 100644 index 0000000..7dcc186 --- /dev/null +++ b/src/demo.ts @@ -0,0 +1,9 @@ +import * as lancedb from "@lancedb/lancedb"; +const db = await lancedb.connect("storage/lance.db"); +// const table = await db.createTable("my_table", [ +// { id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 }, +// { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }, +// ]); +const table = await db.openTable("my_table"); +const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray(); +console.log(results); \ No newline at end of file diff --git a/src/test-lancedb.ts b/src/test-lancedb.ts new file mode 100644 index 0000000..b552e0a --- /dev/null +++ b/src/test-lancedb.ts @@ -0,0 +1,144 @@ +import './config'; +import { provider, textV4 } from './ai'; +import * as lancedb from "@lancedb/lancedb"; +const db = await lancedb.connect("storage/lance.db"); +// const table = await db.createTable("my_table", [ +// { id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 }, +// { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }, +// ]); +// const table = await db.openTable("my_table"); +// const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray(); +// console.log(results); +const data = [ + { + id: 1, + name: "Alice", + age: 30, + city: "New York", + skill: "JavaScript, Python" + }, + { + id: 2, + name: "Bob", + age: 25, + city: "San Francisco", + skill: "React, Node.js, TypeScript" + }, + { + id: 3, + name: "Charlie", + age: 35, + city: "London", + skill: "Java, Spring, MySQL" + }, + { + id: 4, + name: "Diana", + age: 28, + city: "Tokyo", + skill: "Vue.js, PHP, MongoDB, React, TypeScript" + }, + { + id: 5, + name: "Edward", + age: 32, + city: "Berlin", + skill: "Go, Docker, Kubernetes" + }, + { + id: 6, + name: "Fiona", + age: 27, + city: "Sydney", + skill: "Python, Django, PostgreSQL" + }, + { + id: 7, + name: "George", + age: 29, + city: "Toronto", + skill: "C#, .NET, Azure" + }, + { + id: 8, + name: "Helen", + age: 33, + city: "Paris", + skill: "Ruby, Rails, Redis, Python, PostgreSQL" + }, + { + id: 9, + name: "Ivan", + age: 26, + city: "Moscow", + skill: "Rust, WebAssembly, GraphQL" + }, + { + id: 10, + name: "Julia", + age: 31, + city: "São Paulo", + skill: "Kotlin, Android, Firebase" + }, +] + +export const createEmbedding = async () => { + const response = await provider.generateEmbeddingCore(data.map(item => item.skill), { + model: textV4, + }); + const embedding = response.data; + return data.map((item, index) => ({ + ...item, + vector: embedding[index].embedding + })); +} + +export const createTable = async () => { + const table = await db.createTable("employees", await createEmbedding()); + console.log("Table created:", table); +} + +// createTable(); + +export const searchTable = async (query: string) => { + const table = await db.openTable("employees"); + const response = await provider.generateEmbeddingCore([query], { + model: textV4, + }); + const queryEmbedding = response.data[0].embedding; + const results = await table.vectorSearch(queryEmbedding).limit(5).toArray(); + // console.log("Search results for query:", query); + console.log('keys', Object.keys(results[0])); + console.log(results.map(item => { + return { + id: item.id, + name: item.name, + age: item.age, + city: item.city, + skill: item.skill, + distance: item._distance, + similarity: 1 / (1 + item._distance) + } + })); +} + +// await searchTable("Looking for a Python developer with Django experience"); +// await searchTable("找一个会js，也会Python开发的程序员"); + + +export const listTable = async () => { + const table = await db.openTable("employees"); + const results = await table.query().where('id > 2').limit(2).toArray(); + console.log("Listing first 10 records in the table:"); + console.log(results.map(item => { + return { + id: item.id, + name: item.name, + age: item.age, + city: item.city, + skill: item.skill, + } + })); +} + +listTable(); \ No newline at end of file