Overview
When an app performs knowledge Q&A through the RAG API, the system goes through the following process: question decomposition, query rewriting, knowledge retrieval, and retrieval-augmented generation. This process requires multiple interactions with the LLM. You can select either of the following deployment solutions:
-
Self-built on-cloud LLMs
-
On-device Q&A model capability provided by the Kit
The on-device Q&A model solution has the following advantages:
-
No O&M costs for on-cloud LLMs
-
Enhanced data security (data processed on the device)
Constraints
-
Currently, the on-device Q&A model is supported only on PCs/2-in-1 devices.
-
Currently, the on-device Q&A model is available only to enterprise developers.
API Description
The following table lists the key APIs of the on-device Q&A model. For details about the APIs, see API Reference.
| API | Description |
|---|---|
| init(): Promise<boolean> | Initializes the on-device Q&A model and starts the model management application. |
| chat(info: QuestionInfo, config: Config, callback: AsyncCallback<Answer>): Promise<void> | Interacts with the on-device model to implement the Q&A function. |
[h2]Requesting Permissions
Request the ohos.permission.INTERNET permission. For details about the configuration method, see Declaring Permissions.
[h2]Functionality Development
-
Call the init API to start the local AI manager.
-
If local AI manager is started for the first time, the privacy statement screen is displayed. After you agree to the privacy statement, the default model is downloaded.
-
For subsequent launches of the local AI manager, go to Settings > System > Local AI Manager and download the default model.
-
After the model is downloaded, call the chat API to start on-device Q&A.
Sample code
import { BusinessError } from '@kit.BasicServicesKit';
import { localChatModel } from '@kit.DataAugmentationKit'
type MessageType = 'tips' | 'request' | 'response';
interface ChatMessage {
type: MessageType;
content: string;
}
@Entry
@Component
struct Index {
@State title: string = 'On-device LLM Q&A assistant';
@State isStreamMode: boolean = true;
@State messages: ChatMessage[] = [];
@State inputText: string = '';
@State initFlag: boolean = false;
@State isProcessing: boolean = false;
@State responseContent: string = '';
@State chatCounter: number = 0;
// Start the model management application when loading the page.
onPageShow() {
this.initModel();
}
private async initModel(): Promise<void> {
try {
this.initFlag = await localChatModel.init();
// ...
} catch (error) {
this.initFlag = false;
// ...
}
}
private scroller: Scroller = new Scroller();
// ...
private async DoChat(questionId: number): Promise<void> {
if (!this.inputText.trim() || this.isProcessing) {
return;
}
const requestQuestion = this.inputText.trim();
if (!requestQuestion) {
return;
}
this.inputText = '';
// ...
this.responseContent = 'Thinking...';
this.isProcessing = true;
const questionInfo: localChatModel.QuestionInfo = {
questionId: questionId,
content: requestQuestion
};
const localConfig: localChatModel.Config = {
isStream: this.isStreamMode
};
const localChatCallback = async (err: BusinessError, ans: localChatModel.Answer): Promise<void> => {
// ...
if (ans.content && ans.content.trim() !== '') {
if (this.responseContent == 'Thinking...') {
this.responseContent = '';
}
this.responseContent += ans.content;
// ...
}
// ...
if (ans.isFinished) {
// ...
this.isProcessing = false;
}
};
try {
localChatModel.chat(questionInfo, localConfig, localChatCallback);
} catch (error) {
// ...
this.isProcessing = false;
}
}
private clearChat(): void {
this.messages = [];
}
build() {
Stack({ alignContent: Alignment.Top }) {
Column() {
Row() {
// ...
Row() {
Button(this.isStreamMode ? 'Streaming' : 'Non-streaming')
.height(28)
.fontSize(16)
.fontWeight(500)
.backgroundColor('#0c000000')
.fontColor('#0A59F7')
.borderRadius(8)
.onClick(() => {
this.isStreamMode = !this.isStreamMode;
// ...
})
}
.margin({ right: 12 })
}
.width('100%')
.height(56)
.backgroundColor(Color.White)
.borderRadius(8)
.shadow({
radius: 4,
color: '#1a73e888',
offsetX: 0,
offsetY: 2
})
// Chat area.
Scroll(this.scroller) {
Column() {
ForEach(this.messages, (msg: ChatMessage, index: number) => {
if (msg.type === 'tips') {
Row() {
Text(msg.content)
.fontSize(14)
.fontColor('#666')
.textAlign(TextAlign.Center)
.padding(8)
}
.width('100%')
.justifyContent(FlexAlign.Center)
.margin({ top: index === 0 ? 0 : 12 })
} else if (msg.type === 'request') {
Row() {
Blank()
Text(msg.content)
.fontSize(16)
.fontColor(Color.White)
.padding(10)
.backgroundColor('#1a73e8')
.borderRadius(8)
}
.width('100%')
.margin({ top: 12 })
.justifyContent(FlexAlign.End)
} else if (msg.type === 'response') {
Row() {
Column() {
Text(msg.content)
.fontSize(16)
.fontColor('#333')
.lineHeight(20)
.padding(10)
.backgroundColor(Color.White)
.borderRadius(8)
}
.borderRadius(8)
.margin({ left: 8 })
Blank()
}
.width('100%')
.margin({ top: 12 })
.justifyContent(FlexAlign.Start)
}
}, (msg: ChatMessage) => msg.toString())
// Loading indicator.
if (this.isProcessing) {
Row() {
Column() {
Text(this.responseContent)
.fontSize(16)
.fontColor('#333')
.lineHeight(20)
.padding(10)
.backgroundColor(Color.White)
.borderRadius(8)
}
.borderRadius(8)
.margin({ left: 8 })
Blank()
}
.width('100%')
.margin({ top: 12 })
}
}
.padding(12)
.width('100%')
}
.width('100%')
.layoutWeight(1)
.margin({ bottom: 12 })
// Input area.
Column() {
Row() {
TextInput({ text: this.inputText, placeholder: 'Enter your question...' })
.flexGrow(1)
.height(42)
.fontSize(16)
.padding(8)
.backgroundColor(Color.White)
.borderRadius(8)
.width('85%')
.onChange((value: string) => {
this.inputText = value;
})
.onSubmit(() => {
if (!this.isProcessing && this.inputText.trim() !== '') {
const chatId = this.chatCounter++;
this.DoChat(chatId);
}
})
Button('Send')
.height(40)
.fontSize(16)
.fontWeight(500)
.margin({ right: 12 })
.backgroundColor('#0A59F7')
.fontColor(Color.White)
.borderRadius(8)
.onClick(() => {
if (!this.isProcessing && this.inputText.trim() !== '') {
const chatId = this.chatCounter++;
this.DoChat(chatId);
}
})
.opacity(this.isProcessing || this.inputText.trim() === '' ? 0.4 : 1)
Button("Clear")
.height(40)
.fontSize(16)
.fontWeight(500)
.fontColor('#fff')
.backgroundColor('#ea4335')
.borderRadius(8)
.onClick(() => {
this.clearChat();
})
}
.width('100%')
.justifyContent(FlexAlign.SpaceBetween)
.alignItems(VerticalAlign.Center)
}
.width('100%')
.padding(8)
.backgroundColor(Color.White)
.borderRadius(8)
.shadow({
radius: 4,
color: '#1a73e888',
offsetX: 0,
offsetY: 2
})
}
.width('100%')
.height('100%')
.backgroundColor('#f0f5ff')
.padding(20)
}
.width('100%')
.height('100%')
}
}