-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Expand file tree
/
Copy pathModelTestService.ts
More file actions
189 lines (165 loc) · 6.81 KB
/
Copy pathModelTestService.ts
File metadata and controls
189 lines (165 loc) · 6.81 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
/**
* Model Test Service
* Handles testing of model configurations with user API keys
*/
import { BaseService } from './BaseService';
import { AIModels } from '../../agents/inferutils/config.types';
import { infer, InferError } from '../../agents/inferutils/core';
import { createUserMessage } from '../../agents/inferutils/common';
import type { TestResult, ModelTestRequest, ModelTestResult } from '../types';
import { isErrorWithMessage } from '../types';
export class ModelTestService extends BaseService {
/**
* Test a model configuration by making a simple chat request using core inference
*/
async testModelConfig({
modelConfig,
userApiKeys,
testPrompt = "Hello! Please respond with 'Test successful' to confirm the connection is working."
}: ModelTestRequest): Promise<ModelTestResult> {
const startTime = Date.now();
const modelName: string = modelConfig.name;
const cleanModelName = modelName.replace(/\[.*?\]/, ''); // Remove provider prefix for display
try {
const testMessage = createUserMessage(testPrompt);
// Use core inference system to test the model configuration
const response = await infer({
env: this.env,
metadata: { agentId: `test-${Date.now()}`, userId: 'system' }, // Generate unique test ID
messages: [testMessage],
modelName: modelName,
maxTokens: Math.min(modelConfig.max_tokens || 100, 100), // Limit to 100 tokens for test
temperature: modelConfig.temperature || 0.1,
reasoning_effort: modelConfig.reasoning_effort,
actionKey: 'testModelConfig',
runtimeOverrides: {
userApiKeys
}
});
const endTime = Date.now();
const latencyMs = endTime - startTime;
const content = response.string || '';
return {
success: true,
responsePreview: content.length > 100 ? content.substring(0, 100) + '...' : content,
latencyMs,
modelUsed: cleanModelName,
};
} catch (error: unknown) {
const endTime = Date.now();
const latencyMs = endTime - startTime;
// Handle InferError and other errors from core system
let rawError = 'Unknown error occurred';
if (error instanceof InferError) {
rawError = error.message;
} else if (error instanceof Error) {
rawError = error.message;
} else if (isErrorWithMessage(error)) {
// Handle error objects from the core system
if (error.message) {
rawError = error.message;
} else if (error.error?.message) {
rawError = error.error.message;
} else {
rawError = JSON.stringify(error);
}
} else {
rawError = String(error);
}
return {
success: false,
error: rawError,
latencyMs,
modelUsed: cleanModelName
};
}
}
/**
* Test a specific provider's API key using core inference
*/
async testProviderKey(provider: string, apiKey: string): Promise<TestResult> {
const startTime = Date.now();
try {
// Get a simple model for this provider to test with
const testModel = this.getTestModelForProvider(provider);
if (!testModel) {
return {
success: false,
error: `No test model available for provider: ${provider}`
};
}
// Create a userApiKeys map with the test key
const testApiKeys = new Map<string, string>();
testApiKeys.set(provider, apiKey);
// Create test message using core abstractions
const testMessage = createUserMessage('Test connection. Please respond with "OK".');
// Use core inference system to test the provider key
const response = await infer({
env: this.env,
metadata: { agentId: `provider-test-${Date.now()}`, userId: 'system' }, // Generate unique test ID
messages: [testMessage],
modelName: testModel,
maxTokens: 10,
temperature: 0,
actionKey: 'testModelConfig',
runtimeOverrides: {
userApiKeys: Object.fromEntries(testApiKeys)
}
});
const endTime = Date.now();
const cleanModelName = testModel.replace(/\[.*?\]/, '');
if (response.string && response.string.trim()) {
return {
success: true,
model: cleanModelName,
latencyMs: endTime - startTime
};
} else {
return {
success: false,
error: 'No response received from model'
};
}
} catch (error: unknown) {
const endTime = Date.now();
const latencyMs = endTime - startTime;
// Handle InferError and other errors from core system
let rawError = 'Connection test failed';
if (error instanceof InferError) {
rawError = error.message;
} else if (error instanceof Error) {
rawError = error.message;
} else if (isErrorWithMessage(error)) {
// Handle error objects from the core system
if (error.message) {
rawError = error.message;
} else if (error.error?.message) {
rawError = error.error.message;
} else {
rawError = JSON.stringify(error);
}
} else {
rawError = String(error);
}
return {
success: false,
error: rawError,
latencyMs
};
}
}
/**
* Get a simple test model for a given provider
*/
private getTestModelForProvider(provider: string): string | null {
const testModels: Record<string, string> = {
'openai': AIModels.OPENAI_5_MINI,
'anthropic': AIModels.CLAUDE_4_SONNET,
'google-ai-studio': AIModels.GEMINI_2_5_FLASH,
'gemini': AIModels.GEMINI_2_5_FLASH,
// 'openrouter': AIModels.OPENROUTER_QWEN_3_CODER, // Removed - not available
// 'cerebras': AIModels.CEREBRAS_GPT_OSS
};
return testModels[provider] || null;
}
}