import PreProcess from './preprocess.js';
import PostProcess from './postprocess.js';
import AsyncQueue from './asyncqueue.js';
import Mutex from './mutex.js';
import DGError from './dgerror.js';
import StatsDict from './statsdict.js';
import { parser } from './helpers/customParser.js';
import { loadMsgpack } from './msgpack.min.js';
import { loadSocketIo } from './socket.io.esm.min.js';
// Load msgpack library if needed
if (typeof msgpack === 'undefined') {
console.log('cloudservermodel.js: Loading msgpack library');
loadMsgpack();
}
// Load socketio library if needed
if (typeof io === 'undefined') {
console.log('cloudservermodel.js: Loading socket.io library');
loadSocketIo();
}
// Socketio client API:
// Socket Class: Fundamental class for interacting with the server, belonging to a certain namespace and using an underlying Manager for communication.
// Events
// 'connect': Fired upon connection and reconnection.
// 'connect_error': Fired upon connection failure.
// 'disconnect': Fired upon disconnection.
// Attributes
// socket.active: Indicates if the socket will automatically try to reconnect.
// socket.connected: Indicates if the socket is currently connected to the server.
// socket.disconnected: Indicates if the socket is currently disconnected from the server.
// socket.id: Unique identifier for the socket session, set after the connect event.
// socket.io: Reference to the underlying Manager.
// socket.recovered: Indicates if the connection state was successfully recovered during the last reconnection.
// Methods
// socket.close(): Synonym for socket.disconnect().
// socket.compress(value): Sets a modifier for event emission to determine if the data will be compressed.
// socket.connect(): Manually connects or reconnects the socket.
// socket.disconnect(): Manually disconnects the socket.
// socket.emit(eventName[, ...args][, ack]): Emits an event to the socket with the specified name and arguments.
// socket.emitWithAck(eventName[, ...args]): Promised-based version of emitting and expecting an acknowledgment from the server.
// socket.listeners(eventName): Returns the array of listeners for the specified event.
// socket.listenersAny(): Returns the list of registered catch-all listeners.
// socket.listenersAnyOutgoing(): Returns the list of registered catch-all listeners for outgoing packets.
// socket.off([eventName][, listener]): Removes the specified listener from the listener array for the event.
// socket.offAny([listener]): Removes all catch-all listeners or a specified listener.
// socket.offAnyOutgoing([listener]): Removes all catch-all listeners for outgoing packets or a specified listener.
// socket.on(eventName, callback): Registers a new handler for the specified event.
// socket.onAny(callback): Registers a new catch-all listener.
// socket.onAnyOutgoing(callback): Registers a new catch-all listener for outgoing packets.
// socket.once(eventName, callback): Adds a one-time listener function for the specified event.
// socket.open(): Synonym for socket.connect().
// socket.prependAny(callback): Registers a new catch-all listener added to the beginning of the listeners array.
// socket.prependAnyOutgoing(callback): Registers a new catch-all listener for outgoing packets added to the beginning of the listeners array.
// socket.send([...args][, ack]): Sends a message event.
// socket.timeout(value): Sets a modifier for a subsequent event emission to set a callback with an error if no acknowledgment is received within the given time.
/**
* @class
* @classdesc This class handles interactions with a cloud server for running inference.<br>
* It uses socket.io for communication with the server uses internal queues to provide results in order.<br>
* Use this class to send image data to the server, retrieve predictions, and manage model parameters.<br>
* Internally, it connects through socketio as soon as the model instance is created. It only disconnects when the `cleanup` method is called.<br>
* @example <caption>Usage:</caption>
* let model = zoo.loadModel('model_name', {});
* - Use the `predict` method for inference with individual data items or `predict_batch` for multiple items.
* let result = await model.predict(someImage);
* for await (let result of model.predict_batch(someDataGeneratorFn)) { ... }
*/
class CloudServerModel {
/**
* Do not call the constructor directly. Use the `loadModel` method of an AIServerZoo instance to create an AIServerModel.
* @constructor
* @param {Object} options - Options for initializing the model.
* @param {string} options.modelName - The name of the model to load.
* @param {string} options.serverUrl - The URL of the server to connect to.
* @param {Object} options.modelParams - The parameters for the model.
* @param {string} options.token - The authentication token for the model.
* @param {number} [options.max_q_len=80] - The maximum length of the internal queues.
* @param {function} [options.callback=null] - The callback function to call when results are available.
* @param {Array<string>} [options.labels=null] - The labels for the model.
* @param {Object} [additionalParams=null] - Additional parameters for the model.
*/
constructor({ modelName, serverUrl, modelParams, token, max_q_len = 80, callback = null, labels = null, systemDeviceTypes }, additionalParams) {
// console.log(
// 'CloudServerModel: Entered constructor.',
// 'Parameters:',
// 'modelName=', modelName,
// 'serverUrl=', serverUrl,
// 'modelParams=', modelParams,
// 'token=', token,
// 'max_q_len=', max_q_len,
// 'callback=', callback,
// 'labels=', labels,
// 'additionalParams=', additionalParams
// );
this.debugLogsEnabled = true;
this.modelName = modelName;
this.serverUrl = serverUrl;
this.modelParams = modelParams;
this.token = token;
this.max_q_len = max_q_len;
this.callback = callback;
this.labels = labels;
this.systemDeviceTypes = systemDeviceTypes;
if (!this.systemDeviceTypes || this.systemDeviceTypes.length === 0) {
throw new DGError("System Device Types are missing from Zoo upon initialization of CloudServerModel class!", "MISSING_SYSTEM_DEVICE_TYPES", {}, "An error occurred during initialization.", "AIServerZoo should have sent these to the model in loadModel().");
}
this._deviceType = null;
this.additionalParams = additionalParams;
this.dirty = false;
this.preProcessor = null;
this.postProcessor = null;
this.configParamsDirty = true;
this.initialized = false;
this.infoQ = new AsyncQueue(max_q_len, 'infoQ');
this.resultQ = new AsyncQueue(max_q_len, 'resultQ');
this.poison = false;
this.finishedSettingAdditionalParams = false;
this.initMemberValues();
this.initializeSocket();
this.lastProcessedMessage = Promise.resolve();
this.mutex = new Mutex();
// temporary variable to store current connection state.
// instead: use this.socket.connected?
this.socketConnected = false;
this.inputFrameNumber = 0; // Current frame number, increments for each frame sent to the server.
this.unorderedResults = new Map();
this.expectedFrameNo = 0; // Keep track of the next expected frame number
this.MAX_SOCKET_WAIT_MS = 10000; // Max Time to wait for the socket connection to be opened before error.
}
/// Initialize some member values from modelParams
initMemberValues() {
if (this.modelParams && this.modelParams["MODEL_PARAMETERS"] && this.modelParams["MODEL_PARAMETERS"].length > 0) {
const parameters = this.modelParams["MODEL_PARAMETERS"][0];
const preProcessParams = this.modelParams["PRE_PROCESS"][0];
this.modelPath = parameters.ModelPath;
// NCHW info is either under 'MODEL_PARAMETERS' or 'PRE_PROCESS'
if (parameters.ModelInputN) {
this.modelInputN = parameters.ModelInputN;
this.modelInputC = parameters.ModelInputC;
this.modelInputH = parameters.ModelInputH;
this.modelInputW = parameters.ModelInputW;
} else if (preProcessParams.InputN) {
this.modelInputN = preProcessParams.InputN;
this.modelInputC = preProcessParams.InputC;
this.modelInputH = preProcessParams.InputH;
this.modelInputW = preProcessParams.InputW;
} else {
throw new DGError("Model Parameters don't contain input height / width.", "MISSING_PARAMS", { parameters, preProcessParams }, "Ensure model parameters include input height and width.", "Check the model's documentation to provide the required input height and width parameters.");
}
}
// Set internal parameters to default, only if the additionalParams weren't handled yet.
if (!this.finishedSettingAdditionalParams) {
// Internal model pre/post processing and inference parameters, initially set to defaults.
// Display Parameters
this._overlayColor = [255, 0, 0];
this._overlayLineWidth = 2;
this._overlayShowLabels = true;
this._overlayShowProbabilities = false;
this._overlayAlpha = 0.75;
this._overlayFontScale = 1.0;
this._autoScaleDrawing = false;
// Input Handling Parameters
this._inputLetterboxFillColor = [0, 0, 0];
this._inputPadMethod = 'letterbox';
this._saveModelImage = false;
this._inputCropPercentage = 1.0;
// Parameters for stats tracking
this._measureTime = false;
this.timeStats = this._measureTime ? new StatsDict() : null;
}
// Assign additional parameters, using our set/get functions. Try to overwrite them by these values, and
// warn the user if that parameter doesn't exist.
// Only do this ONCE: prior to websocket opening.
if (this.additionalParams !== null && this.additionalParams !== undefined && !this.finishedSettingAdditionalParams) {
for (const [key, value] of Object.entries(this.additionalParams)) {
// console.log('initMemberValues(): Setting additional param key', key, 'and value', value);
// Check for the existence of a setter for 'key'
const descriptor = Object.getOwnPropertyDescriptor(Object.getPrototypeOf(this), key);
const hasSetter = descriptor && typeof descriptor.set === 'function';
if (hasSetter) {
try {
// console.log('Invoking setter to set', key, 'to', value);
this[key] = value; // invoke the setter for 'key'
// console.log('Now, the value for', key, 'is:', this[key]);
} catch (error) {
console.warn(`Error using setter for '${key}': ${error.message}`);
}
} else {
console.warn(`Setter for '${key}' does not exist or cannot be used.`);
}
}
}
if (!this.additionalParams?.deviceType) {
// console.log("User did not specify deviceType in additionalParams. Attempting to infer deviceType from Model Parameters.");
try {
this._deviceType = this.getModelParameter('RuntimeAgent') + '/' + this.getModelParameter('DeviceType');
console.log('Set deviceType from Model Parameters:', this.deviceType);
} catch (error) {
throw new DGError('Failed to infer device type from Model Parameters:' + error, "DEVICE_TYPE_INFERENCE_ERROR", { error }, "An error occurred during device type inference.", "Please check the model parameters to ensure they have RuntimeAgent and DeviceType set");
}
} else {
console.log('User set deviceType using additionalParams:', this.deviceType);
}
this.finishedSettingAdditionalParams = true;
// Now set dirty to false.
// Dirty is set to true when these are modified even if model isn't initialized
// yet, so we set it back to false, so we don't reinitialize the websocket connection for no reason.
this.dirty = false;
}
/**
* Predicts the result for a given image. <br>
*
* @example If callback is provided:
* Onresult handler will invoke the callback directly when the result arrives.
*
* @example If callback is not provided:
* The function waits for the resultQ to get a result, then returns it.
* let result = await model.predict(someImage);
*
* @async
* @param {Blob|File|string|HTMLImageElement|HTMLVideoElement|HTMLCanvasElement|ArrayBuffer|TypedArray|ImageBitmap} imageFile
* @param {string} [info=performance.now()] - Unique frame information provided by user (such as frame num). Used for matching results back to input images within callback.
* @param {boolean} [bypassPreprocessing=false] - Whether to bypass preprocessing. Used to send Blob data directly to the socket without any preprocessing.
* @returns {Promise<Object>} The prediction result.
*/
async predict(imageFile, info = '_DEFAULT_FRAME_INFO_', bypassPreprocessing = false) {
let unlockedAlready = true; // only allow one mutex unlock operation per function call.
// passthrough if the error flag is enabled.
if (this.poison) return;
// Outer try catch finally block for handling unexpected errors with cleanup code (e.g. release mutex)
try {
// Check if the model needs to be reloaded
if (this.dirty) {
this.log('predict(): dirty flag caught. modelParams object prior to reset:', this.modelParams);
await this.handleDirtyFlag();
}
// Generate unique info from the imageFile here if needed
if (info == '_DEFAULT_FRAME_INFO_') {
info = `frame_${performance.now()}`;
}
// Instantly push the frame info to infoQ
await this.infoQ.push({ info });
// Attempt to lock the mutex with a timeout
const mutexLockPromise = this.mutex.lock();
const mutexTimeout_ms = 10000;
const mutexTimeoutPromise = this.timeoutPromise(mutexTimeout_ms, () => this.mutex.cancelLock(mutexLockPromise));
// Wait for either the mutex to be acquired, or for the timeout to occur
await Promise.race([
mutexLockPromise.then(() => { mutexTimeoutPromise.cancel(); }), // Lock acquired
mutexTimeoutPromise // Timeout
]).catch(error => {
throw new DGError("Error during mutex lock / mutex timeout unlock: " + error, "MUTEX_LOCK_TIMEOUT_ERROR", {}, "An error occurred while acquiring the mutex lock.", error);
});
unlockedAlready = false; // Set to false, so that now we are allowed to call unlock
if (bypassPreprocessing) {
// Ensure that the imageFile is a Blob
if (!(imageFile instanceof Blob)) {
throw new DGError("predict(): Bypassed image must be a Blob.", "INVALID_BYPASSED_IMAGE", {}, "An error occurred during image preprocessing.", "Please check the image and try again.");
}
// Attach fake transformationDetails to infoQ.
let fakeTransformationDetails = {
scaleX: 1.0,
scaleY: 1.0,
offsetX: 0,
offsetY: 0
};
let infoQObject = { transformationDetails: fakeTransformationDetails, imageFrame: null, modelImage: null, coreInferenceStartTime: null };
// if measure time and if save image then update infoQ
await this.infoQ.update(
item => item.info === info,
infoQObject
);
// Directly send the image to the socket.
await this.waitForSocketConnection(); // Timeout error here will be caught in outer trycatch
this.log('predict(): Sending bypassed image to socket with info:', info);
this.predictEmit(imageFile, info);
} else {
// Validate / send the image frame
await this.validateAndSendFrame(imageFile, info);
}
this.mutex.unlock();
unlockedAlready = true;
if (this.callback == null) {
return await this.resultQ.pop(); // does not instantly complete the try block. waits for the Promise to resolve before returning the resolved value
}
} catch (error) {
this.poison = true; // Set the error flag.
throw new DGError("An error occurred during predict:" + error, "PREDICT_ERROR", { error }, "An error occurred during predict.");
} finally {
if (!unlockedAlready) this.mutex.unlock();
}
}
// helper function to emit the predict event to the socket
// imageFile: blob
// frameInfo: frame info, not used yet
async predictEmit(imageFile, frameInfo) {
// console.log('Entered predictEmit(). Current frameNumber:', this.inputFrameNumber);
/**
The packet sent in Python is a tuple with two elements: the packed data and a string representation of the frame number
data_packed = msgpack.packb(data if isinstance(data, list) else [data])
packet = (b"0" + data_packed, str(self._input_frame_number))
try:
self._sio.emit("predict", packet)
*/
//-------------------------------------------------------------------------------------
// Flow: First we prepare a packet with the image data and the frame number.
// The image data IS packed with a 0 prefix.
// Then, it goes to the custom encoder serialization to get encoded AGAIN.
// Example emit() input: (Before the final custom encoder serialization):
// From python: {'type': 2, 'data': ['predict', b'0\x91\xc5\xb82\xff\xd8\xff\xe0\x00\x10JFIF\x0... 9f\xff\xd9', '0'], 'nsp': '/'}
// So, the final packet sent to server is encoded with the custom encoder.
//-------------------------------------------------------------------------------------
// emit() input:
// Create a final packet that will be:
// [packetName, packedImage, String(frameNum)];
// 1. Grab the fixed-length raw binary data buffer from the imageFile
// 2. Pack this binary data with msgpack
const arrayBuffer = await imageFile.arrayBuffer();
// eslint-disable-next-line no-undef
const codec = msgpack.createCodec({
binarraybuffer: true,
uint8array: true
});
// eslint-disable-next-line no-undef
let packedImage = msgpack.encode([arrayBuffer], { codec: codec });
// packedImage is now a Uint8Array:
// array-like view of underlying raw binary data buffer. Values are 0-255 and we can now manipulate this object.
// 2.5. Deep clone the Uint8Array.
// packedImage = structuredClone(packedImage);
// 3. Prepend binary form of the ASCII character 0.
const newArray = new Uint8Array(packedImage.length + 1);
newArray[0] = 48; // ASCII 0
newArray.set(packedImage, 1);
// 4. Prepare final data type from the Uint8Array
let finalData = newArray.buffer; // For ArrayBuffer
// Emitting the packet
try {
// If we are measuring time, then we need to start core inference right when we start to send
// out the packet
if (this._measureTime){
await this.infoQ.update(
item => item.info === frameInfo,
{ coreInferenceStartTime: performance.now() }
);
}
// When the emit method is called, the Encoder class's encode method is used to encode the packet.
// The customDumps function is called to serialize the packet using msgpack.
// emit() -> Encoder.encode() -> customDumps()
// debugger;
// console.log('Emitting packet for frame number:', this.inputFrameNumber);
this.socket.emit("predict", finalData, String(this.inputFrameNumber)); // --> data will be Array(3)["predict", ArrayBuffer, "0"]
// this.socket.compress(false).emit("predict", packedImage, String(this.inputFrameNumber));
// console.log('Called socket emit');
} catch (error) {
console.error('Transport error while sending packet to socket:', error);
// transport error will be caught on reconnect..
} finally {
this.inputFrameNumber += 1;
}
}
async validateAndSendFrame(imageFile, info) {
// Input frame validation / conversion
let imageFrame = await this.validateAndConvertInputFrame(imageFile);
if (this.configParamsDirty || !this.preProcessor || !this.postProcessor) {
await this.initPrePostProcessors();
}
await this.preprocessAndSend(imageFrame, info);
}
async preprocessAndSend(imageFrame, info = null) {
if (this.poison) return;
try {
if (!imageFrame || !this.modelInputW || !this.modelInputH) {
throw new DGError("preprocessAndSend(): missing input parameters.", "MISSING_INPUT_PARAMETERS_ERROR", {}, "An error occurred during image preprocessing.", "Please check the input and try again.");
}
const startTime = performance.now(); // PERFORMANCE LOGGING
const { blob: resizedBlob, transformationDetails } = await this.preProcessor.resizeImage(imageFrame);
const preProcTime = performance.now() - startTime;
this.log('resizeImage() took:', preProcTime, 'ms.'); // PERFORMANCE LOGGING
if (!transformationDetails) {
throw new DGError("Transformation details are missing after resizeImage()", "MISSING_TRANSFORMATION_DETAILS_ERROR", {}, "An error occurred during image preprocessing.", "Please check the image and try again.");
}
if (!transformationDetails.scaleX || !transformationDetails.scaleY) {
throw new DGError("Scale factors are missing after resizeImage()", "MISSING_SCALE_FACTORS_ERROR", {}, "An error occurred during image preprocessing.", "Please check the image and try again.");
}
if (typeof transformationDetails.offsetX === 'undefined' || typeof transformationDetails.offsetY === 'undefined') {
throw new DGError("Offset values are missing after resizeImage()", "MISSING_OFFSET_VALUES_ERROR", {}, "An error occurred during image preprocessing.", "Please check the image and try again.");
}
if (!resizedBlob) {
throw new DGError("preprocessAndSend(): resizedBlob is null or undefined", "RESIZED_BLOB_NULL_ERROR", {}, "An error occurred during image preprocessing.", "Please check the image and try again.");
}
if (imageFrame instanceof HTMLVideoElement) {
imageFrame = null;
}
// Adding things to infoQ based on if set true or not
const infoQObject = {transformationDetails, imageFrame: imageFrame, modelImage: null, coreInferenceStartTime: null};
if (this._saveModelImage) {
infoQObject.modelImage = resizedBlob;
}
if (this._measureTime) {
if (!this.timeStats) {
this.timeStats = new StatsDict();
}
this.timeStats.addNewStat("PreprocessDuration_ms", preProcTime);
}
await this.infoQ.update(
item => item.info === info, infoQObject
);
await this.waitForSocketConnection(); // Timeout error here will be caught in outer trycatch
this.log('preprocessAndSend(): Sending blob to socket with info:', info);
this.predictEmit(resizedBlob, info);
} catch (error) {
// Is poison enabled? Then we ignore the error and stop.
if (this.poison) return;
this.poison = true; // Set the error flag.
throw new DGError("preprocessAndSend(): Failed to preprocess the image:" + error, "PREPROCESS_IMAGE_FAILED", { error }, "Failed to preprocess the image.");
}
}
/**
* Predicts results for a batch of data. Will yield results if a callback is not provided.
* @async
* @generator
*
* @example The function asynchronously processes results. If a callback is not provided, it will yield results.
* for await (let result of model.predict_batch(data_source)) { console.log(result); }
* @param {AsyncIterable} data_source - An async iterable data source.
* @param {boolean} [bypassPreprocessing=false] - Whether to bypass preprocessing.
* @yields {Object} The prediction result.
*/
async * predict_batch(data_source, bypassPreprocessing = false) {
if (this.poison) return;
try {
if (this.dirty) {
this.log('predict_batch(): dirty flag caught. modelParams object prior to reset:', this.modelParams);
await this.handleDirtyFlag();
}
let prevInfo = '_'; // Duplicate info check, probably not the best impl but works for now
for await (let [data, info] of data_source) {
if (info === prevInfo) { // Duplicate info check, probably not the best impl but works for now
console.warn('predict_batch(): Duplicate info detected:', info); // Duplicate info check, probably not the best impl but works for now
info = info + '_dup'; // Duplicate info check, probably not the best impl but works for now
} // Duplicate info check, probably not the best impl but works for now
prevInfo = info; // Duplicate info check, probably not the best impl but works for now
await this.infoQ.push({ info });
if (bypassPreprocessing) {
if (!(data instanceof Blob)) {
throw new DGError("predict_batch(): Bypassed image must be a Blob.", "INVALID_BYPASSED_IMAGE", {}, "An error occurred during image preprocessing.", "Please check the image and try again.");
}
let fakeTransformationDetails = {
scaleX: 1.0,
scaleY: 1.0,
offsetX: 0,
offsetY: 0
};
await this.infoQ.update(
item => item.info === info,
{ transformationDetails: fakeTransformationDetails, imageFrame: null }
);
await this.waitForSocketConnection();
this.predictEmit(data, info);
} else {
await this.validateAndSendFrame(data, info);
}
if (this.callback == null) {
if (!this.resultQ.empty()) {
yield await this.resultQ.pop();
}
}
}
if (this.callback == null) {
while (!this.infoQ.empty() || !this.resultQ.empty()) {
yield await this.resultQ.pop();
}
}
} catch (error) {
// Is poison enabled? Then we ignore the error and stop.
if (this.poison) return;
this.poison = true; // Set the error flag.
throw new DGError('An error occurred during predict_batch:' + error, "PREDICT_BATCH_ERROR", { error }, "An error occurred during predict_batch.");
}
}
async initializePreProcessor() {
if (!this.initialized) await this.waitFor(() => this.initialized);
this.log('(re)setting preprocessor...');
this.preProcessor = null;
this.preProcessor = new PreProcess(this.modelParams, this.constructOverlayConfig());
}
async initializePostProcessor() {
if (!this.initialized) await this.waitFor(() => this.initialized);
this.log('(re)setting postprocessor...');
this.postProcessor = null;
this.postProcessor = new PostProcess(this.modelParams, this.constructOverlayConfig());
}
async initPrePostProcessors() {
await this.initializePreProcessor();
await this.initializePostProcessor();
this.configParamsDirty = false;
}
// Constructs a config from all of the internal parameters. This is passed to pre/post processors
// Compiles every internal parameter settable by user
constructOverlayConfig() {
return {
labels: this.labels,
overlayColor: this._overlayColor,
overlayLineWidth: this._overlayLineWidth,
overlayShowLabels: this._overlayShowLabels,
overlayShowProbabilities: this._overlayShowProbabilities,
overlayAlpha: this._overlayAlpha,
overlayFontScale: this._overlayFontScale,
inputLetterboxFillColor: this._inputLetterboxFillColor,
inputPadMethod: this._inputPadMethod,
saveModelImage: this._saveModelImage,
inputCropPercentage: this._inputCropPercentage,
autoScaleDrawing: this._autoScaleDrawing
};
}
async initializeSocket() {
// console.log('cloudservermodel.js: Entered initializeSocket()');
const nameParts = this.modelName.split("/");
// console.log('initializeSocket(): Constructing io object.');
try {
// Debug: connect to cs2.
// this.serverUrl = 'https://cs2.degirum.com';
// this.token = '....';
let path = "/api/v2/socket.io";
// Socket init:
// eslint-disable-next-line no-undef
this.socket = io(this.serverUrl, {
parser: parser,
reconnection: false,
path: path,
transports: ['websocket'],
timeout: 5000,
auth: {
token: this.token,
organization: nameParts[0],
zoo: nameParts[1],
model: nameParts[2],
model_params: JSON.stringify(this.modelParams),
frame_queue_depth: this.max_q_len,
inference_timeout_s: 100.0
}
});
// console.log('Socket io() constructor params:');
// console.log('serverUrl:', this.serverUrl);
// console.log('parser:', parser);
// console.log('reconnection:', false);
// console.log('path:', "/api/v2/socket.io");
// console.log('transports:', ['websocket']);
// console.log('timeout:', 5000);
// console.log('auth:', {
// token: this.token,
// organization: nameParts[0],
// zoo: nameParts[1],
// model: nameParts[2],
// model_params: JSON.stringify(this.modelParams),
// frame_queue_depth: this.max_q_len,
// inference_timeout_s: 100.0
// });
// After initialization, server sends us something like: {'type': 0, 'data': {'sid': 'XNBVcBDMVBc_kjDSFH3'}, 'nsp': '/'}
// console.log('SocketIO Protocol number:', io.protocol); // undefined???
// Manager object events (this.socket.io)
this.socket.io.on("reconnect", (attempt) => {
console.log('socket.io: Reconnected to the server. Attempt:', attempt);
});
this.socket.io.on("error", (error) => {
console.log('socket.io: Error connecting to the server:', error);
});
this.socket.io.on("reconnect_attempt", (attempt) => {
console.log('socket.io: Reconnect attempt:', attempt);
});
this.socket.io.on("reconnect_error", (error) => {
console.log('socket.io: Reconnect error:', error);
});
this.socket.io.on("reconnect_failed", () => {
console.log('socket.io: Reconnect failed.');
});
// Socket object events (this.socket)
this.socket.on("connect", () => {
console.log('socket: Connected to the server.');
this.socketConnected = true;
});
this.socket.on("connect_error", (error) => {
if (this.socket.active) {
// temporary failure, the socket will automatically try to reconnect
console.log('socket: Connection error, will automatically try to reconnect:', error);
} else {
// the connection was denied by the server
// in that case, `socket.connect()` must be manually called in order to reconnect
console.log('socket: Connection error, must manually reconnect:', error);
}
});
this.socket.on("disconnect", (reason, details) => {
console.log('socket: Disconnected from the server. Reason:', reason, 'Details:', details);
this.socketConnected = false;
if (this.socket.active) {
// temporary failure, the socket will automatically try to reconnect
console.log('socket: Connection error, will automatically try to reconnect:', reason);
} else {
// the connection was denied by the server
// in that case, `socket.connect()` must be manually called in order to reconnect
console.log('socket: Connection error, must manually reconnect:', reason);
}
});
this.socket.on("predict_result", (data, frame_no) => {
this.handlePredictResult(data, frame_no);
});
// Log all engine packets
this.socket.io.engine.on('packet', packet => {
if (!packet.type === 'ping') console.log('socket.io.engine: packet received:', packet);
});
// Catch all engine packets with the .type === 'open'
this.socket.io.engine.on('open', () => {
console.log('SocketIO engine open packet received');
// debug log
if (this.socket.connected) { console.log('this.socket.connected is true.'); }
else { console.log('this.socket.connected is false.'); }
});
this.initialized = true;
} catch (error) {
console.error('CloudServerModel: Error initializing socket:', error);
throw new DGError("Failed to initialize socket: " + error, "SOCKET_INIT_ERROR", { error }, "Failed to initialize socket.");
}
}
/////////////////// Internal Parameter Setters / Getters ///////////////////
// Internal parameters can be set / get without explicit getter / setter calling:
// model.overlayShowLabels = false; // This actually calls the setter method
// console.log(model.overlayShowLabels); // This calls the getter method
set deviceType(value) {
console.log('Entered deviceType setter with value:', value);
if (!value || (typeof value !== 'string' && !Array.isArray(value))) {
throw new TypeError("deviceType should be a string or an array of strings. e.g. 'RUNTIME/DEVICE' or ['RUNTIME1/DEVICE1', 'RUNTIME2/DEVICE2'].");
}
let currentDevice = this.modelParams.DEVICE[0]['RuntimeAgent'] + '/' + this.modelParams.DEVICE[0]['DeviceType'];
if (currentDevice === value) {
console.warn('Device type is already set to:', value);
if (!this._deviceType) this._deviceType = value;
return;
}
const checkDeviceType = (deviceType) => {
const agentDevice = deviceType.split('/');
if (agentDevice.length !== 2) {
throw new DGError("deviceType should be in the format 'RUNTIME/DEVICE'.", "INVALID_DEVICE_TYPE", {}, "An error occurred while setting the device type.", "Please check the device type and try again.");
}
if (this.supportedDeviceTypes.includes(deviceType)) {
return agentDevice;
}
return null;
};
const values = Array.isArray(value) ? value : [value];
let agentDevice = null;
for (const deviceType of values) {
agentDevice = checkDeviceType(deviceType);
if (agentDevice !== null) {
break;
}
}
if (agentDevice === null) {
throw new Error(`None of the device types in the list ${values} are supported by the model ${this.modelName}. Supported device types are: ${this.supportedDeviceTypes}.`);
}
this.setModelParameter('RuntimeAgent', agentDevice[0]);
this.setModelParameter('DeviceType', agentDevice[1]);
this._deviceType = agentDevice.join('/');
this.log(`Device type set to ${this._deviceType}`);
}
get deviceType() {
return this._deviceType;
}
matchSupportedDevices(modelSupportedTypes, systemDeviceTypes) {
const matchesWildcard = (pattern, type) => {
const [patternRuntime, patternDevice] = pattern.split('/');
const [typeRuntime, typeDevice] = type.split('/');
const runtimeMatches = patternRuntime === '*' || patternRuntime === typeRuntime;
const deviceMatches = patternDevice === '*' || patternDevice === typeDevice;
return runtimeMatches && deviceMatches;
};
return systemDeviceTypes.filter(systemType =>
modelSupportedTypes.some(modelType => matchesWildcard(modelType, systemType))
);
}
get supportedDeviceTypes() {
let modelSupportedTypes;
try {
modelSupportedTypes = this.getModelParameter('SupportedDeviceTypes');
modelSupportedTypes = modelSupportedTypes.split(',').map(type => type.trim());
} catch (err) {
modelSupportedTypes = this.systemDeviceTypes;
}
return this.matchSupportedDevices(modelSupportedTypes, this.systemDeviceTypes);
}
// labelWhitelist
set labelWhitelist(value) {
if (!Array.isArray(value)) {
throw new TypeError("labelWhitelist should be an array of strings. e.g. ['cat', 'dog'].");
}
for (const label of value) {
if (typeof label !== 'string') {
throw new TypeError("All items in labelWhitelist must be strings. e.g. ['cat', 'dog'].");
}
}
this._labelWhitelist = value;
}
get labelWhitelist() {
return this._labelWhitelist;
}
// labelBlacklist
set labelBlacklist(value) {
if (!Array.isArray(value)) {
throw new TypeError("labelBlacklist should be an array of strings. e.g. ['cat', 'dog'].");
}
for (const label of value) {
if (typeof label !== 'string') {
throw new TypeError("All items in labelBlacklist must be strings. e.g. ['cat', 'dog'].");
}
}
this._labelBlacklist = value;
}
get labelBlacklist() {
return this._labelBlacklist;
}
/////////////////// Display Parameters ///////////////////
// overlayColor
set overlayColor(value) {
if (!Array.isArray(value)) {
throw new TypeError("overlayColor should be an array.");
}
// Validate if it's a list of [R, G, B] triplets or just a single triplet
const isValidTriplet = (triplet) => {
return Array.isArray(triplet) &&
triplet.length === 3 &&
triplet.every(color => typeof color === 'number' && color >= 0 && color <= 255);
};
if (!isValidTriplet(value)) {
if (!value.every(isValidTriplet)) {
throw new TypeError("overlayColor should either be a single [R, G, B] triplet or a list of such triplets.");
}
}
this.configParamsDirty = true;
this._overlayColor = value;
}
get overlayColor() {
return this._overlayColor;
}
// overlayLineWidth
set overlayLineWidth(value) {
if (typeof value !== 'number' || value <= 0) {
throw new TypeError("overlayLineWidth should be a positive number.");
}
this.configParamsDirty = true;
this._overlayLineWidth = value;
}
get overlayLineWidth() {
return this._overlayLineWidth;
}
// overlayShowLabels
set overlayShowLabels(value) {
if (typeof value !== 'boolean') {
throw new TypeError("overlayShowLabels should be a boolean value.");
}
this.configParamsDirty = true;
this._overlayShowLabels = value;
}
get overlayShowLabels() {
return this._overlayShowLabels;
}
// overlayShowProbabilities
set overlayShowProbabilities(value) {
if (typeof value !== 'boolean') {
throw new TypeError("overlayShowProbabilities should be a boolean value.");
}
this.configParamsDirty = true;
this._overlayShowProbabilities = value;
}
get overlayShowProbabilities() {
return this._overlayShowProbabilities;
}
// overlayAlpha
set overlayAlpha(value) {
if (typeof value !== 'number' || value < 0 || value > 1) {
throw new TypeError("overlayAlpha should be a number between 0 and 1.");
}
this.configParamsDirty = true;
this._overlayAlpha = value;
}
get overlayAlpha() {
return this._overlayAlpha;
}
// overlayFontScale
set overlayFontScale(value) {
if (typeof value !== 'number' || value <= 0) {
throw new TypeError("overlayFontScale should be a positive number.");
}
this.configParamsDirty = true;
this._overlayFontScale = value;
}
get overlayFontScale() {
return this._overlayFontScale;
}
/////////////////// Input Handling Parameters ///////////////////
set measureTime(value) {
if (typeof value !== 'boolean') {
throw new DGError(`Value of measureTime (${value}) is not of type Boolean.`, "TYPE_ERROR");
}
this.timeStats = value ? new StatsDict() : null;
this._measureTime = value;
}
get measureTime() {
return this._measureTime;
}
// inputLetterboxFillColor
set inputLetterboxFillColor(value) {
// Validation for single [R, G, B] triplet
if (!Array.isArray(value) ||
value.length !== 3 ||
!value.every(color => typeof color === 'number' && color >= 0 && color <= 255)) {
throw new TypeError("inputLetterboxFillColor should be a single [R, G, B] triplet.");
}
this.configParamsDirty = true;
this._inputLetterboxFillColor = value;
}
get inputLetterboxFillColor() {
return this._inputLetterboxFillColor;
}
// inputPadMethod
set inputPadMethod(value) {
if (typeof value !== 'string' ||
!["stretch", "letterbox", "crop-first", "crop-last"].includes(value)) {
throw new TypeError("inputPadMethod should be one of 'stretch', 'letterbox', 'crop-first', or 'crop-last'.");
}
this.configParamsDirty = true;
this._inputPadMethod = value;
}
get inputPadMethod() {
return this._inputPadMethod;
}
// saveModelImage
set saveModelImage(value) {
if (typeof value !== 'boolean') {
throw new TypeError("saveModelImage should be a boolean value.");
}
this.configParamsDirty = true;
this._saveModelImage = value;
}
get saveModelImage() {
return this._saveModelImage;
}
// inputCropPercentage
set inputCropPercentage(value) {
if (typeof value !== 'number' || value < 0 || value > 1) {
throw new TypeError("inputCropPercentage should be a number between 0 and 1.");
}
this.configParamsDirty = true;
this._inputCropPercentage = value;
}
get inputCropPercentage() {
return this._inputCropPercentage;
}
/**
* Boolean to auto scale font / line width based on image size.
* @type {boolean}
* @private
*/
set autoScaleDrawing(value) {
if (typeof value !== 'boolean') {
throw new TypeError("autoScaleDrawing should be a boolean value.");
}
this.configParamsDirty = true;
this._autoScaleDrawing = value;
console.log('Set autoScaleDrawing:', value);
}
/**
* Gets whether to auto scale font / line width based on image size.
* @type {boolean}
* @private
*/
get autoScaleDrawing() {
return this._autoScaleDrawing;
}
/////////////////// Inference Parameters ///////////////////
// These just wrap setModelParameter() with input handling
// cloudToken
set cloudToken(value) {
if (typeof value !== 'string') {
throw new TypeError("cloudToken should be a string.");
}
this.setModelParameter('CloudToken', value);
}
get cloudToken() {
return this.getModelParameter('CloudToken');
}
// cloudURL
set cloudURL(value) {
// console.log('Setting cloudURL:', value);
if (typeof value !== 'string') {
throw new TypeError("cloudURL should be a string.");
}
this.setModelParameter('CloudURL', value);
// Parse the URL and reconstruct it without the path (patch for HttpServer not expecting a path)
// try {
// const urlObj = new URL(value);
// const urlWithoutPath = urlObj.origin; // origin includes protocol and host
// this.setModelParameter('CloudURL', urlWithoutPath);
// } catch (e) {
// throw new DGError("Invalid URL provided.", "INVALID_URL", {}, "Invalid URL provided.");
// }
}
get cloudURL() {
return this.getModelParameter('CloudURL');
}
// outputConfidenceThreshold
set outputConfidenceThreshold(value) {
if (typeof value !== 'number' || value < 0 || value > 1) {
throw new TypeError("outputConfidenceThreshold should be a number between 0 and 1.");
}
this.setModelParameter('OutputConfThreshold', value);
}
get outputConfidenceThreshold() {
return this.getModelParameter('OutputConfThreshold');
}
// outputMaxDetections
set outputMaxDetections(value) {
if (typeof value !== 'number' || !Number.isInteger(value)) {
throw new TypeError("outputMaxDetections should be an integer.");
}
this.setModelParameter('MaxDetections', value);
}
get outputMaxDetections() {
return this.getModelParameter('MaxDetections');
}
// outputMaxDetectionsPerClass
set outputMaxDetectionsPerClass(value) {
if (typeof value !== 'number' || !Number.isInteger(value)) {
throw new TypeError("outputMaxDetectionsPerClass should be an integer.");
}
this.setModelParameter('MaxDetectionsPerClass', value);
}
get outputMaxDetectionsPerClass() {
return this.getModelParameter('MaxDetectionsPerClass');
}
// outputMaxClassesPerDetection
set outputMaxClassesPerDetection(value) {
if (typeof value !== 'number' || !Number.isInteger(value)) {
throw new TypeError("outputMaxClassesPerDetection should be an integer.");
}
this.setModelParameter('MaxClassesPerDetection', value);
}
get outputMaxClassesPerDetection() {
return this.getModelParameter('MaxClassesPerDetection');
}
// outputNmsThreshold
set outputNmsThreshold(value) {
if (typeof value !== 'number' || value < 0 || value > 1) {
throw new TypeError("outputNmsThreshold should be a number between 0 and 1.");
}
this.setModelParameter('OutputNMSThreshold', value);
}
get outputNmsThreshold() {
return this.getModelParameter('OutputNMSThreshold');
}
// outputPoseThreshold
set outputPoseThreshold(value) {
if (typeof value !== 'number' || value < 0 || value > 1) {
throw new TypeError("outputPoseThreshold should be a number between 0 and 1.");
}
this.setModelParameter('OutputConfThreshold', value); // set OutputConfThreshold with the value (not pose threshold)
}
get outputPoseThreshold() {
return this.getModelParameter('OutputConfThreshold');
}
// outputPostprocessType
set outputPostprocessType(value) {
const validValues = ["Classification", "Detection", "DetectionYolo", "PoseDetection", "HandDetection", "FaceDetect", "Segmentation", "BodyPix", "Python", "None"];
if (typeof value !== 'string' || !validValues.includes(value)) {
throw new TypeError("outputPostprocessType should be one of the specified valid string values.");
}
this.setModelParameter('OutputPostprocessType', value);
}
get outputPostprocessType() {
return this.getModelParameter('OutputPostprocessType');
}
// outputTopK
set outputTopK(value) {
if (typeof value !== 'number' || !Number.isInteger(value)) {
throw new TypeError("outputTopK should be an integer.");
}
this.setModelParameter('OutputTopK', value);
}
get outputTopK() {
return this.getModelParameter('OutputTopK');
}
// outputUseRegularNms
set outputUseRegularNms(value) {
if (typeof value !== 'boolean') {
throw new TypeError("outputUseRegularNms should be a boolean.");
}
this.setModelParameter('UseRegularNMS', value);
}
get outputUseRegularNms() {
return this.getModelParameter('UseRegularNMS');
}
// Helper function to wait for up to 1 second for some condition
waitFor(conditionFunction, timeout = 1000, interval = 10) {
const poll = resolve => {
if (conditionFunction()) resolve();
else if (timeout > 0) setTimeout(() => poll(resolve), interval);
else throw new DGError("Timed out waiting.", "WAIT_TIMEOUT", {}, "Timed out waiting.");
};
return new Promise(poll);
}
// Return the modelParams JSON object - FROM THE CLASS
async getModelParameters() {
if (this.socket && this.modelParams) {
return (this.modelParams);
} else {
throw new DGError("Model parameters are not yet initialized for this model!", "MODEL_PARAMETERS_NOT_INITIALIZED", {}, "Model parameters are not yet initialized for this model!");
}
}
// Setter for updating a key withing modelParams and setting the dirty flag
// Designed to only modify leaf nodes with primitive values in the JSON
async setModelParameter(key, value) {
this.log('setModelParameter(). Attempting to update:', key, 'to value:', value);
let updated = false;
try {
// Ensure modelParams exists
if (!this.modelParams) {
throw new DGError("Model parameters are not initialized!", "MODEL_PARAMETERS_NOT_INITIALIZED", {}, "Model parameters are not initialized. Please initialize the model parameters before updating.");
}
// Check for top-level key
if (Object.prototype.hasOwnProperty.call(this.modelParams, key)) {
this.log('Top-level key found! Updating key to:', value);
this.modelParams[key] = value;
this.dirty = true;
updated = true;
} else if (key === 'CloudToken' || key === 'CloudURL') { // TEMPORARY PATCH - Cloud doesn't return FULL model params like our websocket does, so we manually add cloudURL/token if missing
// We don't need to update cloud token / URL for params to pass to IO.
// They are empty when constructing socket io instance.
// this.modelParams[key] = value;
this.dirty = true;
updated = true;
} else {
// Try setting the value for each top-level key
for (const topLevelKey in this.modelParams) {
if (this.modelParams[topLevelKey] && this.modelParams[topLevelKey][0] && Object.prototype.hasOwnProperty.call(this.modelParams[topLevelKey][0], key)) {
this.log('Key found! Updating key to:', value);
// Updating local copy of model params is now done on confirmation message from websocket in initializeSocket()
// Need to update local copy anyway, even if it will be overwritten by next lazy reload upon predict()
// This is so querying the model params after user changes parameter without performing inference
// will yield expected new model params, not old unchanged params object
this.log('setModelParameter(): Updating local modelParams copy, setting', this.modelParams[topLevelKey][0][key], 'to', value);
this.modelParams[topLevelKey][0][key] = value;
this.dirty = true;
updated = true;
break;
}
}
}
} catch (error) {
throw new DGError(`Failed to set a parameter: ${error}`, "SET_PARAMETER_FAILED", {}, "Failed to set a parameter.");
}
// If not updated, log an error
if (!updated) {
throw new DGError(`Failed to update the parameter. Key "${key}" not found!`, "UPDATE_PARAMETER_FAILED", { key }, `Failed to update the parameter "${key}". Please make sure the key exists.`);
}
}
// Get model parameter from the modelParams JSON associated with this Model instance.
getModelParameter(key) {
this.log('Entered getModelParameter(). Querying value for key:', key);
if (!this.modelParams) {
throw new DGError("Model parameters are not initialized!", "MODEL_PARAMETERS_NOT_INITIALIZED", {}, "Model parameters are not initialized. Please initialize the model parameters before querying.");
}
// Check for top-level key
if (Object.prototype.hasOwnProperty.call(this.modelParams, key)) {
this.log('Top-level key found. Value:', this.modelParams[key]);
return this.modelParams[key];
} else {
// Check in nested structures
for (const topLevelKey in this.modelParams) {
if (this.modelParams[topLevelKey] && this.modelParams[topLevelKey][0] && Object.prototype.hasOwnProperty.call(this.modelParams[topLevelKey][0], key)) {
this.log('Key found in nested structure. Value:', this.modelParams[topLevelKey][0][key]);
return this.modelParams[topLevelKey][0][key];
}
}
}
throw new DGError(`Failed to get the parameter. Key "${key}" not found!`, "GET_PARAMETER_FAILED", { key }, `Failed to get the parameter "${key}". Please make sure the key exists.`);
}
// Method to grab a read only copy of modelParams
modelInfo() {
return JSON.parse(JSON.stringify(this.modelParams));
}
// Method to return label dictionary
labelDictionary() {
return this.labels;
}
// Let's say user loaded a model and later requested to change a parameter.
// If socket.io connection already established, we need to reset the connection in order to
// tell the server about the new parameter
// Close and reinitialize the socket.io connection
async resetSocket() {
// console.log('cloudservermodel.js: Entered resetSocket()');
// We need to also reset the internal frame counters to 0.
this.inputFrameNumber = 0;
this.expectedFrameNo = 0;
if (this.socketConnected) this.socket.disconnect();
this.socketConnected = false;
// Reinitialize the socket (this.modelParams has modifications which will be applied upon reinit)
await this.initializeSocket();
// Reset dirty flag
this.dirty = false;
}
async handleDirtyFlag() {
this.log('handleDirtyFlag(): dirty flag caught. modelParams object prior to reset:', this.modelParams);
if (this.infoQ.empty() && this.resultQ.empty()) {
this.resetSocket();
} else {
await new Promise(resolve => {
const checkQueuesEmptyAndReset = () => {
if (this.infoQ.empty() && this.resultQ.empty()) {
if (this.infoQ.hasEventListener('onPop')) {
this.infoQ.removeEventListener('onPop', checkQueuesEmptyAndReset);
}
if (this.resultQ.hasEventListener('onPop')) {
this.resultQ.removeEventListener('onPop', checkQueuesEmptyAndReset);
}
this.resetSocket();
resolve();
}
};
this.infoQ.addEventListener('onPop', checkQueuesEmptyAndReset);
this.resultQ.addEventListener('onPop', checkQueuesEmptyAndReset);
});
}
}
async handlePredictResult(data, frame_no) {
// Store the result in the unorderedResults map
this.unorderedResults.set(frame_no, data);
// Process any available results in order
this.processOrderedResults();
}
async processOrderedResults() {
while (this.unorderedResults.has(this.expectedFrameNo)) {
// console.log('processOrderedResults(): Processing frame:', this.expectedFrameNo, 'from unorderedResults map.');
let data = this.unorderedResults.get(this.expectedFrameNo);
this.unorderedResults.delete(this.expectedFrameNo);
if (!this.initialized) return;
if (this.poison) return;
if (!data) {
throw new DGError('processOrderedResults(): Prediction result data is null or undefined.', 'DATA_NULL_ERROR', {}, 'Prediction result data is null or undefined.');
}
let combinedResult;
const errorMsg = this.errorCheck(data);
if (errorMsg) {
this.poison = true;
throw new DGError(`Error caught in result object: ${errorMsg}`, "RESULT_ERROR", { error: errorMsg }, "Error caught in result object.");
}
// Here, we won't have an error or anything if modelImage or coreInference aren't instantiated by the popping, it'll just
// be undefined. So as long as undefined checks are performed, we don't have to nest if statements.
// I can change this to just use if statements, if you prefer
const { info, transformationDetails, imageFrame, modelImage, coreInferenceStartTime } = await this.infoQ.pop();
// Logic for filtering objects based on labelWhitelist and labelBlacklist
if (this._labelWhitelist || this._labelBlacklist) {
// dummy check: Does this model even have labels?
if (!this.labels) {
console.warn('labelWhitelist/labelBlacklist is set but this model does not have a label dictionary. Ignoring the labelWhitelist/labelBlacklist.');
} else {
// whitelist set and blacklist set
if (this._labelWhitelist && this._labelBlacklist) {
const filteredData = data.filter(item => this._labelWhitelist.includes(item.label) && !this._labelBlacklist.includes(item.label));
data = filteredData;
}
// whitelist set, blacklist not set
else if (this._labelWhitelist) {
const filteredData = data.filter(item => this._labelWhitelist.includes(item.label));
data = filteredData;
}
// blacklist set, whitelist not set
else {
const filteredData = data.filter(item => !this._labelBlacklist.includes(item.label));
data = filteredData;
}
}
}
if (!transformationDetails && !this.poison) {
// Should be impossible as the infoQ should always have transformation details (we don't send until it's updated...)
// Could be due to duplicate frame info, though.
throw new DGError('processOrderedResults(): Transformation details are null or undefined.', 'TRANSFORMATION_DETAILS_NULL_ERROR', {}, 'Transformation details are null or undefined.');
}
data.scaleX = transformationDetails.scaleX;
data.scaleY = transformationDetails.scaleY;
data.offsetX = transformationDetails.offsetX;
data.offsetY = transformationDetails.offsetY;
// Building out combined result that's sent to resultQ or callback
const resultArray = [data, info];
combinedResult = { result: resultArray, imageFrame };
if (this._measureTime) {
if (coreInferenceStartTime === undefined || coreInferenceStartTime === null) {
throw new DGError("Did not recieve core inference duration start time.", "MISSING_PARAMETER", {}, "", "Please check your internet connection and try again.");
}
if (!this.timeStats) {
this.timeStats = new StatsDict();
}
this.timeStats.addNewStat("CoreInferenceDuration_ms", performance.now() - coreInferenceStartTime);
}
if (this._saveModelImage){
combinedResult.modelImage = modelImage;
}
if (this.callback == null) {
this.resultQ.push(combinedResult);
} else {
this.callback(combinedResult, info);
}
this.expectedFrameNo++; // Increment to expect the next frame
}
}
/**
* Overlay the result onto the image frame and display it on the canvas.
* @async
* @param {Object} combinedResult - The result object combined with the original image frame. This is directly received from `predict` or `predict_batch`
* @param {string|HTMLCanvasElement} outputCanvasName - The canvas to draw the image onto. Either the canvas element or the ID of the canvas element.
* @param {boolean} [justResults=false] - Whether to show only the result overlay without the image frame.
*/
async displayResultToCanvas(combinedResult, outputCanvasName, justResults = false) {
this.log('Entered displayResultToCanvas()');
// Handle incorrect / empty result object
if (!combinedResult || !combinedResult.result) {
throw new DGError('displayResultToCanvas(): Invalid or empty result object, returning', "INVALID_RESULT_OBJECT", {}, "Invalid or empty result object. Please make sure the result object is valid.");
}
// If !combinedResult.imageFrame then it means the input was a video element
// allow it, just set justResults to true
if (!combinedResult.imageFrame) {
justResults = true;
this.log('displayResultToCanvas(): No imageFrame found in combinedResult most likely due to video element inference. Setting justResults to true.');
}
const { result, imageFrame } = combinedResult; // Destructure to extract result and imageFrame
let canvas;
// Input validation for outputCanvasName
if (!outputCanvasName || typeof outputCanvasName !== 'string' || outputCanvasName.trim() === '') {
// also accept HTMLCanvasElement
if (!(outputCanvasName instanceof HTMLCanvasElement)) {
throw new DGError('Invalid outputCanvasName parameter', "INVALID_OUTPUT_CANVAS_NAME", {}, "Invalid outputCanvasName parameter. Please provide a valid outputCanvasName.");
} else {
canvas = outputCanvasName;
}
}
if (!canvas) {
canvas = document.getElementById(outputCanvasName);
}
try {
// Check result for errors
const errorMsg = this.errorCheck(result);
if (errorMsg) {
throw new DGError(`Error in result: ${errorMsg}`, "RESULT_ERROR", { errorMsg }, "Error in result. Please check the result for errors.");
}
// letterbox details attached to result already in onmessage
this.postProcessor.displayResultToCanvas(imageFrame, result, canvas, justResults);
} catch (error) {
throw new DGError("Error in parsing result: ", "PARSE_RESULT_ERROR", {}, "Error in parsing result.");
}
}
async validateAndConvertInputFrame(image) {
if (!image) {
throw new DGError('validateAndConvertInputFrame(): Image must be provided.', "INVALID_IMAGE_INPUT", {}, "Image must be provided.");
}
// Directly passthrough for these image types, as they can be directly used with our resizeImage() implementation:
// HTMLImageElement
// SVGImageElement
// HTMLVideoElement
// HTMLCanvasElement
// ImageBitmap
// OffscreenCanvas
if (image instanceof HTMLImageElement || image instanceof SVGImageElement || image instanceof HTMLVideoElement || image instanceof HTMLCanvasElement || image instanceof ImageBitmap || image instanceof OffscreenCanvas) {
return image;
}
// For Blob, ImageData, and File types, we use createImageBitmap()
if (image instanceof Blob || image instanceof ImageData) {
// Blob and ImageData are valid input types, but we need to convert them to an ImageBitmap
const imageBitmap = await createImageBitmap(image);
return imageBitmap;
}
if (image instanceof File) {
// Check if the file is an image
if (!image.type.startsWith('image/')) {
throw new DGError('validateAndConvertInputFrame(): input image is a File but is not an image.', "INVALID_IMAGE_INPUT", {}, "File is not an image.");
}
// Convert the File to an ImageBitmap
const imageBitmap = await createImageBitmap(image);
return imageBitmap;
}
// Handle Data URLs, image URLs, base64 strings, ArrayBuffers, and typed arrays
if (typeof image === 'string') {
// Data URL
if (image.startsWith('data:')) {
return await this.convertDataUrlToImageBitmap(image);
}
try {
// TODO: Need some more robust way to validate URLs...
new URL(image); // This will throw an error if `image` is not a valid URL
// if (image.startsWith('http'))
} catch (error) {
// If here, the string is neither a Data URL nor a valid URL, so it should be a base64 string
return await this.convertBase64ToImageBitmap(image);
}
// Fetching image from URL
return await this.convertImageURLToImageBitmap(image);
}
if (image instanceof ArrayBuffer) {
return await this.convertArrayBufferToImageBitmap(image);
}
if (image instanceof Uint8Array || image instanceof Uint16Array || image instanceof Float32Array) {
return await this.convertTypedArrayToImageBitmap(image);
}
throw new DGError('Invalid image input type, it is: ' + typeof (image), "INVALID_IMAGE_INPUT", {}, "Invalid image input type.");
}
async convertDataUrlToImageBitmap(dataUrl) {
if (!dataUrl.startsWith('data:')) {
throw new DGError('Invalid data URL: ' + dataUrl, "INVALID_DATA_URL", {}, "Invalid data URL.");
}
const response = await fetch(dataUrl);
const blob = await response.blob();
return await createImageBitmap(blob);
}
async convertImageURLToImageBitmap(imageUrl) {
try {
new URL(imageUrl); // Validates the URL
const response = await fetch(imageUrl);
const blob = await response.blob();
return await createImageBitmap(blob);
} catch (error) {
throw new DGError('Invalid image URL: ' + imageUrl + ' : ' + error, "INVALID_IMAGE_URL", { error }, "Invalid image URL.");
}
}
async convertBase64ToImageBitmap(base64) {
try {
const byteString = atob(base64);
const ab = new ArrayBuffer(byteString.length);
const ia = new Uint8Array(ab);
for (let i = 0; i < byteString.length; i++) {
ia[i] = byteString.charCodeAt(i);
}
let blob = new Blob([ab], { type: 'image/jpeg' });
return await createImageBitmap(blob);
} catch (error) {
throw new DGError('Invalid base64 string: ' + base64 + ' : ' + error, "INVALID_BASE64_STRING", { error }, "Invalid base64 string.");
}
}
async convertArrayBufferToImageBitmap(arrayBuffer) {
if (!(arrayBuffer instanceof ArrayBuffer)) {
throw new DGError('Invalid ArrayBuffer input: ' + arrayBuffer, "INVALID_ARRAY_BUFFER", {}, "Invalid ArrayBuffer input.");
}
const blob = new Blob([arrayBuffer]);
return await createImageBitmap(blob);
}
async convertTypedArrayToImageBitmap(typedArray) {
if (!(typedArray instanceof Uint8Array)) {
throw new DGError('Invalid Uint8Array input: ' + typedArray, "INVALID_TYPED_ARRAY", {}, "Invalid Uint8Array input.");
}
const blob = new Blob([typedArray.buffer]);
return await createImageBitmap(blob);
}
errorCheck(response) {
// console.log('Entered errorCheck with result:', JSON.stringify(response));
if (!response)
return new DGError("response JSON is null!", "RESPONSE_JSON_NULL", {}, "Response JSON is null!");
// Check for the success flag
if (Object.prototype.hasOwnProperty.call(response, 'success')) {
if (!response.success) {
let msg = Object.prototype.hasOwnProperty.call(response, 'msg') ? response.msg : "unspecified error";
return new DGError(msg, "RESPONSE_ERROR", { msg }, "Error in response.");
}
}
// also add check for the string '[ERROR]' inside the first 25 characters of the stringified response
if (JSON.stringify(response).substring(0, 25).includes('[ERROR]')) {
// We have to parse the response to get the error message as well
return new DGError("Error in response: " + response, "RESPONSE_ERROR", { response }, "Error in response.");
}
return ""; // no error
}
waitForSocketConnection() {
return new Promise((resolve, reject) => {
resolve();
// TODO: test this.
// use this.waitFor() to wait for the socket to connect
// this.waitFor(() => this.socket.connected, this.MAX_SOCKET_WAIT_MS, 100)
// .then(() => resolve())
// .catch(error => reject(new DGError("Socket connection timeout: " + error, "SOCKET_CONNECTION_TIMEOUT", { error }, "Socket connection timeout.")))
});
}
// Utility method for timeout which also handles cancellation
// Specifically used for mutex lock timeouts
timeoutPromise(duration, onCancel) {
let timeoutId;
const promise = new Promise((resolve, reject) => {
timeoutId = setTimeout(() => {
reject(new DGError('Mutex lock timeout exceeded', "MUTEX_LOCK_TIMEOUT", {}, "Mutex lock timeout exceeded."));
onCancel();
}, duration);
});
// Attach the cancel method
promise.cancel = () => {
clearTimeout(timeoutId);
};
return promise;
}
log(...args) {
if (this.debugLogsEnabled) {
console.log(...args);
} else {
console.warn('Debug logs are disabled. Enable them by setting debugLogsEnabled to true.');
}
}
/**
* Cleanup the model and release all resources.
* Internally, it does the following:
* - Sets the poison flag to stop further inferences
* - Disconnects the socket
* - Clears the async queues
* - Nullifies references
* - Resets internal states and flags
* Call this method when you are done using the model to free up resources
* @async
*/
async cleanup() {
// Set poison flag to stop further inferences
this.poison = true;
// Dummify predict_result handler.
this.socket.on("predict_result", () => { });
// Close connection
if (this.socket) {
this.socket.disconnect();
}
// Clear Async Queues
if (this.infoQ) await this.infoQ.clear();
if (this.resultQ) await this.resultQ.clear();
// Nullify references
this.preProcessor = null;
this.postProcessor = null;
// this.mutex = null;
// this.infoQ = null;
// this.resultQ = null;
// Reset internal states and flags
this.initialized = false;
}
/**
* Resets the stats dict to an empty dict
*/
resetTimeStats() {
if(this.timeStats) {
this.timeStats = new StatsDict();
} else {
throw new DGError("Time stats object not found.", "INTERNAL_ERROR");
}
}
/**
* Returns the stats dict to the user
*/
getTimeStats() {
if (this.timeStats){
return String(this.timeStats);
}
throw new DGError("Time stats object not found.", "INTERNAL_ERROR");
}
}
export default CloudServerModel;