Sync widgets demo

packages/tasks/src/index.ts

@@ -17,7 +17,7 @@ export {
 export { ALL_DISPLAY_MODEL_LIBRARY_KEYS, ALL_MODEL_LIBRARY_KEYS, MODEL_LIBRARIES_UI_ELEMENTS } from "./model-libraries";
 export type { LibraryUiElement, ModelLibraryKey } from "./model-libraries";
 export type { ModelData, TransformersInfo } from "./model-data";
-export type { SpecialTokensMap, TokenizerConfig } from "./tokenizer-data";
+export type { AddedToken, SpecialTokensMap, TokenizerConfig } from "./tokenizer-data";
 export type {
 	ChatMessage,
 	WidgetExample,

packages/tasks/src/tokenizer-data.ts

@@ -12,8 +12,16 @@ export const SPECIAL_TOKENS_ATTRIBUTES = [
 /**
  * Public interface for a tokenizer's special tokens mapping
  */
+export interface AddedToken {
+	__type: "AddedToken";
+	content?: string;
+	lstrip?: boolean;
+	normalized?: boolean;
+	rstrip?: boolean;
+	single_word?: boolean;
+}
 export type SpecialTokensMap = {
-	[key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string;
+	[key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken;
 };
 /**
  * Public interface for tokenizer config

packages/widgets/src/lib/components/InferenceWidget/widgets/ConversationalWidget/ConversationalWidget.svelte

@@ -10,6 +10,7 @@
 		WidgetExampleOutputText,
 		WidgetExampleChatInput,
 		WidgetExample,
+		AddedToken,
 	} from "@huggingface/tasks";
 	import { SPECIAL_TOKENS_ATTRIBUTES } from "@huggingface/tasks";
 	import { HfInference } from "@huggingface/inference";
@@ -186,6 +187,8 @@
 			const value = tokenizerConfig[key];
 			if (typeof value === "string") {
 				specialTokensMap[key] = value;
+			} else if (typeof value === "object" && value.__type === "AddedToken" && value.content) {
+				specialTokensMap[key] = (value as AddedToken).content;
 			}
 		}
 		return specialTokensMap;