Add new ILlmFunction.ISeparated.validate() function. (#155)

Validator function for separated arguments case.

Author: samchon

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@samchon/openapi",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "OpenAPI definitions and converters for 'typia' and 'nestia'.",
   "main": "./lib/index.js",
   "module": "./lib/index.mjs",
@@ -46,7 +46,7 @@
     "@anthropic-ai/sdk": "^0.32.1",
     "@google/generative-ai": "^0.21.0",
     "@nestia/core": "4.2.0",
-    "@nestia/e2e": "0.8.2",
+    "@nestia/e2e": "0.8.3",
     "@nestia/fetcher": "4.2.0",
     "@nestia/sdk": "4.2.0",
     "@nestjs/common": "^10.4.1",
@@ -82,7 +82,7 @@
     "typedoc-github-theme": "^0.2.1",
     "typescript": "~5.7.2",
     "typescript-transform-paths": "^3.5.2",
-    "typia": "7.6.4",
+    "typia": "^8.0.0",
     "uuid": "^10.0.0"
   },
   "sideEffects": false,

src/HttpLlm.ts

@@ -194,11 +194,11 @@ export namespace HttpLlm {
   /**
    * Properties for the parameters' merging.
    */
-  export interface IMergeProps {
+  export interface IMergeProps<Model extends ILlmSchema.Model> {
     /**
      * Metadata of the target function.
      */
-    function: ILlmFunction<any>;
+    function: ILlmFunction<Model>;
 
     /**
      * Arguments composed by the LLM.
@@ -217,7 +217,7 @@ export namespace HttpLlm {
    * If you've configured the {@link IHttpLlmApplication.IOptions.separate} option,
    * so that the parameters are separated to human and LLM sides, you can merge these
    * humand and LLM sides' parameters into one through this `HttpLlm.mergeParameters()`
-   * function before the actual LLM function call execution.
+   * function before the actual LLM function call wexecution.
    *
    * On contrary, if you've not configured the
    * {@link IHttpLlmApplication.IOptions.separate} option, this function would throw
@@ -226,8 +226,9 @@ export namespace HttpLlm {
    * @param props Properties for the parameters' merging
    * @returns Merged parameter values
    */
-  export const mergeParameters = (props: IMergeProps): object =>
-    LlmDataMerger.parameters(props);
+  export const mergeParameters = <Model extends ILlmSchema.Model>(
+    props: IMergeProps<Model>,
+  ): object => LlmDataMerger.parameters(props);
 
   /**
    * Merge two values.

src/composers/llm/ChatGptSchemaComposer.ts

@@ -8,6 +8,7 @@ import { ChatGptTypeChecker } from "../../utils/ChatGptTypeChecker";
 import { LlmTypeCheckerV3_1 } from "../../utils/LlmTypeCheckerV3_1";
 import { NamingConvention } from "../../utils/NamingConvention";
 import { OpenApiTypeChecker } from "../../utils/OpenApiTypeChecker";
+import { OpenApiValidator } from "../../utils/OpenApiValidator";
 import { LlmDescriptionInverter } from "./LlmDescriptionInverter";
 import { LlmSchemaV3_1Composer } from "./LlmSchemaV3_1Composer";
 
@@ -271,6 +272,18 @@ export namespace ChatGptSchemaComposer {
     for (const key of Object.keys(props.parameters.$defs))
       if (key.endsWith(".Llm") === false && key.endsWith(".Human") === false)
         delete props.parameters.$defs[key];
+    if (Object.keys(output.llm.properties).length !== 0) {
+      const components: OpenApi.IComponents = {};
+      output.validate = OpenApiValidator.create({
+        components,
+        schema: invert({
+          components,
+          schema: output.llm,
+          $defs: output.llm.$defs,
+        }),
+        required: true,
+      });
+    }
     return output;
   };
 

src/composers/llm/LlmSchemaV3Composer.ts

@@ -8,6 +8,7 @@ import { IResult } from "../../structures/IResult";
 import { LlmTypeCheckerV3 } from "../../utils/LlmTypeCheckerV3";
 import { OpenApiConstraintShifter } from "../../utils/OpenApiConstraintShifter";
 import { OpenApiTypeChecker } from "../../utils/OpenApiTypeChecker";
+import { OpenApiValidator } from "../../utils/OpenApiValidator";
 import { LlmDescriptionInverter } from "./LlmDescriptionInverter";
 import { LlmParametersFinder } from "./LlmParametersComposer";
 
@@ -186,6 +187,13 @@ export namespace LlmSchemaV3Composer {
         required: [],
       },
       human: human as ILlmSchemaV3.IParameters | null,
+      validate: llm
+        ? OpenApiValidator.create({
+            components: {},
+            schema: invert({ schema: llm }),
+            required: true,
+          })
+        : undefined,
     };
   };
 

src/composers/llm/LlmSchemaV3_1Composer.ts

@@ -7,6 +7,7 @@ import { LlmTypeCheckerV3_1 } from "../../utils/LlmTypeCheckerV3_1";
 import { NamingConvention } from "../../utils/NamingConvention";
 import { OpenApiConstraintShifter } from "../../utils/OpenApiConstraintShifter";
 import { OpenApiTypeChecker } from "../../utils/OpenApiTypeChecker";
+import { OpenApiValidator } from "../../utils/OpenApiValidator";
 import { JsonDescriptionUtil } from "../../utils/internal/JsonDescriptionUtil";
 import { LlmDescriptionInverter } from "./LlmDescriptionInverter";
 import { LlmParametersFinder } from "./LlmParametersComposer";
@@ -384,6 +385,18 @@ export namespace LlmSchemaV3_1Composer {
     for (const key of Object.keys(props.parameters.$defs))
       if (key.endsWith(".Llm") === false && key.endsWith(".Human") === false)
         delete props.parameters.$defs[key];
+    if (Object.keys(output.llm.properties).length !== 0) {
+      const components: OpenApi.IComponents = {};
+      output.validate = OpenApiValidator.create({
+        components,
+        schema: invert({
+          components,
+          schema: output.llm,
+          $defs: output.llm.$defs,
+        }),
+        required: true,
+      });
+    }
     return output;
   };
 

src/structures/IHttpLlmFunction.ts

@@ -193,8 +193,16 @@ export interface IHttpLlmFunction<Model extends ILlmSchema.Model> {
    * And in my experience, OpenAI's `gpt-4o-mini` model tends to construct
    * an invalid function calling arguments at the first trial about 50% of
    * the time. However, if correct it through this `validate` function,
-   * the success rate soars to 99% at the second trial, and I've never failed
-   * at the third trial.
+   * the success rate soars to 99% at the second trial, and I've never
+   * failed at the third trial.
+   *
+   * > If you've {@link separated} parameters, use the
+   * > {@link IHttpLlmFunction.ISeparated.validate} function instead when
+   * > validating the LLM composed arguments.
+   * >
+   * > In that case, This `validate` function would be meaningful only
+   * > when you've merged the LLM and human composed arguments by
+   * > {@link HttpLlm.mergeParameters} function.
    *
    * @param args Arguments to validate.
    * @returns Validation result
@@ -236,5 +244,32 @@ export namespace IHttpLlmFunction {
      * Parameters that would be composed by the human.
      */
     human: ILlmSchema.ModelParameters[Model] | null;
+
+    /**
+     * Validate function of the separated arguments.
+     *
+     * If LLM part of separated parameters has some properties,
+     * this `validate` function will be filled for the {@link llm}
+     * type validation.
+     *
+     * > You know what? LLM (Large Language Model) like OpenAI takes a lot of
+     * > mistakes when composing arguments in function calling. Even though
+     * > `number` like simple type is defined in the {@link parameters} schema,
+     * > LLM often fills it just by a `string` typed value.
+     * >
+     * > In that case, you have to give a validation feedback to the LLM by
+     * > using this `validate` function. The `validate` function will return
+     * > detailed information about every type errors about the arguments.
+     * >
+     * > And in my experience, OpenAI's `gpt-4o-mini` model tends to construct
+     * > an invalid function calling arguments at the first trial about 50% of
+     * > the time. However, if correct it through this `validate` function,
+     * > the success rate soars to 99% at the second trial, and I've never
+     * > failed at the third trial.
+     *
+     * @param args Arguments to validate
+     * @returns Validate result
+     */
+    validate?: ((args: unknown) => IValidation<unknown>) | undefined;
   }
 }

src/structures/ILlmFunction.ts

@@ -76,7 +76,7 @@ export interface ILlmFunction<Model extends ILlmSchema.Model> {
    *
    * You can fill this property by the `@tag ${name}` comment tag.
    */
-  tags?: string[];
+  tags?: string[] | undefined;
 
   /**
    * Validate function of the arguments.
@@ -93,10 +93,18 @@ export interface ILlmFunction<Model extends ILlmSchema.Model> {
    * And in my experience, OpenAI's `gpt-4o-mini` model tends to construct
    * an invalid function calling arguments at the first trial about 50% of
    * the time. However, if correct it through this `validate` function,
-   * the success rate soars to 99% at the second trial, and I've never failed
-   * at the third trial.
+   * the success rate soars to 99% at the second trial, and I've never
+   * failed at the third trial.
    *
-   * @param args Arguments to validate.
+   * > If you've {@link separated} parameters, use the
+   * > {@link ILlmFunction.ISeparated.validate} function instead when
+   * > validating the LLM composed arguments.
+   * >
+   * > In that case, This `validate` function would be meaningful only
+   * > when you've merged the LLM and human composed arguments by
+   * > {@link HttpLlm.mergeParameters} function.
+   *
+   * @param args Arguments to validate
    * @returns Validation result
    */
   validate: (args: unknown) => IValidation<unknown>;
@@ -118,5 +126,32 @@ export namespace ILlmFunction {
      * Parameters that would be composed by the human.
      */
     human: ILlmSchema.ModelParameters[Model] | null;
+
+    /**
+     * Validate function of the separated arguments.
+     *
+     * If LLM part of separated parameters has some properties,
+     * this `validate` function will be filled for the {@link llm}
+     * type validation.
+     *
+     * > You know what? LLM (Large Language Model) like OpenAI takes a lot of
+     * > mistakes when composing arguments in function calling. Even though
+     * > `number` like simple type is defined in the {@link parameters} schema,
+     * > LLM often fills it just by a `string` typed value.
+     * >
+     * > In that case, you have to give a validation feedback to the LLM by
+     * > using this `validate` function. The `validate` function will return
+     * > detailed information about every type errors about the arguments.
+     * >
+     * > And in my experience, OpenAI's `gpt-4o-mini` model tends to construct
+     * > an invalid function calling arguments at the first trial about 50% of
+     * > the time. However, if correct it through this `validate` function,
+     * > the success rate soars to 99% at the second trial, and I've never
+     * > failed at the third trial.
+     *
+     * @param args Arguments to validate
+     * @returns Validate result
+     */
+    validate?: ((args: unknown) => IValidation<unknown>) | undefined;
   }
 }

src/utils/LlmDataMerger.ts

@@ -10,11 +10,11 @@ export namespace LlmDataMerger {
   /**
    * Properties of {@link parameters} function.
    */
-  export interface IProps {
+  export interface IProps<Model extends ILlmSchema.Model> {
     /**
      * Target function to call.
      */
-    function: ILlmFunction<ILlmSchema.Model>;
+    function: ILlmFunction<Model>;
 
     /**
      * Arguments composed by LLM (Large Language Model).
@@ -42,7 +42,9 @@ export namespace LlmDataMerger {
    * @param props Properties to combine LLM and human arguments with metadata.
    * @returns Combined arguments
    */
-  export const parameters = (props: IProps): object => {
+  export const parameters = <Model extends ILlmSchema.Model>(
+    props: IProps<Model>,
+  ): object => {
     const separated = props.function.separated;
     if (separated === undefined)
       throw new Error(

test/features/llm/validate_llm_parameters_separate_validate.ts

@@ -0,0 +1,66 @@
+import { TestValidator } from "@nestia/e2e";
+import {
+  ILlmFunction,
+  ILlmSchema,
+  IOpenApiSchemaError,
+  IResult,
+  OpenApi,
+  OpenApiTypeChecker,
+} from "@samchon/openapi";
+import { LlmSchemaComposer } from "@samchon/openapi/lib/composers/LlmSchemaComposer";
+import typia from "typia";
+
+export const test_chatgpt_parameters_separate_validate = (): void =>
+  validate_llm_parameters_separate_validate("chatgpt");
+
+export const test_claude_parameters_separate_validate = (): void =>
+  validate_llm_parameters_separate_validate("claude");
+
+export const test_gemini_parameters_separate_validate = (): void =>
+  validate_llm_parameters_separate_validate("gemini");
+
+export const test_llama_parameters_separate_validate = (): void =>
+  validate_llm_parameters_separate_validate("llama");
+
+export const test_llm_v30_parameters_separate_validate = (): void =>
+  validate_llm_parameters_separate_validate("3.0");
+
+export const test_llm_v31_parameters_separate_validate = (): void =>
+  validate_llm_parameters_separate_validate("3.1");
+
+const validate_llm_parameters_separate_validate = <
+  Model extends ILlmSchema.Model,
+>(
+  model: Model,
+): void => {
+  const collection = typia.json.schemas<[ISeparatable, IHumanOnly]>();
+  const validate = (schema: OpenApi.IJsonSchema, exists: boolean) => {
+    const result: IResult<
+      ILlmSchema.IParameters<Model>,
+      IOpenApiSchemaError
+    > = LlmSchemaComposer.parameters(model)({
+      $defs: {},
+      components: collection.components,
+      schema: schema as OpenApi.IJsonSchema.IReference,
+      config: LlmSchemaComposer.defaultConfig(model),
+    } as any) as IResult<ILlmSchema.IParameters<Model>, IOpenApiSchemaError>;
+    if (result.success === false) throw new Error("Failed to convert");
+
+    const separated: ILlmFunction.ISeparated<Model> =
+      LlmSchemaComposer.separateParameters(model)({
+        parameters: result.value as ILlmSchema.IParameters<Model>,
+        predicate: (s: OpenApi.IJsonSchema) => OpenApiTypeChecker.isNumber(s),
+      } as any) as ILlmFunction.ISeparated<Model>;
+    TestValidator.equals("validate")(!!separated.validate)(exists);
+  };
+  validate(collection.schemas[0], true);
+  validate(collection.schemas[1], false);
+};
+
+interface ISeparatable {
+  title: string;
+  value: number;
+}
+interface IHumanOnly {
+  value: number;
+}