From 5a060a7e13c1e51c7f2330225bfff81a302c2974 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Fri, 15 Nov 2024 16:35:41 +0100 Subject: [PATCH 01/50] chore(deps): add tekton and kustomize resources Signed-off-by: Ruben Romero Montes --- .gitignore | 3 + .tekton/on-pull-request.yaml | 116 +++++++++++++++ .tekton/on-push.yaml | 110 ++++++++++++++ .tekton/on-tag.yaml | 138 ++++++++++++++++++ kustomize/README.md | 40 +++++ kustomize/base/agent_morpheus-RH.yaml | 135 +++++++++++++++++ kustomize/base/agent_morpheus_client.yaml | 68 +++++++++ kustomize/base/agent_morpheus_client_db.yaml | 64 ++++++++ kustomize/base/anyuid-scc-permission.yaml | 11 ++ .../base/image-pull-secret-permission.yaml | 0 kustomize/base/ips-patch-client.json | 6 + kustomize/base/ips-patch.json | 6 + kustomize/base/kustomization.yaml | 36 +++++ kustomize/base/service-account.yaml | 4 + kustomize/network-policy.yaml | 17 +++ .../agent-morpheus-config.json | 75 ++++++++++ .../agent-morpheus-patch.yaml | 29 ++++ .../nvidia-llm-service/kustomization.yaml | 13 ++ .../agent-morpheus-config.json | 73 +++++++++ .../our-llm-service/agent-morpheus-patch.yaml | 31 ++++ .../our-llm-service/kustomization.yaml | 13 ++ 21 files changed, 988 insertions(+) create mode 100644 .tekton/on-pull-request.yaml create mode 100644 .tekton/on-push.yaml create mode 100644 .tekton/on-tag.yaml create mode 100644 kustomize/README.md create mode 100644 kustomize/base/agent_morpheus-RH.yaml create mode 100644 kustomize/base/agent_morpheus_client.yaml create mode 100644 kustomize/base/agent_morpheus_client_db.yaml create mode 100644 kustomize/base/anyuid-scc-permission.yaml create mode 100644 kustomize/base/image-pull-secret-permission.yaml create mode 100644 kustomize/base/ips-patch-client.json create mode 100644 kustomize/base/ips-patch.json create mode 100644 kustomize/base/kustomization.yaml create mode 100644 kustomize/base/service-account.yaml create mode 100644 kustomize/network-policy.yaml create mode 100644 kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json create mode 100644 kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml create mode 100644 kustomize/overlays/nvidia-llm-service/kustomization.yaml create mode 100644 kustomize/overlays/our-llm-service/agent-morpheus-config.json create mode 100644 kustomize/overlays/our-llm-service/agent-morpheus-patch.yaml create mode 100644 kustomize/overlays/our-llm-service/kustomization.yaml diff --git a/.gitignore b/.gitignore index aa5db81..6e527f5 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,9 @@ ENV/ env.bak/ venv.bak/ +# Env files +*.env + # Spyder project settings .spyderproject .spyproject diff --git a/.tekton/on-pull-request.yaml b/.tekton/on-pull-request.yaml new file mode 100644 index 0000000..75b8df2 --- /dev/null +++ b/.tekton/on-pull-request.yaml @@ -0,0 +1,116 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: vulnerability-analysis-on-pr + annotations: + # The event we are targeting as seen from the webhook payload + # this can be an array too, i.e: [pull_request, push] + pipelinesascode.tekton.dev/on-event: "[pull_request]" + + # The branch or tag we are targeting (ie: main, refs/tags/*) + pipelinesascode.tekton.dev/on-target-branch: "[main, rh-main]" + + # Fetch the git-clone task from hub, we are able to reference later on it + # with taskRef and it will automatically be embedded into our pipeline. + pipelinesascode.tekton.dev/task: "git-clone" + + # How many runs we want to keep. + pipelinesascode.tekton.dev/max-keep-runs: "5" +spec: + params: + # The variable with brackets are special to Pipelines as Code + # They will automatically be expanded with the events from Github. + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + - name: image-expires-after + value: 5d + - name: output-image + value: quay.io/ecosystem-appeng/agent-morpheus-rh:on-pr-{{revision}} + - name: path-context + value: . + - name: dockerfile + value: ./Dockerfile + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: output-image + description: Fully Qualified Output Image + type: string + - name: path-context + default: . + description: Path to the source code of an application's component from where to build image. + type: string + - name: dockerfile + default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + type: string + - name: image-expires-after + default: "" + description: Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively. + workspaces: + - name: source + - name: basic-auth + tasks: + - name: fetch-repository + taskRef: + name: git-clone + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: url + value: $(params.repo_url) + - name: revision + value: $(params.revision) + - name: buildah + runAfter: + - fetch-repository + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: BUILDER_IMAGE + value: >- + registry.redhat.io/rhel8/buildah@sha256:aac6629389db17e99894c5bee0da01d4c8065d11d8c6f6e1602f9484290baa70 + - name: STORAGE_DRIVER + value: vfs + - name: FORMAT + value: docker + - name: BUILD_EXTRA_ARGS + value: >- + --target base + --label=quay.expires-after=$(params.image-expires-after) + taskRef: + kind: ClusterTask + name: buildah + workspaces: + - name: source + workspace: source + - name: dockerconfig + workspace: dockerconfig-ws + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + # This workspace will inject secret to help the git-clone task to be able to + # checkout the private repositories + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: dockerconfig-ws + secret: + secretName: ecosystem-appeng-morpheus-quay diff --git a/.tekton/on-push.yaml b/.tekton/on-push.yaml new file mode 100644 index 0000000..e12c000 --- /dev/null +++ b/.tekton/on-push.yaml @@ -0,0 +1,110 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: vulnerability-analysis-on-push + annotations: + # The event we are targeting as seen from the webhook payload + # this can be an array too, i.e: [pull_request, push] + pipelinesascode.tekton.dev/on-event: "[push]" + + # The branch or tag we are targeting (ie: main, refs/tags/*) + pipelinesascode.tekton.dev/on-target-branch: "[rh-main]" + + # Fetch the git-clone task from hub, we are able to reference later on it + # with taskRef and it will automatically be embedded into our pipeline. + pipelinesascode.tekton.dev/task: "git-clone" + + # How many runs we want to keep. + pipelinesascode.tekton.dev/max-keep-runs: "5" +spec: + params: + # The variable with brackets are special to Pipelines as Code + # They will automatically be expanded with the events from Github. + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + - name: output-image + value: quay.io/ecosystem-appeng/agent-morpheus-rh:latest + - name: path-context + value: . + - name: dockerfile + value: ./Dockerfile + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: output-image + description: Fully Qualified Output Image + type: string + - name: path-context + default: . + description: Path to the source code of an application's component from where to build image. + type: string + - name: dockerfile + default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + type: string + workspaces: + - name: source + - name: basic-auth + tasks: + - name: fetch-repository + taskRef: + name: git-clone + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: url + value: $(params.repo_url) + - name: revision + value: $(params.revision) + - name: buildah + runAfter: + - fetch-repository + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: BUILDER_IMAGE + value: >- + registry.redhat.io/rhel8/buildah@sha256:aac6629389db17e99894c5bee0da01d4c8065d11d8c6f6e1602f9484290baa70 + - name: STORAGE_DRIVER + value: vfs + - name: FORMAT + value: docker + - name: BUILD_EXTRA_ARGS + value: >- + --target base + taskRef: + kind: ClusterTask + name: buildah + workspaces: + - name: source + workspace: source + - name: dockerconfig + workspace: dockerconfig-ws + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + # This workspace will inject secret to help the git-clone task to be able to + # checkout the private repositories + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: dockerconfig-ws + secret: + secretName: ecosystem-appeng-morpheus-quay diff --git a/.tekton/on-tag.yaml b/.tekton/on-tag.yaml new file mode 100644 index 0000000..bad954e --- /dev/null +++ b/.tekton/on-tag.yaml @@ -0,0 +1,138 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: vulnerability-analysis-on-tag + annotations: + # The event we are targeting as seen from the webhook payload + # this can be an array too, i.e: [pull_request, push] + pipelinesascode.tekton.dev/on-event: "[push]" + + # The branch or tag we are targeting (ie: main, refs/tags/*) + pipelinesascode.tekton.dev/on-target-branch: "[refs/tags/*]" + + # Fetch the git-clone task from hub, we are able to reference later on it + # with taskRef and it will automatically be embedded into our pipeline. + pipelinesascode.tekton.dev/task: "git-clone" + + # How many runs we want to keep. + pipelinesascode.tekton.dev/max-keep-runs: "5" +spec: + params: + # The variable with brackets are special to Pipelines as Code + # They will automatically be expanded with the events from Github. + - name: repo_url + value: "{{ repo_url }}" + - name: revision + value: "{{ revision }}" + - name: output-image + value: 'quay.io/ecosystem-appeng/agent-morpheus-rh' + - name: tag-name + value: "{{ target_branch }}" + - name: path-context + value: . + - name: dockerfile + value: ./Dockerfile + pipelineSpec: + params: + - name: repo_url + - name: revision + - name: tag-name + - name: output-image + description: Fully Qualified Output Image + type: string + - name: path-context + default: . + description: Path to the source code of an application's component from where to build image. + type: string + - name: dockerfile + default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + type: string + workspaces: + - name: source + - name: basic-auth + tasks: + - name: fetch-repository + taskRef: + name: git-clone + workspaces: + - name: output + workspace: source + - name: basic-auth + workspace: basic-auth + params: + - name: url + value: $(params.repo_url) + - name: revision + value: $(params.revision) + - name: format-image-name + params: + - name: image_name + value: $(params.output-image) + - name: tag_name + value: $(params.tag-name) + taskSpec: + params: + - name: image_name + type: string + - name: tag_name + type: string + results: + - name: target-image + steps: + - name: format-image + image: registry.redhat.io/ubi9/ubi-minimal:9.4 + script: | + #!/usr/bin/env bash + IMAGE="$(params.image_name):$(echo $(params.tag_name) | sed 's|refs/tags/||')" + echo "Building image: $IMAGE" + + # Set the formatted image as an output + echo -n "${IMAGE}" > "$(results.target-image.path)" + - name: buildah + runAfter: + - fetch-repository + - format-image-name + params: + - name: IMAGE + value: $(tasks.format-image-name.results.target-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: BUILDER_IMAGE + value: >- + registry.redhat.io/rhel8/buildah@sha256:aac6629389db17e99894c5bee0da01d4c8065d11d8c6f6e1602f9484290baa70 + - name: STORAGE_DRIVER + value: vfs + - name: FORMAT + value: docker + - name: BUILD_EXTRA_ARGS + value: >- + --target base + taskRef: + kind: ClusterTask + name: buildah + workspaces: + - name: source + workspace: source + - name: dockerconfig + workspace: dockerconfig-ws + workspaces: + - name: source + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + # This workspace will inject secret to help the git-clone task to be able to + # checkout the private repositories + - name: basic-auth + secret: + secretName: "{{ git_auth_secret }}" + - name: dockerconfig-ws + secret: + secretName: ecosystem-appeng-morpheus-quay diff --git a/kustomize/README.md b/kustomize/README.md new file mode 100644 index 0000000..a4bd2a2 --- /dev/null +++ b/kustomize/README.md @@ -0,0 +1,40 @@ +# Procedure to Deploy + +1. Set all your api keys and tokens in the following command. Note that all the keys are not required. +Look at this [Readme](../README.md) for more details. + +```shell +cat > base/secrets.env << EOF +nvd_api_key=you_api_key +serpapi_api_key=your_api_key +tavily_api_key=your_api_key +nvidia_api_key=your_api_key +ghsa_api_key=your_api_key +EOF +``` + +2. If you don't have namespace , create it +```shell +export YOUR_NAMESPACE_NAME=yourNamespaceNameHere +oc new-project $YOUR_NAMESPACE_NAME +``` + +3. Create an image pull secret that is authorized to pull agent morpheus image: +```shell +oc create secret generic morpheus-pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson +``` + +4. Deploy agent-morpheus + agent-morpheus-client to your namespace +```shell +oc kustomize overlays/nvidia-llm-service | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` + +5. Alternatively, if you want to deploy agent-morpheus with our self-hosted LLM, then run +```shell +# Enable ingress traffic into our LLM Model service in the cluster +oc apply -f network-policy.yaml +# label namespace with application=agent-morpheus to open communication from your namespace to LLM Model service on our Cluster +oc label namespace $YOUR_NAMESPACE_NAME application=agent-morpheus +# Deploy agent-morpheus integrated with our self-hosted LLM +oc kustomize overlays/our-llm-service | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` diff --git a/kustomize/base/agent_morpheus-RH.yaml b/kustomize/base/agent_morpheus-RH.yaml new file mode 100644 index 0000000..e4c7814 --- /dev/null +++ b/kustomize/base/agent_morpheus-RH.yaml @@ -0,0 +1,135 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-morpheus-rh + labels: + app: agent-morpheus + component: agent-morpheus-rh +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: agent-morpheus + component: agent-morpheus-rh + template: + metadata: + labels: + app: agent-morpheus + component: agent-morpheus-rh + spec: + nodeSelector: + nvidia.com/gpu.deploy.driver: "true" + imagePullSecrets: [] + tolerations: + - key: p4-gpu + operator: Exists + effect: NoSchedule + serviceAccountName: morpheus-sa + containers: + - name: agent-morpheus + image: quay.io/ecosystem-appeng/agent-morpheus-rh:latest + imagePullPolicy: Always + workingDir: /workspace/ + args: + - "python" + - "/workspace/src/main.py" + - "--log_level" + - "DEBUG" + - "cve" + - "pipeline" + - "--config_file" + - "/configs/agent-morpheus-config.json" + securityContext: + runAsUser: 0 + ports: + - name: http + protocol: TCP + containerPort: 8080 + resources: + limits: + memory: "8Gi" + cpu: "500m" + nvidia.com/gpu: "1" + requests: + memory: "1Gi" + cpu: "100m" + nvidia.com/gpu: "1" + env: + - name: SERPAPI_API_KEY + valueFrom: + secretKeyRef: + key: serpapi_api_key + name: agent-morpheus-secret + - name: NVD_API_KEY + valueFrom: + secretKeyRef: + key: nvd_api_key + name: agent-morpheus-secret + - name: NVIDIA_API_KEY + valueFrom: + secretKeyRef: + key: nvidia_api_key + name: agent-morpheus-secret + - name: GHSA_API_KEY + valueFrom: + secretKeyRef: + key: ghsa_api_key + name: agent-morpheus-secret + volumeMounts: + - name: config + mountPath: /configs + - name: cache + mountPath: /morpheus-data + volumes: + - name: config + configMap: + name: agent-morpheus-config + - name: cache + persistentVolumeClaim: + claimName: agent-morpheus-rh-data +--- +apiVersion: v1 +kind: Service +metadata: + name: agent-morpheus-rh + labels: + app: agent-morpheus + component: agent-morpheus-rh +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: agent-morpheus + component: agent-morpheus-rh +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: agent-morpheus-rh + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + port: + targetPort: 8080 + to: + kind: Service + name: agent-morpheus-rh +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: agent-morpheus-rh-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi \ No newline at end of file diff --git a/kustomize/base/agent_morpheus_client.yaml b/kustomize/base/agent_morpheus_client.yaml new file mode 100644 index 0000000..9a04519 --- /dev/null +++ b/kustomize/base/agent_morpheus_client.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-morpheus-client + labels: + app: agent-morpheus + component: agent-morpheus-client +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: agent-morpheus + component: agent-morpheus-client + template: + metadata: + labels: + app: agent-morpheus + component: agent-morpheus-client + spec: + imagePullSecrets: [] + containers: + - name: agent-morpheus-client + image: quay.io/ecosystem-appeng/agent-morpheus-client:latest + imagePullPolicy: Always + ports: + - name: http + protocol: TCP + containerPort: 8080 + env: + - name: QUARKUS_REST-CLIENT_MORPHEUS_URL + value: http://agent-morpheus-rh:8080/scan + - name: QUARKUS_MONGODB_HOSTS + value: agent-morpheus-client-db:27017 + - name: QUARKUS_MONGODB_DATABASE + value: agent-morpheus-client +--- +apiVersion: v1 +kind: Service +metadata: + name: agent-morpheus-client + labels: + app: agent-morpheus + component: agent-morpheus-client +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: agent-morpheus + component: agent-morpheus-client +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: agent-morpheus-client +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + port: + targetPort: 8080 + to: + kind: Service + name: agent-morpheus-client diff --git a/kustomize/base/agent_morpheus_client_db.yaml b/kustomize/base/agent_morpheus_client_db.yaml new file mode 100644 index 0000000..d41e192 --- /dev/null +++ b/kustomize/base/agent_morpheus_client_db.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-morpheus-client-db + labels: + app: agent-morpheus + component: agent-morpheus-client-db +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: agent-morpheus + component: agent-morpheus-client-db + template: + metadata: + labels: + app: agent-morpheus + component: agent-morpheus-client-db + spec: + containers: + - name: mongodb + image: mongodb/mongodb-community-server:8.0.3-ubi8 + imagePullPolicy: IfNotPresent + ports: + - name: db + protocol: TCP + containerPort: 27017 + volumeMounts: + - name: data + mountPath: /data/db + volumes: + - name: data + persistentVolumeClaim: + claimName: agent-morpheus-client-data +--- +apiVersion: v1 +kind: Service +metadata: + name: agent-morpheus-client-db + labels: + app: agent-morpheus + component: agent-morpheus-client-db +spec: + ports: + - name: db + port: 27017 + protocol: TCP + targetPort: 27017 + selector: + app: agent-morpheus + component: agent-morpheus-client-db +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: agent-morpheus-client-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Mi diff --git a/kustomize/base/anyuid-scc-permission.yaml b/kustomize/base/anyuid-scc-permission.yaml new file mode 100644 index 0000000..7a5cb0b --- /dev/null +++ b/kustomize/base/anyuid-scc-permission.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:openshift:scc:anyuid +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:anyuid +subjects: + - kind: ServiceAccount + name: morpheus-sa diff --git a/kustomize/base/image-pull-secret-permission.yaml b/kustomize/base/image-pull-secret-permission.yaml new file mode 100644 index 0000000..e69de29 diff --git a/kustomize/base/ips-patch-client.json b/kustomize/base/ips-patch-client.json new file mode 100644 index 0000000..ccaaf1a --- /dev/null +++ b/kustomize/base/ips-patch-client.json @@ -0,0 +1,6 @@ +[{ + "op": "add", + "path": "/spec/template/spec/imagePullSecrets/0", + "value": {"name": "morpheus-client-pull-secret"} +} +] \ No newline at end of file diff --git a/kustomize/base/ips-patch.json b/kustomize/base/ips-patch.json new file mode 100644 index 0000000..cf4d87c --- /dev/null +++ b/kustomize/base/ips-patch.json @@ -0,0 +1,6 @@ +[{ + "op": "add", + "path": "/spec/template/spec/imagePullSecrets/0", + "value": {"name": "morpheus-pull-secret"} +} +] \ No newline at end of file diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml new file mode 100644 index 0000000..935978d --- /dev/null +++ b/kustomize/base/kustomization.yaml @@ -0,0 +1,36 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - service-account.yaml + - anyuid-scc-permission.yaml + - agent_morpheus-RH.yaml + - agent_morpheus_client.yaml + +generatorOptions: + disableNameSuffixHash: false + +secretGenerator: + - name: agent-morpheus-secret + envs: + - secrets.env + +patches: + - path: ips-patch.json + + target: + name: agent-morpheus-rh + kind: Deployment + + - path: ips-patch-client.json + + target: + name: agent-morpheus-client + kind: Deployment + +images: + - name: quay.io/ecosystem-appeng/agent-morpheus-rh + newTag: rh-blueprint + + - name: quay.io/ecosystem-appeng/agent-morpheus-client + newTag: latest diff --git a/kustomize/base/service-account.yaml b/kustomize/base/service-account.yaml new file mode 100644 index 0000000..d1368c9 --- /dev/null +++ b/kustomize/base/service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: morpheus-sa \ No newline at end of file diff --git a/kustomize/network-policy.yaml b/kustomize/network-policy.yaml new file mode 100644 index 0000000..5ea2ba7 --- /dev/null +++ b/kustomize/network-policy.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + networking.knative.dev/ingress-provider: istio + name: allow-from-agent-morpheus-namespaces + namespace: morpheus-cn-models-nim +spec: + ingress: + - from: + - namespaceSelector: + matchLabels: + application: agent-morpheus + podSelector: {} + policyTypes: + - Ingress + diff --git a/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json b/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json new file mode 100644 index 0000000..662cd27 --- /dev/null +++ b/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json @@ -0,0 +1,75 @@ +{ + "$schema": "./schemas/config.schema.json", + "engine": { + "agent": { + "model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "verbose": false + }, + "checklist_model": { + "service": { + "_type": "nvfoundation" + }, + "model_name": "meta/llama3-70b-instruct", + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "justification_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "rag_embedding": { + "_type": "nim", + "model": "nvidia/nv-embedqa-e5-v5", + "truncate": "END", + "max_batch_size": 128 + }, + "summary_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + } + }, + "general": { + "cache_dir": null, + "base_vdb_dir": "/morpheus-data/vdbs", + "base_git_dir": "/morpheus-data/repos", + "max_retries": 5, + "model_max_batch_size": 64, + "pipeline_batch_size": 1024, + "use_uvloop": true + }, + "input": { + "_type": "http", + "port": 8080, + "address": "0.0.0.0" + }, + "output": { + "_type": "plugin", + "plugin_name": "src.cve.data_models.plugins.http_output_plugin.HttpOutputPlugin", + "plugin_config": { + "callback_url": "http://agent-morpheus-client:8080/reports" + } + } +} diff --git a/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml b/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml new file mode 100644 index 0000000..f74d131 --- /dev/null +++ b/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-morpheus-rh + labels: + app: agent-morpheus + component: agent-morpheus-rh +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: agent-morpheus + component: agent-morpheus-rh + template: + metadata: + labels: + app: agent-morpheus + component: agent-morpheus-rh + spec: + containers: + - name: agent-morpheus + env: + - name: NVIDIA_API_KEY + valueFrom: + secretKeyRef: + key: nvidia_api_key + name: agent-morpheus-secret \ No newline at end of file diff --git a/kustomize/overlays/nvidia-llm-service/kustomization.yaml b/kustomize/overlays/nvidia-llm-service/kustomization.yaml new file mode 100644 index 0000000..2b423f7 --- /dev/null +++ b/kustomize/overlays/nvidia-llm-service/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +configMapGenerator: + - name: agent-morpheus-config + files: + - agent-morpheus-config.json + +patchesStrategicMerge: + - agent-morpheus-patch.yaml \ No newline at end of file diff --git a/kustomize/overlays/our-llm-service/agent-morpheus-config.json b/kustomize/overlays/our-llm-service/agent-morpheus-config.json new file mode 100644 index 0000000..31bf704 --- /dev/null +++ b/kustomize/overlays/our-llm-service/agent-morpheus-config.json @@ -0,0 +1,73 @@ +{ + "$schema": "./schemas/config.schema.json", + "engine": { + "agent": { + "model": { + "model_name": "meta/llama3-8b-instruct", + "service": { + "_type": "openai" + }, + "max_tokens": 2000 + }, + "verbose": true, + "version_compare_tool": true + }, + "checklist_model": { + "service": { + "_type": "openai" + }, + "model_name": "meta/llama3-8b-instruct", + "temperature": 0, + "max_tokens": 2000, + "top_p": 0.01, + "seed": 42 + }, + "justification_model": { + "model_name": "meta/llama3-8b-instruct", + "service": { + "_type": "openai" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "rag_embedding": { + "_type": "nim", + "model": "nvidia/nv-embedqa-e5-v5", + "truncate": "END", + "max_batch_size": 128 + }, + "summary_model": { + "model_name": "meta/llama3-8b-instruct", + "service": { + "_type": "openai" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + } + }, + "general": { + "cache_dir": null, + "base_vdb_dir": "/morpheus-data/vdbs", + "base_git_dir": "/morpheus-data/repos", + "max_retries": 5, + "model_max_batch_size": 64, + "pipeline_batch_size": 128, + "use_uvloop": true + }, + "input": { + "_type": "http", + "port": 8080, + "address": "0.0.0.0" + }, + "output": { + "_type": "plugin", + "plugin_name": "src.cve.data_models.plugins.http_output_plugin.HttpOutputPlugin", + "plugin_config": { + "callback_url": "http://agent-morpheus-client:8080/reports" + } + } +} diff --git a/kustomize/overlays/our-llm-service/agent-morpheus-patch.yaml b/kustomize/overlays/our-llm-service/agent-morpheus-patch.yaml new file mode 100644 index 0000000..a056093 --- /dev/null +++ b/kustomize/overlays/our-llm-service/agent-morpheus-patch.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-morpheus-rh + labels: + app: agent-morpheus + component: agent-morpheus-rh +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: agent-morpheus + component: agent-morpheus-rh + template: + metadata: + labels: + app: agent-morpheus + component: agent-morpheus-rh + spec: + containers: + - name: agent-morpheus + env: + + - name: OPENAI_BASE_URL + value: http://llama3-8b-instruct-1xgpu-predictor-00001-private.morpheus-cn-models-nim/v1 + + - name: OPENAI_API_KEY + value: dummy-string + diff --git a/kustomize/overlays/our-llm-service/kustomization.yaml b/kustomize/overlays/our-llm-service/kustomization.yaml new file mode 100644 index 0000000..2b423f7 --- /dev/null +++ b/kustomize/overlays/our-llm-service/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +configMapGenerator: + - name: agent-morpheus-config + files: + - agent-morpheus-config.json + +patchesStrategicMerge: + - agent-morpheus-patch.yaml \ No newline at end of file From 76ab83f94f08704015bccdff03bdf6c294aa4b7f Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 14 Nov 2024 21:25:30 +0100 Subject: [PATCH 02/50] feat: add id and timestamps to output file Signed-off-by: Ruben Romero Montes --- src/cve/pipeline/pipeline.py | 15 ++++++++++++++- src/cve/stages/convert_to_output_object.py | 4 ++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/cve/pipeline/pipeline.py b/src/cve/pipeline/pipeline.py index e5cf872..5e3f716 100644 --- a/src/cve/pipeline/pipeline.py +++ b/src/cve/pipeline/pipeline.py @@ -18,6 +18,7 @@ import logging import os import time +import uuid import langchain import langchain.globals @@ -37,7 +38,7 @@ from morpheus_llm.stages.llm.llm_engine_stage import LLMEngineStage from ..data_models.config import RunConfig -from ..data_models.input import AgentMorpheusEngineInput +from ..data_models.input import AgentMorpheusEngineInput, AgentMorpheusInput from ..stages.convert_to_output_object import convert_to_output_object from .engine import build_engine from .input import build_input @@ -169,6 +170,18 @@ def convert_input_to_df(message: AgentMorpheusEngineInput) -> ControlMessage: pipe.add_stage(convert_input_to_df(config)) + @stage + def add_scan_id(message: ControlMessage) -> ControlMessage: + input: AgentMorpheusInput = message.get_metadata("input") + if input.scan.id is not None: + message.set_metadata("scan_id", input.scan.id) + else: + message.set_metadata("scan_id", uuid.uuid4()) + + return message + + pipe.add_stage(add_scan_id(config=config)) + @stage def add_start_timestamp(message: ControlMessage) -> ControlMessage: message.set_timestamp("start_time", datetime.datetime.now()) diff --git a/src/cve/stages/convert_to_output_object.py b/src/cve/stages/convert_to_output_object.py index 149bb65..457041f 100644 --- a/src/cve/stages/convert_to_output_object.py +++ b/src/cve/stages/convert_to_output_object.py @@ -105,6 +105,10 @@ def convert_to_output_object(message: ControlMessage) -> AgentMorpheusOutput: # Pull input and info objects from control message metadata input: AgentMorpheusInput = message.get_metadata("input") +<<<<<<< HEAD +======= + input.scan.id = message.get_metadata("scan_id") +>>>>>>> 6be9369 (feat: add id and timestamps to output file) input.scan.started_at = message.get_timestamp("start_time") input.scan.completed_at = message.get_timestamp("end_time") From 7ef4977210eefc76d9fdcc42ef817475b7234a13 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 14 Nov 2024 21:29:34 +0100 Subject: [PATCH 03/50] feat: add http output plugin Signed-off-by: Ruben Romero Montes --- configs/from_http_http.json | 75 +++++++++++++++++++ src/cve/data_models/plugin.py | 2 +- .../data_models/plugins/http_output_plugin.py | 74 ++++++++++++++++++ 3 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 configs/from_http_http.json create mode 100644 src/cve/data_models/plugins/http_output_plugin.py diff --git a/configs/from_http_http.json b/configs/from_http_http.json new file mode 100644 index 0000000..b447ff3 --- /dev/null +++ b/configs/from_http_http.json @@ -0,0 +1,75 @@ +{ + "$schema": "./schemas/config.schema.json", + "engine": { + "agent": { + "model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "verbose": false + }, + "checklist_model": { + "service": { + "_type": "nvfoundation" + }, + "model_name": "meta/llama3-70b-instruct", + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "justification_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "rag_embedding": { + "_type": "nim", + "model": "nvidia/nv-embedqa-e5-v5", + "truncate": "END", + "max_batch_size": 128 + }, + "summary_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + } + }, + "general": { + "cache_dir": null, + "base_vdb_dir": ".cache/am_cache/vdb", + "base_git_dir": ".cache/am_cache/git", + "max_retries": 5, + "model_max_batch_size": 64, + "pipeline_batch_size": 1024, + "use_uvloop": true + }, + "input": { + "_type": "http", + "port": 26466, + "address": "0.0.0.0" + }, + "output": { + "_type": "plugin", + "plugin_name": "src.cve.data_models.plugins.http_output_plugin.HttpOutputPlugin", + "plugin_config": { + "callback_url": "http://agent-morpheus-client:8080/reports" + } + } +} diff --git a/src/cve/data_models/plugin.py b/src/cve/data_models/plugin.py index 79f3aa3..055dd9d 100644 --- a/src/cve/data_models/plugin.py +++ b/src/cve/data_models/plugin.py @@ -37,7 +37,7 @@ def locate(cls: type[_T], plugin_name: str) -> _T: if not pluginClass: raise ValueError(f"Plugin not found: {plugin_name}") if not issubclass(pluginClass, cls): - raise ValueError("Plugin object must be a subclass of {cls}") + raise ValueError(f"{pluginClass} object must be a subclass of {cls}") return pluginClass() diff --git a/src/cve/data_models/plugins/http_output_plugin.py b/src/cve/data_models/plugins/http_output_plugin.py new file mode 100644 index 0000000..1be7782 --- /dev/null +++ b/src/cve/data_models/plugins/http_output_plugin.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +import logging +from pydantic import BaseModel +import requests + +from morpheus.config import Config +from morpheus.pipeline.linear_pipeline import LinearPipeline +from morpheus.pipeline.stage_decorator import stage + +from ...data_models.config import RunConfig +from ...data_models.output import AgentMorpheusOutput +from ..plugin import OutputPluginSchema + +logger = logging.getLogger(__name__) + + +class HttpOutputPluginConfig(BaseModel): + + callback_url: str + + +class HttpOutputPlugin(OutputPluginSchema): + """ + OutputPlugin that sends and HTTP Post with the current output message + as an application/json to a callback URL defined in the configuration. + + Usage: + + "output": { + "_type": "plugin", + "plugin_name": "agent_morpheus.cve.data_models.plugins.http_output_plugin.HttpOutputPlugin", + "plugin_config": { + "callback_url": "http://agent-morpheus-client:8080/reports" + } + } + """ + + def build_output(self, pipe: LinearPipeline, config: Config, run_config: RunConfig): + + http_config = HttpOutputPluginConfig(**run_config.output.plugin_config) + if not http_config.callback_url: + raise ValueError( + "No callback URL provided in HTTP Output plugin config") + + @stage + def http_callback_stage(message: AgentMorpheusOutput) -> AgentMorpheusOutput: + headers = {'Content-type': 'application/json'} + model_json = message.model_dump_json(by_alias=True) + try: + response = requests.post( + http_config.callback_url, data=model_json.encode('utf-8'), headers=headers) + response.raise_for_status() + except requests.exceptions.RequestException as e: + logger.error( + f"Error sending output to {http_config.callback_url}: {e}") + if not response.ok: + logger.error( + f'Unable to send output response to {http_config.callback_url}. Error: {response.status_code} - {response.reason}') + else: + logger.info( + f'Successfully sent output to {http_config.callback_url}') + # Make stage a pass-through + return message + + pipe.add_stage(http_callback_stage(config)) From f8acd5f1e20fcb2a07b6ad9364c2a6a4da26d74c Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 14 Nov 2024 21:32:33 +0100 Subject: [PATCH 04/50] feat: log input file only for non http requests Signed-off-by: Ruben Romero Montes --- src/cve/pipeline/input.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/cve/pipeline/input.py b/src/cve/pipeline/input.py index 0186478..625bb4c 100644 --- a/src/cve/pipeline/input.py +++ b/src/cve/pipeline/input.py @@ -309,5 +309,17 @@ async def _calc_dep(cve_intel: CveIntel): pipe.add_stage(check_vulnerable_dependencies(config)) + if run_config.input.type != 'http': + @stage + def debug_input(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + + # Save the input object to disk for debugging + with open("./.tmp/input_object.json", "w") as f: + f.write(message.model_dump_json(indent=2)) + + return message + + pipe.add_stage(debug_input(config)) + # Return embedding model to be used later in the pipeline return embedding From c5b34f685989f265d70354303fc1a225185545c1 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Mon, 18 Nov 2024 09:19:00 +0100 Subject: [PATCH 05/50] chore: deploy latest tag Signed-off-by: Ruben Romero Montes --- kustomize/base/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 935978d..4fe2ada 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -30,7 +30,7 @@ patches: images: - name: quay.io/ecosystem-appeng/agent-morpheus-rh - newTag: rh-blueprint + newTag: latest - name: quay.io/ecosystem-appeng/agent-morpheus-client newTag: latest From f02499b25f4e4e8f1fff6c537eeb0952f6d7e17d Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Mon, 18 Nov 2024 14:04:11 +0100 Subject: [PATCH 06/50] chore(deps): add nginx cache Signed-off-by: Ruben Romero Montes --- kustomize/base/agent_morpheus-RH.yaml | 44 ++++-- kustomize/base/kustomization.yaml | 17 +++ kustomize/base/nginx.yaml | 141 ++++++++++++++++++ kustomize/base/nginx/nginx_cache.conf | 132 ++++++++++++++++ kustomize/base/nginx/nginx_ssl.conf | 24 +++ .../templates/routes/intel.conf.template | 123 +++++++++++++++ .../nginx/templates/routes/nemo.conf.template | 24 +++ .../nginx/templates/routes/nim.conf.template | 42 ++++++ .../templates/routes/nvidia.conf.template | 47 ++++++ .../templates/routes/openai.conf.template | 23 +++ .../template-variables.conf.template | 57 +++++++ .../agent-morpheus-config.json | 2 +- .../agent-morpheus-patch.yaml | 27 ++++ 13 files changed, 686 insertions(+), 17 deletions(-) create mode 100644 kustomize/base/nginx.yaml create mode 100644 kustomize/base/nginx/nginx_cache.conf create mode 100644 kustomize/base/nginx/nginx_ssl.conf create mode 100644 kustomize/base/nginx/templates/routes/intel.conf.template create mode 100644 kustomize/base/nginx/templates/routes/nemo.conf.template create mode 100644 kustomize/base/nginx/templates/routes/nim.conf.template create mode 100644 kustomize/base/nginx/templates/routes/nvidia.conf.template create mode 100644 kustomize/base/nginx/templates/routes/openai.conf.template create mode 100644 kustomize/base/nginx/templates/variables/template-variables.conf.template diff --git a/kustomize/base/agent_morpheus-RH.yaml b/kustomize/base/agent_morpheus-RH.yaml index e4c7814..5cefa2a 100644 --- a/kustomize/base/agent_morpheus-RH.yaml +++ b/kustomize/base/agent_morpheus-RH.yaml @@ -77,6 +77,34 @@ spec: secretKeyRef: key: ghsa_api_key name: agent-morpheus-secret + - name: CVE_DETAILS_BASE_URL + value: http://nginx-cache:8080/cve-details + - name: CWE_DETAILS_BASE_URL + value: http://nginx-cache:8080/cwe-details + - name: DEPSDEV_BASE_URL + value: http://nginx-cache:8080/depsdev + - name: FIRST_BASE_URL + value: http://nginx-cache:8080/first + - name: GHSA_BASE_URL + value: http://nginx-cache:8080/ghsa + - name: NGC_API_BASE + value: http://nginx-cache:8080/nemo/v1 + - name: NIM_EMBED_BASE_URL + value: http://nginx-cache:8080/nim_embed/v1 + - name: NVD_BASE_URL + value: http://nginx-cache:8080/nvd + - name: NVIDIA_API_BASE + value: http://nginx-cache:8080/nim_llm/v1 + - name: OPENAI_API_BASE + value: http://nginx-cache:8080/openai/v1 + - name: OPENAI_BASE_URL + value: http://nginx-cache:8080/openai/v1 + - name: RHSA_BASE_URL + value: http://nginx-cache:8080/rhsa + - name: SERPAPI_BASE_URL + value: http://nginx-cache:8080/serpapi + - name: UBUNTU_BASE_URL + value: http://nginx-cache:8080/ubuntu volumeMounts: - name: config mountPath: /configs @@ -107,22 +135,6 @@ spec: app: agent-morpheus component: agent-morpheus-rh --- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: agent-morpheus-rh - annotations: - haproxy.router.openshift.io/timeout: 5m -spec: - tls: - insecureEdgeTerminationPolicy: Redirect - termination: edge - port: - targetPort: 8080 - to: - kind: Service - name: agent-morpheus-rh ---- apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 4fe2ada..ab8cd24 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -4,7 +4,9 @@ kind: Kustomization resources: - service-account.yaml - anyuid-scc-permission.yaml + - nginx.yaml - agent_morpheus-RH.yaml + - agent_morpheus_client_db.yaml - agent_morpheus_client.yaml generatorOptions: @@ -15,6 +17,21 @@ secretGenerator: envs: - secrets.env +configMapGenerator: + - name: nginx-cache-config + files: + - nginx.conf=nginx/nginx_cache.conf + - name: nginx-cache-variables + files: + - nginx/templates/variables/template-variables.conf.template + - name: nginx-cache-routes + files: + - nginx/templates/routes/intel.conf.template + - nginx/templates/routes/nemo.conf.template + - nginx/templates/routes/nim.conf.template + - nginx/templates/routes/nvidia.conf.template + - nginx/templates/routes/openai.conf.template + patches: - path: ips-patch.json diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml new file mode 100644 index 0000000..f7e41c4 --- /dev/null +++ b/kustomize/base/nginx.yaml @@ -0,0 +1,141 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache + labels: + app: agent-morpheus + component: nginx-cache +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: agent-morpheus + component: nginx-cache + template: + metadata: + labels: + app: agent-morpheus + component: nginx-cache + spec: + imagePullSecrets: [] + containers: + - name: nginx + image: docker.io/nginxinc/nginx-unprivileged:1.27 + imagePullPolicy: IfNotPresent + ports: + - name: http + protocol: TCP + containerPort: 8080 + resources: + limits: + memory: "256Mi" + cpu: "250m" + requests: + memory: "100Mi" + cpu: "100m" + env: + - name: SERPAPI_API_KEY + valueFrom: + secretKeyRef: + key: serpapi_api_key + name: agent-morpheus-secret + - name: NVD_API_KEY + valueFrom: + secretKeyRef: + key: nvd_api_key + name: agent-morpheus-secret + - name: NVIDIA_API_KEY + valueFrom: + secretKeyRef: + key: nvidia_api_key + name: agent-morpheus-secret + - name: GHSA_API_KEY + valueFrom: + secretKeyRef: + key: ghsa_api_key + name: agent-morpheus-secret + - name: OPENAI_API_KEY + value: "" + - name: NGC_API_KEY + value: "" + - name: NGC_ORG_ID + value: "" + - name: NGINX_UPSTREAM_NVAI + value: https://api.nvcf.nvidia.com + - name: NGINX_UPSTREAM_NIM_LLM + value: https://integrate.api.nvidia.com + - name: NGINX_UPSTREAM_NIM_EMBED + value: https://integrate.api.nvidia.com + volumeMounts: + - name: routes + mountPath: /etc/nginx/templates/routes + - name: variables + mountPath: /etc/nginx/templates/variables + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: extra-config + mountPath: /etc/nginx/conf.d + - name: cache + mountPath: /server_cache + volumes: + - name: config + configMap: + name: nginx-cache-config + - name: extra-config + emptyDir: {} + - name: routes + configMap: + name: nginx-cache-routes + - name: variables + configMap: + name: nginx-cache-variables + - name: cache + persistentVolumeClaim: + claimName: nginx-cache-server +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-cache + labels: + app: agent-morpheus + component: nginx-cache +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: agent-morpheus + component: nginx-cache +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: agent-morpheus-rh + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + port: + targetPort: 8080 + to: + kind: Service + name: nginx-cache +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nginx-cache-server +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/kustomize/base/nginx/nginx_cache.conf b/kustomize/base/nginx/nginx_cache.conf new file mode 100644 index 0000000..837625d --- /dev/null +++ b/kustomize/base/nginx/nginx_cache.conf @@ -0,0 +1,132 @@ +pid /tmp/nginx.pid; + +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + proxy_ssl_server_name on; + + proxy_cache_path /server_cache/llm levels=1:2 keys_zone=llm_cache:10m max_size=20g inactive=14d use_temp_path=off; + + proxy_cache_path /server_cache/intel levels=1:2 keys_zone=intel_cache:10m max_size=20g inactive=14d use_temp_path=off; + + error_log /dev/stdout info; + + log_format upstream_time '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"' + 'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; + + log_format cache_log '[$time_local] ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format no_cache_log '[$time_local] (BYPASSED) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format mirror_log '[$time_local] (MIRROR) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present'; + + log_format nvai_cache_log '[$time_local] ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization_present. Final Auth: $http_authorization_present'; + + include /etc/nginx/conf.d/variables/*.conf; + + map $http_cache_control $cache_bypass { + no-cache 1; + } + + # Log to stdout + access_log /dev/stdout cache_log; + + error_log /dev/stdout info; + + server { + listen 8080; + server_name localhost; + + proxy_http_version 1.1; + + # Headers to Add + # proxy_set_header Host $host; + proxy_set_header Connection ''; + + # Headers to Remove + proxy_ignore_headers Cache-Control; + proxy_ignore_headers "Set-Cookie"; + proxy_hide_header "Set-Cookie"; + + # Proxy Buffer Config + proxy_busy_buffers_size 1024k; + proxy_buffers 4 512k; + proxy_buffer_size 1024k; + + # Proxy validity + proxy_cache_valid 200 202 14d; + proxy_read_timeout 8m; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_bypass $cache_bypass; + + set $http_authorization_present '[NOT PROVIDED]'; # Default to '[NOT PROVIDED]' + + if ($http_authorization) { + set $http_authorization_present '[REDACTED]'; # Set to '[REDACTED]' when the Authorization header is present + } + + # Configure a resolver to use for DNS resolution. This uses the Docker DNS resolver + # See https://tenzer.dk/nginx-with-dynamic-upstreams/ for why this is necessary + # When considering what the "base_url" should be, consider the following: + # - The base_url should be the unchangable part of the URL for any request tho that API + # - If the API uses versioning, the version should be included in the base_url + # - If the API is a subpath of a larger API, the base_url should be the path to the API + # - Examples: + # - GET `https://api.first.org/data/v1/epss` => base_url=`https://api.first.org/data/v1` + # - GET `https://services.nvd.nist.gov/rest/json/cves/2.0` => base_url=`https://services.nvd.nist.gov/rest` + + # resolver 127.0.0.11 [::1]:5353 valid=60s; + + # Use kube-dns DNS resolver and public resolvers as fallback + # resolver kube-dns.kube-system.svc.cluster.local 8.8.8.8 8.8.4.4 valid=60s; + + # Use OpenShift default DNS resolver and public resolvers as fallback + resolver dns-default.openshift-dns.svc.cluster.local 8.8.8.8 8.8.4.4 valid=60s; + + # rewrite_log on; + + ################ Docker Compose Services ################# + + # Force nginx to resolve morpheus-vuln-analysis each call to allow starting this before starting the service + set $morpheus_vuln_analysis_upstream "http://morpheus-vuln-analysis:26466"; + + location /scan { + proxy_pass $morpheus_vuln_analysis_upstream; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } + + # Include any additional routes from the routes directory + include /etc/nginx/conf.d/routes/*.conf; + + + ################### Redirect Handling #################### + + location @handle_redirects { + # store the current state of the world so we can reuse it in a minute + # We need to capture these values now, because as soon as we invoke + # the proxy_* directives, these will disappear + set $original_uri $uri; + set $orig_loc $upstream_http_location; + + # nginx goes to fetch the value from the upstream Location header + proxy_pass $orig_loc; + proxy_cache llm_cache; + + # But we store the result with the cache key of the original request URI + # so that future clients don't need to follow the redirect too + proxy_cache_key $original_uri; + proxy_cache_valid 200 206 14d; + } + } +} diff --git a/kustomize/base/nginx/nginx_ssl.conf b/kustomize/base/nginx/nginx_ssl.conf new file mode 100644 index 0000000..a3dfae3 --- /dev/null +++ b/kustomize/base/nginx/nginx_ssl.conf @@ -0,0 +1,24 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + server { + listen 443 ssl; + listen [::]:443 ssl; + server_name localhost; + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/key.pem; + + + location / { + proxy_pass http://nginx-cache:80; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +} diff --git a/kustomize/base/nginx/templates/routes/intel.conf.template b/kustomize/base/nginx/templates/routes/intel.conf.template new file mode 100644 index 0000000..f848a64 --- /dev/null +++ b/kustomize/base/nginx/templates/routes/intel.conf.template @@ -0,0 +1,123 @@ +####################### Intel APIs ####################### + +set $serpapi_upstream https://serpapi.com; + +location /serpapi { + rewrite ^\/serpapi(\/.*)$ $1 break; + proxy_pass $serpapi_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $nvd_upstream https://services.nvd.nist.gov; + +location /nvd { + rewrite ^\/nvd(\/.*)$ /rest$1 break; + proxy_pass $nvd_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $cve_details_upstream https://www.cvedetails.com; + +location /cve-details { + rewrite ^\/cve-details(\/.*)$ $1 break; + proxy_pass $cve_details_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $cwe_details_upstream https://cwe.mitre.org; + +location /cwe-details { + rewrite ^\/cwe-details(\/.*)$ $1 break; + proxy_pass $cwe_details_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + + error_page 301 302 307 = @handle_redirects; +} + +set $epss_upstream https://api.first.org; + +location /first { + rewrite ^\/first(\/.*)$ /data/v1$1 break; + proxy_pass $epss_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $recf_upstream https://api.recordedfuture.com; + +location /recf { + rewrite ^\/recf(\/.*)$ /v2$1 break; + proxy_pass $recf_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $ghsa_upstream https://api.github.com; + +location /ghsa { + rewrite ^\/ghsa(\/.*)$ $1 break; + proxy_pass $ghsa_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $rhsa_upstream https://access.redhat.com; + +location /rhsa { + rewrite ^\/rhsa(\/.*)$ /hydra/rest$1 break; + proxy_pass $rhsa_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $ubuntu_upstream https://ubuntu.com; + +location /ubuntu { + rewrite ^\/ubuntu(\/.*)$ $1 break; + proxy_pass $ubuntu_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $depsdev_upstream https://api.deps.dev; + +location /depsdev { + rewrite ^\/depsdev(\/.*)$ $1 break; + proxy_pass $depsdev_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} diff --git a/kustomize/base/nginx/templates/routes/nemo.conf.template b/kustomize/base/nginx/templates/routes/nemo.conf.template new file mode 100644 index 0000000..a3c7aef --- /dev/null +++ b/kustomize/base/nginx/templates/routes/nemo.conf.template @@ -0,0 +1,24 @@ +set $nemo_upstream https://api.llm.ngc.nvidia.com; + +location /nemo { + + location ~* ^\/nemo\/v1\/models(\/.+\/completions)?$ { + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass $nemo_upstream; + proxy_set_header Connection ''; + proxy_set_header Authorization $nemo_http_authorization; + proxy_set_header Organization-ID $nemo_http_organization_id; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /nemo/v1 { + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass $nemo_upstream; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/base/nginx/templates/routes/nim.conf.template b/kustomize/base/nginx/templates/routes/nim.conf.template new file mode 100644 index 0000000..f6ab6ef --- /dev/null +++ b/kustomize/base/nginx/templates/routes/nim.conf.template @@ -0,0 +1,42 @@ + +set $upstream_nim_llm ${NGINX_UPSTREAM_NIM_LLM}; +set $upstream_nim_embed ${NGINX_UPSTREAM_NIM_EMBED}; + +location ~* ^\/nim_llm\/v1\/(?:chat\/completions|completions|edits|moderations|answers)$ { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass $upstream_nim_llm; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nim_llm/v1 { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass $upstream_nim_llm; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; +} + + +location ~* ^/nim_embed/v1/embeddings$ { + rewrite ^/nim_embed(.*)$ $1 break; + proxy_pass $upstream_nim_embed; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +location /nim_embed/v1 { + rewrite ^/nim_embed(.*)$ $1 break; + proxy_pass $upstream_nim_embed; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; +} diff --git a/kustomize/base/nginx/templates/routes/nvidia.conf.template b/kustomize/base/nginx/templates/routes/nvidia.conf.template new file mode 100644 index 0000000..2bdf18b --- /dev/null +++ b/kustomize/base/nginx/templates/routes/nvidia.conf.template @@ -0,0 +1,47 @@ + +set $nvai_upstream ${NGINX_UPSTREAM_NVAI}; + +location /nvai { + + location ~* ^\/nvai\/v2\/nvcf\/((pexec\/functions\/.+)|(functions))$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass $nvai_upstream; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + + # Only cache 200/202, not 404. + proxy_cache_valid 200 14d; + + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + + access_log /dev/stdout nvai_cache_log; + access_log /var/log/nginx/access.log nvai_cache_log; + } + + location ~* ^\/nvai\/v2\/nvcf\/(pexec\/status\/.+)$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass $nvai_upstream; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + + # Only cache 200. Not 202 as it will cause a never ending loop + proxy_cache_valid 200 202 14d; + + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + } + + location /nvai/v2 { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass $nvai_upstream; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/base/nginx/templates/routes/openai.conf.template b/kustomize/base/nginx/templates/routes/openai.conf.template new file mode 100644 index 0000000..8a8a0cb --- /dev/null +++ b/kustomize/base/nginx/templates/routes/openai.conf.template @@ -0,0 +1,23 @@ +set $openai_upstream https://api.openai.com; + +location /openai { + + location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass $openai_upstream; + proxy_set_header Connection ''; + proxy_set_header Authorization $openai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /openai/v1 { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass $openai_upstream; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/kustomize/base/nginx/templates/variables/template-variables.conf.template b/kustomize/base/nginx/templates/variables/template-variables.conf.template new file mode 100644 index 0000000..0af689d --- /dev/null +++ b/kustomize/base/nginx/templates/variables/template-variables.conf.template @@ -0,0 +1,57 @@ +map $http_authorization $nemo_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${NGC_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${NGC_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NGC_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${NGC_API_KEY}'; + default $http_authorization; +} + +map $http_organization_id $nemo_http_organization_id { + 'AGENT_MORPHEUS' '${NGC_ORG_ID}'; + '"AGENT_MORPHEUS"' '${NGC_ORG_ID}'; + 'CYBER_DEV_DAY' '${NGC_ORG_ID}'; + '"CYBER_DEV_DAY"' '${NGC_ORG_ID}'; + default $http_organization_id; +} + +map $http_authorization $nim_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' '${NVIDIA_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' '${NVIDIA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $nvai_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer nvapi-AGENT_MORPHEUS' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "nvapi-AGENT_MORPHEUS"' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer nvapi-CYBER_DEV_DAY' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "nvapi-CYBER_DEV_DAY"' 'Bearer ${NVIDIA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $openai_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${OPENAI_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $ghsa_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${GHSA_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${GHSA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${GHSA_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${GHSA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $nvd_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${NVD_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${NVD_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NVD_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${NVD_API_KEY}'; + default $http_authorization; +} diff --git a/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json b/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json index 662cd27..3f4ee51 100644 --- a/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json +++ b/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json @@ -12,7 +12,7 @@ "top_p": 0.01, "seed": 42 }, - "verbose": false + "verbose": true }, "checklist_model": { "service": { diff --git a/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml b/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml index f74d131..394a7c0 100644 --- a/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml +++ b/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml @@ -21,6 +21,33 @@ spec: spec: containers: - name: agent-morpheus + env: + - name: NVIDIA_API_KEY + valueFrom: + secretKeyRef: + key: nvidia_api_key + name: agent-morpheus-secret +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache + labels: + app: agent-morpheus + component: nginx-cache +spec: + selector: + matchLabels: + app: agent-morpheus + component: nginx-cache + template: + metadata: + labels: + app: agent-morpheus + component: nginx-cache + spec: + containers: + - name: nginx env: - name: NVIDIA_API_KEY valueFrom: From c87a41f80ad6ebd3c691623d7cac398857ceec94 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Mon, 18 Nov 2024 17:16:11 +0100 Subject: [PATCH 07/50] fix: depsdev url encoding in the golang packages Signed-off-by: Ruben Romero Montes --- kustomize/base/nginx/nginx_cache.conf | 2 +- kustomize/base/nginx/nginx_ssl.conf | 6 +- .../templates/routes/intel.conf.template | 146 +++++++++--------- 3 files changed, 78 insertions(+), 76 deletions(-) diff --git a/kustomize/base/nginx/nginx_cache.conf b/kustomize/base/nginx/nginx_cache.conf index 837625d..b96c842 100644 --- a/kustomize/base/nginx/nginx_cache.conf +++ b/kustomize/base/nginx/nginx_cache.conf @@ -96,7 +96,7 @@ http { ################ Docker Compose Services ################# # Force nginx to resolve morpheus-vuln-analysis each call to allow starting this before starting the service - set $morpheus_vuln_analysis_upstream "http://morpheus-vuln-analysis:26466"; + set $morpheus_vuln_analysis_upstream "http://agent-morpheus-rh:8080"; location /scan { proxy_pass $morpheus_vuln_analysis_upstream; diff --git a/kustomize/base/nginx/nginx_ssl.conf b/kustomize/base/nginx/nginx_ssl.conf index a3dfae3..edec88a 100644 --- a/kustomize/base/nginx/nginx_ssl.conf +++ b/kustomize/base/nginx/nginx_ssl.conf @@ -6,15 +6,15 @@ events { http { server { - listen 443 ssl; - listen [::]:443 ssl; + listen 8443 ssl; + listen [::]:8443 ssl; server_name localhost; ssl_certificate /etc/nginx/ssl/cert.pem; ssl_certificate_key /etc/nginx/ssl/key.pem; location / { - proxy_pass http://nginx-cache:80; + proxy_pass http://nginx-cache:8080; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/kustomize/base/nginx/templates/routes/intel.conf.template b/kustomize/base/nginx/templates/routes/intel.conf.template index f848a64..2150fbf 100644 --- a/kustomize/base/nginx/templates/routes/intel.conf.template +++ b/kustomize/base/nginx/templates/routes/intel.conf.template @@ -3,121 +3,123 @@ set $serpapi_upstream https://serpapi.com; location /serpapi { - rewrite ^\/serpapi(\/.*)$ $1 break; - proxy_pass $serpapi_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; + rewrite ^\/serpapi(\/.*)$ $1 break; + proxy_pass $serpapi_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; } set $nvd_upstream https://services.nvd.nist.gov; location /nvd { - rewrite ^\/nvd(\/.*)$ /rest$1 break; - proxy_pass $nvd_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; + rewrite ^\/nvd(\/.*)$ /rest$1 break; + proxy_pass $nvd_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; } set $cve_details_upstream https://www.cvedetails.com; location /cve-details { - rewrite ^\/cve-details(\/.*)$ $1 break; - proxy_pass $cve_details_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; + rewrite ^\/cve-details(\/.*)$ $1 break; + proxy_pass $cve_details_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; } set $cwe_details_upstream https://cwe.mitre.org; location /cwe-details { - rewrite ^\/cwe-details(\/.*)$ $1 break; - proxy_pass $cwe_details_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; - - error_page 301 302 307 = @handle_redirects; + rewrite ^\/cwe-details(\/.*)$ $1 break; + proxy_pass $cwe_details_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + + error_page 301 302 307 = @handle_redirects; } set $epss_upstream https://api.first.org; location /first { - rewrite ^\/first(\/.*)$ /data/v1$1 break; - proxy_pass $epss_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; + rewrite ^\/first(\/.*)$ /data/v1$1 break; + proxy_pass $epss_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; } set $recf_upstream https://api.recordedfuture.com; location /recf { - rewrite ^\/recf(\/.*)$ /v2$1 break; - proxy_pass $recf_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; + rewrite ^\/recf(\/.*)$ /v2$1 break; + proxy_pass $recf_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; } set $ghsa_upstream https://api.github.com; location /ghsa { - rewrite ^\/ghsa(\/.*)$ $1 break; - proxy_pass $ghsa_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; + rewrite ^\/ghsa(\/.*)$ $1 break; + proxy_pass $ghsa_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; } set $rhsa_upstream https://access.redhat.com; location /rhsa { - rewrite ^\/rhsa(\/.*)$ /hydra/rest$1 break; - proxy_pass $rhsa_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; + rewrite ^\/rhsa(\/.*)$ /hydra/rest$1 break; + proxy_pass $rhsa_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; } set $ubuntu_upstream https://ubuntu.com; location /ubuntu { - rewrite ^\/ubuntu(\/.*)$ $1 break; - proxy_pass $ubuntu_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; + rewrite ^\/ubuntu(\/.*)$ $1 break; + proxy_pass $ubuntu_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; } set $depsdev_upstream https://api.deps.dev; location /depsdev { - rewrite ^\/depsdev(\/.*)$ $1 break; - proxy_pass $depsdev_upstream; - proxy_cache intel_cache; - proxy_cache_methods GET; - proxy_cache_key "$request_method|$request_uri"; - add_header X-Cache-Status $upstream_cache_status; - client_body_buffer_size 4m; + rewrite ^ $request_uri; + rewrite ^\/depsdev\/(.*) $1 break; + return 400; + proxy_pass $depsdev_upstream/$uri; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; } From da3ea4dd9efcd22be8668f6cdd864b2fd20572fc Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 21 Nov 2024 11:09:48 +0100 Subject: [PATCH 08/50] feat: use pydantic default factory Signed-off-by: Ruben Romero Montes --- src/cve/pipeline/pipeline.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/cve/pipeline/pipeline.py b/src/cve/pipeline/pipeline.py index 5e3f716..d08f77f 100644 --- a/src/cve/pipeline/pipeline.py +++ b/src/cve/pipeline/pipeline.py @@ -18,7 +18,6 @@ import logging import os import time -import uuid import langchain import langchain.globals @@ -173,10 +172,7 @@ def convert_input_to_df(message: AgentMorpheusEngineInput) -> ControlMessage: @stage def add_scan_id(message: ControlMessage) -> ControlMessage: input: AgentMorpheusInput = message.get_metadata("input") - if input.scan.id is not None: - message.set_metadata("scan_id", input.scan.id) - else: - message.set_metadata("scan_id", uuid.uuid4()) + message.set_metadata("scan_id", input.scan.id) return message From 17cdaf2eaec6b9dff002159e35663566a4e69401 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 21 Nov 2024 11:21:57 +0100 Subject: [PATCH 09/50] fix: dns was not properly being resolved Signed-off-by: Ruben Romero Montes --- kustomize/base/nginx/nginx_cache.conf | 13 +----- .../templates/routes/intel.conf.template | 42 ++++++------------- .../nginx/templates/routes/nemo.conf.template | 6 +-- .../nginx/templates/routes/nim.conf.template | 11 ++--- .../templates/routes/nvidia.conf.template | 8 ++-- .../templates/routes/openai.conf.template | 6 +-- 6 files changed, 24 insertions(+), 62 deletions(-) diff --git a/kustomize/base/nginx/nginx_cache.conf b/kustomize/base/nginx/nginx_cache.conf index b96c842..1ea6ac1 100644 --- a/kustomize/base/nginx/nginx_cache.conf +++ b/kustomize/base/nginx/nginx_cache.conf @@ -13,8 +13,6 @@ http { proxy_cache_path /server_cache/intel levels=1:2 keys_zone=intel_cache:10m max_size=20g inactive=14d use_temp_path=off; - error_log /dev/stdout info; - log_format upstream_time '$remote_addr - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent"' @@ -85,21 +83,12 @@ http { # resolver 127.0.0.11 [::1]:5353 valid=60s; - # Use kube-dns DNS resolver and public resolvers as fallback - # resolver kube-dns.kube-system.svc.cluster.local 8.8.8.8 8.8.4.4 valid=60s; - - # Use OpenShift default DNS resolver and public resolvers as fallback - resolver dns-default.openshift-dns.svc.cluster.local 8.8.8.8 8.8.4.4 valid=60s; - # rewrite_log on; ################ Docker Compose Services ################# - # Force nginx to resolve morpheus-vuln-analysis each call to allow starting this before starting the service - set $morpheus_vuln_analysis_upstream "http://agent-morpheus-rh:8080"; - location /scan { - proxy_pass $morpheus_vuln_analysis_upstream; + proxy_pass http://agent-morpheus-rh:8080; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; diff --git a/kustomize/base/nginx/templates/routes/intel.conf.template b/kustomize/base/nginx/templates/routes/intel.conf.template index 2150fbf..f5d9d0b 100644 --- a/kustomize/base/nginx/templates/routes/intel.conf.template +++ b/kustomize/base/nginx/templates/routes/intel.conf.template @@ -1,10 +1,8 @@ ####################### Intel APIs ####################### -set $serpapi_upstream https://serpapi.com; - location /serpapi { rewrite ^\/serpapi(\/.*)$ $1 break; - proxy_pass $serpapi_upstream; + proxy_pass https://serpapi.com; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -12,11 +10,9 @@ location /serpapi { client_body_buffer_size 4m; } -set $nvd_upstream https://services.nvd.nist.gov; - location /nvd { rewrite ^\/nvd(\/.*)$ /rest$1 break; - proxy_pass $nvd_upstream; + proxy_pass https://services.nvd.nist.gov; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -24,11 +20,9 @@ location /nvd { client_body_buffer_size 4m; } -set $cve_details_upstream https://www.cvedetails.com; - location /cve-details { rewrite ^\/cve-details(\/.*)$ $1 break; - proxy_pass $cve_details_upstream; + proxy_pass https://www.cvedetails.com; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -36,11 +30,9 @@ location /cve-details { client_body_buffer_size 4m; } -set $cwe_details_upstream https://cwe.mitre.org; - location /cwe-details { rewrite ^\/cwe-details(\/.*)$ $1 break; - proxy_pass $cwe_details_upstream; + proxy_pass https://cwe.mitre.org; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -50,11 +42,9 @@ location /cwe-details { error_page 301 302 307 = @handle_redirects; } -set $epss_upstream https://api.first.org; - location /first { rewrite ^\/first(\/.*)$ /data/v1$1 break; - proxy_pass $epss_upstream; + proxy_pass https://api.first.org; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -62,11 +52,9 @@ location /first { client_body_buffer_size 4m; } -set $recf_upstream https://api.recordedfuture.com; - location /recf { rewrite ^\/recf(\/.*)$ /v2$1 break; - proxy_pass $recf_upstream; + proxy_pass https://api.recordedfuture.com; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -74,11 +62,9 @@ location /recf { client_body_buffer_size 4m; } -set $ghsa_upstream https://api.github.com; - location /ghsa { rewrite ^\/ghsa(\/.*)$ $1 break; - proxy_pass $ghsa_upstream; + proxy_pass https://api.github.com; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -86,11 +72,9 @@ location /ghsa { client_body_buffer_size 4m; } -set $rhsa_upstream https://access.redhat.com; - location /rhsa { rewrite ^\/rhsa(\/.*)$ /hydra/rest$1 break; - proxy_pass $rhsa_upstream; + proxy_pass https://access.redhat.com; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -98,11 +82,9 @@ location /rhsa { client_body_buffer_size 4m; } -set $ubuntu_upstream https://ubuntu.com; - location /ubuntu { rewrite ^\/ubuntu(\/.*)$ $1 break; - proxy_pass $ubuntu_upstream; + proxy_pass https://ubuntu.com; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; @@ -110,16 +92,16 @@ location /ubuntu { client_body_buffer_size 4m; } -set $depsdev_upstream https://api.deps.dev; - location /depsdev { rewrite ^ $request_uri; rewrite ^\/depsdev\/(.*) $1 break; return 400; - proxy_pass $depsdev_upstream/$uri; + proxy_pass https://api.deps.dev/$uri; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; add_header X-Cache-Status $upstream_cache_status; client_body_buffer_size 4m; + + resolver dns-default.openshift-dns.svc.cluster.local 8.8.8.8 valid=60s; } diff --git a/kustomize/base/nginx/templates/routes/nemo.conf.template b/kustomize/base/nginx/templates/routes/nemo.conf.template index a3c7aef..117dd7b 100644 --- a/kustomize/base/nginx/templates/routes/nemo.conf.template +++ b/kustomize/base/nginx/templates/routes/nemo.conf.template @@ -1,10 +1,8 @@ -set $nemo_upstream https://api.llm.ngc.nvidia.com; - location /nemo { location ~* ^\/nemo\/v1\/models(\/.+\/completions)?$ { rewrite ^\/nemo(\/.*)$ $1 break; - proxy_pass $nemo_upstream; + proxy_pass https://api.llm.ngc.nvidia.com; proxy_set_header Connection ''; proxy_set_header Authorization $nemo_http_authorization; proxy_set_header Organization-ID $nemo_http_organization_id; @@ -17,7 +15,7 @@ location /nemo { location /nemo/v1 { rewrite ^\/nemo(\/.*)$ $1 break; - proxy_pass $nemo_upstream; + proxy_pass https://api.llm.ngc.nvidia.com; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } diff --git a/kustomize/base/nginx/templates/routes/nim.conf.template b/kustomize/base/nginx/templates/routes/nim.conf.template index f6ab6ef..13bd684 100644 --- a/kustomize/base/nginx/templates/routes/nim.conf.template +++ b/kustomize/base/nginx/templates/routes/nim.conf.template @@ -1,10 +1,7 @@ -set $upstream_nim_llm ${NGINX_UPSTREAM_NIM_LLM}; -set $upstream_nim_embed ${NGINX_UPSTREAM_NIM_EMBED}; - location ~* ^\/nim_llm\/v1\/(?:chat\/completions|completions|edits|moderations|answers)$ { rewrite ^\/nim_llm(\/.*)$ $1 break; - proxy_pass $upstream_nim_llm; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; proxy_set_header Connection ''; proxy_set_header Authorization $nim_http_authorization; proxy_cache llm_cache; @@ -16,7 +13,7 @@ location ~* ^\/nim_llm\/v1\/(?:chat\/completions|completions|edits|moderations|a location /nim_llm/v1 { rewrite ^\/nim_llm(\/.*)$ $1 break; - proxy_pass $upstream_nim_llm; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } @@ -24,7 +21,7 @@ location /nim_llm/v1 { location ~* ^/nim_embed/v1/embeddings$ { rewrite ^/nim_embed(.*)$ $1 break; - proxy_pass $upstream_nim_embed; + proxy_pass ${NGINX_UPSTREAM_NIM_EMBED}; proxy_set_header Connection ''; proxy_set_header Authorization $nim_http_authorization; proxy_cache llm_cache; @@ -36,7 +33,7 @@ location ~* ^/nim_embed/v1/embeddings$ { location /nim_embed/v1 { rewrite ^/nim_embed(.*)$ $1 break; - proxy_pass $upstream_nim_embed; + proxy_pass ${NGINX_UPSTREAM_NIM_EMBED}; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } diff --git a/kustomize/base/nginx/templates/routes/nvidia.conf.template b/kustomize/base/nginx/templates/routes/nvidia.conf.template index 2bdf18b..0f68066 100644 --- a/kustomize/base/nginx/templates/routes/nvidia.conf.template +++ b/kustomize/base/nginx/templates/routes/nvidia.conf.template @@ -1,11 +1,9 @@ -set $nvai_upstream ${NGINX_UPSTREAM_NVAI}; - location /nvai { location ~* ^\/nvai\/v2\/nvcf\/((pexec\/functions\/.+)|(functions))$ { rewrite ^\/nvai(\/.*)$ $1 break; - proxy_pass $nvai_upstream; + proxy_pass ${NGINX_UPSTREAM_NVAI}; proxy_set_header Connection ''; proxy_set_header Authorization $nvai_http_authorization; proxy_cache llm_cache; @@ -24,7 +22,7 @@ location /nvai { location ~* ^\/nvai\/v2\/nvcf\/(pexec\/status\/.+)$ { rewrite ^\/nvai(\/.*)$ $1 break; - proxy_pass $nvai_upstream; + proxy_pass ${NGINX_UPSTREAM_NVAI}; proxy_set_header Connection ''; proxy_set_header Authorization $nvai_http_authorization; proxy_cache llm_cache; @@ -40,7 +38,7 @@ location /nvai { location /nvai/v2 { rewrite ^\/nvai(\/.*)$ $1 break; - proxy_pass $nvai_upstream; + proxy_pass ${NGINX_UPSTREAM_NVAI}; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } diff --git a/kustomize/base/nginx/templates/routes/openai.conf.template b/kustomize/base/nginx/templates/routes/openai.conf.template index 8a8a0cb..6d986cb 100644 --- a/kustomize/base/nginx/templates/routes/openai.conf.template +++ b/kustomize/base/nginx/templates/routes/openai.conf.template @@ -1,10 +1,8 @@ -set $openai_upstream https://api.openai.com; - location /openai { location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass $openai_upstream; + proxy_pass https://api.openai.com; proxy_set_header Connection ''; proxy_set_header Authorization $openai_http_authorization; proxy_cache llm_cache; @@ -16,7 +14,7 @@ location /openai { location /openai/v1 { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass $openai_upstream; + proxy_pass https://api.openai.com; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } From 5e93255e2263dc6cf973c94a23c1a78fd5312dd6 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 21 Nov 2024 22:40:58 +0100 Subject: [PATCH 10/50] chore(deps): add nim-llm and nim-embed deployments Signed-off-by: Ruben Romero Montes --- kustomize/README.md | 10 ++- kustomize/base/agent_morpheus-RH.yaml | 4 + kustomize/base/agent_morpheus_client.yaml | 2 +- kustomize/base/nginx.yaml | 2 +- .../agent-morpheus-config.json | 73 +++++++++++++++ .../local-nim-service/kustomization.yaml | 17 ++++ .../local-nim-service/nginx-patch.yaml | 21 +++++ .../nim-embed-image-pull-secret-patch.yaml | 9 ++ .../overlays/local-nim-service/nim-embed.yaml | 88 +++++++++++++++++++ .../nim-llm-image-pull-secret-patch.yaml | 9 ++ .../overlays/local-nim-service/nim-llm.yaml | 88 +++++++++++++++++++ 11 files changed, 319 insertions(+), 4 deletions(-) create mode 100644 kustomize/overlays/local-nim-service/agent-morpheus-config.json create mode 100644 kustomize/overlays/local-nim-service/kustomization.yaml create mode 100644 kustomize/overlays/local-nim-service/nginx-patch.yaml create mode 100644 kustomize/overlays/local-nim-service/nim-embed-image-pull-secret-patch.yaml create mode 100644 kustomize/overlays/local-nim-service/nim-embed.yaml create mode 100644 kustomize/overlays/local-nim-service/nim-llm-image-pull-secret-patch.yaml create mode 100644 kustomize/overlays/local-nim-service/nim-llm.yaml diff --git a/kustomize/README.md b/kustomize/README.md index a4bd2a2..7acb388 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -10,6 +10,7 @@ serpapi_api_key=your_api_key tavily_api_key=your_api_key nvidia_api_key=your_api_key ghsa_api_key=your_api_key +ngc_api_key=your_api_key EOF ``` @@ -24,12 +25,17 @@ oc new-project $YOUR_NAMESPACE_NAME oc create secret generic morpheus-pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson ``` -4. Deploy agent-morpheus + agent-morpheus-client to your namespace +4. Create an image pull secret that is authorized to pull images from ngrc.io: +```shell +oc create secret generic ngrc-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson +``` + +5. Deploy agent-morpheus + agent-morpheus-client to your namespace ```shell oc kustomize overlays/nvidia-llm-service | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` -5. Alternatively, if you want to deploy agent-morpheus with our self-hosted LLM, then run +6. Alternatively, if you want to deploy agent-morpheus with our self-hosted LLM, then run ```shell # Enable ingress traffic into our LLM Model service in the cluster oc apply -f network-policy.yaml diff --git a/kustomize/base/agent_morpheus-RH.yaml b/kustomize/base/agent_morpheus-RH.yaml index 5cefa2a..ca4498a 100644 --- a/kustomize/base/agent_morpheus-RH.yaml +++ b/kustomize/base/agent_morpheus-RH.yaml @@ -77,6 +77,10 @@ spec: secretKeyRef: key: ghsa_api_key name: agent-morpheus-secret + - name: OPENAI_API_KEY + value: "" + - name: NGC_API_KEY + value: "" - name: CVE_DETAILS_BASE_URL value: http://nginx-cache:8080/cve-details - name: CWE_DETAILS_BASE_URL diff --git a/kustomize/base/agent_morpheus_client.yaml b/kustomize/base/agent_morpheus_client.yaml index 9a04519..85bb9a8 100644 --- a/kustomize/base/agent_morpheus_client.yaml +++ b/kustomize/base/agent_morpheus_client.yaml @@ -30,7 +30,7 @@ spec: containerPort: 8080 env: - name: QUARKUS_REST-CLIENT_MORPHEUS_URL - value: http://agent-morpheus-rh:8080/scan + value: http://nginx-cache:8080/scan - name: QUARKUS_MONGODB_HOSTS value: agent-morpheus-client-db:27017 - name: QUARKUS_MONGODB_DATABASE diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml index f7e41c4..cb7e277 100644 --- a/kustomize/base/nginx.yaml +++ b/kustomize/base/nginx.yaml @@ -138,4 +138,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 1Gi + storage: 20Gi diff --git a/kustomize/overlays/local-nim-service/agent-morpheus-config.json b/kustomize/overlays/local-nim-service/agent-morpheus-config.json new file mode 100644 index 0000000..454a33e --- /dev/null +++ b/kustomize/overlays/local-nim-service/agent-morpheus-config.json @@ -0,0 +1,73 @@ +{ + "$schema": "./schemas/config.schema.json", + "engine": { + "agent": { + "model": { + "model_name": "meta/llama3.1-8b-instruct", + "service": { + "_type": "openai" + }, + "max_tokens": 2000 + }, + "verbose": true, + "version_compare_tool": true + }, + "checklist_model": { + "service": { + "_type": "openai" + }, + "model_name": "meta/llama3.1-8b-instruct", + "temperature": 0, + "max_tokens": 2000, + "top_p": 0.01, + "seed": 42 + }, + "justification_model": { + "model_name": "meta/llama3.1-8b-instruct", + "service": { + "_type": "openai" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "rag_embedding": { + "_type": "nim", + "model": "nvidia/nv-embedqa-e5-v5", + "truncate": "END", + "max_batch_size": 128 + }, + "summary_model": { + "model_name": "meta/llama3.1-8b-instruct", + "service": { + "_type": "openai" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + } + }, + "general": { + "cache_dir": null, + "base_vdb_dir": "/morpheus-data/vdbs", + "base_git_dir": "/morpheus-data/repos", + "max_retries": 5, + "model_max_batch_size": 64, + "pipeline_batch_size": 128, + "use_uvloop": true + }, + "input": { + "_type": "http", + "port": 8080, + "address": "0.0.0.0" + }, + "output": { + "_type": "plugin", + "plugin_name": "src.cve.data_models.plugins.http_output_plugin.HttpOutputPlugin", + "plugin_config": { + "callback_url": "http://agent-morpheus-client:8080/reports" + } + } +} diff --git a/kustomize/overlays/local-nim-service/kustomization.yaml b/kustomize/overlays/local-nim-service/kustomization.yaml new file mode 100644 index 0000000..415e2fe --- /dev/null +++ b/kustomize/overlays/local-nim-service/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + - nim-embed.yaml + - nim-llm.yaml + +configMapGenerator: + - name: agent-morpheus-config + files: + - agent-morpheus-config.json + +patchesStrategicMerge: + - nginx-patch.yaml + - nim-embed-image-pull-secret-patch.yaml + - nim-llm-image-pull-secret-patch.yaml \ No newline at end of file diff --git a/kustomize/overlays/local-nim-service/nginx-patch.yaml b/kustomize/overlays/local-nim-service/nginx-patch.yaml new file mode 100644 index 0000000..6bd86d2 --- /dev/null +++ b/kustomize/overlays/local-nim-service/nginx-patch.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache +spec: + template: + spec: + containers: + - name: nginx + env: + - name: NGINX_UPSTREAM_NVAI + value: http://nim-llm:8000 + - name: NGINX_UPSTREAM_NIM_LLM + value: http://nim-llm:8000 + - name: NGINX_UPSTREAM_NIM_EMBED + value: http://nim-embed:8000 + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + key: ngc_api_key + name: agent-morpheus-secret \ No newline at end of file diff --git a/kustomize/overlays/local-nim-service/nim-embed-image-pull-secret-patch.yaml b/kustomize/overlays/local-nim-service/nim-embed-image-pull-secret-patch.yaml new file mode 100644 index 0000000..92ac824 --- /dev/null +++ b/kustomize/overlays/local-nim-service/nim-embed-image-pull-secret-patch.yaml @@ -0,0 +1,9 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nim-embed +spec: + template: + spec: + imagePullSecrets: + - name: nvcr-secret diff --git a/kustomize/overlays/local-nim-service/nim-embed.yaml b/kustomize/overlays/local-nim-service/nim-embed.yaml new file mode 100644 index 0000000..5ed187f --- /dev/null +++ b/kustomize/overlays/local-nim-service/nim-embed.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nim-embed + labels: + app: agent-morpheus + component: nim-embed +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: agent-morpheus + component: nim-embed + template: + metadata: + labels: + app: agent-morpheus + component: nim-embed + spec: + nodeSelector: + nvidia.com/gpu.deploy.driver: "true" + imagePullSecrets: [] + tolerations: + - key: p4-gpu + operator: Exists + effect: NoSchedule + containers: + - name: nim-embed + image: nvcr.io/nim/nvidia/nv-embedqa-e5-v5:1 + ports: + - name: http + protocol: TCP + containerPort: 8000 + resources: + limits: + memory: "16Gi" + cpu: "500m" + nvidia.com/gpu: "1" + requests: + memory: "8Gi" + cpu: "100m" + nvidia.com/gpu: "1" + env: + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + key: ngc_api_key + name: agent-morpheus-secret + volumeMounts: + - name: cache + mountPath: /opt/nim/.cache + volumes: + - name: cache + persistentVolumeClaim: + claimName: nim-embed-cache +--- +apiVersion: v1 +kind: Service +metadata: + name: nim-embed + labels: + app: agent-morpheus + component: nim-embed +spec: + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: 8000 + selector: + app: agent-morpheus + component: nim-embed +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nim-embed-cache + labels: + app: agent-morpheus + component: nim-embed +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi \ No newline at end of file diff --git a/kustomize/overlays/local-nim-service/nim-llm-image-pull-secret-patch.yaml b/kustomize/overlays/local-nim-service/nim-llm-image-pull-secret-patch.yaml new file mode 100644 index 0000000..74cf4c8 --- /dev/null +++ b/kustomize/overlays/local-nim-service/nim-llm-image-pull-secret-patch.yaml @@ -0,0 +1,9 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nim-llm +spec: + template: + spec: + imagePullSecrets: + - name: nvcr-secret diff --git a/kustomize/overlays/local-nim-service/nim-llm.yaml b/kustomize/overlays/local-nim-service/nim-llm.yaml new file mode 100644 index 0000000..1808ced --- /dev/null +++ b/kustomize/overlays/local-nim-service/nim-llm.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nim-llm + labels: + app: agent-morpheus + component: nim-llm +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: agent-morpheus + component: nim-llm + template: + metadata: + labels: + app: agent-morpheus + component: nim-llm + spec: + nodeSelector: + nvidia.com/gpu.deploy.driver: "true" + imagePullSecrets: [] + tolerations: + - key: p4-gpu + operator: Exists + effect: NoSchedule + containers: + - name: nim-llm + image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.3 + ports: + - name: http + protocol: TCP + containerPort: 8000 + resources: + limits: + memory: "16Gi" + cpu: "500m" + nvidia.com/gpu: "1" + requests: + memory: "8Gi" + cpu: "100m" + nvidia.com/gpu: "1" + env: + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + key: ngc_api_key + name: agent-morpheus-secret + volumeMounts: + - name: cache + mountPath: /opt/nim/.cache + volumes: + - name: cache + persistentVolumeClaim: + claimName: nim-llm-cache +--- +apiVersion: v1 +kind: Service +metadata: + name: nim-llm + labels: + app: agent-morpheus + component: nim-llm +spec: + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: 8000 + selector: + app: agent-morpheus + component: nim-llm +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nim-llm-cache + labels: + app: agent-morpheus + component: nim-llm +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi \ No newline at end of file From 6fabe337027503c871a12fcdfd6d1e7db7d14822 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 21 Nov 2024 22:42:22 +0100 Subject: [PATCH 11/50] feat!: support zip artifact resources Signed-off-by: Ruben Romero Montes --- .pytest.ini | 3 + configs/from_manual.json | 6 +- configs/schemas/config.schema.json | 179 ++++++++++++------ .../morpheus:23.11-runtime.json | 6 +- .../morpheus:24.03-runtime.json | 6 +- src/cve/data_models/__init__.py | 14 ++ src/cve/data_models/input.py | 41 +++- src/cve/data_models/tests/__init__.py | 14 ++ src/cve/data_models/tests/test_input.py | 69 +++++++ src/cve/pipeline/engine.py | 2 +- src/cve/utils/document_embedding.py | 36 ++-- src/cve/utils/source_code_artifact_loader.py | 170 +++++++++++++++++ src/cve/utils/tests/__init__.py | 14 ++ src/cve/utils/tests/test_source_loaders.py | 162 ++++++++++++++++ 14 files changed, 634 insertions(+), 88 deletions(-) create mode 100644 .pytest.ini create mode 100644 src/cve/data_models/__init__.py create mode 100644 src/cve/data_models/tests/__init__.py create mode 100644 src/cve/data_models/tests/test_input.py create mode 100644 src/cve/utils/source_code_artifact_loader.py create mode 100644 src/cve/utils/tests/__init__.py create mode 100644 src/cve/utils/tests/test_source_loaders.py diff --git a/.pytest.ini b/.pytest.ini new file mode 100644 index 0000000..dcf87ef --- /dev/null +++ b/.pytest.ini @@ -0,0 +1,3 @@ +# pytest.ini +[pytest] +asyncio_default_fixture_loop_scope = function \ No newline at end of file diff --git a/configs/from_manual.json b/configs/from_manual.json index d204bb8..27b2bc0 100644 --- a/configs/from_manual.json +++ b/configs/from_manual.json @@ -71,7 +71,8 @@ "tag": "23.11-runtime", "source_info": [ { - "type": "code", + "type": "git", + "source_type": "code", "git_repo": "https://github.com/nv-morpheus/Morpheus.git", "ref": "v23.11.01", "include": [ @@ -89,7 +90,8 @@ ] }, { - "type": "doc", + "type": "git", + "source_type": "doc", "git_repo": "https://github.com/nv-morpheus/Morpheus.git", "ref": "v23.11.01", "include": [ diff --git a/configs/schemas/config.schema.json b/configs/schemas/config.schema.json index 7a3b490..37451f3 100644 --- a/configs/schemas/config.schema.json +++ b/configs/schemas/config.schema.json @@ -17,6 +17,59 @@ "title": "AgentMorpheusInput", "type": "object" }, + "ArtifactSourceInfo": { + "description": "Information about the Artifact source documents for the container image.\n\n- path: remote path that will be downloaded and unzipped.", + "properties": { + "type": { + "const": "artifact", + "default": "artifact", + "title": "Type" + }, + "source_type": { + "enum": [ + "code", + "doc" + ], + "title": "Source Type", + "type": "string" + }, + "include": { + "default": [ + "*.py", + "*.ipynb" + ], + "items": { + "type": "string" + }, + "title": "Include", + "type": "array" + }, + "exclude": { + "default": [], + "items": { + "type": "string" + }, + "title": "Exclude", + "type": "array" + }, + "path": { + "minLength": 1, + "title": "Path", + "type": "string" + }, + "compression": { + "const": "zip", + "default": "zip", + "title": "Compression" + } + }, + "required": [ + "source_type", + "path" + ], + "title": "ArtifactSourceInfo", + "type": "object" + }, "EngineAgentConfig": { "properties": { "model": { @@ -264,6 +317,60 @@ "title": "GeneralConfig", "type": "object" }, + "GitSourceInfo": { + "description": "Information about the Git source documents for the container image.\n\n- git_repo: git repo URL where the source documents can be cloned.\n- ref: git reference. i.e. tag/branch/commit_id", + "properties": { + "type": { + "const": "git", + "default": "git", + "title": "Type" + }, + "source_type": { + "enum": [ + "code", + "doc" + ], + "title": "Source Type", + "type": "string" + }, + "include": { + "default": [ + "*.py", + "*.ipynb" + ], + "items": { + "type": "string" + }, + "title": "Include", + "type": "array" + }, + "exclude": { + "default": [], + "items": { + "type": "string" + }, + "title": "Exclude", + "type": "array" + }, + "git_repo": { + "minLength": 1, + "title": "Git Repo", + "type": "string" + }, + "ref": { + "minLength": 1, + "title": "Ref", + "type": "string" + } + }, + "required": [ + "source_type", + "git_repo", + "ref" + ], + "title": "GitSourceInfo", + "type": "object" + }, "HTTPMethod": { "description": "Not a complete list of HTTP methods, just the ones we use.", "enum": [ @@ -409,7 +516,14 @@ }, "source_info": { "items": { - "$ref": "#/$defs/SourceDocumentsInfo" + "anyOf": [ + { + "$ref": "#/$defs/GitSourceInfo" + }, + { + "$ref": "#/$defs/ArtifactSourceInfo" + } + ] }, "title": "Source Info", "type": "array" @@ -1010,16 +1124,8 @@ "description": "Information about a unique scan for a container image against a list of vulnerabilies.", "properties": { "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Id" + "title": "Id", + "type": "string" }, "type": { "anyOf": [ @@ -1072,55 +1178,6 @@ "title": "ScanInfoInput", "type": "object" }, - "SourceDocumentsInfo": { - "description": "Information about the source documents for the container image.\n\n- type: document type.\n- git_repo: git repo URL where the source documents can be cloned.\n- ref: git reference, such as tag/branch/commit_id\n- include: file extensions to include when indexing the source documents.\n- exclude: file extensions to exclude when indexing the source documents.", - "properties": { - "type": { - "enum": [ - "code", - "doc" - ], - "title": "Type", - "type": "string" - }, - "git_repo": { - "minLength": 1, - "title": "Git Repo", - "type": "string" - }, - "ref": { - "minLength": 1, - "title": "Ref", - "type": "string" - }, - "include": { - "default": [ - "*.py", - "*.ipynb" - ], - "items": { - "type": "string" - }, - "title": "Include", - "type": "array" - }, - "exclude": { - "default": [], - "items": { - "type": "string" - }, - "title": "Exclude", - "type": "array" - } - }, - "required": [ - "type", - "git_repo", - "ref" - ], - "title": "SourceDocumentsInfo", - "type": "object" - }, "VulnInfo": { "additionalProperties": true, "description": "Information about a vulnerability.", @@ -1284,7 +1341,7 @@ "ignore_build_vdb_errors": false, "max_retries": 10, "model_max_batch_size": 64, - "num_threads": 64, + "num_threads": 5, "pipeline_batch_size": 1024, "use_uvloop": true, "code_search_tool": false diff --git a/data/input_messages/morpheus:23.11-runtime.json b/data/input_messages/morpheus:23.11-runtime.json index 2805c60..d5635eb 100644 --- a/data/input_messages/morpheus:23.11-runtime.json +++ b/data/input_messages/morpheus:23.11-runtime.json @@ -4,7 +4,8 @@ "tag": "23.11-runtime", "source_info": [ { - "type": "code", + "type": "git", + "source_type": "code", "git_repo": "https://github.com/nv-morpheus/Morpheus.git", "ref": "v23.11.01", "include": [ @@ -22,7 +23,8 @@ ] }, { - "type": "doc", + "type": "git", + "source_type": "doc", "git_repo": "https://github.com/nv-morpheus/Morpheus.git", "ref": "v23.11.01", "include": [ diff --git a/data/input_messages/morpheus:24.03-runtime.json b/data/input_messages/morpheus:24.03-runtime.json index feb3c20..6eb96db 100644 --- a/data/input_messages/morpheus:24.03-runtime.json +++ b/data/input_messages/morpheus:24.03-runtime.json @@ -4,7 +4,8 @@ "tag": "v24.03.02-runtime", "source_info": [ { - "type": "code", + "type": "git", + "source_type": "code", "git_repo": "https://github.com/nv-morpheus/Morpheus.git", "ref": "v24.03.02", "include": [ @@ -22,7 +23,8 @@ ] }, { - "type": "doc", + "type": "git", + "source_type": "doc", "git_repo": "https://github.com/nv-morpheus/Morpheus.git", "ref": "v24.03.02", "include": [ diff --git a/src/cve/data_models/__init__.py b/src/cve/data_models/__init__.py new file mode 100644 index 0000000..28d3a3a --- /dev/null +++ b/src/cve/data_models/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cve/data_models/input.py b/src/cve/data_models/input.py index 1554c2f..cf10790 100644 --- a/src/cve/data_models/input.py +++ b/src/cve/data_models/input.py @@ -14,6 +14,7 @@ # limitations under the License. +from abc import ABC import typing from uuid import uuid4 @@ -32,22 +33,18 @@ from .info import SBOMPackage -class SourceDocumentsInfo(HashableModel): +class SourceInfo(HashableModel, ABC): """ Information about the source documents for the container image. - - type: document type. - - git_repo: git repo URL where the source documents can be cloned. - - ref: git reference, such as tag/branch/commit_id + - type: the source type. It can be Git or Artifact + - source_type: document type. It can be code or doc - include: file extensions to include when indexing the source documents. - exclude: file extensions to exclude when indexing the source documents. """ type: typing.Literal["code", "doc"] - - git_repo: typing.Annotated[str, Field(min_length=1)] - ref: typing.Annotated[str, Field(min_length=1, validation_alias=AliasChoices( - "ref", "tag"))] # Support "tag" as alias for backward compatibility + source_type: typing.Literal["code", "doc"] include: list[str] = ["*.py", "*.ipynb"] exclude: list[str] = [] @@ -58,6 +55,34 @@ def sort_lists(cls, v: list[str]) -> list[str]: return list(sorted(v)) +class GitSourceInfo(SourceInfo): + """ + Information about the Git source documents for the container image. + + - git_repo: git repo URL where the source documents can be cloned. + - ref: git reference. i.e. tag/branch/commit_id + """ + type: typing.Literal["git"] = "git" + git_repo: typing.Annotated[str, Field(min_length=1)] + ref: typing.Annotated[str, Field(min_length=1, validation_alias=AliasChoices( + "ref", "tag"))] # Support "tag" as alias for backward compatibility + + +class ArtifactSourceInfo(SourceInfo): + + """ + Information about the Artifact source documents for the container image. + + - path: remote path that will be downloaded and unzipped. + """ + type: typing.Literal["artifact"] = "artifact" + path: typing.Annotated[str, Field(min_length=1)] + compression: typing.Literal["zip"] = "zip" + + +SourceDocumentsInfo = typing.Union[GitSourceInfo, ArtifactSourceInfo] + + class VulnInfo(HashableModel): """ Information about a vulnerability. diff --git a/src/cve/data_models/tests/__init__.py b/src/cve/data_models/tests/__init__.py new file mode 100644 index 0000000..28d3a3a --- /dev/null +++ b/src/cve/data_models/tests/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cve/data_models/tests/test_input.py b/src/cve/data_models/tests/test_input.py new file mode 100644 index 0000000..4babfad --- /dev/null +++ b/src/cve/data_models/tests/test_input.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ..input import ArtifactSourceInfo, GitSourceInfo, ImageInfoInput, ManualSBOMInfoInput + +def test_list(): + # Test data with a GitSourceInfo and an ArtifactSourceInfo + data = { + "name": "example-image", + "sbom_info": { + "_type": "manual", + "packages": [] + }, + "source_info": [ + { + "_type": "git", + "source_type": "code", + "git_repo": "https://github.com/example/repo.git", + "ref": "main", + "include": ["*.py"], + "exclude": ["*.md"] + }, + { + "_type": "artifact", + "source_type": "doc", + "path": "/path/to/artifact.zip", + "compression": "zip", + "include": ["*.pdf"], + "exclude": ["*.tmp"] + } + ] + } + + # Create an Example instance + example = ImageInfoInput(**data) + + # Assertions to verify correct parsing + assert example.name == "example-image" + assert len(example.source_info) == 2 + + # Check the first item (GitSourceInfo) + assert isinstance(example.source_info[0], GitSourceInfo) + assert not isinstance(example.source_info[0], ArtifactSourceInfo) + assert example.source_info[0].type == "git" + assert example.source_info[0].git_repo == "https://github.com/example/repo.git" + assert example.source_info[0].ref == "main" + assert example.source_info[0].include == ["*.py"] + assert example.source_info[0].exclude == ["*.md"] + + # Check the second item (ArtifactSourceInfo) + assert isinstance(example.source_info[1], ArtifactSourceInfo) + assert example.source_info[1].type == "artifact" + assert example.source_info[1].path == "/path/to/artifact.zip" + assert example.source_info[1].compression == "none" + assert example.source_info[1].include == ["*.pdf"] + assert example.source_info[1].exclude == ["*.tmp"] \ No newline at end of file diff --git a/src/cve/pipeline/engine.py b/src/cve/pipeline/engine.py index 3e97b5e..fb78932 100644 --- a/src/cve/pipeline/engine.py +++ b/src/cve/pipeline/engine.py @@ -114,7 +114,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d documents = [] sources = context.message().get_metadata("input").image.source_info for source_info in sources: - if source_info.type == 'code': + if source_info.source_type == 'code': documents.extend(embedder.collect_documents(source_info)) if len(documents) > 0: diff --git a/src/cve/utils/document_embedding.py b/src/cve/utils/document_embedding.py index 3403513..fc6866f 100644 --- a/src/cve/utils/document_embedding.py +++ b/src/cve/utils/document_embedding.py @@ -36,7 +36,9 @@ from langchain_community.document_loaders.parsers.language.language_parser import LANGUAGE_SEGMENTERS from langchain_core.document_loaders.blob_loaders import Blob -from ..data_models.input import SourceDocumentsInfo + +from ..data_models.input import ArtifactSourceInfo, GitSourceInfo, SourceDocumentsInfo +from .source_code_artifact_loader import SourceCodeArtifactLoader from .source_code_git_loader import SourceCodeGitLoader if typing.TYPE_CHECKING: @@ -326,14 +328,24 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: list[Document] Returns a list of documents collected from the source document info. """ - - repo_path = self.get_repo_path(source_info) - - blob_loader = SourceCodeGitLoader(repo_path=repo_path, - clone_url=source_info.git_repo, - ref=source_info.ref, - include=source_info.include, - exclude=source_info.exclude) + if isinstance(source_info, GitSourceInfo): + + path = self.get_repo_path(source_info) + + blob_loader = SourceCodeGitLoader(repo_path=path, + clone_url=source_info.git_repo, + ref=source_info.ref, + include=source_info.include, + exclude=source_info.exclude) + + elif isinstance(source_info, ArtifactSourceInfo): + + path = self._git_directory / PurePath(source_info.path) + blob_loader = SourceCodeArtifactLoader(local_path=path, + remote_path=source_info.path, + compression=source_info.compression, + include=source_info.include, + exclude=source_info.exclude) blob_parser = ExtendedLanguageParser() @@ -341,7 +353,7 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: documents = loader.load() - logger.debug("Collected documents for '%s', Document count: %d", repo_path, len(documents)) + logger.debug("Collected documents for '%s', Document count: %d", path, len(documents)) return documents @@ -363,7 +375,7 @@ def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathL Returns an instance of the FAISS database. """ - logger.debug("Collecting documents from git repos. Source Infos: %s", + logger.debug("Collecting documents from external sources. Source Infos: %s", json.dumps([x.model_dump(mode="json") for x in source_infos])) output_path = Path(output_path) @@ -448,7 +460,7 @@ def build_vdbs(self, continue # Filter the source documents - source_infos = [source_info for source_info in input_sources if source_info.type == source_type] + source_infos = [source_info for source_info in input_sources if source_info.source_type == source_type] if source_infos: diff --git a/src/cve/utils/source_code_artifact_loader.py b/src/cve/utils/source_code_artifact_loader.py new file mode 100644 index 0000000..aba79e5 --- /dev/null +++ b/src/cve/utils/source_code_artifact_loader.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import fnmatch +from io import BytesIO +import logging +import os +import shutil +import typing +from pathlib import Path +from zipfile import ZipFile + +from langchain_community.document_loaders.blob_loaders.schema import BlobLoader +from langchain_core.document_loaders.blob_loaders import Blob +import requests +from tqdm import tqdm + +PathLike = typing.Union[str, os.PathLike] + +logger = logging.getLogger(__name__) + + +class SourceCodeArtifactLoader(BlobLoader): + """ + Load `Artifact` source files. + + The Artifact can be local on disk available at `local_path`, + or remote at `remote_path` that will be cloned to `local_path`. + Currently, supports only text files. + + Each document represents one file. The `local_path` points to + the local extracted artifact folder. + """ + + def __init__( + self, + local_path: PathLike, + remote_path: str | None = None, + compression: str | None = None, + include: typing.Optional[typing.Iterable[str]] = None, + exclude: typing.Optional[typing.Iterable[str]] = None, + ): + """ + Initialize the Git loader. + + Parameters + ---------- + local_path : PathLike + Path to the local extracted artifact. + remote_path : str | None, optional + URL to the remote Artifact to be downloaded and extracted, by default None + compression : compression type of the artifact + include : typing.Optional[typing.Iterable[str]], optional + A list of file patterns to include. Uses the glob syntax, by default None + exclude : typing.Optional[typing.Iterable[str]], optional + A list of file patterns to exclude. Uses the glob syntax, by default None + """ + + self.local_path = Path(local_path) + self.remote_path = remote_path + self.compression = compression + + self.include = include + self.exclude = exclude + + + def load_artifact(self): + """ + Downloads the Artifact and return the location of the extracted artifact folder. + + Returns + ------- + str + Location of the extracted folder. + + Raises + ------ + ValueError + If only the path is provided but does not exist. + """ + + if not os.path.exists(self.local_path) and self.remote_path is None: + raise ValueError(f"Path {self.local_path} does not exist") + + elif self.remote_path: + if not os.path.isdir(self.local_path): + logger.debug("Downloading artifact from URL: '%s'", self.remote_path) + + with requests.get(self.remote_path, stream=True) as remote_source: + if self.compression == "zip": + with ZipFile(BytesIO(remote_source.content)) as zfile: + zfile.extractall(self.local_path) + + subdirs = os.listdir(self.local_path) + if len(subdirs) == 1: + src_dir = os.path.join(self.local_path, subdirs[0]) + contents = os.listdir(src_dir) + for f in contents: + src_path = os.path.join(src_dir, f) + shutil.move(src_path, os.path.join(self.local_path, f)) + else: + raise ValueError(f"Unsupported compression type {self.compression}") + + logger.debug("Loaded Artifact at path: '%s' ", self.local_path) + + else: + logger.debug("Using existing Artifact at path: '%s'", self.local_path) + + return self.local_path + + def yield_blobs(self) -> typing.Iterator[Blob]: + """ + Yield the blobs from the downloaded artifact folder. The retrieved files will be + filtered using the includes and excludes expressions. + + Returns + ------- + typing.Iterator[Blob] + An iterator of `Blob` objects representing the files in the local folder. + + Yields + ------ + Iterator[typing.Iterator[Blob]] + An iterator of `Blob` objects representing the files in the local folder. + """ + + path = self.load_artifact() + + logger.debug("Scanning documents for path: '%s'", path) + + base_path = Path(path) + + final_files = [] + + for include_pattern in self.include: + for file in base_path.rglob(include_pattern): + rel_file = file.relative_to(path) + # If the file matches any exclude pattern, skip it + if file.is_file() and not any(fnmatch.fnmatch(str(rel_file), exclude_pattern) for exclude_pattern in self.exclude): + final_files.append(rel_file) + + for f in tqdm(final_files): + + file_path = Path(f) + + abs_file_path = base_path / file_path + + rel_file_path = str(file_path) + + metadata = { + "source": rel_file_path, + "file_path": rel_file_path, + "file_name": file_path.name, + "file_type": file_path.suffix, + } + + yield Blob.from_path(abs_file_path, metadata=metadata) diff --git a/src/cve/utils/tests/__init__.py b/src/cve/utils/tests/__init__.py new file mode 100644 index 0000000..28d3a3a --- /dev/null +++ b/src/cve/utils/tests/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cve/utils/tests/test_source_loaders.py b/src/cve/utils/tests/test_source_loaders.py new file mode 100644 index 0000000..768faac --- /dev/null +++ b/src/cve/utils/tests/test_source_loaders.py @@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import PurePath +import shutil +import tempfile + +import pytest + +from ..source_code_git_loader import SourceCodeGitLoader +from ...data_models.input import ArtifactSourceInfo, GitSourceInfo +from ..source_code_artifact_loader import SourceCodeArtifactLoader + + +test_artifact_data = [ + ({ + "type": "artifact", + "source_type": "code", + "path": "https://github.com/openshift/cluster-resource-override-admission-operator/archive/41e0b0161c9f2a08ebfbacbb9a712a5746b971df.zip", + "compress": "zip", + "include": [ + "**/*.go", + "go.*", + "**/*.py", + "requirements.txt", + "Pipfile", + "Pipfile.lock", + "pyproject.toml", + "setup.py", + "setup.cfg", + "Dockerfile*", + "docker-compose.yml", + "*.dockerfile", + "*.dockerignore", + "docker-compose.*.yml", + "*.sh", + "scripts/**/*", + "*.env", + "*.yaml", + "*.yml", + "*.json", + "config/**/*", + "conf.d/**/*" + ], + "exclude": [ + "test/**/*", + "vendor/**/*", + "tests/**/*", + "test/**/*", + "venv/**/*", + ".venv/**/*", + "env/**/*", + "build/**/*", + "dist/**/*", + ".mypy_cache/**/*", + ".pytest_cache/**/*", + "__pycache__/**/*", + "*.pyc", + "*.pyo", + "*.pyd", + ".github/**/*" + ] + }, 156), + ({ + "type": "artifact", + "source_type": "doc", + "path": "https://github.com/openshift/cluster-resource-override-admission-operator/archive/41e0b0161c9f2a08ebfbacbb9a712a5746b971df.zip", + "compress": "zip", + "include": [ + "**/*.md", + "docs/**/*.rst" + ], + "exclude": [] + }, 119) +] + +@pytest.mark.parametrize("data, expected", test_artifact_data) +def test_artifact_loader(data, expected): + + source_info = ArtifactSourceInfo(**data) + tmp_path = tempfile.mkdtemp() + path = tmp_path / PurePath(source_info.path) + + blob_loader = SourceCodeArtifactLoader(local_path=path, + remote_path=source_info.path, + compression=source_info.compression, + include=source_info.include, + exclude=source_info.exclude) + try: + blobs = blob_loader.yield_blobs() + + count = 0 + for _ in blobs: + count = count + 1 + + assert count == expected + finally: + shutil.rmtree(tmp_path) + +test_git_data = [ + ({ + "_type": "git", + "source_type": "code", + "git_repo": "https://github.com/openshift/telemeter", + "ref": "14489f7dc656175e11a3ef962fcbcd113b3651a9", + "include": [ + "**/*.go", + "go.*" + ], + "exclude": [ + "test/**/*", + "vendor/**/*" + ] + }, 121), + ({ + "_type": "git", + "source_type": "code", + "git_repo": "https://github.com/openshift/telemeter", + "ref": "14489f7dc656175e11a3ef962fcbcd113b3651a9", + "include": [ + "docs/**/*.md" + ], + "exclude": [ ] + }, 7) +] + + +@pytest.mark.parametrize("data, expected", test_git_data) +def test_git_loader(data, expected): + + source_info = GitSourceInfo(**data) + tmp_path = tempfile.mkdtemp() + path = tmp_path / PurePath(source_info.git_repo) + + blob_loader = SourceCodeGitLoader(repo_path=path, + clone_url=source_info.git_repo, + ref=source_info.ref, + include=source_info.include, + exclude=source_info.exclude) + try: + blobs = blob_loader.yield_blobs() + + count = 0 + for _ in blobs: + count = count + 1 + + assert count == expected + finally: + shutil.rmtree(tmp_path) + From ac547494998d1413a37eacef28a4c00f3f17f896 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Sun, 24 Nov 2024 19:01:22 +0200 Subject: [PATCH 12/50] fix: configure local Llama 3-1-8b LLM to work with nginx Signed-off-by: Zvi Grinberg --- kustomize/base/agent_morpheus-RH.yaml | 2 +- .../base/nginx/templates/routes/openai.conf.template | 5 ++--- .../overlays/local-nim-service/agent-morpheus-config.json | 8 ++++---- kustomize/overlays/local-nim-service/kustomization.yaml | 4 ++-- kustomize/overlays/local-nim-service/nginx-patch.yaml | 4 ++-- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/kustomize/base/agent_morpheus-RH.yaml b/kustomize/base/agent_morpheus-RH.yaml index ca4498a..0958f48 100644 --- a/kustomize/base/agent_morpheus-RH.yaml +++ b/kustomize/base/agent_morpheus-RH.yaml @@ -78,7 +78,7 @@ spec: key: ghsa_api_key name: agent-morpheus-secret - name: OPENAI_API_KEY - value: "" + value: "dummy-key" - name: NGC_API_KEY value: "" - name: CVE_DETAILS_BASE_URL diff --git a/kustomize/base/nginx/templates/routes/openai.conf.template b/kustomize/base/nginx/templates/routes/openai.conf.template index 6d986cb..73cf2a6 100644 --- a/kustomize/base/nginx/templates/routes/openai.conf.template +++ b/kustomize/base/nginx/templates/routes/openai.conf.template @@ -2,9 +2,8 @@ location /openai { location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass https://api.openai.com; + proxy_pass http://nim-llm.nim:8000; proxy_set_header Connection ''; - proxy_set_header Authorization $openai_http_authorization; proxy_cache llm_cache; proxy_cache_methods POST; proxy_cache_key "$request_method|$request_uri|$request_body"; @@ -14,7 +13,7 @@ location /openai { location /openai/v1 { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass https://api.openai.com; + proxy_pass http://nim-llm.nim:8000; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } diff --git a/kustomize/overlays/local-nim-service/agent-morpheus-config.json b/kustomize/overlays/local-nim-service/agent-morpheus-config.json index 454a33e..636d359 100644 --- a/kustomize/overlays/local-nim-service/agent-morpheus-config.json +++ b/kustomize/overlays/local-nim-service/agent-morpheus-config.json @@ -3,7 +3,7 @@ "engine": { "agent": { "model": { - "model_name": "meta/llama3.1-8b-instruct", + "model_name": "meta/llama-3.1-8b-instruct", "service": { "_type": "openai" }, @@ -16,14 +16,14 @@ "service": { "_type": "openai" }, - "model_name": "meta/llama3.1-8b-instruct", + "model_name": "meta/llama-3.1-8b-instruct", "temperature": 0, "max_tokens": 2000, "top_p": 0.01, "seed": 42 }, "justification_model": { - "model_name": "meta/llama3.1-8b-instruct", + "model_name": "meta/llama-3.1-8b-instruct", "service": { "_type": "openai" }, @@ -39,7 +39,7 @@ "max_batch_size": 128 }, "summary_model": { - "model_name": "meta/llama3.1-8b-instruct", + "model_name": "meta/llama-3.1-8b-instruct", "service": { "_type": "openai" }, diff --git a/kustomize/overlays/local-nim-service/kustomization.yaml b/kustomize/overlays/local-nim-service/kustomization.yaml index 415e2fe..a997128 100644 --- a/kustomize/overlays/local-nim-service/kustomization.yaml +++ b/kustomize/overlays/local-nim-service/kustomization.yaml @@ -4,7 +4,7 @@ kind: Kustomization resources: - ../../base - nim-embed.yaml - - nim-llm.yaml +# - nim-llm.yaml configMapGenerator: - name: agent-morpheus-config @@ -14,4 +14,4 @@ configMapGenerator: patchesStrategicMerge: - nginx-patch.yaml - nim-embed-image-pull-secret-patch.yaml - - nim-llm-image-pull-secret-patch.yaml \ No newline at end of file +# - nim-llm-image-pull-secret-patch.yaml \ No newline at end of file diff --git a/kustomize/overlays/local-nim-service/nginx-patch.yaml b/kustomize/overlays/local-nim-service/nginx-patch.yaml index 6bd86d2..c0ea890 100644 --- a/kustomize/overlays/local-nim-service/nginx-patch.yaml +++ b/kustomize/overlays/local-nim-service/nginx-patch.yaml @@ -9,9 +9,9 @@ spec: - name: nginx env: - name: NGINX_UPSTREAM_NVAI - value: http://nim-llm:8000 + value: http://nim-llm.nim:8000 - name: NGINX_UPSTREAM_NIM_LLM - value: http://nim-llm:8000 + value: http://nim-llm.nim:8000 - name: NGINX_UPSTREAM_NIM_EMBED value: http://nim-embed:8000 - name: NGC_API_KEY From ba4b0f81aebd175cbea5351c5313ec202222bcc9 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Nov 2024 09:41:53 +0200 Subject: [PATCH 13/50] refactor: parameterize openai_upstream from environment Signed-off-by: Zvi Grinberg --- kustomize/base/nginx/templates/routes/openai.conf.template | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kustomize/base/nginx/templates/routes/openai.conf.template b/kustomize/base/nginx/templates/routes/openai.conf.template index 73cf2a6..23efd0b 100644 --- a/kustomize/base/nginx/templates/routes/openai.conf.template +++ b/kustomize/base/nginx/templates/routes/openai.conf.template @@ -1,8 +1,10 @@ +set $openai_upstream ${NGINX_UPSTREAM_NIM_LLM}; + location /openai { location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass http://nim-llm.nim:8000; + proxy_pass $openai_upstream; proxy_set_header Connection ''; proxy_cache llm_cache; proxy_cache_methods POST; @@ -13,7 +15,7 @@ location /openai { location /openai/v1 { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass http://nim-llm.nim:8000; + proxy_pass $openai_upstream; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } From 2d26749de24153f342f65508bddc24811fd25353 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 25 Nov 2024 23:58:59 +0200 Subject: [PATCH 14/50] fix: address required changes Signed-off-by: Zvi Grinberg --- kustomize/base/nginx/templates/routes/openai.conf.template | 6 +++--- kustomize/overlays/local-nim-service/kustomization.yaml | 4 +--- kustomize/overlays/local-nim-service/nginx-patch.yaml | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/kustomize/base/nginx/templates/routes/openai.conf.template b/kustomize/base/nginx/templates/routes/openai.conf.template index 23efd0b..5d7522d 100644 --- a/kustomize/base/nginx/templates/routes/openai.conf.template +++ b/kustomize/base/nginx/templates/routes/openai.conf.template @@ -1,10 +1,10 @@ -set $openai_upstream ${NGINX_UPSTREAM_NIM_LLM}; + location /openai { location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass $openai_upstream; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; proxy_set_header Connection ''; proxy_cache llm_cache; proxy_cache_methods POST; @@ -15,7 +15,7 @@ location /openai { location /openai/v1 { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass $openai_upstream; + proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } diff --git a/kustomize/overlays/local-nim-service/kustomization.yaml b/kustomize/overlays/local-nim-service/kustomization.yaml index a997128..7d80507 100644 --- a/kustomize/overlays/local-nim-service/kustomization.yaml +++ b/kustomize/overlays/local-nim-service/kustomization.yaml @@ -4,7 +4,6 @@ kind: Kustomization resources: - ../../base - nim-embed.yaml -# - nim-llm.yaml configMapGenerator: - name: agent-morpheus-config @@ -13,5 +12,4 @@ configMapGenerator: patchesStrategicMerge: - nginx-patch.yaml - - nim-embed-image-pull-secret-patch.yaml -# - nim-llm-image-pull-secret-patch.yaml \ No newline at end of file + - nim-embed-image-pull-secret-patch.yaml \ No newline at end of file diff --git a/kustomize/overlays/local-nim-service/nginx-patch.yaml b/kustomize/overlays/local-nim-service/nginx-patch.yaml index c0ea890..3d519f3 100644 --- a/kustomize/overlays/local-nim-service/nginx-patch.yaml +++ b/kustomize/overlays/local-nim-service/nginx-patch.yaml @@ -11,7 +11,7 @@ spec: - name: NGINX_UPSTREAM_NVAI value: http://nim-llm.nim:8000 - name: NGINX_UPSTREAM_NIM_LLM - value: http://nim-llm.nim:8000 + value: http://nim-llm.nim:8000 - name: NGINX_UPSTREAM_NIM_EMBED value: http://nim-embed:8000 - name: NGC_API_KEY From 84e14ab9a25992334ba2fdc3814219a554b4fdee Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 26 Nov 2024 00:26:11 +0200 Subject: [PATCH 15/50] docs: add instruction to deploy helm chart of nim-meta/llama3.1-8b chore: delete non-working nim-llm' manifest + patch Signed-off-by: Zvi Grinberg --- kustomize/README.md | 1 + .../nim-llm-image-pull-secret-patch.yaml | 9 -- .../overlays/local-nim-service/nim-llm.yaml | 88 ------------------- 3 files changed, 1 insertion(+), 97 deletions(-) delete mode 100644 kustomize/overlays/local-nim-service/nim-llm-image-pull-secret-patch.yaml delete mode 100644 kustomize/overlays/local-nim-service/nim-llm.yaml diff --git a/kustomize/README.md b/kustomize/README.md index 7acb388..c8bf551 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -44,3 +44,4 @@ oc label namespace $YOUR_NAMESPACE_NAME application=agent-morpheus # Deploy agent-morpheus integrated with our self-hosted LLM oc kustomize overlays/our-llm-service | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` +7. Deploying self-hosted LLM Instructions can be found [here](https://github.com/zvigrinberg/nim-deploy/blob/main/helm/README.md#launching-a-nim-llama31-8b-instruct-including-toleration-to-schedule-on-a100-node-on-openshift) \ No newline at end of file diff --git a/kustomize/overlays/local-nim-service/nim-llm-image-pull-secret-patch.yaml b/kustomize/overlays/local-nim-service/nim-llm-image-pull-secret-patch.yaml deleted file mode 100644 index 74cf4c8..0000000 --- a/kustomize/overlays/local-nim-service/nim-llm-image-pull-secret-patch.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nim-llm -spec: - template: - spec: - imagePullSecrets: - - name: nvcr-secret diff --git a/kustomize/overlays/local-nim-service/nim-llm.yaml b/kustomize/overlays/local-nim-service/nim-llm.yaml deleted file mode 100644 index 1808ced..0000000 --- a/kustomize/overlays/local-nim-service/nim-llm.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nim-llm - labels: - app: agent-morpheus - component: nim-llm -spec: - strategy: - type: Recreate - replicas: 1 - selector: - matchLabels: - app: agent-morpheus - component: nim-llm - template: - metadata: - labels: - app: agent-morpheus - component: nim-llm - spec: - nodeSelector: - nvidia.com/gpu.deploy.driver: "true" - imagePullSecrets: [] - tolerations: - - key: p4-gpu - operator: Exists - effect: NoSchedule - containers: - - name: nim-llm - image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.3 - ports: - - name: http - protocol: TCP - containerPort: 8000 - resources: - limits: - memory: "16Gi" - cpu: "500m" - nvidia.com/gpu: "1" - requests: - memory: "8Gi" - cpu: "100m" - nvidia.com/gpu: "1" - env: - - name: NGC_API_KEY - valueFrom: - secretKeyRef: - key: ngc_api_key - name: agent-morpheus-secret - volumeMounts: - - name: cache - mountPath: /opt/nim/.cache - volumes: - - name: cache - persistentVolumeClaim: - claimName: nim-llm-cache ---- -apiVersion: v1 -kind: Service -metadata: - name: nim-llm - labels: - app: agent-morpheus - component: nim-llm -spec: - ports: - - name: http - port: 8000 - protocol: TCP - targetPort: 8000 - selector: - app: agent-morpheus - component: nim-llm ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: nim-llm-cache - labels: - app: agent-morpheus - component: nim-llm -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 30Gi \ No newline at end of file From 680271246606fc5ab945ba307147d3e094fcc738 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 26 Nov 2024 13:28:04 +0200 Subject: [PATCH 16/50] docs: add instructions to deploy llama-3.1-8b-instruct LLM refactor: use less confusing env name NGINX_UPSTREAM_OPENAI for self hosted LLM base url Signed-off-by: Zvi Grinberg --- kustomize/README.md | 31 ++++++++++++++++++- .../templates/routes/openai.conf.template | 4 +-- .../local-nim-service/nginx-patch.yaml | 4 +-- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index c8bf551..f1dc738 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -44,4 +44,33 @@ oc label namespace $YOUR_NAMESPACE_NAME application=agent-morpheus # Deploy agent-morpheus integrated with our self-hosted LLM oc kustomize overlays/our-llm-service | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` -7. Deploying self-hosted LLM Instructions can be found [here](https://github.com/zvigrinberg/nim-deploy/blob/main/helm/README.md#launching-a-nim-llama31-8b-instruct-including-toleration-to-schedule-on-a100-node-on-openshift) \ No newline at end of file +7. Deploy self-hosted Llama-3.1-8b LLM + 1. Clone the following repo + ```shell + clone git@github.com:zvigrinberg/nim-deploy.git + cd nim-deploy/helm + ``` + 2. Create new namespace + ```shell + oc new-project nim + ``` + 3. Create service account for the deployment + ```shell + export SERVICE_ACCOUNT_NAME=nim-llm-sa + oc create serviceaccount $SERVICE_ACCOUNT_NAME + ``` + 4. add `anyuid` `SecurityContextConstraint` privilege to the created service account + ```shell + oc adm policy add-scc-to-user -z $SERVICE_ACCOUNT_NAME anyuid + ``` + 5. Deploy the chart + ```shell + helm install nim-llm nim-llm/ --set persistence.enabled=true \ + --set model.ngcAPIKey=$NGC_API_KEY \ + --set image.repository=nvcr.io/nim/meta/llama-3.1-8b-instruct \ + --set image.tag=latest \ + --set serviceAccount.name=$SERVICE_ACCOUNT_NAME \ + --set tolerations[0].key=p4-gpu \ + --set tolerations[0].operator=Exists \ + --set tolerations[0].effect=NoSchedule + ``` \ No newline at end of file diff --git a/kustomize/base/nginx/templates/routes/openai.conf.template b/kustomize/base/nginx/templates/routes/openai.conf.template index 5d7522d..3bcea88 100644 --- a/kustomize/base/nginx/templates/routes/openai.conf.template +++ b/kustomize/base/nginx/templates/routes/openai.conf.template @@ -4,7 +4,7 @@ location /openai { location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; + proxy_pass ${NGINX_UPSTREAM_OPENAI}; proxy_set_header Connection ''; proxy_cache llm_cache; proxy_cache_methods POST; @@ -15,7 +15,7 @@ location /openai { location /openai/v1 { rewrite ^\/openai(\/.*)$ $1 break; - proxy_pass ${NGINX_UPSTREAM_NIM_LLM}; + proxy_pass ${NGINX_UPSTREAM_OPENAI}; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } diff --git a/kustomize/overlays/local-nim-service/nginx-patch.yaml b/kustomize/overlays/local-nim-service/nginx-patch.yaml index 3d519f3..48724c0 100644 --- a/kustomize/overlays/local-nim-service/nginx-patch.yaml +++ b/kustomize/overlays/local-nim-service/nginx-patch.yaml @@ -8,9 +8,7 @@ spec: containers: - name: nginx env: - - name: NGINX_UPSTREAM_NVAI - value: http://nim-llm.nim:8000 - - name: NGINX_UPSTREAM_NIM_LLM + - name: NGINX_UPSTREAM_OPENAI value: http://nim-llm.nim:8000 - name: NGINX_UPSTREAM_NIM_EMBED value: http://nim-embed:8000 From 90b1b42b8ed053d4a0288824af399eb5ffbe1b09 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Wed, 27 Nov 2024 13:00:42 +0100 Subject: [PATCH 17/50] feat: remove unnecessary stage Signed-off-by: Ruben Romero Montes --- src/cve/pipeline/pipeline.py | 11 +---------- src/cve/stages/convert_to_output_object.py | 5 +---- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/cve/pipeline/pipeline.py b/src/cve/pipeline/pipeline.py index d08f77f..e5cf872 100644 --- a/src/cve/pipeline/pipeline.py +++ b/src/cve/pipeline/pipeline.py @@ -37,7 +37,7 @@ from morpheus_llm.stages.llm.llm_engine_stage import LLMEngineStage from ..data_models.config import RunConfig -from ..data_models.input import AgentMorpheusEngineInput, AgentMorpheusInput +from ..data_models.input import AgentMorpheusEngineInput from ..stages.convert_to_output_object import convert_to_output_object from .engine import build_engine from .input import build_input @@ -169,15 +169,6 @@ def convert_input_to_df(message: AgentMorpheusEngineInput) -> ControlMessage: pipe.add_stage(convert_input_to_df(config)) - @stage - def add_scan_id(message: ControlMessage) -> ControlMessage: - input: AgentMorpheusInput = message.get_metadata("input") - message.set_metadata("scan_id", input.scan.id) - - return message - - pipe.add_stage(add_scan_id(config=config)) - @stage def add_start_timestamp(message: ControlMessage) -> ControlMessage: message.set_timestamp("start_time", datetime.datetime.now()) diff --git a/src/cve/stages/convert_to_output_object.py b/src/cve/stages/convert_to_output_object.py index 457041f..abee908 100644 --- a/src/cve/stages/convert_to_output_object.py +++ b/src/cve/stages/convert_to_output_object.py @@ -105,10 +105,7 @@ def convert_to_output_object(message: ControlMessage) -> AgentMorpheusOutput: # Pull input and info objects from control message metadata input: AgentMorpheusInput = message.get_metadata("input") -<<<<<<< HEAD -======= - input.scan.id = message.get_metadata("scan_id") ->>>>>>> 6be9369 (feat: add id and timestamps to output file) + input.scan.started_at = message.get_timestamp("start_time") input.scan.completed_at = message.get_timestamp("end_time") From 024b6418ba4ea92b364e275f63ead465323cc0bd Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 21 Nov 2024 18:28:44 +0200 Subject: [PATCH 18/50] feat: make VulnerableDependencyChecker facility pluggable Signed-off-by: Zvi Grinberg fix: adjust description of new switch field Signed-off-by: Zvi Grinberg chore: change pipeline configuration to disable usage of dependency checker tool Signed-off-by: Zvi Grinberg --- configs/schemas/config.schema.json | 5 +++++ .../overlays/local-nim-service/agent-morpheus-config.json | 3 ++- .../overlays/nvidia-llm-service/agent-morpheus-config.json | 3 ++- src/cve/pipeline/input.py | 3 ++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/configs/schemas/config.schema.json b/configs/schemas/config.schema.json index 37451f3..643ffed 100644 --- a/configs/schemas/config.schema.json +++ b/configs/schemas/config.schema.json @@ -312,6 +312,11 @@ "default": false, "title": "Code Search Tool", "type": "boolean" + }, + "use_dependency_tool": { + "default": true, + "title": "Whether to use VulnerableDependencyChecker tool in the pipeline or not", + "type": "boolean" } }, "title": "GeneralConfig", diff --git a/kustomize/overlays/local-nim-service/agent-morpheus-config.json b/kustomize/overlays/local-nim-service/agent-morpheus-config.json index 636d359..ee4b17d 100644 --- a/kustomize/overlays/local-nim-service/agent-morpheus-config.json +++ b/kustomize/overlays/local-nim-service/agent-morpheus-config.json @@ -56,7 +56,8 @@ "max_retries": 5, "model_max_batch_size": 64, "pipeline_batch_size": 128, - "use_uvloop": true + "use_uvloop": true, + "use_dependency_tool": false }, "input": { "_type": "http", diff --git a/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json b/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json index 3f4ee51..ce0dd5b 100644 --- a/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json +++ b/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json @@ -58,7 +58,8 @@ "max_retries": 5, "model_max_batch_size": 64, "pipeline_batch_size": 1024, - "use_uvloop": true + "use_uvloop": true, + "use_dependency_tool": false }, "input": { "_type": "http", diff --git a/src/cve/pipeline/input.py b/src/cve/pipeline/input.py index 625bb4c..57aae0d 100644 --- a/src/cve/pipeline/input.py +++ b/src/cve/pipeline/input.py @@ -307,7 +307,8 @@ async def _calc_dep(cve_intel: CveIntel): message.info.vulnerable_dependencies = asyncio.run(_inner()) return message - pipe.add_stage(check_vulnerable_dependencies(config)) + if run_config.general.use_dependency_tool: + pipe.add_stage(check_vulnerable_dependencies(config)) if run_config.input.type != 'http': @stage From d28307babd778ad5365caf260612730d97cd3170 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 27 Nov 2024 08:48:50 +0200 Subject: [PATCH 19/50] refactor: rename to use_dependency_checker fix: add a field which was omitted by mistake from class Signed-off-by: Zvi Grinberg --- configs/schemas/config.schema.json | 2 +- kustomize/overlays/local-nim-service/agent-morpheus-config.json | 2 +- .../overlays/nvidia-llm-service/agent-morpheus-config.json | 2 +- src/cve/data_models/config.py | 1 + src/cve/pipeline/input.py | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/configs/schemas/config.schema.json b/configs/schemas/config.schema.json index 643ffed..49e21f3 100644 --- a/configs/schemas/config.schema.json +++ b/configs/schemas/config.schema.json @@ -313,7 +313,7 @@ "title": "Code Search Tool", "type": "boolean" }, - "use_dependency_tool": { + "use_dependency_checker": { "default": true, "title": "Whether to use VulnerableDependencyChecker tool in the pipeline or not", "type": "boolean" diff --git a/kustomize/overlays/local-nim-service/agent-morpheus-config.json b/kustomize/overlays/local-nim-service/agent-morpheus-config.json index ee4b17d..8ec63c8 100644 --- a/kustomize/overlays/local-nim-service/agent-morpheus-config.json +++ b/kustomize/overlays/local-nim-service/agent-morpheus-config.json @@ -57,7 +57,7 @@ "model_max_batch_size": 64, "pipeline_batch_size": 128, "use_uvloop": true, - "use_dependency_tool": false + "use_dependency_checker": false }, "input": { "_type": "http", diff --git a/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json b/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json index ce0dd5b..fe6f293 100644 --- a/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json +++ b/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json @@ -59,7 +59,7 @@ "model_max_batch_size": 64, "pipeline_batch_size": 1024, "use_uvloop": true, - "use_dependency_tool": false + "use_dependency_checker": false }, "input": { "_type": "http", diff --git a/src/cve/data_models/config.py b/src/cve/data_models/config.py index dc852d1..742912c 100644 --- a/src/cve/data_models/config.py +++ b/src/cve/data_models/config.py @@ -61,6 +61,7 @@ class GeneralConfig(BaseModel): """ code_search_tool: bool = False + use_dependency_checker: bool = Field(default=True, description="Whether to use the VulnerableDependencyChecker in the pipeline or not.") @field_validator("num_threads") @classmethod diff --git a/src/cve/pipeline/input.py b/src/cve/pipeline/input.py index 57aae0d..073d704 100644 --- a/src/cve/pipeline/input.py +++ b/src/cve/pipeline/input.py @@ -307,7 +307,7 @@ async def _calc_dep(cve_intel: CveIntel): message.info.vulnerable_dependencies = asyncio.run(_inner()) return message - if run_config.general.use_dependency_tool: + if run_config.general.use_dependency_checker: pipe.add_stage(check_vulnerable_dependencies(config)) if run_config.input.type != 'http': From e02c491abc070eadcfa5a18b5aa63aa2f92f32dd Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 27 Nov 2024 09:01:52 +0200 Subject: [PATCH 20/50] docs: new general configuration setting - use_dependency_checker Signed-off-by: Zvi Grinberg --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 64d01a9..1981fd1 100644 --- a/README.md +++ b/README.md @@ -527,6 +527,7 @@ The configuration defines how the workflow operates, including model settings, i - `pipeline_batch_size`: Determines the number of messages per batch for the pipeline. - `use_uvloop`: Toggles the use of `uvloop`, an optimized event loop for improved performance. - `code_search_tool`: Enables or disables the use of the code search tool. + - `use_dependency_checker`: Whether to use the VulnerableDependencyChecker in the pipeline or not. 4. **Input configuration**: The `input` section defines how and where the input data (container images and vulnerabilities) is sourced. - `_type`: Defines the input type - `manual`: input data is provided manually in the config file From f73f2009b9ec7faafc015759bef2aa9c011d4938 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg <75700623+zvigrinberg@users.noreply.github.com> Date: Wed, 27 Nov 2024 20:45:53 +0200 Subject: [PATCH 21/50] Update configs/schemas/config.schema.json update the schema' new field addition to the auto-generated version. Co-authored-by: Ashley Song <165685692+ashsong-nv@users.noreply.github.com> --- configs/schemas/config.schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/schemas/config.schema.json b/configs/schemas/config.schema.json index 49e21f3..920a7c5 100644 --- a/configs/schemas/config.schema.json +++ b/configs/schemas/config.schema.json @@ -315,7 +315,8 @@ }, "use_dependency_checker": { "default": true, - "title": "Whether to use VulnerableDependencyChecker tool in the pipeline or not", + "description": "Whether to use the VulnerableDependencyChecker in the pipeline or not.", + "title": "Use Dependency Checker", "type": "boolean" } }, From c3a37a40e99bce01cbaf8fe3132fd887c2decead Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Fri, 29 Nov 2024 16:10:15 +0100 Subject: [PATCH 22/50] fix: vulnerable_dependencies are expected in the dataFrame Signed-off-by: Ruben Romero Montes --- src/cve/pipeline/input.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cve/pipeline/input.py b/src/cve/pipeline/input.py index 073d704..bd249e4 100644 --- a/src/cve/pipeline/input.py +++ b/src/cve/pipeline/input.py @@ -235,6 +235,10 @@ def process_sbom(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: @stage def check_vulnerable_dependencies(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: """Check for vulnerable packages in the dependency graph and update the message object.""" + if not run_config.general.use_dependency_checker: + message.info.vulnerable_dependencies = [VulnerableDependencies( + vuln_id=cve_intel.cve_id, vuln_package_intel_sources=[], vulnerable_sbom_packages=[]) for cve_intel in message.info.intel] + return message sbom = message.info.sbom.packages image = f"{message.input.image.name}:{message.input.image.tag}" @@ -307,8 +311,7 @@ async def _calc_dep(cve_intel: CveIntel): message.info.vulnerable_dependencies = asyncio.run(_inner()) return message - if run_config.general.use_dependency_checker: - pipe.add_stage(check_vulnerable_dependencies(config)) + pipe.add_stage(check_vulnerable_dependencies(config)) if run_config.input.type != 'http': @stage From 64f15b2df7d17d82f1987ede024de13d56ed3682 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Thu, 28 Nov 2024 06:02:01 +0200 Subject: [PATCH 23/50] fix: make broken 'openai' model type to work with the agent Signed-off-by: Zvi Grinberg --- src/cve/data_models/config.py | 2 +- src/cve/pipeline/engine.py | 47 +++++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/cve/data_models/config.py b/src/cve/data_models/config.py index 742912c..ff035e1 100644 --- a/src/cve/data_models/config.py +++ b/src/cve/data_models/config.py @@ -134,7 +134,7 @@ class OpenAIModelConfig(BaseModel): top_p: float = 1.0 seed: int | None = None max_retries: int = 10 - json_output: bool = Field(False, alias='json') + # json_output: bool = Field(False, alias='json') model_config = ConfigDict(protected_namespaces=(), populate_by_name=True) diff --git a/src/cve/pipeline/engine.py b/src/cve/pipeline/engine.py index fb78932..0ad96b2 100644 --- a/src/cve/pipeline/engine.py +++ b/src/cve/pipeline/engine.py @@ -21,7 +21,10 @@ from langchain.agents import initialize_agent from langchain.chains.retrieval_qa.base import RetrievalQA from langchain.vectorstores.faiss import FAISS +from langchain_community.adapters.openai import ChatCompletion +from langchain_community.chat_models import ChatOpenAI from langchain_core.embeddings import Embeddings +from langchain_core.exceptions import OutputParserException from morpheus_llm.llm import LLMContext from morpheus_llm.llm import LLMEngine @@ -41,17 +44,19 @@ from ..utils.prompting import agent_examples_for_prompt from ..utils.serp_api_wrapper import MorpheusSerpAPIWrapper +OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER = "Parsing LLM output produced both a final answer and a parse-able action" + logger = logging.getLogger(__name__) def _build_dynamic_agent_fn(run_config: RunConfig, embeddings: Embeddings): - chat_service = LLMService.create(run_config.engine.agent.model.service.type, **run_config.engine.agent.model.service.model_dump(exclude={"type"}, by_alias=True)) chat_client = chat_service.get_client(**run_config.engine.agent.model.model_dump(exclude={"service", "type"}, by_alias=True)) langchain_llm = LangchainLLMClientWrapper(client=chat_client) + openai_llm = ChatOpenAI(client=ChatCompletion(), model_name=run_config.engine.agent.model.model_name) # Initialize a SerpAPIWrapper object to perform internet searches. search = MorpheusSerpAPIWrapper(max_retries=run_config.general.max_retries) @@ -93,7 +98,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d # load code vector DB code_vector_db = FAISS.load_local(vdb_map.code_vdb_path, embeddings, allow_dangerous_deserialization=True) code_qa_tool = RetrievalQA.from_chain_type( - llm=langchain_llm, + llm=choose_llm_based_on_model_type(), chain_type="stuff", retriever=code_vector_db.as_retriever(), return_source_documents=run_config.engine.agent.return_source_documents) @@ -133,7 +138,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d if (vdb_map.doc_vdb_path is not None): guide_vector_db = FAISS.load_local(vdb_map.doc_vdb_path, embeddings, allow_dangerous_deserialization=True) guide_qa_tool = RetrievalQA.from_chain_type( - llm=langchain_llm, + llm=choose_llm_based_on_model_type(), chain_type="stuff", retriever=guide_vector_db.as_retriever(), return_source_documents=run_config.engine.agent.return_source_documents) @@ -155,10 +160,10 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d # This agent is designed to handle zero-shot reaction descriptions and parse errors. agent = initialize_agent( tools, - langchain_llm, + choose_llm_based_on_model_type(), agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=run_config.engine.agent.verbose, - handle_parsing_errors="Check your output and make sure it conforms, use the Action/Action Input syntax", + handle_parsing_errors=notify_llm_to_rephrase, max_iterations=10, early_stopping_method="generate", return_intermediate_steps=run_config.engine.agent.return_intermediate_steps) @@ -170,21 +175,43 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d ("If the input is not a question, formulate it into a question first. Include intermediate thought in the " "final answer."), 1).replace( - "Use the following format:", - "Use the following format (start each response with one of the following prefixes: " - "[Question, Thought, Action, Action Input, Final Answer]):", - 1) + "Use the following format:", + "Use the following format (start each response with one of the following prefixes: " + "Question, Thought, Action, Action Input, Final Answer):", + 1) + if run_config.engine.summary_model.service.type.lower() == 'openai': + prompt_template = prompt_template.replace("Final Answer: the final answer to the original input question", + "Final Answer: the final answer to the original input question. \n" + "Do not include Final Answer together" + " with Action or with Action Input in the same response" + , 1) + if run_config.engine.agent.prompt_examples: prompt_template = prompt_template.replace("Begin!\n\n", agent_examples_for_prompt + "Begin!\n\n") agent.agent.llm_chain.prompt.template = f'{sys_prompt} {prompt_template}' return agent + def choose_llm_based_on_model_type(): + if run_config.engine.summary_model.service.type.lower() == 'openai': + return openai_llm + else: + return langchain_llm + + def notify_llm_to_rephrase(exception: OutputParserException) -> str: + if run_config.engine.summary_model.service.type.lower() == 'openai': + logger.debug(f"Exception intercepted in notify {exception}") + exception_string = str(exception) + if OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER in exception_string: + return (f"Check you output and make sure it conforms! Do not output an action " + f"and a final answer at the same time") + + return "Check your output and make sure it conforms, use the Action/Action Input syntax" + return inner_create_agent_fn def build_engine(*, run_config: RunConfig, embeddings: Embeddings): - summary_service = LLMService.create(run_config.engine.summary_model.service.type, **run_config.engine.summary_model.service.model_dump(exclude={"type"})) justification_service = LLMService.create( From 2b31095059beb83ba1ed5ff56b64da40765f300d Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 28 Nov 2024 15:41:29 +0100 Subject: [PATCH 24/50] chore(deps): use shared nim and refactor kustomize resources Signed-off-by: Ruben Romero Montes --- .../agent-morpheus-config.json | 19 ++-- kustomize/base/agent_morpheus-RH.yaml | 24 ++--- kustomize/base/kustomization.yaml | 3 + kustomize/base/nginx.yaml | 7 +- .../templates/routes/intel.conf.template | 1 + .../templates/routes/openai.conf.template | 2 + .../agent-morpheus-config.json | 8 +- .../kustomization.yaml | 3 +- .../overlays/local-nim-all/nginx-patch.yaml | 14 +++ .../local-nim-service/kustomization.yaml | 15 ---- .../local-nim-service/nginx-patch.yaml | 19 ---- .../nim-embed-image-pull-secret-patch.yaml | 9 -- .../overlays/local-nim-service/nim-embed.yaml | 88 ------------------- .../agent-morpheus-patch.yaml | 56 ------------ .../our-llm-service/agent-morpheus-patch.yaml | 31 ------- .../agent-morpheus-config.json | 18 ++-- .../kustomization.yaml | 3 +- .../overlays/remote-nim-all/nginx-patch.yaml | 16 ++++ 18 files changed, 74 insertions(+), 262 deletions(-) rename kustomize/{overlays/our-llm-service => base}/agent-morpheus-config.json (77%) rename kustomize/overlays/{local-nim-service => local-nim-all}/agent-morpheus-config.json (92%) rename kustomize/overlays/{nvidia-llm-service => local-nim-all}/kustomization.yaml (83%) create mode 100644 kustomize/overlays/local-nim-all/nginx-patch.yaml delete mode 100644 kustomize/overlays/local-nim-service/kustomization.yaml delete mode 100644 kustomize/overlays/local-nim-service/nginx-patch.yaml delete mode 100644 kustomize/overlays/local-nim-service/nim-embed-image-pull-secret-patch.yaml delete mode 100644 kustomize/overlays/local-nim-service/nim-embed.yaml delete mode 100644 kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml delete mode 100644 kustomize/overlays/our-llm-service/agent-morpheus-patch.yaml rename kustomize/overlays/{nvidia-llm-service => remote-nim-all}/agent-morpheus-config.json (82%) rename kustomize/overlays/{our-llm-service => remote-nim-all}/kustomization.yaml (83%) create mode 100644 kustomize/overlays/remote-nim-all/nginx-patch.yaml diff --git a/kustomize/overlays/our-llm-service/agent-morpheus-config.json b/kustomize/base/agent-morpheus-config.json similarity index 77% rename from kustomize/overlays/our-llm-service/agent-morpheus-config.json rename to kustomize/base/agent-morpheus-config.json index 31bf704..57436e3 100644 --- a/kustomize/overlays/our-llm-service/agent-morpheus-config.json +++ b/kustomize/base/agent-morpheus-config.json @@ -3,9 +3,9 @@ "engine": { "agent": { "model": { - "model_name": "meta/llama3-8b-instruct", + "model_name": "meta/llama-3.1-70b-instruct", "service": { - "_type": "openai" + "_type": "nvfoundation" }, "max_tokens": 2000 }, @@ -14,18 +14,18 @@ }, "checklist_model": { "service": { - "_type": "openai" + "_type": "nvfoundation" }, - "model_name": "meta/llama3-8b-instruct", + "model_name": "meta/llama-3.1-70b-instruct", "temperature": 0, "max_tokens": 2000, "top_p": 0.01, "seed": 42 }, "justification_model": { - "model_name": "meta/llama3-8b-instruct", + "model_name": "meta/llama-3.1-70b-instruct", "service": { - "_type": "openai" + "_type": "nvfoundation" }, "max_tokens": 1024, "temperature": 0, @@ -39,9 +39,9 @@ "max_batch_size": 128 }, "summary_model": { - "model_name": "meta/llama3-8b-instruct", + "model_name": "meta/llama-3.1-70b-instruct", "service": { - "_type": "openai" + "_type": "nvfoundation" }, "max_tokens": 1024, "temperature": 0, @@ -56,7 +56,8 @@ "max_retries": 5, "model_max_batch_size": 64, "pipeline_batch_size": 128, - "use_uvloop": true + "use_uvloop": true, + "use_dependency_checker": false }, "input": { "_type": "http", diff --git a/kustomize/base/agent_morpheus-RH.yaml b/kustomize/base/agent_morpheus-RH.yaml index 0958f48..036278e 100644 --- a/kustomize/base/agent_morpheus-RH.yaml +++ b/kustomize/base/agent_morpheus-RH.yaml @@ -58,29 +58,17 @@ spec: nvidia.com/gpu: "1" env: - name: SERPAPI_API_KEY - valueFrom: - secretKeyRef: - key: serpapi_api_key - name: agent-morpheus-secret + value: AGENT_MORPHEUS - name: NVD_API_KEY - valueFrom: - secretKeyRef: - key: nvd_api_key - name: agent-morpheus-secret + value: AGENT_MORPHEUS - name: NVIDIA_API_KEY - valueFrom: - secretKeyRef: - key: nvidia_api_key - name: agent-morpheus-secret + value: AGENT_MORPHEUS - name: GHSA_API_KEY - valueFrom: - secretKeyRef: - key: ghsa_api_key - name: agent-morpheus-secret + value: AGENT_MORPHEUS - name: OPENAI_API_KEY - value: "dummy-key" + value: AGENT_MORPHEUS - name: NGC_API_KEY - value: "" + value: AGENT_MORPHEUS - name: CVE_DETAILS_BASE_URL value: http://nginx-cache:8080/cve-details - name: CWE_DETAILS_BASE_URL diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index ab8cd24..53ff801 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -31,6 +31,9 @@ configMapGenerator: - nginx/templates/routes/nim.conf.template - nginx/templates/routes/nvidia.conf.template - nginx/templates/routes/openai.conf.template + - name: agent-morpheus-config + files: + - agent-morpheus-config.json patches: - path: ips-patch.json diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml index cb7e277..574d873 100644 --- a/kustomize/base/nginx.yaml +++ b/kustomize/base/nginx.yaml @@ -57,7 +57,10 @@ spec: key: ghsa_api_key name: agent-morpheus-secret - name: OPENAI_API_KEY - value: "" + valueFrom: + secretKeyRef: + key: nvidia_api_key + name: agent-morpheus-secret - name: NGC_API_KEY value: "" - name: NGC_ORG_ID @@ -67,6 +70,8 @@ spec: - name: NGINX_UPSTREAM_NIM_LLM value: https://integrate.api.nvidia.com - name: NGINX_UPSTREAM_NIM_EMBED + value: http://nim-embed.nim.svc.cluster.local:8000 + - name: NGINX_UPSTREAM_OPENAI value: https://integrate.api.nvidia.com volumeMounts: - name: routes diff --git a/kustomize/base/nginx/templates/routes/intel.conf.template b/kustomize/base/nginx/templates/routes/intel.conf.template index f5d9d0b..cfebf4b 100644 --- a/kustomize/base/nginx/templates/routes/intel.conf.template +++ b/kustomize/base/nginx/templates/routes/intel.conf.template @@ -65,6 +65,7 @@ location /recf { location /ghsa { rewrite ^\/ghsa(\/.*)$ $1 break; proxy_pass https://api.github.com; + proxy_set_header Authorization $ghsa_http_authorization; proxy_cache intel_cache; proxy_cache_methods GET; proxy_cache_key "$request_method|$request_uri"; diff --git a/kustomize/base/nginx/templates/routes/openai.conf.template b/kustomize/base/nginx/templates/routes/openai.conf.template index 3bcea88..8a59026 100644 --- a/kustomize/base/nginx/templates/routes/openai.conf.template +++ b/kustomize/base/nginx/templates/routes/openai.conf.template @@ -6,6 +6,7 @@ location /openai { rewrite ^\/openai(\/.*)$ $1 break; proxy_pass ${NGINX_UPSTREAM_OPENAI}; proxy_set_header Connection ''; + proxy_set_header Authorization $openai_http_authorization; proxy_cache llm_cache; proxy_cache_methods POST; proxy_cache_key "$request_method|$request_uri|$request_body"; @@ -16,6 +17,7 @@ location /openai { location /openai/v1 { rewrite ^\/openai(\/.*)$ $1 break; proxy_pass ${NGINX_UPSTREAM_OPENAI}; + proxy_set_header Authorization $openai_http_authorization; access_log /dev/stdout no_cache_log; access_log /var/log/nginx/access.log no_cache_log; } diff --git a/kustomize/overlays/local-nim-service/agent-morpheus-config.json b/kustomize/overlays/local-nim-all/agent-morpheus-config.json similarity index 92% rename from kustomize/overlays/local-nim-service/agent-morpheus-config.json rename to kustomize/overlays/local-nim-all/agent-morpheus-config.json index 8ec63c8..41ac9ca 100644 --- a/kustomize/overlays/local-nim-service/agent-morpheus-config.json +++ b/kustomize/overlays/local-nim-all/agent-morpheus-config.json @@ -5,7 +5,7 @@ "model": { "model_name": "meta/llama-3.1-8b-instruct", "service": { - "_type": "openai" + "_type": "nvfoundation" }, "max_tokens": 2000 }, @@ -14,7 +14,7 @@ }, "checklist_model": { "service": { - "_type": "openai" + "_type": "nvfoundation" }, "model_name": "meta/llama-3.1-8b-instruct", "temperature": 0, @@ -25,7 +25,7 @@ "justification_model": { "model_name": "meta/llama-3.1-8b-instruct", "service": { - "_type": "openai" + "_type": "nvfoundation" }, "max_tokens": 1024, "temperature": 0, @@ -41,7 +41,7 @@ "summary_model": { "model_name": "meta/llama-3.1-8b-instruct", "service": { - "_type": "openai" + "_type": "nvfoundation" }, "max_tokens": 1024, "temperature": 0, diff --git a/kustomize/overlays/nvidia-llm-service/kustomization.yaml b/kustomize/overlays/local-nim-all/kustomization.yaml similarity index 83% rename from kustomize/overlays/nvidia-llm-service/kustomization.yaml rename to kustomize/overlays/local-nim-all/kustomization.yaml index 2b423f7..6f5a243 100644 --- a/kustomize/overlays/nvidia-llm-service/kustomization.yaml +++ b/kustomize/overlays/local-nim-all/kustomization.yaml @@ -6,8 +6,9 @@ resources: configMapGenerator: - name: agent-morpheus-config + behavior: replace files: - agent-morpheus-config.json patchesStrategicMerge: - - agent-morpheus-patch.yaml \ No newline at end of file + - nginx-patch.yaml \ No newline at end of file diff --git a/kustomize/overlays/local-nim-all/nginx-patch.yaml b/kustomize/overlays/local-nim-all/nginx-patch.yaml new file mode 100644 index 0000000..3281654 --- /dev/null +++ b/kustomize/overlays/local-nim-all/nginx-patch.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache +spec: + template: + spec: + containers: + - name: nginx + env: + - name: NGINX_UPSTREAM_NIM_LLM + value: http://nim-llm.nim.svc.cluster.local:8000 + - name: NGINX_UPSTREAM_OPENAI + value: http://nim-llm.nim.svc.cluster.local:8000 diff --git a/kustomize/overlays/local-nim-service/kustomization.yaml b/kustomize/overlays/local-nim-service/kustomization.yaml deleted file mode 100644 index 7d80507..0000000 --- a/kustomize/overlays/local-nim-service/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ../../base - - nim-embed.yaml - -configMapGenerator: - - name: agent-morpheus-config - files: - - agent-morpheus-config.json - -patchesStrategicMerge: - - nginx-patch.yaml - - nim-embed-image-pull-secret-patch.yaml \ No newline at end of file diff --git a/kustomize/overlays/local-nim-service/nginx-patch.yaml b/kustomize/overlays/local-nim-service/nginx-patch.yaml deleted file mode 100644 index 48724c0..0000000 --- a/kustomize/overlays/local-nim-service/nginx-patch.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-cache -spec: - template: - spec: - containers: - - name: nginx - env: - - name: NGINX_UPSTREAM_OPENAI - value: http://nim-llm.nim:8000 - - name: NGINX_UPSTREAM_NIM_EMBED - value: http://nim-embed:8000 - - name: NGC_API_KEY - valueFrom: - secretKeyRef: - key: ngc_api_key - name: agent-morpheus-secret \ No newline at end of file diff --git a/kustomize/overlays/local-nim-service/nim-embed-image-pull-secret-patch.yaml b/kustomize/overlays/local-nim-service/nim-embed-image-pull-secret-patch.yaml deleted file mode 100644 index 92ac824..0000000 --- a/kustomize/overlays/local-nim-service/nim-embed-image-pull-secret-patch.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nim-embed -spec: - template: - spec: - imagePullSecrets: - - name: nvcr-secret diff --git a/kustomize/overlays/local-nim-service/nim-embed.yaml b/kustomize/overlays/local-nim-service/nim-embed.yaml deleted file mode 100644 index 5ed187f..0000000 --- a/kustomize/overlays/local-nim-service/nim-embed.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nim-embed - labels: - app: agent-morpheus - component: nim-embed -spec: - strategy: - type: Recreate - replicas: 1 - selector: - matchLabels: - app: agent-morpheus - component: nim-embed - template: - metadata: - labels: - app: agent-morpheus - component: nim-embed - spec: - nodeSelector: - nvidia.com/gpu.deploy.driver: "true" - imagePullSecrets: [] - tolerations: - - key: p4-gpu - operator: Exists - effect: NoSchedule - containers: - - name: nim-embed - image: nvcr.io/nim/nvidia/nv-embedqa-e5-v5:1 - ports: - - name: http - protocol: TCP - containerPort: 8000 - resources: - limits: - memory: "16Gi" - cpu: "500m" - nvidia.com/gpu: "1" - requests: - memory: "8Gi" - cpu: "100m" - nvidia.com/gpu: "1" - env: - - name: NGC_API_KEY - valueFrom: - secretKeyRef: - key: ngc_api_key - name: agent-morpheus-secret - volumeMounts: - - name: cache - mountPath: /opt/nim/.cache - volumes: - - name: cache - persistentVolumeClaim: - claimName: nim-embed-cache ---- -apiVersion: v1 -kind: Service -metadata: - name: nim-embed - labels: - app: agent-morpheus - component: nim-embed -spec: - ports: - - name: http - port: 8000 - protocol: TCP - targetPort: 8000 - selector: - app: agent-morpheus - component: nim-embed ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: nim-embed-cache - labels: - app: agent-morpheus - component: nim-embed -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 30Gi \ No newline at end of file diff --git a/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml b/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml deleted file mode 100644 index 394a7c0..0000000 --- a/kustomize/overlays/nvidia-llm-service/agent-morpheus-patch.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: agent-morpheus-rh - labels: - app: agent-morpheus - component: agent-morpheus-rh -spec: - strategy: - type: Recreate - replicas: 1 - selector: - matchLabels: - app: agent-morpheus - component: agent-morpheus-rh - template: - metadata: - labels: - app: agent-morpheus - component: agent-morpheus-rh - spec: - containers: - - name: agent-morpheus - env: - - name: NVIDIA_API_KEY - valueFrom: - secretKeyRef: - key: nvidia_api_key - name: agent-morpheus-secret ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-cache - labels: - app: agent-morpheus - component: nginx-cache -spec: - selector: - matchLabels: - app: agent-morpheus - component: nginx-cache - template: - metadata: - labels: - app: agent-morpheus - component: nginx-cache - spec: - containers: - - name: nginx - env: - - name: NVIDIA_API_KEY - valueFrom: - secretKeyRef: - key: nvidia_api_key - name: agent-morpheus-secret \ No newline at end of file diff --git a/kustomize/overlays/our-llm-service/agent-morpheus-patch.yaml b/kustomize/overlays/our-llm-service/agent-morpheus-patch.yaml deleted file mode 100644 index a056093..0000000 --- a/kustomize/overlays/our-llm-service/agent-morpheus-patch.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: agent-morpheus-rh - labels: - app: agent-morpheus - component: agent-morpheus-rh -spec: - strategy: - type: Recreate - replicas: 1 - selector: - matchLabels: - app: agent-morpheus - component: agent-morpheus-rh - template: - metadata: - labels: - app: agent-morpheus - component: agent-morpheus-rh - spec: - containers: - - name: agent-morpheus - env: - - - name: OPENAI_BASE_URL - value: http://llama3-8b-instruct-1xgpu-predictor-00001-private.morpheus-cn-models-nim/v1 - - - name: OPENAI_API_KEY - value: dummy-string - diff --git a/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json b/kustomize/overlays/remote-nim-all/agent-morpheus-config.json similarity index 82% rename from kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json rename to kustomize/overlays/remote-nim-all/agent-morpheus-config.json index fe6f293..8620d88 100644 --- a/kustomize/overlays/nvidia-llm-service/agent-morpheus-config.json +++ b/kustomize/overlays/remote-nim-all/agent-morpheus-config.json @@ -3,29 +3,27 @@ "engine": { "agent": { "model": { - "model_name": "meta/llama3-70b-instruct", + "model_name": "meta/llama-3.1-70b-instruct", "service": { "_type": "nvfoundation" }, - "max_tokens": 2000, - "temperature": 0, - "top_p": 0.01, - "seed": 42 + "max_tokens": 2000 }, - "verbose": true + "verbose": true, + "version_compare_tool": true }, "checklist_model": { "service": { "_type": "nvfoundation" }, - "model_name": "meta/llama3-70b-instruct", - "max_tokens": 2000, + "model_name": "meta/llama-3.1-70b-instruct", "temperature": 0, + "max_tokens": 2000, "top_p": 0.01, "seed": 42 }, "justification_model": { - "model_name": "meta/llama3-70b-instruct", + "model_name": "meta/llama-3.1-70b-instruct", "service": { "_type": "nvfoundation" }, @@ -41,7 +39,7 @@ "max_batch_size": 128 }, "summary_model": { - "model_name": "meta/llama3-70b-instruct", + "model_name": "meta/llama-3.1-70b-instruct", "service": { "_type": "nvfoundation" }, diff --git a/kustomize/overlays/our-llm-service/kustomization.yaml b/kustomize/overlays/remote-nim-all/kustomization.yaml similarity index 83% rename from kustomize/overlays/our-llm-service/kustomization.yaml rename to kustomize/overlays/remote-nim-all/kustomization.yaml index 2b423f7..6f5a243 100644 --- a/kustomize/overlays/our-llm-service/kustomization.yaml +++ b/kustomize/overlays/remote-nim-all/kustomization.yaml @@ -6,8 +6,9 @@ resources: configMapGenerator: - name: agent-morpheus-config + behavior: replace files: - agent-morpheus-config.json patchesStrategicMerge: - - agent-morpheus-patch.yaml \ No newline at end of file + - nginx-patch.yaml \ No newline at end of file diff --git a/kustomize/overlays/remote-nim-all/nginx-patch.yaml b/kustomize/overlays/remote-nim-all/nginx-patch.yaml new file mode 100644 index 0000000..4afc4ae --- /dev/null +++ b/kustomize/overlays/remote-nim-all/nginx-patch.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache +spec: + template: + spec: + containers: + - name: nginx + env: + - name: NGINX_UPSTREAM_NIM_LLM + value: https://integrate.api.nvidia.com + - name: NGINX_UPSTREAM_NIM_EMBED + value: https://integrate.api.nvidia.com + - name: NGINX_UPSTREAM_OPENAI + value: https://integrate.api.nvidia.com From cf04d4dcfdba58b75ba488eb9ffc670488742cc2 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Mon, 2 Dec 2024 17:26:42 +0100 Subject: [PATCH 25/50] fix: lock httpx dependency version and fix incompatibility with other models Signed-off-by: Ruben Romero Montes --- requirements.yaml | 1 + src/cve/pipeline/engine.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.yaml b/requirements.yaml index a3035e8..b1dcdeb 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -38,6 +38,7 @@ dependencies: # dependencies after 12.6.2 is available on both channels. - cuda-python!=12.6.1 - openai=1.13 + - httpx<0.28.0 # lock version because >=0.28.0 introduces an incompatibility - packaging>=23.2,<24.0 # versions required by langchain-core - pydantic=2.6 - python>=3.10.15,<3.11 diff --git a/src/cve/pipeline/engine.py b/src/cve/pipeline/engine.py index 0ad96b2..5fa8ba8 100644 --- a/src/cve/pipeline/engine.py +++ b/src/cve/pipeline/engine.py @@ -176,7 +176,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d "final answer."), 1).replace( "Use the following format:", - "Use the following format (start each response with one of the following prefixes: " + "Use the following format (start each response with one of the following prefixes followed by a semicolon and a line break: " "Question, Thought, Action, Action Input, Final Answer):", 1) if run_config.engine.summary_model.service.type.lower() == 'openai': From a0f05654e0fae63b9ef4cddac7ec3c144abf318c Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Tue, 26 Nov 2024 17:09:02 +0100 Subject: [PATCH 26/50] feat: implement plugin based intel retrieval Signed-off-by: Ruben Romero Montes --- src/cve/data_models/config.py | 24 +++------ src/cve/data_models/cve_intel.py | 9 ++++ src/cve/data_models/plugin.py | 23 +++++++-- src/cve/data_models/plugins/intel_plugin.py | 57 +++++++++++++++++++++ src/cve/pipeline/input.py | 7 ++- src/cve/pipeline/pipeline.py | 8 ++- src/cve/utils/intel_retriever.py | 25 ++++++++- src/cve/utils/prompting.py | 1 + 8 files changed, 127 insertions(+), 27 deletions(-) create mode 100644 src/cve/data_models/plugins/intel_plugin.py diff --git a/src/cve/data_models/config.py b/src/cve/data_models/config.py index ff035e1..dfa721b 100644 --- a/src/cve/data_models/config.py +++ b/src/cve/data_models/config.py @@ -40,6 +40,11 @@ def _llm_discriminator(v: typing.Any) -> str | None: return v.get("service").get("_type") return getattr(getattr(v, "service"), "_type") +class PluginConfig(TypedBaseModel[typing.Literal["plugin"]]): + + plugin_name: str + + plugin_config: dict[str, typing.Any] = {} class GeneralConfig(BaseModel): @@ -62,6 +67,7 @@ class GeneralConfig(BaseModel): code_search_tool: bool = False use_dependency_checker: bool = Field(default=True, description="Whether to use the VulnerableDependencyChecker in the pipeline or not.") + intel_plugins: list[PluginConfig] | None = None @field_validator("num_threads") @classmethod @@ -173,17 +179,10 @@ class HttpInputConfig(TypedBaseModel[typing.Literal["http"]]): stop_after: int = 0 -class PluginInputConfig(TypedBaseModel[typing.Literal["plugin"]]): - - plugin_name: str - - plugin_config: dict[str, typing.Any] = {} - - InputConfig = typing.Annotated[typing.Annotated[ManualInputConfig, Tag(ManualInputConfig.static_type())] | typing.Annotated[FileInputConfig, Tag(FileInputConfig.static_type())] | typing.Annotated[HttpInputConfig, Tag(HttpInputConfig.static_type())] - | typing.Annotated[PluginInputConfig, Tag(PluginInputConfig.static_type())], + | typing.Annotated[PluginConfig, Tag(PluginConfig.static_type())], Discriminator(TypedBaseModel.discriminator)] @@ -270,16 +269,9 @@ class OutputElasticsearchConfig(TypedBaseModel[typing.Literal["elasticsearch"]]) raise_on_exception: bool = False -class OutputPluginConfig(TypedBaseModel[typing.Literal["plugin"]]): - - plugin_name: str - - plugin_config: dict[str, typing.Any] = {} - - OutputConfig = typing.Annotated[typing.Annotated[OutputPrintConfig, Tag(OutputPrintConfig.static_type())] | typing.Annotated[OutputFileConfig, Tag(OutputFileConfig.static_type())] - | typing.Annotated[OutputPluginConfig, Tag(OutputPluginConfig.static_type())], + | typing.Annotated[PluginConfig, Tag(PluginConfig.static_type())], Discriminator(TypedBaseModel.discriminator)] diff --git a/src/cve/data_models/cve_intel.py b/src/cve/data_models/cve_intel.py index 50c0aa5..c9b9fd4 100644 --- a/src/cve/data_models/cve_intel.py +++ b/src/cve/data_models/cve_intel.py @@ -157,6 +157,10 @@ class CveIntelEpss(BaseModel): percentile: float | None = None date: str | None = None +class IntelPluginData(BaseModel): + + label: str + description: str class CveIntel(BaseModel): """ @@ -173,6 +177,7 @@ class CveIntel(BaseModel): rhsa: CveIntelRhsa | None = None ubuntu: CveIntelUbuntu | None = None epss: CveIntelEpss | None = None + plugin_data: list[IntelPluginData] = [] @property def cve_id(self): @@ -256,3 +261,7 @@ def get_ghsa_id(self): return self.vuln_id return None + + @property + def plugins_intel_data(self) -> str: + return [f"{pd.label}: {pd.description}" for pd in self.plugin_data] \ No newline at end of file diff --git a/src/cve/data_models/plugin.py b/src/cve/data_models/plugin.py index 055dd9d..6f03670 100644 --- a/src/cve/data_models/plugin.py +++ b/src/cve/data_models/plugin.py @@ -18,10 +18,13 @@ from abc import ABC from abc import abstractmethod from pydoc import locate +import aiohttp from morpheus.config import Config from morpheus.pipeline.linear_pipeline import LinearPipeline +from .cve_intel import IntelPluginData + from ..data_models.config import RunConfig _T = typing.TypeVar('_T', bound='PluginSchema') @@ -30,8 +33,8 @@ class PluginSchema(ABC): @classmethod - def locate(cls: type[_T], plugin_name: str) -> _T: - '''Locate input plugin''' + def locate(cls: type[_T], plugin_name: str, **kwargs) -> _T: + '''Locate plugin''' pluginClass: type | None = locate(plugin_name) if not pluginClass: @@ -39,7 +42,7 @@ def locate(cls: type[_T], plugin_name: str) -> _T: if not issubclass(pluginClass, cls): raise ValueError(f"{pluginClass} object must be a subclass of {cls}") - return pluginClass() + return pluginClass(**kwargs) class InputPluginSchema(PluginSchema): @@ -56,3 +59,17 @@ class OutputPluginSchema(PluginSchema): def build_output(self, pipe: LinearPipeline, config: Config, run_config: RunConfig): # add the plugin specific output building logic here pass + + +class IntelPluginSchema(PluginSchema): + + def __init__(self, session: aiohttp.ClientSession, config: dict = {}): + self._session = session + self._config = config + + @abstractmethod + async def retrieve(self, vuln_id: str) -> IntelPluginData: + """ + Asynchronously retrieve the details of a given CVE Id + """ + pass diff --git a/src/cve/data_models/plugins/intel_plugin.py b/src/cve/data_models/plugins/intel_plugin.py new file mode 100644 index 0000000..0e0633b --- /dev/null +++ b/src/cve/data_models/plugins/intel_plugin.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +import logging +from pydantic import BaseModel +import requests +from ..plugin import IntelPluginSchema +from ..cve_intel import IntelPluginData + +logger = logging.getLogger(__name__) + + +class SimpleHttpIntelPluginConfig(BaseModel): + source: str + endpoint: str + api_key: str | None = None + + +class SimpleHttpIntelPlugin(IntelPluginSchema): + """ + This is a simple Http intel retriever that sends an http GET to the configured + endpoint and returns the text content as a result. + The configuration expects an endpoint with the `{vuln_id}` template parameter + that will be replaced by the current vuln_id. e.g. https://example.com/{vuln_id}/details + Additionally if the `api_key` is provided, it will be sent as an HTTP Authorization Header + with value: Bearer + """ + + def __init__(self, session, config): + self._session = session + self._config = SimpleHttpIntelPluginConfig(**config) + + async def retrieve(self, vuln_id: str) -> IntelPluginData: + endpoint = self._config.endpoint.format(vuln_id=vuln_id) + try: + headers = {'Accept': 'text/plain'} + if self._config.api_key: + headers['Authorization'] = f'Bearer {self._config.api_key}' + response = requests.get(endpoint, headers=headers) + response.raise_for_status() + if not response.ok: + logger.error( + f'Unable to request intel data from {endpoint}. Error: {response.status_code} - {response.reason}') + else: + logger.debug( + f'Successfully requested intel data to: {endpoint}') + return IntelPluginData(label=self._config.source, description=response.text) + except requests.exceptions.RequestException as e: + logger.error(f"Error fetching intel data from {endpoint}: {e}") + return IntelPluginData(label=self._config.source, description='No data available for this intel source') \ No newline at end of file diff --git a/src/cve/pipeline/input.py b/src/cve/pipeline/input.py index bd249e4..cda7eaa 100644 --- a/src/cve/pipeline/input.py +++ b/src/cve/pipeline/input.py @@ -30,10 +30,9 @@ from morpheus.pipeline.stage_decorator import source from morpheus.pipeline.stage_decorator import stage -from ..data_models.config import FileInputConfig +from ..data_models.config import FileInputConfig, PluginConfig from ..data_models.config import HttpInputConfig from ..data_models.config import ManualInputConfig -from ..data_models.config import PluginInputConfig from ..data_models.config import RunConfig from ..data_models.cve_intel import CveIntel from ..data_models.dependencies import VulnerableDependencies @@ -150,7 +149,7 @@ def emit_input_object(subscription: mrc.Subscription) -> typing.Generator[AgentM build_http_input(pipe, config, run_config.input, AgentMorpheusInput) - elif (run_config.input.type == PluginInputConfig.static_type()): + elif (run_config.input.type == PluginConfig.static_type()): # Set source based on plugins plugin = InputPluginSchema.locate(run_config.input.plugin_name) plugin.build_input(pipe, config, run_config) @@ -179,7 +178,7 @@ async def _inner(): async with aiohttp.ClientSession() as session: - intel_retriever = IntelRetriever(session=session) + intel_retriever = IntelRetriever(session=session, plugins_config=run_config.general.intel_plugins) intel_coros = [intel_retriever.retrieve(vuln_id=cve.vuln_id) for cve in message.input.scan.vulns] diff --git a/src/cve/pipeline/pipeline.py b/src/cve/pipeline/pipeline.py index e5cf872..a90b3b9 100644 --- a/src/cve/pipeline/pipeline.py +++ b/src/cve/pipeline/pipeline.py @@ -92,7 +92,7 @@ def convert_input_to_df(message: AgentMorpheusEngineInput) -> ControlMessage: assert message.info.intel is not None, "The input message must have intel information" - input_names = ['vuln_id', 'ghsa_id', 'cve_id'] + input_names = ['vuln_id', 'ghsa_id', 'cve_id', 'plugins_intel_data'] # Determine the input columns that need to be set from the engine (based on checklist_prompt fields) input_names.extend([name for name in engine.get_input_names() if name not in input_names]) @@ -100,7 +100,11 @@ def convert_input_to_df(message: AgentMorpheusEngineInput) -> ControlMessage: # Convert intel object to DataFrame # Using pandas since cudf doesn't support json_normalize() full_df = pd.json_normalize([{ - "vuln_id": x.vuln_id, "ghsa_id": x.get_ghsa_id(), "cve_id": x.get_cve_id(), **x.model_dump(mode="json") + "vuln_id": x.vuln_id, + "ghsa_id": x.get_ghsa_id(), + "cve_id": x.get_cve_id(), + "plugins_intel_data": x.plugins_intel_data, + **x.model_dump(mode="json") } for x in message.info.intel], sep="_") diff --git a/src/cve/utils/intel_retriever.py b/src/cve/utils/intel_retriever.py index 44e6cb5..38101a1 100644 --- a/src/cve/utils/intel_retriever.py +++ b/src/cve/utils/intel_retriever.py @@ -21,7 +21,11 @@ import aiohttp -from ..data_models.cve_intel import CveIntel +from ..data_models.plugin import IntelPluginSchema + +from ..data_models.config import PluginConfig + +from ..data_models.cve_intel import CveIntel, IntelPluginData from ..data_models.cve_intel import CveIntelEpss from ..data_models.cve_intel import CveIntelNvd from ..data_models.cve_intel import CveIntelRhsa @@ -45,7 +49,8 @@ def __init__(self, nist_api_key: str | None = None, ghsa_api_key: str | None = None, lang_code: str = 'en', - max_retries: int = 10): + max_retries: int = 10, + plugins_config: list[PluginConfig] | None = None): """ Initialize the NISTCVERetriever with URL templates for vulnerability and CVE details. """ @@ -63,6 +68,10 @@ def __init__(self, retry_count=max_retries) self._rhsa_client = RHSAClient(session=self._session, retry_count=max_retries) self._ubuntu_client = UbuntuClient(session=self._session, retry_count=max_retries) + self._intel_plugins = [] + if plugins_config is not None: + self._intel_plugins = [IntelPluginSchema.locate( + plugin_name=cfg.plugin_name, session=session, config=cfg.plugin_config) for cfg in plugins_config] @asynccontextmanager async def _get_session(self, session: aiohttp.ClientSession | None = None): @@ -153,6 +162,16 @@ async def _get_epss_score(self, intel: CveIntel) -> CveIntelEpss | None: return None + async def append_plugin_data(self, plugin: IntelPluginSchema, intel: CveIntel) -> IntelPluginData: + + try: + intel.plugin_data.append(await plugin.retrieve(intel.vuln_id)) + return intel.plugin_data + + except Exception as e: + logger.error("Error fetching data from plugin %s : %s", intel.vuln_id, e, exc_info=True) + return None + async def retrieve(self, vuln_id: str) -> CveIntel: """ Asynchronously retrieve all relevant details for a given CVE ID. @@ -186,6 +205,8 @@ async def retrieve(self, vuln_id: str) -> CveIntel: self._get_epss_score(intel=intel) ] + coros.extend([self.append_plugin_data(intel=intel, plugin=plugin) for plugin in self._intel_plugins]) + await asyncio.gather(*coros) return intel diff --git a/src/cve/utils/prompting.py b/src/cve/utils/prompting.py index 5ecfba9..ca34d60 100644 --- a/src/cve/utils/prompting.py +++ b/src/cve/utils/prompting.py @@ -139,6 +139,7 @@ def build_prompt(self) -> str: # IfPromptBuilder('ubuntu_notices', 'Ubuntu Priority Reason: '), # Disabling for now since its very long IfPromptBuilder('ubuntu_ubuntu_description', 'Ubuntu Security Note: '), IfPromptBuilder('vulnerable_dependencies', 'Identified Vulnerable Dependencies: '), + IfPromptBuilder('plugins_intel_data', 'Extra information: ') ] additional_intel_prompting = '\n'.join([pb.build_prompt() for pb in additional_intel_fields]) From 162f664f4af7c9dfa2174f8239b8e35233330dd3 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 4 Dec 2024 18:09:27 +0200 Subject: [PATCH 27/50] fix: a little typo Signed-off-by: Zvi Grinberg --- src/cve/pipeline/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cve/pipeline/engine.py b/src/cve/pipeline/engine.py index 5fa8ba8..9b607f7 100644 --- a/src/cve/pipeline/engine.py +++ b/src/cve/pipeline/engine.py @@ -203,7 +203,7 @@ def notify_llm_to_rephrase(exception: OutputParserException) -> str: logger.debug(f"Exception intercepted in notify {exception}") exception_string = str(exception) if OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER in exception_string: - return (f"Check you output and make sure it conforms! Do not output an action " + return (f"Check your output and make sure it conforms! Do not output an action " f"and a final answer at the same time") return "Check your output and make sure it conforms, use the Action/Action Input syntax" From d6d131453209a396c8265633901000fd0e4aad69 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Wed, 4 Dec 2024 18:41:23 +0200 Subject: [PATCH 28/50] feat: add new overlay of llama-3.1-70b-4bit Signed-off-by: Zvi Grinberg --- .dockerignore | 200 ++++++++++++++++++ kustomize/base/nginx.yaml | 2 +- .../agent-morpheus-config.json | 74 +++++++ .../kustomization.yaml | 15 ++ .../local-llama3.1-70b-4bit/nginx-patch.yaml | 12 ++ 5 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 kustomize/overlays/local-llama3.1-70b-4bit/agent-morpheus-config.json create mode 100644 kustomize/overlays/local-llama3.1-70b-4bit/kustomization.yaml create mode 100644 kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6e527f5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,200 @@ +###### Place new entries directly below this line! ###### + +# Ignore anything in the ./.tmp directory +.tmp/ + +# Ignore user-generated fiass databases +vdb/ + +# Explicitly ignore .vscode/. Shared settings should go in morpheus.code-workspace +# and user settings will go in .vscode/ +.vscode/ + +# Ignore .sqlite cache files +*.sqlite + + +##### Do not alter the items below this line! ##### +########## They are managed by a script! ########## + +# Created by https://www.gitignore.io/api/vim,c++,cmake,python,synology + +### C++ ### +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +### CMake ### +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Env files +*.env + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### Synology ### +# Thumbnails +@eaDir +# Recycle bin +\#recycle + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + + +# End of https://www.gitignore.io/api/vim,c++,cmake,python,synology diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml index 574d873..1b1e20b 100644 --- a/kustomize/base/nginx.yaml +++ b/kustomize/base/nginx.yaml @@ -70,7 +70,7 @@ spec: - name: NGINX_UPSTREAM_NIM_LLM value: https://integrate.api.nvidia.com - name: NGINX_UPSTREAM_NIM_EMBED - value: http://nim-embed.nim.svc.cluster.local:8000 + value: http://nim-embed.agent-morpheus-models.svc.cluster.local:8000 - name: NGINX_UPSTREAM_OPENAI value: https://integrate.api.nvidia.com volumeMounts: diff --git a/kustomize/overlays/local-llama3.1-70b-4bit/agent-morpheus-config.json b/kustomize/overlays/local-llama3.1-70b-4bit/agent-morpheus-config.json new file mode 100644 index 0000000..c43120d --- /dev/null +++ b/kustomize/overlays/local-llama3.1-70b-4bit/agent-morpheus-config.json @@ -0,0 +1,74 @@ +{ + "$schema": "./schemas/config.schema.json", + "engine": { + "agent": { + "model": { + "model_name": "hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4", + "service": { + "_type": "openai" + }, + "max_tokens": 2000 + }, + "verbose": true, + "version_compare_tool": true + }, + "checklist_model": { + "service": { + "_type": "openai" + }, + "model_name": "hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4", + "temperature": 0, + "max_tokens": 2000, + "top_p": 0.01, + "seed": 42 + }, + "justification_model": { + "model_name": "hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4", + "service": { + "_type": "openai" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "rag_embedding": { + "_type": "nim", + "model": "nvidia/nv-embedqa-e5-v5", + "truncate": "END", + "max_batch_size": 128 + }, + "summary_model": { + "model_name": "hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4", + "service": { + "_type": "openai" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + } + }, + "general": { + "cache_dir": null, + "base_vdb_dir": "/morpheus-data/vdbs", + "base_git_dir": "/morpheus-data/repos", + "max_retries": 5, + "model_max_batch_size": 64, + "pipeline_batch_size": 128, + "use_uvloop": true, + "use_dependency_checker": false + }, + "input": { + "_type": "http", + "port": 8080, + "address": "0.0.0.0" + }, + "output": { + "_type": "plugin", + "plugin_name": "src.cve.data_models.plugins.http_output_plugin.HttpOutputPlugin", + "plugin_config": { + "callback_url": "http://agent-morpheus-client:8080/reports" + } + } +} diff --git a/kustomize/overlays/local-llama3.1-70b-4bit/kustomization.yaml b/kustomize/overlays/local-llama3.1-70b-4bit/kustomization.yaml new file mode 100644 index 0000000..513020d --- /dev/null +++ b/kustomize/overlays/local-llama3.1-70b-4bit/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +configMapGenerator: + - name: agent-morpheus-config + behavior: replace + files: + - agent-morpheus-config.json + +patchesStrategicMerge: + - nginx-patch.yaml + diff --git a/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml b/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml new file mode 100644 index 0000000..4c393fe --- /dev/null +++ b/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-cache +spec: + template: + spec: + containers: + - name: nginx + env: + - name: NGINX_UPSTREAM_OPENAI + value: http://llama3-1-70b-instruct.agent-morpheus-models.svc.cluster.local:8000 From f5f8ad6db95cf0cee6f36ed140acc00008db1586 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 00:09:05 +0200 Subject: [PATCH 29/50] fix: only create openai_llm for openai model service type and reference the service type of agent model in config Signed-off-by: Zvi Grinberg --- src/cve/pipeline/engine.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/cve/pipeline/engine.py b/src/cve/pipeline/engine.py index 9b607f7..fc66369 100644 --- a/src/cve/pipeline/engine.py +++ b/src/cve/pipeline/engine.py @@ -56,7 +56,10 @@ def _build_dynamic_agent_fn(run_config: RunConfig, embeddings: Embeddings): chat_client = chat_service.get_client(**run_config.engine.agent.model.model_dump(exclude={"service", "type"}, by_alias=True)) langchain_llm = LangchainLLMClientWrapper(client=chat_client) - openai_llm = ChatOpenAI(client=ChatCompletion(), model_name=run_config.engine.agent.model.model_name) + if run_config.engine.agent.model.type.lower() == 'openai': + openai_llm = ChatOpenAI(client=ChatCompletion(), model_name=run_config.engine.agent.model.model_name) + else: + openai_llm = None # Initialize a SerpAPIWrapper object to perform internet searches. search = MorpheusSerpAPIWrapper(max_retries=run_config.general.max_retries) @@ -179,7 +182,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d "Use the following format (start each response with one of the following prefixes followed by a semicolon and a line break: " "Question, Thought, Action, Action Input, Final Answer):", 1) - if run_config.engine.summary_model.service.type.lower() == 'openai': + if run_config.engine.agent.model.type.lower() == 'openai': prompt_template = prompt_template.replace("Final Answer: the final answer to the original input question", "Final Answer: the final answer to the original input question. \n" "Do not include Final Answer together" @@ -193,13 +196,13 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d return agent def choose_llm_based_on_model_type(): - if run_config.engine.summary_model.service.type.lower() == 'openai': + if run_config.engine.agent.model.type.lower() == 'openai': return openai_llm else: return langchain_llm def notify_llm_to_rephrase(exception: OutputParserException) -> str: - if run_config.engine.summary_model.service.type.lower() == 'openai': + if run_config.engine.agent.model.type.lower() == 'openai': logger.debug(f"Exception intercepted in notify {exception}") exception_string = str(exception) if OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER in exception_string: From f3681de97c4541d4de5aeb83867b744891f1116f Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 00:43:14 +0200 Subject: [PATCH 30/50] docs: point to agent-morpheus-models module to deploy self hosted LLMs Signed-off-by: Zvi Grinberg --- .gitmodules | 4 ++++ agent-morpheus-models | 1 + kustomize/README.md | 31 +------------------------------ 3 files changed, 6 insertions(+), 30 deletions(-) create mode 100644 .gitmodules create mode 160000 agent-morpheus-models diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..eb0dda7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "agent-morpheus-models"] + path = agent-morpheus-models + url = ../agent-morpheus-models/ + branch = main diff --git a/agent-morpheus-models b/agent-morpheus-models new file mode 160000 index 0000000..9c7f739 --- /dev/null +++ b/agent-morpheus-models @@ -0,0 +1 @@ +Subproject commit 9c7f73940947ad3572f04c40716fb9e423a08901 diff --git a/kustomize/README.md b/kustomize/README.md index f1dc738..c7cf909 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -44,33 +44,4 @@ oc label namespace $YOUR_NAMESPACE_NAME application=agent-morpheus # Deploy agent-morpheus integrated with our self-hosted LLM oc kustomize overlays/our-llm-service | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` -7. Deploy self-hosted Llama-3.1-8b LLM - 1. Clone the following repo - ```shell - clone git@github.com:zvigrinberg/nim-deploy.git - cd nim-deploy/helm - ``` - 2. Create new namespace - ```shell - oc new-project nim - ``` - 3. Create service account for the deployment - ```shell - export SERVICE_ACCOUNT_NAME=nim-llm-sa - oc create serviceaccount $SERVICE_ACCOUNT_NAME - ``` - 4. add `anyuid` `SecurityContextConstraint` privilege to the created service account - ```shell - oc adm policy add-scc-to-user -z $SERVICE_ACCOUNT_NAME anyuid - ``` - 5. Deploy the chart - ```shell - helm install nim-llm nim-llm/ --set persistence.enabled=true \ - --set model.ngcAPIKey=$NGC_API_KEY \ - --set image.repository=nvcr.io/nim/meta/llama-3.1-8b-instruct \ - --set image.tag=latest \ - --set serviceAccount.name=$SERVICE_ACCOUNT_NAME \ - --set tolerations[0].key=p4-gpu \ - --set tolerations[0].operator=Exists \ - --set tolerations[0].effect=NoSchedule - ``` \ No newline at end of file +7. If you didn't deploy yet self-hosted embedding model and LLM, follow the instructions [here](../agent-morpheus-models/README.md) \ No newline at end of file From a1b7fb9d91ce642e856c4f1cd75fde0371bbda36 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 00:50:55 +0200 Subject: [PATCH 31/50] docs: add documentation to install helm chart of self hosted models Signed-off-by: Zvi Grinberg --- kustomize/README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index c7cf909..cbfa440 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -37,11 +37,13 @@ oc kustomize overlays/nvidia-llm-service | oc apply -f - -n $YOUR_NAMESPACE_NAM 6. Alternatively, if you want to deploy agent-morpheus with our self-hosted LLM, then run ```shell -# Enable ingress traffic into our LLM Model service in the cluster -oc apply -f network-policy.yaml -# label namespace with application=agent-morpheus to open communication from your namespace to LLM Model service on our Cluster -oc label namespace $YOUR_NAMESPACE_NAME application=agent-morpheus -# Deploy agent-morpheus integrated with our self-hosted LLM -oc kustomize overlays/our-llm-service | oc apply -f - -n $YOUR_NAMESPACE_NAME +# Deploy agent morpheus with Nim meta/llama-3.1-8b-16bit LLM +oc kustomize overlays/local-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME +# Deploy agent morpheus with llama-3.1-70b-4bit LLM +oc kustomize overlays/local-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` -7. If you didn't deploy yet self-hosted embedding model and LLM, follow the instructions [here](../agent-morpheus-models/README.md) \ No newline at end of file +7. If you didn't deploy yet self-hosted embedding model and one of the above LLMs, first fetch latest version of agent-morpheus git submodule +```shell +git submodule update --recursive +``` +8. Follow the instructions [here](../agent-morpheus-models/README.md) \ No newline at end of file From c4efce921c2004ff15bfc8311d235ba77e53340d Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 01:49:34 +0200 Subject: [PATCH 32/50] chore: change to remote submodule Signed-off-by: Zvi Grinberg --- .gitmodules | 4 ++-- agent-morpheus-models | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index eb0dda7..8216571 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "agent-morpheus-models"] path = agent-morpheus-models - url = ../agent-morpheus-models/ - branch = main + url = ../agent-morpheus-models + branch= main diff --git a/agent-morpheus-models b/agent-morpheus-models index 9c7f739..c994516 160000 --- a/agent-morpheus-models +++ b/agent-morpheus-models @@ -1 +1 @@ -Subproject commit 9c7f73940947ad3572f04c40716fb9e423a08901 +Subproject commit c9945167f216ba93c73772d88ed53e511ccae7a7 From b78a865445a5828091fb120dbcd904ccb4dcd27f Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 02:49:26 +0200 Subject: [PATCH 33/50] docs: fix link for installing LLMs and models helm chart Signed-off-by: Zvi Grinberg --- kustomize/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/README.md b/kustomize/README.md index cbfa440..5b2d197 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -46,4 +46,4 @@ oc kustomize overlays/local-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPAC ```shell git submodule update --recursive ``` -8. Follow the instructions [here](../agent-morpheus-models/README.md) \ No newline at end of file +8. Follow the instructions [here from local](../agent-morpheus-models/README.md) and [here from remote](https://github.com/RHEcosystemAppEng/agent-morpheus-models) \ No newline at end of file From 5c0fb62b819eaf8f93a72b74c197a2973b33eb39 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 07:59:19 +0200 Subject: [PATCH 34/50] docs: refine the instruction to install chart Signed-off-by: Zvi Grinberg --- .gitmodules | 2 +- agent-morpheus-models | 1 - kustomize/README.md | 8 +++++++- 3 files changed, 8 insertions(+), 3 deletions(-) delete mode 160000 agent-morpheus-models diff --git a/.gitmodules b/.gitmodules index 8216571..0ea904d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "agent-morpheus-models"] path = agent-morpheus-models - url = ../agent-morpheus-models + url = ../agent-morpheus-models/ branch= main diff --git a/agent-morpheus-models b/agent-morpheus-models deleted file mode 160000 index c994516..0000000 --- a/agent-morpheus-models +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c9945167f216ba93c73772d88ed53e511ccae7a7 diff --git a/kustomize/README.md b/kustomize/README.md index 5b2d197..11bd54b 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -46,4 +46,10 @@ oc kustomize overlays/local-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPAC ```shell git submodule update --recursive ``` -8. Follow the instructions [here from local](../agent-morpheus-models/README.md) and [here from remote](https://github.com/RHEcosystemAppEng/agent-morpheus-models) \ No newline at end of file +8. Follow the instructions below + 1. from your local repository + ```shell + cd ../agent-morpheus-models + ``` + [open instructions locally](../agent-morpheus-models/README.md) + 2. [Open instruction in browser](https://github.com/RHEcosystemAppEng/agent-morpheus-models) \ No newline at end of file From 45effa4960a41a8d2862532621110dd7b3f48b18 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 08:47:14 +0200 Subject: [PATCH 35/50] fix: pointer of git submodule to latest one Signed-off-by: Zvi Grinberg --- .gitmodules | 2 +- agent-morpheus-models | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 160000 agent-morpheus-models diff --git a/.gitmodules b/.gitmodules index 0ea904d..8216571 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "agent-morpheus-models"] path = agent-morpheus-models - url = ../agent-morpheus-models/ + url = ../agent-morpheus-models branch= main diff --git a/agent-morpheus-models b/agent-morpheus-models new file mode 160000 index 0000000..a624658 --- /dev/null +++ b/agent-morpheus-models @@ -0,0 +1 @@ +Subproject commit a6246589761848e8d3d28e908478b1a5ad9498ef From 68ea029c6bf68cd431d7da09b68c2b17725b0224 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 08:56:24 +0200 Subject: [PATCH 36/50] chore: fix LLM service name for upstream OPENAI Signed-off-by: Zvi Grinberg --- kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml b/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml index 4c393fe..c37f87a 100644 --- a/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml +++ b/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml @@ -9,4 +9,4 @@ spec: - name: nginx env: - name: NGINX_UPSTREAM_OPENAI - value: http://llama3-1-70b-instruct.agent-morpheus-models.svc.cluster.local:8000 + value: http://llama3-1-70b-instruct-4bit.agent-morpheus-models.svc.cluster.local:8000 From 8c2ddb69ed4c4dec738c7f666a38714329a09f94 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 10:22:47 +0200 Subject: [PATCH 37/50] docs: add instruction to force update to latest version of git submodule Signed-off-by: Zvi Grinberg --- agent-morpheus-models | 2 +- kustomize/README.md | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/agent-morpheus-models b/agent-morpheus-models index a624658..932cbb9 160000 --- a/agent-morpheus-models +++ b/agent-morpheus-models @@ -1 +1 @@ -Subproject commit a6246589761848e8d3d28e908478b1a5ad9498ef +Subproject commit 932cbb9a0024b0179f0b9d22f523d7e31da6becd diff --git a/kustomize/README.md b/kustomize/README.md index 11bd54b..d788d97 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -44,7 +44,10 @@ oc kustomize overlays/local-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPAC ``` 7. If you didn't deploy yet self-hosted embedding model and one of the above LLMs, first fetch latest version of agent-morpheus git submodule ```shell -git submodule update --recursive +git submodule init --recursive +git submodule update --merge --recursive +# Or if the local submodule sha is not updated to latest one, you can fetch it from remote tracking branch +git submodule update --remote --merge --recursive ``` 8. Follow the instructions below 1. from your local repository From 3026da2ec7d398b2e454c3f9278c3e9c6463684d Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 13:11:54 +0200 Subject: [PATCH 38/50] refactor: use a single variable for both types of llm service objects Signed-off-by: Zvi Grinberg --- src/cve/pipeline/engine.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/cve/pipeline/engine.py b/src/cve/pipeline/engine.py index fc66369..665bdc6 100644 --- a/src/cve/pipeline/engine.py +++ b/src/cve/pipeline/engine.py @@ -50,16 +50,16 @@ def _build_dynamic_agent_fn(run_config: RunConfig, embeddings: Embeddings): - chat_service = LLMService.create(run_config.engine.agent.model.service.type, - **run_config.engine.agent.model.service.model_dump(exclude={"type"}, - by_alias=True)) - chat_client = chat_service.get_client(**run_config.engine.agent.model.model_dump(exclude={"service", "type"}, - by_alias=True)) - langchain_llm = LangchainLLMClientWrapper(client=chat_client) + if run_config.engine.agent.model.type.lower() == 'openai': - openai_llm = ChatOpenAI(client=ChatCompletion(), model_name=run_config.engine.agent.model.model_name) + wrapper_llm = ChatOpenAI(client=ChatCompletion(), model_name=run_config.engine.agent.model.model_name) else: - openai_llm = None + chat_service = LLMService.create(run_config.engine.agent.model.service.type, + **run_config.engine.agent.model.service.model_dump(exclude={"type"}, + by_alias=True)) + chat_client = chat_service.get_client(**run_config.engine.agent.model.model_dump(exclude={"service", "type"}, + by_alias=True)) + wrapper_llm = LangchainLLMClientWrapper(client=chat_client) # Initialize a SerpAPIWrapper object to perform internet searches. search = MorpheusSerpAPIWrapper(max_retries=run_config.general.max_retries) @@ -101,7 +101,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d # load code vector DB code_vector_db = FAISS.load_local(vdb_map.code_vdb_path, embeddings, allow_dangerous_deserialization=True) code_qa_tool = RetrievalQA.from_chain_type( - llm=choose_llm_based_on_model_type(), + llm=wrapper_llm, chain_type="stuff", retriever=code_vector_db.as_retriever(), return_source_documents=run_config.engine.agent.return_source_documents) @@ -141,7 +141,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d if (vdb_map.doc_vdb_path is not None): guide_vector_db = FAISS.load_local(vdb_map.doc_vdb_path, embeddings, allow_dangerous_deserialization=True) guide_qa_tool = RetrievalQA.from_chain_type( - llm=choose_llm_based_on_model_type(), + llm=wrapper_llm, chain_type="stuff", retriever=guide_vector_db.as_retriever(), return_source_documents=run_config.engine.agent.return_source_documents) @@ -195,12 +195,6 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d return agent - def choose_llm_based_on_model_type(): - if run_config.engine.agent.model.type.lower() == 'openai': - return openai_llm - else: - return langchain_llm - def notify_llm_to_rephrase(exception: OutputParserException) -> str: if run_config.engine.agent.model.type.lower() == 'openai': logger.debug(f"Exception intercepted in notify {exception}") From 4bed0c6050e671522e31f9894c6db76a7030a0a2 Mon Sep 17 00:00:00 2001 From: Ilona Shishov Date: Mon, 9 Dec 2024 16:27:37 +0200 Subject: [PATCH 39/50] fix: ensure agent checks for vulnerable functions/methods if mentioned in CVE --- src/cve/nodes/cve_checklist_node.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cve/nodes/cve_checklist_node.py b/src/cve/nodes/cve_checklist_node.py index defa696..33ca368 100644 --- a/src/cve/nodes/cve_checklist_node.py +++ b/src/cve/nodes/cve_checklist_node.py @@ -34,6 +34,7 @@ cve_prompt1 = ( MOD_FEW_SHOT.format(examples=get_mod_examples()) + additional_intel_prompting + + "\n\nIf a vulnerable function or method is mentioned in the CVE description, ensure the first checklist item verifies whether this function or method is being called from the code or used by the code." + "\nThe vulnerable version of the vulnerable package is already verified to be installed within the container. Check only the other factors that affect exploitability, no need to verify version again." ) From 0f7e5d67db0f27e0cb94f2174a4b66a32d4da93d Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 15:57:19 +0200 Subject: [PATCH 40/50] fix: agent crash when startuping pipeline Signed-off-by: Zvi Grinberg --- src/cve/pipeline/engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cve/pipeline/engine.py b/src/cve/pipeline/engine.py index 665bdc6..16b3516 100644 --- a/src/cve/pipeline/engine.py +++ b/src/cve/pipeline/engine.py @@ -51,7 +51,7 @@ def _build_dynamic_agent_fn(run_config: RunConfig, embeddings: Embeddings): - if run_config.engine.agent.model.type.lower() == 'openai': + if run_config.engine.agent.model.service.type.lower() == 'openai': wrapper_llm = ChatOpenAI(client=ChatCompletion(), model_name=run_config.engine.agent.model.model_name) else: chat_service = LLMService.create(run_config.engine.agent.model.service.type, @@ -182,7 +182,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d "Use the following format (start each response with one of the following prefixes followed by a semicolon and a line break: " "Question, Thought, Action, Action Input, Final Answer):", 1) - if run_config.engine.agent.model.type.lower() == 'openai': + if run_config.engine.agent.model.service.type.lower() == 'openai': prompt_template = prompt_template.replace("Final Answer: the final answer to the original input question", "Final Answer: the final answer to the original input question. \n" "Do not include Final Answer together" @@ -196,7 +196,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d return agent def notify_llm_to_rephrase(exception: OutputParserException) -> str: - if run_config.engine.agent.model.type.lower() == 'openai': + if run_config.engine.agent.model.service.type.lower() == 'openai': logger.debug(f"Exception intercepted in notify {exception}") exception_string = str(exception) if OUTPUT_CONTAIN_BOTH_ACTION_AND_FINAL_ANSWER in exception_string: From 80906d9bfa08723539ab3251e87f13606aa595e4 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 10 Dec 2024 16:15:09 +0200 Subject: [PATCH 41/50] fix: agent loop not working Signed-off-by: Zvi Grinberg --- src/cve/pipeline/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cve/pipeline/engine.py b/src/cve/pipeline/engine.py index 16b3516..768bf4f 100644 --- a/src/cve/pipeline/engine.py +++ b/src/cve/pipeline/engine.py @@ -163,7 +163,7 @@ def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | d # This agent is designed to handle zero-shot reaction descriptions and parse errors. agent = initialize_agent( tools, - choose_llm_based_on_model_type(), + wrapper_llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=run_config.engine.agent.verbose, handle_parsing_errors=notify_llm_to_rephrase, From 808cc83052e50c67c6cb05d3ef6ffd8cfec32295 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Tue, 10 Dec 2024 18:03:22 +0100 Subject: [PATCH 42/50] feat: remove local nim configuration in favor of the 7b config Signed-off-by: Ruben Romero Montes --- .../local-llama3.1-70b-4bit/nginx-patch.yaml | 2 + .../local-nim-all/agent-morpheus-config.json | 74 ------------------- .../overlays/local-nim-all/kustomization.yaml | 14 ---- .../overlays/local-nim-all/nginx-patch.yaml | 14 ---- 4 files changed, 2 insertions(+), 102 deletions(-) delete mode 100644 kustomize/overlays/local-nim-all/agent-morpheus-config.json delete mode 100644 kustomize/overlays/local-nim-all/kustomization.yaml delete mode 100644 kustomize/overlays/local-nim-all/nginx-patch.yaml diff --git a/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml b/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml index c37f87a..59aff2f 100644 --- a/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml +++ b/kustomize/overlays/local-llama3.1-70b-4bit/nginx-patch.yaml @@ -8,5 +8,7 @@ spec: containers: - name: nginx env: + - name: NGINX_UPSTREAM_NIM_LLM + value: http://llama3-1-70b-instruct-4bit.agent-morpheus-models.svc.cluster.local:8000 - name: NGINX_UPSTREAM_OPENAI value: http://llama3-1-70b-instruct-4bit.agent-morpheus-models.svc.cluster.local:8000 diff --git a/kustomize/overlays/local-nim-all/agent-morpheus-config.json b/kustomize/overlays/local-nim-all/agent-morpheus-config.json deleted file mode 100644 index 41ac9ca..0000000 --- a/kustomize/overlays/local-nim-all/agent-morpheus-config.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "$schema": "./schemas/config.schema.json", - "engine": { - "agent": { - "model": { - "model_name": "meta/llama-3.1-8b-instruct", - "service": { - "_type": "nvfoundation" - }, - "max_tokens": 2000 - }, - "verbose": true, - "version_compare_tool": true - }, - "checklist_model": { - "service": { - "_type": "nvfoundation" - }, - "model_name": "meta/llama-3.1-8b-instruct", - "temperature": 0, - "max_tokens": 2000, - "top_p": 0.01, - "seed": 42 - }, - "justification_model": { - "model_name": "meta/llama-3.1-8b-instruct", - "service": { - "_type": "nvfoundation" - }, - "max_tokens": 1024, - "temperature": 0, - "top_p": 0.01, - "seed": 42 - }, - "rag_embedding": { - "_type": "nim", - "model": "nvidia/nv-embedqa-e5-v5", - "truncate": "END", - "max_batch_size": 128 - }, - "summary_model": { - "model_name": "meta/llama-3.1-8b-instruct", - "service": { - "_type": "nvfoundation" - }, - "max_tokens": 1024, - "temperature": 0, - "top_p": 0.01, - "seed": 42 - } - }, - "general": { - "cache_dir": null, - "base_vdb_dir": "/morpheus-data/vdbs", - "base_git_dir": "/morpheus-data/repos", - "max_retries": 5, - "model_max_batch_size": 64, - "pipeline_batch_size": 128, - "use_uvloop": true, - "use_dependency_checker": false - }, - "input": { - "_type": "http", - "port": 8080, - "address": "0.0.0.0" - }, - "output": { - "_type": "plugin", - "plugin_name": "src.cve.data_models.plugins.http_output_plugin.HttpOutputPlugin", - "plugin_config": { - "callback_url": "http://agent-morpheus-client:8080/reports" - } - } -} diff --git a/kustomize/overlays/local-nim-all/kustomization.yaml b/kustomize/overlays/local-nim-all/kustomization.yaml deleted file mode 100644 index 6f5a243..0000000 --- a/kustomize/overlays/local-nim-all/kustomization.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ../../base - -configMapGenerator: - - name: agent-morpheus-config - behavior: replace - files: - - agent-morpheus-config.json - -patchesStrategicMerge: - - nginx-patch.yaml \ No newline at end of file diff --git a/kustomize/overlays/local-nim-all/nginx-patch.yaml b/kustomize/overlays/local-nim-all/nginx-patch.yaml deleted file mode 100644 index 3281654..0000000 --- a/kustomize/overlays/local-nim-all/nginx-patch.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-cache -spec: - template: - spec: - containers: - - name: nginx - env: - - name: NGINX_UPSTREAM_NIM_LLM - value: http://nim-llm.nim.svc.cluster.local:8000 - - name: NGINX_UPSTREAM_OPENAI - value: http://nim-llm.nim.svc.cluster.local:8000 From 0a6e9cc480cea3897e3ada968d3a07f77b5dfddf Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 12 Dec 2024 14:13:34 +0100 Subject: [PATCH 43/50] chore: update deployment instructions Signed-off-by: Ruben Romero Montes --- kustomize/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index d788d97..85451ef 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -30,17 +30,17 @@ oc create secret generic morpheus-pull-secret --from-file=.dockerconfigjson= --type=kubernetes.io/dockerconfigjson ``` -5. Deploy agent-morpheus + agent-morpheus-client to your namespace +5. Deploy agent-morpheus + agent-morpheus-client to your namespace. The default configuration will use local embeddings and remote NIM LLM ```shell -oc kustomize overlays/nvidia-llm-service | oc apply -f - -n $YOUR_NAMESPACE_NAME +oc kustomize kustomize/base | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` -6. Alternatively, if you want to deploy agent-morpheus with our self-hosted LLM, then run +6. Alternatively, if you want to deploy agent-morpheus with our self-hosted LLM or fully remote services, then run ```shell -# Deploy agent morpheus with Nim meta/llama-3.1-8b-16bit LLM -oc kustomize overlays/local-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME -# Deploy agent morpheus with llama-3.1-70b-4bit LLM -oc kustomize overlays/local-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME +# Deploy agent morpheus with local llama3.1 LLM +oc kustomize kustomize/overlays/local-llama3.1-70b-4bit | oc apply -f - -n $YOUR_NAMESPACE_NAME +# Deploy agent morpheus with remote llama-3.1-70b-4bit LLM +oc kustomize kustomize/overlays/remote-nim-all | oc apply -f - -n $YOUR_NAMESPACE_NAME ``` 7. If you didn't deploy yet self-hosted embedding model and one of the above LLMs, first fetch latest version of agent-morpheus git submodule ```shell From f2271612940b6b2165516fbc490c81ff8f0a8918 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 12 Dec 2024 14:27:45 +0100 Subject: [PATCH 44/50] chore: update deployment instructions Signed-off-by: Ruben Romero Montes --- kustomize/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 85451ef..12d29b2 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -32,15 +32,15 @@ oc create secret generic ngrc-secret --from-file=.dockerconfigjson= Date: Fri, 13 Dec 2024 12:30:10 +0100 Subject: [PATCH 45/50] feat: do not retry for client exceptions Signed-off-by: Ruben Romero Montes --- src/cve/utils/async_http_utils.py | 7 +++++-- src/cve/utils/vulnerable_dependency_checker.py | 1 - 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cve/utils/async_http_utils.py b/src/cve/utils/async_http_utils.py index 93fc349..89dba20 100644 --- a/src/cve/utils/async_http_utils.py +++ b/src/cve/utils/async_http_utils.py @@ -31,11 +31,12 @@ async def request_with_retry(session: aiohttp.ClientSession, max_retries: int = 10, sleep_time: float = 0.1, respect_retry_after_header: bool = True, - log_on_error=True) -> typing.AsyncIterator[aiohttp.ClientResponse]: + log_on_error=True, + retry_on_client_errors = False) -> typing.AsyncIterator[aiohttp.ClientResponse]: """ Async version of `morpheus.utils.http_utils.request_with_retry` """ - assert not request_kwargs.get('raise_for_status'), "raise_for_status is cincompatible with `request_with_retry`" + assert not request_kwargs.get('raise_for_status'), "raise_for_status is incompatible with `request_with_retry`" try_count = 0 done = False while try_count <= max_retries and not done: @@ -61,6 +62,8 @@ async def request_with_retry(session: aiohttp.ClientSession, actual_sleep_time = max(int(response_headers["Retry-After"]), actual_sleep_time) elif respect_retry_after_header and 'X-RateLimit-Reset' in response_headers: actual_sleep_time = max(int(response_headers["X-RateLimit-Reset"]) - time.time(), actual_sleep_time) + elif not retry_on_client_errors and response.status < 500: + raise e logger.warning("Error requesting [%d/%d]: (Retry %.1f sec) %s: %s", try_count, diff --git a/src/cve/utils/vulnerable_dependency_checker.py b/src/cve/utils/vulnerable_dependency_checker.py index 134395f..f173c80 100644 --- a/src/cve/utils/vulnerable_dependency_checker.py +++ b/src/cve/utils/vulnerable_dependency_checker.py @@ -29,7 +29,6 @@ from ..data_models.cve_intel import CveIntelNvd from ..data_models.dependencies import DependencyPackage -from .async_http_utils import request_with_retry from .clients.intel_client import IntelClient from .string_utils import package_names_match from .url_utils import url_join From e136c12ad5f70d7959e572806df142fd02bd6290 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Fri, 13 Dec 2024 14:43:33 +0100 Subject: [PATCH 46/50] fix: escape backslashes in intel descriptions Signed-off-by: Ruben Romero Montes --- src/cve/nodes/cve_checklist_node.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cve/nodes/cve_checklist_node.py b/src/cve/nodes/cve_checklist_node.py index defa696..16b73b3 100644 --- a/src/cve/nodes/cve_checklist_node.py +++ b/src/cve/nodes/cve_checklist_node.py @@ -75,6 +75,9 @@ async def _parse_list(text: list[str]) -> list[list[str]]: # Remove newline characters that can cause incorrect string escaping in the next step x = x.replace("\n", "") + # Ensure backslashes are escaped + x = x.replace("\\", "\\\\") + # Try to do some very basic string cleanup to fix unescaped quotes x = attempt_fix_list_string(x) From 52f149030abfe797f045f0ec89842e2d88a8feee Mon Sep 17 00:00:00 2001 From: Vladimir Belousov Date: Tue, 17 Dec 2024 17:49:12 +0200 Subject: [PATCH 47/50] fix: add ES module support to JavaScript parser Signed-off-by: Vladimir Belousov --- src/cve/utils/document_embedding.py | 32 +++-- src/cve/utils/js_extended_parser.py | 84 +++++++++++++ .../utils/tests/test_java_script_extended.py | 113 ++++++++++++++++++ 3 files changed, 217 insertions(+), 12 deletions(-) create mode 100644 src/cve/utils/js_extended_parser.py create mode 100644 src/cve/utils/tests/test_java_script_extended.py diff --git a/src/cve/utils/document_embedding.py b/src/cve/utils/document_embedding.py index fc6866f..fbd671e 100644 --- a/src/cve/utils/document_embedding.py +++ b/src/cve/utils/document_embedding.py @@ -36,10 +36,10 @@ from langchain_community.document_loaders.parsers.language.language_parser import LANGUAGE_SEGMENTERS from langchain_core.document_loaders.blob_loaders import Blob - -from ..data_models.input import ArtifactSourceInfo, GitSourceInfo, SourceDocumentsInfo +from .js_extended_parser import ExtendedJavaScriptSegmenter from .source_code_artifact_loader import SourceCodeArtifactLoader from .source_code_git_loader import SourceCodeGitLoader +from ..data_models.input import ArtifactSourceInfo, GitSourceInfo, SourceDocumentsInfo if typing.TYPE_CHECKING: from langchain_core.embeddings import Embeddings # pragma: no cover @@ -55,9 +55,9 @@ class MultiLanguageRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter """ def __init__( - self, - keep_separator: bool = True, - **kwargs, + self, + keep_separator: bool = True, + **kwargs, ) -> None: """Create a new RecursiveCharacterTextSplitter.""" super().__init__(is_separator_regex=True, keep_separator=keep_separator, **kwargs) @@ -102,8 +102,17 @@ class ExtendedLanguageParser(LanguageParser): "hpp": "cpp", } + additional_segmenters: dict[str, type[CodeSegmenter]] = {} + + if os.environ.get('ENABLE_EXTENDED_JS_PARSERS'): + additional_segmenters = { + "javascript": ExtendedJavaScriptSegmenter, + "js": ExtendedJavaScriptSegmenter, + } + LANGUAGE_SEGMENTERS: dict[str, type[CodeSegmenter]] = { **LANGUAGE_SEGMENTERS, + **additional_segmenters, } def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: @@ -329,7 +338,7 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: Returns a list of documents collected from the source document info. """ if isinstance(source_info, GitSourceInfo): - + path = self.get_repo_path(source_info) blob_loader = SourceCodeGitLoader(repo_path=path, @@ -339,13 +348,13 @@ def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: exclude=source_info.exclude) elif isinstance(source_info, ArtifactSourceInfo): - + path = self._git_directory / PurePath(source_info.path) blob_loader = SourceCodeArtifactLoader(local_path=path, - remote_path=source_info.path, - compression=source_info.compression, - include=source_info.include, - exclude=source_info.exclude) + remote_path=source_info.path, + compression=source_info.compression, + include=source_info.include, + exclude=source_info.exclude) blob_parser = ExtendedLanguageParser() @@ -382,7 +391,6 @@ def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathL # Warn if the output path already exists and we will overwrite it if (output_path.exists()): - logger.warning("Vector Database already exists and will be overwritten: %s", output_path) documents = [] diff --git a/src/cve/utils/js_extended_parser.py b/src/cve/utils/js_extended_parser.py new file mode 100644 index 0000000..166fa1b --- /dev/null +++ b/src/cve/utils/js_extended_parser.py @@ -0,0 +1,84 @@ +import logging +from typing import List, Any, Tuple + +import esprima +from langchain_community.document_loaders.parsers.language.javascript import JavaScriptSegmenter + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class ExtendedJavaScriptSegmenter(JavaScriptSegmenter): + """Extended JavaScript segmenter that handles shebang and ES optional chaining.""" + + def __init__(self, code: str): + """Initialize the segmenter with preprocessed code.""" + super().__init__(code) + + if code.startswith("#!"): + logger.warning("File contains a shebang line. Skipping parsing.") + self.skip_file = True + else: + self.skip_file = False + self.code = self.code.replace("?.", ".") + + def _parse_with_fallback(self) -> Any: + """Try to parse code as script first, then as module if that fails.""" + try: + logger.debug("Attempting to parse as a script...") + return esprima.parseScript(self.code, loc=True) + except esprima.Error: + logger.debug("Script parsing failed. Trying module parsing...") + try: + return esprima.parseModule(self.code, loc=True) + except esprima.Error as e: + logger.error("Module parsing failed: %s", str(e)) + return None + + def extract_functions_classes(self) -> List[str]: + """Extract functions, classes and exports from the code.""" + if self.skip_file: + return [] + + tree = self._parse_with_fallback() + if tree is None: + return [] + + functions_classes = [] + for node in tree.body: + # Handle direct function/class declarations + if isinstance(node, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): + functions_classes.append(self._extract_code(node)) + # Handle exported declarations + elif isinstance(node, esprima.nodes.ExportNamedDeclaration): + if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): + functions_classes.append(self._extract_code(node)) + + return functions_classes + + def simplify_code(self) -> str: + """Simplify the code by replacing function/class bodies with comments.""" + if self.skip_file: + return self.code + + tree = self._parse_with_fallback() + if tree is None: + return self.code + + simplified_lines = self.source_lines[:] + indices_to_del: List[Tuple[int, int]] = [] + + for node in tree.body: + if isinstance(node, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): + start, end = node.loc.start.line - 1, node.loc.end.line + simplified_lines[start] = f"// Code for: {simplified_lines[start]}" + indices_to_del.append((start + 1, end)) + elif isinstance(node, esprima.nodes.ExportNamedDeclaration): + if isinstance(node.declaration, (esprima.nodes.FunctionDeclaration, esprima.nodes.ClassDeclaration)): + start, end = node.loc.start.line - 1, node.loc.end.line + simplified_lines[start] = f"// Code for: {simplified_lines[start]}" + indices_to_del.append((start + 1, end)) + + for start, end in reversed(indices_to_del): + del simplified_lines[start:end] + + return "\n".join(line for line in simplified_lines) diff --git a/src/cve/utils/tests/test_java_script_extended.py b/src/cve/utils/tests/test_java_script_extended.py new file mode 100644 index 0000000..ff8da26 --- /dev/null +++ b/src/cve/utils/tests/test_java_script_extended.py @@ -0,0 +1,113 @@ +import pytest + +from ..js_extended_parser import ExtendedJavaScriptSegmenter + +TEST_CASES = [ + { + "name": "regular_script", + "code": """ +function hello() { + console.log('Hello'); +} + +class MyClass { + constructor() { + this.value = 42; + } +} +""", + "expected_functions": 2, # One function and one class + "should_parse": True, + }, + { + "name": "es_module", + "code": """ +import { something } from './module'; +export function exportedFunc() { + return 'exported'; +} +export class ExportedClass { + method() {} +} +""", + "expected_functions": 2, # One exported function and one exported class + "should_parse": True, + }, + { + "name": "optional_chaining", + "code": """ +function processUser(user) { + return user?.profile?.name; +} +class UserManager { + getAddress() { + return this.user?.address?.street; + } +} +""", + "expected_functions": 2, + "should_parse": True, + }, + { + "name": "shebang_file", + "code": """#!/usr/bin/env node +function main() { + console.log('Main'); +} +""", + "expected_functions": 0, # Should skip due to shebang + "should_parse": False, + } +] + + +@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) +def test_function_extraction(test_case): + """Test that functions and classes are correctly extracted.""" + segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) + functions = segmenter.extract_functions_classes() + assert len(functions) == test_case["expected_functions"] + if not test_case["should_parse"]: + assert segmenter.skip_file + + +@pytest.mark.parametrize("test_case", TEST_CASES, ids=lambda x: x["name"]) +def test_code_simplification(test_case): + """Test that code is correctly simplified.""" + segmenter = ExtendedJavaScriptSegmenter(test_case["code"]) + simplified = segmenter.simplify_code() + + # Basic checks + assert isinstance(simplified, str) + if not test_case["should_parse"]: + assert segmenter.skip_file + assert simplified == test_case["code"] # Return original code for unparseable files + else: + # Should have fewer or equal lines than original + assert len(simplified.splitlines()) <= len(test_case["code"].splitlines()) + # Should contain "// Code for:" for each function/class + if test_case["expected_functions"] > 0: + assert "// Code for:" in simplified + + +def test_optional_chaining_replacement(): + """Test that optional chaining is correctly replaced.""" + code = "const name = user?.profile?.name;" + segmenter = ExtendedJavaScriptSegmenter(code) + assert not segmenter.skip_file + assert "?." not in segmenter.code + assert "." in segmenter.code + + +def test_invalid_js(): + """Test handling of invalid JavaScript code.""" + code = "this is not valid javascript" + segmenter = ExtendedJavaScriptSegmenter(code) + assert not segmenter.skip_file # Should attempt to parse + + # Should handle errors gracefully + functions = segmenter.extract_functions_classes() + assert functions == [] + + simplified = segmenter.simplify_code() + assert simplified == code # Return original code for invalid JS From efd6f2a95235cac4199ab7d7bcd00b206bacb8c4 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Wed, 18 Dec 2024 13:03:38 +0100 Subject: [PATCH 48/50] chore: increase nginx cache pvc Signed-off-by: Ruben Romero Montes --- kustomize/base/nginx.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/base/nginx.yaml b/kustomize/base/nginx.yaml index 1b1e20b..3826c78 100644 --- a/kustomize/base/nginx.yaml +++ b/kustomize/base/nginx.yaml @@ -143,4 +143,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 20Gi + storage: 100Gi From 6a69faa02ce564dcd897de5b1f56ef485d7f9282 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 19 Dec 2024 16:57:21 +0100 Subject: [PATCH 49/50] feat: make retry_on_client_errors configurable Signed-off-by: Ruben Romero Montes --- configs/schemas/config.schema.json | 6 ++++++ kustomize/base/agent-morpheus-config.json | 3 ++- .../agent-morpheus-config.json | 3 ++- .../remote-nim-all/agent-morpheus-config.json | 3 ++- src/cve/data_models/config.py | 1 + src/cve/pipeline/input.py | 4 +++- src/cve/utils/async_http_utils.py | 2 +- src/cve/utils/clients/first_client.py | 6 ++++-- src/cve/utils/clients/ghsa_client.py | 6 ++++-- src/cve/utils/clients/intel_client.py | 7 +++++-- src/cve/utils/clients/nvd_client.py | 9 ++++++--- src/cve/utils/clients/rhsa_client.py | 6 ++++-- src/cve/utils/clients/ubuntu_client.py | 6 ++++-- src/cve/utils/intel_retriever.py | 18 ++++++++++++------ src/cve/utils/vulnerable_dependency_checker.py | 6 ++++-- 15 files changed, 60 insertions(+), 26 deletions(-) diff --git a/configs/schemas/config.schema.json b/configs/schemas/config.schema.json index 920a7c5..24da982 100644 --- a/configs/schemas/config.schema.json +++ b/configs/schemas/config.schema.json @@ -286,6 +286,11 @@ "title": "Max Retries", "type": "integer" }, + "retry_on_client_errors": { + "default": true, + "title": "Retry On Client Errors", + "type": "boolean" + }, "model_max_batch_size": { "default": 64, "exclusiveMinimum": 0, @@ -1346,6 +1351,7 @@ "cache_dir": null, "ignore_build_vdb_errors": false, "max_retries": 10, + "retry_on_client_errors": true, "model_max_batch_size": 64, "num_threads": 5, "pipeline_batch_size": 1024, diff --git a/kustomize/base/agent-morpheus-config.json b/kustomize/base/agent-morpheus-config.json index 57436e3..48dda48 100644 --- a/kustomize/base/agent-morpheus-config.json +++ b/kustomize/base/agent-morpheus-config.json @@ -57,7 +57,8 @@ "model_max_batch_size": 64, "pipeline_batch_size": 128, "use_uvloop": true, - "use_dependency_checker": false + "use_dependency_checker": false, + "retry_on_client_errors": false }, "input": { "_type": "http", diff --git a/kustomize/overlays/local-llama3.1-70b-4bit/agent-morpheus-config.json b/kustomize/overlays/local-llama3.1-70b-4bit/agent-morpheus-config.json index c43120d..53b0241 100644 --- a/kustomize/overlays/local-llama3.1-70b-4bit/agent-morpheus-config.json +++ b/kustomize/overlays/local-llama3.1-70b-4bit/agent-morpheus-config.json @@ -57,7 +57,8 @@ "model_max_batch_size": 64, "pipeline_batch_size": 128, "use_uvloop": true, - "use_dependency_checker": false + "use_dependency_checker": false, + "retry_on_client_errors": false }, "input": { "_type": "http", diff --git a/kustomize/overlays/remote-nim-all/agent-morpheus-config.json b/kustomize/overlays/remote-nim-all/agent-morpheus-config.json index 8620d88..de4e7d7 100644 --- a/kustomize/overlays/remote-nim-all/agent-morpheus-config.json +++ b/kustomize/overlays/remote-nim-all/agent-morpheus-config.json @@ -57,7 +57,8 @@ "model_max_batch_size": 64, "pipeline_batch_size": 1024, "use_uvloop": true, - "use_dependency_checker": false + "use_dependency_checker": false, + "retry_on_client_errors": false }, "input": { "_type": "http", diff --git a/src/cve/data_models/config.py b/src/cve/data_models/config.py index dfa721b..8f7e655 100644 --- a/src/cve/data_models/config.py +++ b/src/cve/data_models/config.py @@ -56,6 +56,7 @@ class GeneralConfig(BaseModel): cache_dir: str | None = None ignore_build_vdb_errors: bool = False max_retries: NonNegativeInt = 10 + retry_on_client_errors: bool = True model_max_batch_size: PositiveInt = 64 num_threads: PositiveInt = Field(default_factory=os.cpu_count) pipeline_batch_size: PositiveInt = 1024 diff --git a/src/cve/pipeline/input.py b/src/cve/pipeline/input.py index cda7eaa..b52809e 100644 --- a/src/cve/pipeline/input.py +++ b/src/cve/pipeline/input.py @@ -178,7 +178,9 @@ async def _inner(): async with aiohttp.ClientSession() as session: - intel_retriever = IntelRetriever(session=session, plugins_config=run_config.general.intel_plugins) + intel_retriever = IntelRetriever(session=session, + plugins_config=run_config.general.intel_plugins, + retry_on_client_errors=run_config.general.retry_on_client_errors) intel_coros = [intel_retriever.retrieve(vuln_id=cve.vuln_id) for cve in message.input.scan.vulns] diff --git a/src/cve/utils/async_http_utils.py b/src/cve/utils/async_http_utils.py index 89dba20..9ecd79e 100644 --- a/src/cve/utils/async_http_utils.py +++ b/src/cve/utils/async_http_utils.py @@ -32,7 +32,7 @@ async def request_with_retry(session: aiohttp.ClientSession, sleep_time: float = 0.1, respect_retry_after_header: bool = True, log_on_error=True, - retry_on_client_errors = False) -> typing.AsyncIterator[aiohttp.ClientResponse]: + retry_on_client_errors = True) -> typing.AsyncIterator[aiohttp.ClientResponse]: """ Async version of `morpheus.utils.http_utils.request_with_retry` """ diff --git a/src/cve/utils/clients/first_client.py b/src/cve/utils/clients/first_client.py index 7e62560..b4fd771 100644 --- a/src/cve/utils/clients/first_client.py +++ b/src/cve/utils/clients/first_client.py @@ -37,13 +37,15 @@ def __init__(self, session: aiohttp.ClientSession | None = None, retry_count: int = 10, sleep_time: float = 0.1, - respect_retry_after_header: bool = True): + respect_retry_after_header: bool = True, + retry_on_client_errors: bool = True): super().__init__(session=session, base_url=base_url or os.environ.get('FIRST_BASE_URL'), retry_count=retry_count, sleep_time=sleep_time, - respect_retry_after_header=respect_retry_after_header) + respect_retry_after_header=respect_retry_after_header, + retry_on_client_errors=retry_on_client_errors) self._headers = {'Accept': 'application/json'} diff --git a/src/cve/utils/clients/ghsa_client.py b/src/cve/utils/clients/ghsa_client.py index 05997b0..8f02237 100644 --- a/src/cve/utils/clients/ghsa_client.py +++ b/src/cve/utils/clients/ghsa_client.py @@ -41,12 +41,14 @@ def __init__(self, session: aiohttp.ClientSession | None = None, retry_count: int = 10, sleep_time: float = 0.1, - respect_retry_after_header: bool = True): + respect_retry_after_header: bool = True, + retry_on_client_errors: bool = True): super().__init__(session=session, base_url=base_url or os.environ.get('GHSA_BASE_URL'), retry_count=retry_count, sleep_time=sleep_time, - respect_retry_after_header=respect_retry_after_header) + respect_retry_after_header=respect_retry_after_header, + retry_on_client_errors=retry_on_client_errors) self._api_key = api_key or os.environ.get('GHSA_API_KEY', None) diff --git a/src/cve/utils/clients/intel_client.py b/src/cve/utils/clients/intel_client.py index 5d52589..0ff733a 100644 --- a/src/cve/utils/clients/intel_client.py +++ b/src/cve/utils/clients/intel_client.py @@ -30,7 +30,8 @@ def __init__(self, base_url: str | None = None, retry_count: int = 10, sleep_time: float = 0.1, - respect_retry_after_header: bool = True): + respect_retry_after_header: bool = True, + retry_on_client_errors: bool = True): if (session is None): session = aiohttp.ClientSession() @@ -44,6 +45,7 @@ def __init__(self, self._retry_count = retry_count self._sleep_time = sleep_time self._respect_retry_after_header = respect_retry_after_header + self._retry_on_client_errors = retry_on_client_errors @classmethod @abstractmethod @@ -80,6 +82,7 @@ async def request(self, max_retries=self._retry_count, sleep_time=self._sleep_time, respect_retry_after_header=self._respect_retry_after_header, - log_on_error=log_on_error) as response: + log_on_error=log_on_error, + retry_on_client_errors=self._retry_on_client_errors) as response: return await response.json() diff --git a/src/cve/utils/clients/nvd_client.py b/src/cve/utils/clients/nvd_client.py index d84c0b1..4956093 100644 --- a/src/cve/utils/clients/nvd_client.py +++ b/src/cve/utils/clients/nvd_client.py @@ -51,13 +51,15 @@ def __init__(self, session: aiohttp.ClientSession | None = None, retry_count: int = 10, sleep_time: float = 0.1, - respect_retry_after_header: bool = True): + respect_retry_after_header: bool = True, + retry_on_client_errors: bool = True): super().__init__(session=session, base_url=base_url or os.environ.get('NVD_BASE_URL'), retry_count=retry_count, sleep_time=sleep_time, - respect_retry_after_header=respect_retry_after_header) + respect_retry_after_header=respect_retry_after_header, + retry_on_client_errors=retry_on_client_errors) self._api_key = api_key or os.environ.get('NVD_API_KEY', None) @@ -87,7 +89,8 @@ async def _get_soup(self, url: str) -> BeautifulSoup: 'url': url, "skip_auto_headers": {"User-Agent"}, }, - max_retries=self._retry_count) as response: + max_retries=self._retry_count, + retry_on_client_errors=self._retry_on_client_errors) as response: return BeautifulSoup(await response.text(), 'html.parser') def _get_cvss_vector_from_metric(self, metrics: dict, metric_version: str) -> str | None: diff --git a/src/cve/utils/clients/rhsa_client.py b/src/cve/utils/clients/rhsa_client.py index 0a2f888..190deb8 100644 --- a/src/cve/utils/clients/rhsa_client.py +++ b/src/cve/utils/clients/rhsa_client.py @@ -37,13 +37,15 @@ def __init__(self, session: aiohttp.ClientSession | None = None, retry_count: int = 10, sleep_time: float = 0.1, - respect_retry_after_header: bool = True): + respect_retry_after_header: bool = True, + retry_on_client_errors: bool = True): super().__init__(session=session, base_url=base_url or os.environ.get('RHSA_BASE_URL'), retry_count=retry_count, sleep_time=sleep_time, - respect_retry_after_header=respect_retry_after_header) + respect_retry_after_header=respect_retry_after_header, + retry_on_client_errors=retry_on_client_errors) @classmethod def default_base_url(cls) -> str: diff --git a/src/cve/utils/clients/ubuntu_client.py b/src/cve/utils/clients/ubuntu_client.py index 4a9f319..732ee6d 100644 --- a/src/cve/utils/clients/ubuntu_client.py +++ b/src/cve/utils/clients/ubuntu_client.py @@ -35,13 +35,15 @@ def __init__(self, session: aiohttp.ClientSession | None = None, retry_count: int = 10, sleep_time: float = 0.1, - respect_retry_after_header: bool = True): + respect_retry_after_header: bool = True, + retry_on_client_errors: bool = True): super().__init__(session=session, base_url=base_url or os.environ.get('UBUNTU_BASE_URL'), retry_count=retry_count, sleep_time=sleep_time, - respect_retry_after_header=respect_retry_after_header) + respect_retry_after_header=respect_retry_after_header, + retry_on_client_errors=retry_on_client_errors) @classmethod def default_base_url(cls) -> str: diff --git a/src/cve/utils/intel_retriever.py b/src/cve/utils/intel_retriever.py index 38101a1..18a57af 100644 --- a/src/cve/utils/intel_retriever.py +++ b/src/cve/utils/intel_retriever.py @@ -50,7 +50,8 @@ def __init__(self, ghsa_api_key: str | None = None, lang_code: str = 'en', max_retries: int = 10, - plugins_config: list[PluginConfig] | None = None): + plugins_config: list[PluginConfig] | None = None, + retry_on_client_errors: bool = True): """ Initialize the NISTCVERetriever with URL templates for vulnerability and CVE details. """ @@ -61,13 +62,18 @@ def __init__(self, self._nvd_client = NVDClient(api_key=os.environ.get('NVD_API_KEY', nist_api_key), session=self._session, lang_code=lang_code, - retry_count=max_retries) - self._first_client = FirstClient(session=self._session, retry_count=max_retries) + retry_count=max_retries, + retry_on_client_errors=retry_on_client_errors) + self._first_client = FirstClient(session=self._session, retry_count=max_retries, + retry_on_client_errors=retry_on_client_errors) self._ghsa_client = GHSAClient(api_key=os.environ.get('GHSA_API_KEY', ghsa_api_key), session=self._session, - retry_count=max_retries) - self._rhsa_client = RHSAClient(session=self._session, retry_count=max_retries) - self._ubuntu_client = UbuntuClient(session=self._session, retry_count=max_retries) + retry_count=max_retries, + retry_on_client_errors=retry_on_client_errors) + self._rhsa_client = RHSAClient(session=self._session, retry_count=max_retries, + retry_on_client_errors=retry_on_client_errors) + self._ubuntu_client = UbuntuClient(session=self._session, retry_count=max_retries, + retry_on_client_errors=retry_on_client_errors) self._intel_plugins = [] if plugins_config is not None: self._intel_plugins = [IntelPluginSchema.locate( diff --git a/src/cve/utils/vulnerable_dependency_checker.py b/src/cve/utils/vulnerable_dependency_checker.py index f173c80..11040b3 100644 --- a/src/cve/utils/vulnerable_dependency_checker.py +++ b/src/cve/utils/vulnerable_dependency_checker.py @@ -125,13 +125,15 @@ def __init__(self, image: str, sbom_list: list, session: aiohttp.ClientSession | None = None, - respect_retry_after_header: bool = True): + respect_retry_after_header: bool = True, + retry_on_client_errors: bool = True): super().__init__( session=session, base_url=base_url or os.environ.get('DEPSDEV_BASE_URL'), retry_count=1, # Service returns 404 if the package is not found. So dont retry - respect_retry_after_header=respect_retry_after_header) + respect_retry_after_header=respect_retry_after_header, + retry_on_client_errors=retry_on_client_errors) self._semaphore = asyncio.Semaphore(25) From fe0eda97bdd984862f806996841ef4c5839a1fa3 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 19 Dec 2024 17:42:04 +0100 Subject: [PATCH 50/50] chore: create jupyter-lab deployment Signed-off-by: Ruben Romero Montes --- kustomize/README.md | 14 +++++++- .../agent-morpheus-jupyter-network.yaml | 32 +++++++++++++++++++ .../jupyter-lab/agent-morpheus-patch.yaml | 18 +++++++++++ .../overlays/jupyter-lab/kustomization.yaml | 9 ++++++ 4 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 kustomize/overlays/jupyter-lab/agent-morpheus-jupyter-network.yaml create mode 100644 kustomize/overlays/jupyter-lab/agent-morpheus-patch.yaml create mode 100644 kustomize/overlays/jupyter-lab/kustomization.yaml diff --git a/kustomize/README.md b/kustomize/README.md index 12d29b2..32f0930 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -55,4 +55,16 @@ git submodule update --remote --merge --recursive cd ../agent-morpheus-models ``` [open instructions locally](../agent-morpheus-models/README.md) - 2. [Open instruction in browser](https://github.com/RHEcosystemAppEng/agent-morpheus-models) \ No newline at end of file + 2. [Open instruction in browser](https://github.com/RHEcosystemAppEng/agent-morpheus-models) + +## Jupyter Lab + +In order to deploy the vulnerability analysis using Jupyter Lab you can use the +`jupyter-lab` kustomize folder. It will pull the `dev` tag which includes the tools +to start the Jupyter Lab on port `8000`. The overlay will also create the route and +service. It is important to know that the kustomization will not start the HTTP +endpoint meaning that it will not be able to receive external requests. + +```shell +oc kustomize overlays/jupyter-lab | oc apply -f - -n $YOUR_NAMESPACE_NAME +``` \ No newline at end of file diff --git a/kustomize/overlays/jupyter-lab/agent-morpheus-jupyter-network.yaml b/kustomize/overlays/jupyter-lab/agent-morpheus-jupyter-network.yaml new file mode 100644 index 0000000..29617af --- /dev/null +++ b/kustomize/overlays/jupyter-lab/agent-morpheus-jupyter-network.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: agent-morpheus-jupyter + labels: + app: agent-morpheus + component: agent-morpheus-rh +spec: + ports: + - name: jupyter + port: 8000 + protocol: TCP + targetPort: 8000 + selector: + app: agent-morpheus + component: agent-morpheus-rh +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: agent-morpheus-jupyter + annotations: + haproxy.router.openshift.io/timeout: 5m +spec: + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + port: + targetPort: 8000 + to: + kind: Service + name: agent-morpheus-jupyter \ No newline at end of file diff --git a/kustomize/overlays/jupyter-lab/agent-morpheus-patch.yaml b/kustomize/overlays/jupyter-lab/agent-morpheus-patch.yaml new file mode 100644 index 0000000..e8678c1 --- /dev/null +++ b/kustomize/overlays/jupyter-lab/agent-morpheus-patch.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-morpheus-rh + labels: + app: agent-morpheus + component: agent-morpheus-rh +spec: + template: + spec: + containers: + - name: agent-morpheus + image: quay.io/ecosystem-appeng/agent-morpheus-rh:dev + args: ["jupyter-lab", "--no-browser", "--allow-root", "--ip='0.0.0.0'", "--port=8000", "--NotebookApp.token=''", "--NotebookApp.password=''"] + ports: + - name: jupyter + protocol: TCP + containerPort: 8000 diff --git a/kustomize/overlays/jupyter-lab/kustomization.yaml b/kustomize/overlays/jupyter-lab/kustomization.yaml new file mode 100644 index 0000000..c53c903 --- /dev/null +++ b/kustomize/overlays/jupyter-lab/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../local-llama3.1-70b-4bit + - agent-morpheus-jupyter-network.yaml + +patchesStrategicMerge: + - agent-morpheus-patch.yaml