Skip to content

Commit 3bbfd41

Browse files
optimize langchain genai integration (#2715)
1 parent 4797e76 commit 3bbfd41

File tree

1 file changed

+3
-9
lines changed

1 file changed

+3
-9
lines changed

notebooks/llm-rag-langchain/ov_langchain_helper.py

+3-9
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,6 @@ def from_model_path(
6262
model_path: str,
6363
device: str = "CPU",
6464
tokenizer: Any = None,
65-
draft_model_path: Optional[str] = None,
66-
draft_model_device: Optional[str] = "CPU",
6765
**kwargs: Any,
6866
) -> OpenVINOLLM:
6967
"""Construct the oepnvino object from model_path"""
@@ -206,11 +204,7 @@ def put(self, token_id: int) -> bool:
206204
return False
207205
return super().put(token_id)
208206

209-
if draft_model_path is not None:
210-
draft_model = openvino_genai.draft_model(draft_model_path, draft_model_device)
211-
pipe = openvino_genai.LLMPipeline(model_path, device, draft_model=draft_model)
212-
else:
213-
pipe = openvino_genai.LLMPipeline(model_path, device)
207+
pipe = openvino_genai.LLMPipeline(model_path, device, **kwargs)
214208

215209
config = pipe.get_generation_config()
216210
if tokenizer is None:
@@ -245,7 +239,7 @@ def _call(
245239
input_ids = tokens["input_ids"]
246240
attention_mask = tokens["attention_mask"]
247241
prompt = openvino_genai.TokenizedInputs(ov.Tensor(input_ids), ov.Tensor(attention_mask))
248-
output = self.pipe.generate(prompt, self.config)
242+
output = self.pipe.generate(prompt, self.config, **kwargs)
249243
if not isinstance(self.tokenizer, openvino_genai.Tokenizer):
250244
output = self.tokenizer.batch_decode(output.tokens, skip_special_tokens=True)[0]
251245
return output
@@ -280,7 +274,7 @@ def generate_and_signal_complete() -> None:
280274
genration function for single thread
281275
"""
282276
self.streamer.reset()
283-
self.pipe.generate(prompt, self.config, self.streamer)
277+
self.pipe.generate(prompt, self.config, self.streamer, **kwargs)
284278
stream_complete.set()
285279
self.streamer.end()
286280

0 commit comments

Comments
 (0)