Merge branch 'main' into openai-longprobs

Author: Amnah199

docs-website/docs/concepts/pipelines/asyncpipeline.mdx

@@ -52,37 +52,85 @@ You can find more details in our [API Reference](/reference/pipeline-api#asyncpi
 
 ```python
 import asyncio
-from haystack import AsyncPipeline
-from haystack.components.embedders import SentenceTransformersTextEmbedder
-from haystack.components.retrievers import InMemoryEmbeddingRetriever, InMemoryBM25Retriever
-from haystack.components.joiners import DocumentJoiner
+
+from haystack import AsyncPipeline, Document
 from haystack.components.builders import ChatPromptBuilder
+from haystack.components.embedders import (
+    SentenceTransformersDocumentEmbedder,
+    SentenceTransformersTextEmbedder,
+)
 from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.joiners import DocumentJoiner
+from haystack.components.retrievers import InMemoryBM25Retriever, InMemoryEmbeddingRetriever
+from haystack.dataclasses import ChatMessage
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+
+documents = [
+    Document(content="Khufu is the largest pyramid."),
+    Document(content="Khafre is the middle pyramid."),
+    Document(content="Menkaure is the smallest pyramid."),
+]
+
+docs_embedder = SentenceTransformersDocumentEmbedder()
+docs_embedder.warm_up()
+
+document_store = InMemoryDocumentStore()
+document_store.write_documents(docs_embedder.run(documents=documents)["documents"])
+
+prompt_template = [
+    ChatMessage.from_system(
+        """
+        You are a precise, factual QA assistant.
+        According to the following documents:
+        {% for document in documents %}
+        {{document.content}}
+        {% endfor %}
+
+        If an answer cannot be deduced from the documents, say "I don't know based on these documents".
+
+        When answering:
+        - be concise
+        - list the documents that support your answer
+
+        Answer the given question.
+        """
+    ),
+    ChatMessage.from_user("{{query}}"),
+    ChatMessage.from_system("Answer:"),
+]
 
 hybrid_rag_retrieval = AsyncPipeline()
 hybrid_rag_retrieval.add_component("text_embedder", SentenceTransformersTextEmbedder())
-hybrid_rag_retrieval.add_component("embedding_retriever", InMemoryEmbeddingRetriever(document_store=document_store))
-hybrid_rag_retrieval.add_component("bm25_retriever", InMemoryBM25Retriever(document_store=document_store))
+hybrid_rag_retrieval.add_component(
+    "embedding_retriever", InMemoryEmbeddingRetriever(document_store=document_store, top_k=3)
+)
+hybrid_rag_retrieval.add_component("bm25_retriever", InMemoryBM25Retriever(document_store=document_store, top_k=3))
 hybrid_rag_retrieval.add_component("document_joiner", DocumentJoiner())
 hybrid_rag_retrieval.add_component("prompt_builder", ChatPromptBuilder(template=prompt_template))
 hybrid_rag_retrieval.add_component("llm", OpenAIChatGenerator())
 
-hybrid_rag_retrieval.connect("text_embedder", "embedding_retriever")
-hybrid_rag_retrieval.connect("bm25_retriever", "document_joiner")
-hybrid_rag_retrieval.connect("embedding_retriever", "document_joiner")
-hybrid_rag_retrieval.connect("document_joiner", "prompt_builder.documents")
-hybrid_rag_retrieval.connect("prompt_builder", "llm")
+hybrid_rag_retrieval.connect("text_embedder.embedding", "embedding_retriever.query_embedding")
+hybrid_rag_retrieval.connect("bm25_retriever.documents", "document_joiner.documents")
+hybrid_rag_retrieval.connect("embedding_retriever.documents", "document_joiner.documents")
+hybrid_rag_retrieval.connect("document_joiner.documents", "prompt_builder.documents")
+hybrid_rag_retrieval.connect("prompt_builder.prompt", "llm.messages")
+
+question = "Which pyramid is neither the smallest nor the biggest?"
+
+data = {
+    "prompt_builder": {"query": question},
+    "text_embedder": {"text": question},
+    "bm25_retriever": {"query": question},
+}
 
 async def process_results():
     async for partial_output in hybrid_rag_retrieval.run_async_generator(
-            data=data,
-            include_outputs_from={"document_joiner", "llm"}
+        data=data, include_outputs_from={"document_joiner", "llm"}
     ):
-        # Each partial_output contains the results from a completed component
-        if "retriever" in partial_output:
+        if "document_joiner" in partial_output:
             print("Retrieved documents:", len(partial_output["document_joiner"]["documents"]))
         if "llm" in partial_output:
             print("Generated answer:", partial_output["llm"]["replies"][0])
 
 asyncio.run(process_results())
-```
+```

docs-website/docs/concepts/pipelines/visualizing-pipelines.mdx

@@ -84,6 +84,6 @@ This is an example of what a pipeline graph may look like:
 
 ## Importing a Pipeline to deepset Studio
 
-YYou can import your Haystack pipeline into deepset Studio and continue visually building your pipeline
+You can import your Haystack pipeline into deepset Studio and continue visually building your pipeline
 
 To do that, follow the steps described in our deepset AI Platform [documentation](https://docs.cloud.deepset.ai/docs/import-a-pipeline#import-your-pipeline).

docs-website/reference/integrations-api/mcp.md

@@ -353,6 +353,8 @@ async def connect() -> list[types.Tool]
 
 Connect to an MCP server using SSE transport.
 
+Note: If both custom headers and token are provided, custom headers take precedence.
+
 **Raises**:
 
 - `MCPConnectionError`: If connection to the server fails
@@ -396,6 +398,8 @@ async def connect() -> list[types.Tool]
 
 Connect to an MCP server using streamable HTTP transport.
 
+Note: If both custom headers and token are provided, custom headers take precedence.
+
 **Raises**:
 
 - `MCPConnectionError`: If connection to the server fails
@@ -476,11 +480,31 @@ server_info = SSEServerInfo(
 )
 ```
 
+For custom headers (e.g., non-standard authentication):
+
+```python
+# Single custom header with Secret
+server_info = SSEServerInfo(
+    url="https://my-mcp-server.com",
+    headers={"X-API-Key": Secret.from_env_var("API_KEY")},
+)
+
+# Multiple headers (mix of Secret and plain strings)
+server_info = SSEServerInfo(
+    url="https://my-mcp-server.com",
+    headers={
+        "X-API-Key": Secret.from_env_var("API_KEY"),
+        "X-Client-ID": "my-client-id",
+    },
+)
+```
+
 **Arguments**:
 
 - `url`: Full URL of the MCP server (including /sse endpoint)
 - `base_url`: Base URL of the MCP server (deprecated, use url instead)
-- `token`: Authentication token for the server (optional)
+- `token`: Authentication token for the server (optional, generates "Authorization: Bearer `<token>`" header)
+- `headers`: Custom HTTP headers (optional, takes precedence over token parameter if provided)
 - `timeout`: Connection timeout in seconds
 
 <a id="haystack_integrations.tools.mcp.mcp_tool.SSEServerInfo.base_url"></a>
@@ -529,10 +553,30 @@ server_info = StreamableHttpServerInfo(
 )
 ```
 
+For custom headers (e.g., non-standard authentication):
+
+```python
+# Single custom header with Secret
+server_info = StreamableHttpServerInfo(
+    url="https://my-mcp-server.com",
+    headers={"X-API-Key": Secret.from_env_var("API_KEY")},
+)
+
+# Multiple headers (mix of Secret and plain strings)
+server_info = StreamableHttpServerInfo(
+    url="https://my-mcp-server.com",
+    headers={
+        "X-API-Key": Secret.from_env_var("API_KEY"),
+        "X-Client-ID": "my-client-id",
+    },
+)
+```
+
 **Arguments**:
 
 - `url`: Full URL of the MCP server (streamable HTTP endpoint)
-- `token`: Authentication token for the server (optional)
+- `token`: Authentication token for the server (optional, generates "Authorization: Bearer `<token>`" header)
+- `headers`: Custom HTTP headers (optional, takes precedence over token parameter if provided)
 - `timeout`: Connection timeout in seconds
 
 <a id="haystack_integrations.tools.mcp.mcp_tool.StreamableHttpServerInfo.__post_init__"></a>
@@ -795,27 +839,6 @@ def __del__()
 
 Cleanup resources when the tool is garbage collected.
 
-<a id="haystack_integrations.tools.mcp.mcp_tool.MCPTool.tool_spec"></a>
-
-#### MCPTool.tool\_spec
-
-```python
-@property
-def tool_spec() -> dict[str, Any]
-```
-
-Return the Tool specification to be used by the Language Model.
-
-<a id="haystack_integrations.tools.mcp.mcp_tool.MCPTool.invoke"></a>
-
-#### MCPTool.invoke
-
-```python
-def invoke(**kwargs: Any) -> Any
-```
-
-Invoke the Tool with the provided keyword arguments.
-
 <a id="haystack_integrations.tools.mcp.mcp_tool._MCPClientSessionManager"></a>
 
 ### \_MCPClientSessionManager
@@ -1030,126 +1053,3 @@ def close()
 
 Close the underlying MCP client safely.
 
-<a id="haystack_integrations.tools.mcp.mcp_toolset.MCPToolset.__post_init__"></a>
-
-#### MCPToolset.\_\_post\_init\_\_
-
-```python
-def __post_init__()
-```
-
-Validate and set up the toolset after initialization.
-
-This handles the case when tools are provided during initialization.
-
-<a id="haystack_integrations.tools.mcp.mcp_toolset.MCPToolset.__iter__"></a>
-
-#### MCPToolset.\_\_iter\_\_
-
-```python
-def __iter__() -> Iterator[Tool]
-```
-
-Return an iterator over the Tools in this Toolset.
-
-This allows the Toolset to be used wherever a list of Tools is expected.
-
-**Returns**:
-
-An iterator yielding Tool instances
-
-<a id="haystack_integrations.tools.mcp.mcp_toolset.MCPToolset.__contains__"></a>
-
-#### MCPToolset.\_\_contains\_\_
-
-```python
-def __contains__(item: Any) -> bool
-```
-
-Check if a tool is in this Toolset.
-
-Supports checking by:
-- Tool instance: tool in toolset
-- Tool name: "tool_name" in toolset
-
-**Arguments**:
-
-- `item`: Tool instance or tool name string
-
-**Returns**:
-
-True if contained, False otherwise
-
-<a id="haystack_integrations.tools.mcp.mcp_toolset.MCPToolset.add"></a>
-
-#### MCPToolset.add
-
-```python
-def add(tool: Union[Tool, "Toolset"]) -> None
-```
-
-Add a new Tool or merge another Toolset.
-
-**Arguments**:
-
-- `tool`: A Tool instance or another Toolset to add
-
-**Raises**:
-
-- `ValueError`: If adding the tool would result in duplicate tool names
-- `TypeError`: If the provided object is not a Tool or Toolset
-
-<a id="haystack_integrations.tools.mcp.mcp_toolset.MCPToolset.__add__"></a>
-
-#### MCPToolset.\_\_add\_\_
-
-```python
-def __add__(other: Union[Tool, "Toolset", list[Tool]]) -> "Toolset"
-```
-
-Concatenate this Toolset with another Tool, Toolset, or list of Tools.
-
-**Arguments**:
-
-- `other`: Another Tool, Toolset, or list of Tools to concatenate
-
-**Raises**:
-
-- `TypeError`: If the other parameter is not a Tool, Toolset, or list of Tools
-- `ValueError`: If the combination would result in duplicate tool names
-
-**Returns**:
-
-A new Toolset containing all tools
-
-<a id="haystack_integrations.tools.mcp.mcp_toolset.MCPToolset.__len__"></a>
-
-#### MCPToolset.\_\_len\_\_
-
-```python
-def __len__() -> int
-```
-
-Return the number of Tools in this Toolset.
-
-**Returns**:
-
-Number of Tools
-
-<a id="haystack_integrations.tools.mcp.mcp_toolset.MCPToolset.__getitem__"></a>
-
-#### MCPToolset.\_\_getitem\_\_
-
-```python
-def __getitem__(index)
-```
-
-Get a Tool by index.
-
-**Arguments**:
-
-- `index`: Index of the Tool to get
-
-**Returns**:
-
-The Tool at the specified index