from haystack.document_stores import InMemoryDocumentStore
from haystack.nodes.file_classifier import FileTypeClassifier
from haystack.nodes.file_converter import TextConverter
from haystack.nodes.preprocessor import PreProcessor
from haystack.pipelines import Pipeline

# Initialize a DocumentStore
document_store = InMemoryDocumentStore()

# Indexing Pipeline
indexing_pipeline = Pipeline()

# Makes sure the file is a TXT file (FileTypeClassifier node)
classifier = FileTypeClassifier()
indexing_pipeline.add_node(classifier, name="Classifier", inputs=["File"])

# Converts a file into text and performs basic cleaning (TextConverter node)
text_converter = TextConverter(remove_numeric_tables=True)
indexing_pipeline.add_node(text_converter, name="Text_converter", inputs=["Classifier.output_1"])

# Pre-processes the text by performing splits and adding metadata to the text (Preprocessor node)
preprocessor = PreProcessor(
    clean_whitespace=True,
    clean_empty_lines=True,
    split_length=100,
    split_overlap=50,
    split_respect_sentence_boundary=True,
)
indexing_pipeline.add_node(preprocessor, name="Preprocessor", inputs=["Text_converter"])

# - Writes the resulting documents into the document store
indexing_pipeline.add_node(document_store, name="Document_Store", inputs=["Preprocessor"])

# Then we run it with the documents and their metadata as input
result = indexing_pipeline.run(file_paths=file_paths, meta=files_metadata)

from haystack import Pipeline
from haystack.components.routers import FileTypeRouter
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.components.converters import TextFileToDocument
from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter
from haystack.components.writers import DocumentWriter

# Initialize a DocumentStore
document_store = InMemoryDocumentStore()

# Indexing Pipeline
indexing_pipeline = Pipeline()

# Makes sure the file is a TXT file (FileTypeRouter component)
classifier = FileTypeRouter(mime_types=["text/plain"])
indexing_pipeline.add_component("file_type_router", classifier)

# Converts a file into a Document (TextFileToDocument component)
text_converter = TextFileToDocument()
indexing_pipeline.add_component("text_converter", text_converter)

# Performs basic cleaning (DocumentCleaner component)
cleaner = DocumentCleaner(
    remove_empty_lines=True,
    remove_extra_whitespaces=True,
)
indexing_pipeline.add_component("cleaner", cleaner)
 
# Pre-processes the text by performing splits and adding metadata to the text (DocumentSplitter component)
preprocessor = DocumentSplitter(
    split_by="passage", 
    split_length=100, 
    split_overlap=50
)
indexing_pipeline.add_component("preprocessor", preprocessor)

# - Writes the resulting documents into the document store
indexing_pipeline.add_component("writer", DocumentWriter(document_store))

# Connect all the components
indexing_pipeline.connect("file_type_router.text/plain", "text_converter")
indexing_pipeline.connect("text_converter", "cleaner")
indexing_pipeline.connect("cleaner", "preprocessor")
indexing_pipeline.connect("preprocessor", "writer")

# Then we run it with the documents and their metadata as input
result = indexing_pipeline.run({"file_type_router": {"sources": file_paths}})

from haystack.document_stores import InMemoryDocumentStore
from haystack.pipelines import ExtractiveQAPipeline
from haystack import Document
from haystack.nodes import BM25Retriever
from haystack.nodes import FARMReader

document_store = InMemoryDocumentStore(use_bm25=True)
document_store.write_documents([
    Document(content="Paris is the capital of France."),
    Document(content="Berlin is the capital of Germany."),
    Document(content="Rome is the capital of Italy."),
    Document(content="Madrid is the capital of Spain."),
])

retriever = BM25Retriever(document_store=document_store)
reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2")
extractive_qa_pipeline = ExtractiveQAPipeline(reader, retriever)

query = "What is the capital of France?"
result = extractive_qa_pipeline.run(
	query=query, 
	params={
		"Retriever": {"top_k": 10}, 
		"Reader": {"top_k": 5}
	}
)

from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack import Document, Pipeline
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
from haystack.components.readers import ExtractiveReader

document_store = InMemoryDocumentStore()
document_store.write_documents([
    Document(content="Paris is the capital of France."),
    Document(content="Berlin is the capital of Germany."),
    Document(content="Rome is the capital of Italy."),
    Document(content="Madrid is the capital of Spain."),
])

retriever = InMemoryBM25Retriever(document_store)
reader = ExtractiveReader(model="deepset/roberta-base-squad2")
extractive_qa_pipeline = Pipeline()
extractive_qa_pipeline.add_component("retriever", retriever)
extractive_qa_pipeline.add_component("reader", reader)
extractive_qa_pipeline.connect("retriever", "reader")

query = "What is the capital of France?"
result = extractive_qa_pipeline.run(data={
	"retriever": {"query": query, "top_k": 3}, 
	"reader": {"query": query, "top_k": 2}
})

from datasets import load_dataset

from haystack.pipelines import Pipeline
from haystack.document_stores import InMemoryDocumentStore
from haystack.nodes import EmbeddingRetriever, PromptNode, PromptTemplate, AnswerParser

document_store = InMemoryDocumentStore(embedding_dim=384)
dataset = load_dataset("bilgeyucel/seven-wonders", split="train")
document_store.write_documents(dataset)
retriever = EmbeddingRetriever(embedding_model="sentence-transformers/all-MiniLM-L6-v2", document_store=document_store, top_k=2)
document_store.update_embeddings(retriever)

rag_prompt = PromptTemplate(
    prompt="""Synthesize a comprehensive answer from the following text for the given question.
                             Provide a clear and concise response that summarizes the key points and information presented in the text.
                             Your answer should be in your own words and be no longer than 50 words.
                             \n\n Related text: {join(documents)} \n\n Question: {query} \n\n Answer:""",
    output_parser=AnswerParser(),
)

prompt_node = PromptNode(model_name_or_path="gpt-3.5-turbo", api_key=OPENAI_API_KEY, default_prompt_template=rag_prompt)

pipe = Pipeline()
pipe.add_node(component=retriever, name="retriever", inputs=["Query"])
pipe.add_node(component=prompt_node, name="prompt_node", inputs=["retriever"])

output = pipe.run(query="What does Rhodes Statue look like?")

from datasets import load_dataset

from haystack import Document, Pipeline
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.components.builders import PromptBuilder
from haystack.components.generators import OpenAIGenerator
from haystack.components.embedders import SentenceTransformersDocumentEmbedder
from haystack.components.embedders import SentenceTransformersTextEmbedder
from haystack.components.retrievers import InMemoryEmbeddingRetriever

document_store = InMemoryDocumentStore()
dataset = load_dataset("bilgeyucel/seven-wonders", split="train")
embedder = SentenceTransformersDocumentEmbedder("sentence-transformers/all-MiniLM-L6-v2")
embedder.warm_up()
output = embedder.run([Document(**ds) for ds in dataset])
document_store.write_documents(output.get("documents"))

template = """
Given the following information, answer the question.

Context:
{% for document in documents %}
    {{ document.content }}
{% endfor %}

Question: {{question}}
Answer:
"""
prompt_builder = PromptBuilder(template=template)

retriever = InMemoryEmbeddingRetriever(document_store=document_store, top_k=2)
generator = OpenAIGenerator(model="gpt-3.5-turbo")
query_embedder = SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")

basic_rag_pipeline = Pipeline()
basic_rag_pipeline.add_component("text_embedder", query_embedder)
basic_rag_pipeline.add_component("retriever", retriever)
basic_rag_pipeline.add_component("prompt_builder", prompt_builder)
basic_rag_pipeline.add_component("llm", generator)

basic_rag_pipeline.connect("text_embedder.embedding", "retriever.query_embedding")
basic_rag_pipeline.connect("retriever", "prompt_builder.documents")
basic_rag_pipeline.connect("prompt_builder", "llm")

query = "What does Rhodes Statue look like?"
output = basic_rag_pipeline.run({"text_embedder": {"text": query}, "prompt_builder": {"question": query}})