Upload 4 files

app.py

@@ -1,171 +1,59 @@
-import streamlit as st
-from transformers import pipeline
-from sentence_transformers import SentenceTransformer, util
-import pdfplumber
-import re
-
-# ---- App Setup ----
-st.set_page_config(page_title='Gender Strategy Chatbot', layout='wide', initial_sidebar_state='expanded')
-st.title("Chatbot for Gender Strategy Document")
-
-# ---- Helper Functions ----
-def extract_text_from_pdf(pdf_path):
-    """Extracts text from a PDF file."""
-    text = ""
-    with pdfplumber.open(pdf_path) as pdf:
-        for page in pdf.pages:
-            text += page.extract_text()
-    return text
-
-def preprocess_text(document_text):
-    """Processes the text, removes hard line breaks, and ensures clean paragraphs."""
-    # 1. Remove hyphenation and line breaks, but keep the word intact
-    document_text = re.sub(r'(?<=\S)-\n(?=\S)', '', document_text)  # Remove hyphenation and \n
-    # 2. Merge hard line breaks that occur between two words without hyphenation into a single space
-    document_text = re.sub(r'(?<=\S)\n(?=\S)', ' ', document_text)
-    # 3. Remove unnecessary whitespace at the beginning and end of the text
-    document_text = document_text.strip()
-    # 4. Optional: Reduce multiple consecutive spaces to a single space
-    document_text = re.sub(r'\s{2,}', ' ', document_text)
-    # Return the processed text
-    standardized_text = document_text
-    return standardized_text
-
-
-def semantic_search(query, corpus, model):
-    """Performs semantic search to find the most relevant text in the corpus."""
-    query_embedding = model.encode(query, convert_to_tensor=True)
-    corpus_embeddings = model.encode(corpus, convert_to_tensor=True)
-    
-    scores = util.pytorch_cos_sim(query_embedding, corpus_embeddings)[0]
-    best_match_idx = scores.argmax().item()
-    return corpus[best_match_idx], scores[best_match_idx].item()
-
-# ---- Load PDF and Extract Text ----
-@st.cache_data
-def load_pdf_and_prepare_embeddings(pdf_path):
-    """Loads a PDF, extracts text, standardizes formatting, splits into chunks, and prepares embeddings."""
-    document_text = extract_text_from_pdf(pdf_path)
-    standardized_text = preprocess_text(document_text)
-    chunks = standardized_text.split("\n\n")  # Splitting text into chunks by paragraphs
-    model = SentenceTransformer('all-MiniLM-L6-v2')
-    return chunks, model
+"""This is an example of a simple chatbot that uses the RAG model to answer questions
+about GIZ with Streamlit."""
 
-pdf_path = "giz-2019-en-gender-strategy-web-version-with-bookmarks.pdf"
-chunks, embedding_model = load_pdf_and_prepare_embeddings(pdf_path)
-
-# ---- User Input Section ----
-st.sidebar.header("Ask a Question")
-query = st.sidebar.text_area("Type your question here:")
-
-if st.sidebar.button("Submit"):
-    if query.strip() == "":
-        st.sidebar.error("Please enter a question.")
-    else:
-        with st.spinner("Searching for the best answer..."):
-            answer, score = semantic_search(query, chunks, embedding_model)
-            st.write("### Your Question:")
-            st.write(query)
-            st.write("### Best Match:")
-            st.write(answer)
-            st.write(f"**Relevance Score:** {score:.2f}")
-
-# ---- Info Section ----
-with st.expander("ℹ️ - About this app"):
-    st.write(
-        """
-        This chatbot allows users to ask questions about the Gender Strategy document. 
-        It uses a semantic search model (`all-MiniLM-L6-v2`) to find the most relevant passages from the document.
-
-        - The document is pre-loaded and processed into searchable chunks.
-        - The model ranks the relevance of the results based on cosine similarity.
+import streamlit as st
 
-        For feedback or improvements, please contact the developer.
+# Here we import the rag_pipeline function from the rag.py file
+from rag import rag_pipeline
+
+
+# We use the st.cache decorator to load the RAG pipeline only once and cache it
+@st.cache_resource
+def load_rag_pipeline():
+    rag = rag_pipeline()
+    return rag
+
+
+rag = load_rag_pipeline()
+
+with st.chat_message(name="ai", avatar=":material/smart_toy:"):
+    st.html(
+        """Hi, I am an assistant who can answer questions about our organization's gender strategy. 
+        Please ask me a question.
+        <br>
+        <b> Note that I cannot memorize our course of our conversation. 
+        So please always include all the necessary information in your questions and
+        be as specific as possible.</b>
+        <br>
+        <ul>Here are a few examples of possible questions:
+        <li> What are the main targets of the gender strategy? </li>
+        <li> How does GIZ define “gender” and what understanding is the strategy based on? </li>
+        <li> How is the success of the strategy measured and reviewed? </li>
+        </ul>
         """
     )
 
-# import streamlit as st
-# from transformers import pipeline
-# from sentence_transformers import SentenceTransformer, util
-# import pdfplumber
-
-# # ---- App Setup ----
-# st.set_page_config(page_title='Gender Strategy Chatbot', layout='wide', initial_sidebar_state='expanded')
-# st.title("Chatbot for Gender Strategy Document")
-
-# # ---- Helper Functions ----
-# def extract_text_from_pdf(pdf_path):
-#     """Extracts text from a PDF file."""
-#     text = ""
-#     with pdfplumber.open(pdf_path) as pdf:
-#         for page in pdf.pages:
-#             text += page.extract_text()
-#     return text
-
-# def preprocess_text(document_text):
-#     """Processes the text, removes hard line breaks, and ensures clean paragraphs."""
-#     # 1. Remove hyphenation and line breaks, but keep the word intact
-#     document_text = re.sub(r'(?<=\S)-\n(?=\S)', '', document_text)  # Remove hyphenation and \n
-#     # 2. Merge hard line breaks that occur between two words without hyphenation into a single space
-#     document_text = re.sub(r'(?<=\S)\n(?=\S)', ' ', document_text)
-#     # 3. Remove unnecessary whitespace at the beginning and end of the text
-#     document_text = document_text.strip()
-#     # 4. Optional: Reduce multiple consecutive spaces to a single space
-#     document_text = re.sub(r'\s{2,}', ' ', document_text)
-#     # Return the processed text
-#     standardized_text = document_text
-#     return standardized_text
-
-
-# def semantic_search(query, corpus, model):
-#     """Performs semantic search to find the most relevant text in the corpus."""
-#     query_embedding = model.encode(query, convert_to_tensor=True)
-#     corpus_embeddings = model.encode(corpus, convert_to_tensor=True)
-    
-#     scores = util.pytorch_cos_sim(query_embedding, corpus_embeddings)[0]
-#     best_match_idx = scores.argmax().item()
-#     return corpus[best_match_idx], scores[best_match_idx].item()
-
-# # ---- Load PDF and Extract Text ----
-# @st.cache_data
-# def load_pdf_and_prepare_embeddings(pdf_path):
-#     """Loads a PDF, extracts text, standardizes formatting, splits into chunks, and prepares embeddings."""
-#     document_text = extract_text_from_pdf(pdf_path)
-#     standardized_text = preprocess_text(document_text)
-#     chunks = standardized_text.split("\n\n")  # Splitting text into chunks by paragraphs
-#     model = SentenceTransformer('all-MiniLM-L6-v2')
-#     return chunks, model
-
-# pdf_path = "giz-2019-en-gender-strategy-web-version-with-bookmarks.pdf"
-# chunks, embedding_model = load_pdf_and_prepare_embeddings(pdf_path)
-
-# # ---- User Input Section ----
-# st.sidebar.header("Ask a Question")
-# query = st.sidebar.text_area("Type your question here:")
-
-# if st.sidebar.button("Submit"):
-#     if query.strip() == "":
-#         st.sidebar.error("Please enter a question.")
-#     else:
-#         with st.spinner("Searching for the best answer..."):
-#             answer, score = semantic_search(query, chunks, embedding_model)
-#             st.write("### Your Question:")
-#             st.write(query)
-#             st.write("### Best Match:")
-#             st.write(answer)
-#             st.write(f"**Relevance Score:** {score:.2f}")
-
-# # ---- Info Section ----
-# with st.expander("ℹ️ - About this app"):
-#     st.write(
-#         """
-#         This chatbot allows users to ask questions about the Gender Strategy document. 
-#         It uses a semantic search model (`all-MiniLM-L6-v2`) to find the most relevant passages from the document.
-
-#         - The document is pre-loaded and processed into searchable chunks.
-#         - The model ranks the relevance of the results based on cosine similarity.
-
-#         For feedback or improvements, please contact the developer.
-#         """
-#     )
-
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+for message in st.session_state.messages:
+    with st.chat_message(name=message["role"], avatar=message["avatar"]):
+        st.markdown(message["content"])
+
+prompt = st.chat_input("Say something")
+if prompt:
+    with st.chat_message(name="user", avatar=":material/person:"):
+        st.write(prompt)
+        st.session_state.messages.append(
+            {"role": "user", "content": prompt, "avatar": ":material/person:"}
+        )
+    with st.chat_message(name="ai", avatar=":material/smart_toy:"):
+        result = rag.run(
+            {"prompt_builder": {"query": prompt}, "text_embedder": {"text": prompt}},
+        )
+        result = result["llm"]["replies"][0]
+        result = result.split("Question:")[0]
+        st.write(result)
+        st.session_state.messages.append(
+            {"role": "ai", "content": result, "avatar": ":material/smart_toy:"}
+        )

gender_document.md

@@ -0,0 +1,448 @@
+# GIZ Gender Strategy
+
+## Gender reloaded:
+Vision needs Attitude –
+Attitude meets Action
+
+giz Deutsche Gesellschaft
+    für Internationale
+    Zusammenarbeit (GIZ) GmbH
+---
+GIZ Gender Strategy
+
+As a federally owned enterprise, GIZ supports the German Government
+in achieving its objectives in the field of international cooperation for
+sustainable development.
+
+Published by:
+Deutsche Gesellschaft für
+Internationale Zusammenarbeit (GIZ) GmbH
+
+Registered offices
+Bonn and Eschborn, Germany
+
+| | |
+|---------------------------|---------------------------|
+| Friedrich-Ebert-Allee 36 + 40 | Dag-Hammarskjöld-Weg 1 - 5 |
+| 53113 Bonn, Germany | 65760 Eschborn, Germany |
+| T +49 228 44 60-0 | T +49 6196 79-0 |
+| F +49 228 4460-17 66 | F +49 6196 79-11 15 |
+
+E [email protected]
+I www.giz.de
+
+Responsible:
+Dr. Dirk Aßmann, GIZ Gender Ambassador and Director-General, Sectoral Department
+
+Author/Editor:
+Dr. Angela Langenkamp, GIZ Gender Commissioner
+
+Design/Layout:
+Ira Olaleye, Eschborn
+
+GIZ is responsible for the content of this publication.
+
+Printing and distribution:
+Name, Ort
+
+Printed on 100% recycled paper, certified to FSC standards.
+
+Eschborn, January 2019
+
+2
+---
+# Foreword
+
+Dear colleagues,
+commissioning parties and clients,
+partners in Germany and in the field,
+
+'Gender reloaded: Vision needs Attitude – Attitude meets Action' is the motto behind GIZ's
+new Gender Strategy. With it we want to tackle and address current challenges in fostering
+equality of opportunity and rights for all individuals, regardless of their gender, sexual orientation or gender identity. It is about fostering human dignity for all.
+
+In May 2016, the GIZ management commissioned a revision of the 2012 Gender Strategy. This
+work was to be based on the findings and recommendations of the corporate strategic evaluation of the 2012 Gender Strategy and its implementation, and on the growing relevance
+of gender equality and women's rights in both the national and the international context, as
+demonstrated by the 2030 Agenda for Sustainable Development and the new European Consensus on Development. They clearly highlight gender equality as a goal in its own right, a
+value, a principle for action and a central objective across sectors and areas of action. In the
+European Consensus on Development it reads 'Gender equality is at the core of the EU values
+and is enshrined in its legal and political framework. It is vital for achieving the SDGs and
+cuts across the whole 2030 Agenda. The EU and its Member States will promote women's
+and girls' rights, gender equality and the empowerment of women and girls and their protection as a priority across all areas of action.'
+
+Gender equality is an objective to which we are committed both as a company and as individuals working to shape a future worth living around the world. The GIZ Gender Strategy
+embodies the company's clear position on this, both internally and with regard to the wider
+world. The Gender Strategy is an integral part of our corporate and strategic direction and is
+therefore binding for all members of staff.
+
+'Vision needs Attitude – Attitude meets Action' – this is where you have a part to play. Only
+by working together and contributing each individual's potential and ideas, experience and
+commitment can we combat gender-specific disadvantages and discrimination and make
+gender equality around the world a true-life reality as part of a dignified future.
+
+We are prepared and ready – and we look forward to working with you.
+
+Dr. Christoph Beier
+Managing Director
+
+Dr. Dirk Aßmann
+Director-General, Sectoral Department
+
+[Signatures]
+---
+GIZ Gender Strategy
+
+# Table of contents
+
+Foreword | 3
+I. Introduction | 5
+   Gender equality as a benchmark and quality feature at GIZ | 5
+II. Gender strategy: vision, objectives, strategic elements,
+    addressees and scope of application | 7
+   a. Objectives | 7
+   b. Strategic elements | 8
+   c. Addressees and scope of application | 8
+III. Implementation | 9
+    a. Organisation | 9
+    b. Resources | 9
+    c. Monitoring | 10
+Annex | 11
+I GIZ's Gender Architecture | 11
+II ToR for the GIZ Gender Commissioner
+   Role: Coordinator and GIZ spokesperson for gender | 14
+III ToR of the gender focal points in the departments and corporate units
+    Role: To coordinate implementation of the gender strategy within the
+    organisational units and to act as representatives of these organisational units | 15
+IV Glossary | 16
+---
+| Gender reloaded: Vision needs Attitude – Attitude meets Action
+
+# I. Introduction
+
+## Gender equality as a benchmark and quality feature at GIZ
+
+Gender equality is one of the key values of our company and of the work we do. It is a prerequisite for and driver of sustainable development and a viable future of our society, both at national and international level. At GIZ, we take a gender-sensitive and wherever needed a gender-differentiated approach and consistent action to eliminate existing gender-based discrimination and to foster equal rights and opportunities for everyone, regardless of their gender, sexual orientation and gender identity. This and our targeted promotion of gender equality are quality features of the Deutsche Gesellschaft für Internationale Zusammenarbeit (GIZ) GmbH.
+
+Gender equality is a human right whose realisation is neither a given nor something that happens and will be realised all on its own. It entails a conscious approach to the transformation of gender relations and, above all, it calls for active engagement on the part of us all. Gender equality is enshrined in Article 3 of the Basic Law of the Federal Republic of Germany, to which we feel deeply committed.
+
+(1) All persons shall be equal before the law.
+
+(2) Men and women have equal rights. The state promotes the actual implementation of equal rights for women and men and take steps to eliminate disadvantages that now exist.
+
+(3) No person shall be favoured or disadvantaged because of sex, parentage, ethnicity, language, homeland and origin, faith, or religious or political opinions. No person shall be disfavoured because of disability.
+
+Gender equality and the elimination of gender-based discrimination and disadvantages is a core objective and guiding principle of international, European and German (development) policies. By aligning our work, strategies and processes with these objective and principles, and advising and working with our partners and commissioning parties on how to design and implement measures to realise gender equality, we are contributing to a number of national and international agreements, in particular the 2030 Agenda, the European Consensus on Development and the German Sustainable Development Strategy.
+---
+GIZ Gender Strategy |
+
+The promotion of gender equality and the elimination of gender-based disadvantages and discrimination are two strategic pillars of our corporate-policy orientation, which is reflected in our services and in our equal opportunity policy within the company. The GIZ Gender Strategy embodies those two pillars and actively communicates them inside and outside the company. By turning the Gender Policy into action and documenting its implementation we intend to further enhance and consolidate the perception of GIZ as a value-based and responsible company, a reliable partner and an attractive employer. We consider the implementation of the GIZ Internal Plan for Equal Opportunities and the Women's Empowerment Principles on the one hand, and the consistent application of the Safeguards+Gender Management System to new and follow-on projects on the other, as a key foundation for our corporate sustainability, our credibility and our ability to deliver results, both in Germany and in our partner countries.
+
+The Safeguards+Gender Management System is applied for all of GIZ's commissioning parties, clients and business areas. It assesses commissions, projects and programmes in their specific context, identifies opportunities to promote gender equality and also potential risks and unintended negative impacts at an early stage, draws up specific measures – in the case of risks and unintended negative impacts known as 'safeguards' – and follows them up in the project cycle.
+
+Equal opportunity within the company is both a declaration of what we believe in and a human resources policy directive. We work with other people without any distinction in terms of gender, marital status, skin colour, religion or world view, culture, education, ethnicity, disability, age, sexual orientation, gender identity or nationality.
+
+We do not tolerate sexual harassment inside GIZ or through employees and members of our workforce.
+
+GIZ's Gender Strategy, with its comprehensive understanding of gender, contributes to our diversity management within the company.
+---
+| Gender reloaded: Vision needs Attitude – Attitude meets Action
+
+## II. Gender strategy: vision, objectives, strategic elements, addressees and scope of application
+
+GIZ's Gender Strategy provides us with a binding framework for the entire company. It nevertheless provides the needed flexibility to design and implement measures in line with the specific mandates and tasks of every organisational unit and position within the company. The gender strategy is based on GIZ's vision, corporate values and guiding principles.
+
+> Our vision: We work to shape a future worth living around the world.
+
+The objectives and design of specific contributions to the strategy's implementation are guided both by the policy requirements of the German Government and by GIZ's Corporate Strategy.
+
+### a. Objectives
+
+**Effectiveness** We enhance the effectiveness and sustainability of our service delivery by taking action consistently to eliminate existing gender-based discrimination and to achieve equal rights and opportunities for everyone, regardless of their gender, sexual orientation and gender identity, both within the company and in connection with our commissions.
+
+**Business development** We convince our commissioning parties and clients through our proven gender competence. We secure our share of markets by making effective contributions to the gender equality objectives of international, European and German agreements, such as the 2030 Agenda, in line with our clients' needs. We advise our commissioning parties and clients on how to increase the number and quality of measures that focus on gender equality or that enshrine gender equality as a clearly defined area of action in their objectives.
+
+**Skills and alliances** We continuously expand our sector-specific and cross-sectoral gender competence. Through our measures to promote equal opportunities within the company, we secure the future of our company and harness the potentials and opportunities of gender-diverse teams for creative and high-quality performance and service delivery.
+
+**Economic efficiency** We meet the gender-related requirements of our commissioning parties and clients by applying our gender knowledge and competence efficiently and by standardising existing approaches and processes to the greatest extent possible.
+
+7
+---
+GIZ Gender Strategy |
+
+## b. Strategic elements
+
+The five complementary strategic elements are of key importance for achieving
+GIZ's Vision and objectives of the GIZ Gender Strategy. They serve as yardsticks of
+our performance:
+
+| No. | Element | Description |
+|-----|---------|-------------|
+| 1 | Political will and accountability | Clear positioning of the company and of the manner in which managers commit to realising gender equality and promote and follow up on the implementation of the gender strategy in their area of responsibility. |
+| 2 | Corporate culture | Make the patterns of behaviour, codes of conduct and processes that promote and enhance gender equality within the company visible and represent them inside and outside of GIZ. |
+| 3 | Gender competence | Skills and gender knowledge of the workforce to apply and use the relevant instruments and approaches for actively contributing to gender equality and the elimination of gender-based disadvantages and discrimination. |
+| 4 | Process adjustment | Gender-sensitive and gender-differentiated design of all procedures and instruments at Head Office and in the field structure, especially HR management, commission and quality management as well as results and impact monitoring. |
+| 5 | Equal opportunities within the company | Promotion of potentials, equal rights and opportunities for all employees and members of the workforce, irrespective of their gender, sexual orientation and gender identity. Creation of a balanced gender ratio for the different job categories and assignments within the company. |
+
+## c. Addressees and scope of application
+
+GIZ's Gender Strategy is the guiding framework for all managers, employees and
+other members of GIZ's workforce. It therefore applies to all Head Office and field
+staff, national personnel, development workers and integrated experts. They stand
+up for gender equality and in doing so help to bring this strategy to life and translate it into specific actions. Managers act as role models in this context. The
+strategy also serves as a basic source of reference and guidance for our commissioning parties and partners and as a set of binding instructions for our subcontractors.
+
+8
+---
+| Gender reloaded: Vision needs Attitude – Attitude meets Action
+
+## III. Implementation
+
+### a. Organisation
+
+The strategic framework of the gender strategy is binding for all organisational units and everyone who works for GIZ. Taking five strategic elements as the starting point, the strategy is implemented at decentralised level in a manner suited to the mandates, fields and forms of work of the individual organisational units. Part 2 of GIZ's Gender Strategy (for internal use only) provides clear guidelines for its operationalisation.
+
+In order to achieve the objectives of the gender strategy, the individual organisational units, including the HR Department for equal opportunity within the company, define, adopt and document their own specific action documents, measures and, where necessary or appropriate, key figures/indicators. They are supported in this by the gender focal points they appoint. The action documents, measures and key figures/indicators are used for steering and annual reporting. For transparency and accountability within GIZ they are available on the GIZ Intranet, in the Document Management System (DMS) and/or on the Integrated Digital Applications (IDA).
+
+Managers at all levels are responsible for the implementation of this policy. Key bodies and office holders are named and briefly described in the gender architecture (see Annex I). The special roles of the GIZ Gender Commissioner and the gender focal points at departmental and corporate unit level are shown in Annex II and III.
+
+### b. Resources
+
+GIZ's company management provides an annual budget to facilitate company-wide measures to implement the gender strategy. This applies to costs, for instance, of the Gender Competition, the implementation of the Gender Week and company-wide GIZ gender network meetings, the digital gender platform and all costs of the GIZ Gender Commissioner (the coordinator and GIZ spokesperson for gender).
+
+To ensure the successful implementation of the Gender Strategy at all levels, managers secure the required human resources, time and financial resources in their respective organisational units. They also support the gender focal points in performing their duties and promote their capacity development so that they can discharge their responsibilities in a professional manner.
+
+The Human Resources Department plans, establishes and provides the resources for equal opportunity measures within the company and for the gender focal point in the Human Resources Department. The Equal Opportunity Commissioners are released from their other duties.
+
+9
+---
+GIZ Gender Strategy |
+
+## c. Monitoring
+
+In line with GIZ's internal Guidelines for Operationalisation (part 2) and the specific action documents, measures and key figures/indicators, managers in the departments and corporate units are responsible for implementing, monitoring and reporting on the current implementation status of the gender strategy in their area of responsibility. The implementation of GIZ's Gender Strategy is reflected in the corporate annual objectives. The GIZ Gender Commissioner examines the implementation status of the Gender Strategy and is responsible for the budget provisions by corporate management. In cooperation with the gender focal points of the individual organisational units the Gender Commissioner collates a consolidated company-wide implementation report (including recommendations) and submits it to GIZ's Management Committee and Strategy Committee on an annual basis. The key results are incorporated into the Integrated Company Report and into other reports and declarations (e.g. UN Global Compact, German Sustainability Code).
+
+The annual reports from the organisational units are complemented by the results of an online survey among the gender focal points.
+
+Monitoring and any recommendations for action on equal opportunities within the company fall under the responsibility of the Human Resources Department. The GIZ Internal Plan for Equal Opportunities is reviewed every two years. The Equal Opportunity commissioners continuously monitor compliance with the relevant legal and HR policy requirements, which are set out in employer/staff council agreements, for example, and call on the Human Resources Department or the Managing Director responsible for the Human Resources Department to ensure they are put into practice. The Management Board reports to the Supervisory Board once a year on the status of equal opportunities and gender equality within GIZ. The key results are incorporated into the report on the implementation of the GIZ Gender Strategy.
+
+10
+---
+| Gender reloaded: Vision needs Attitude – Attitude meets Action
+
+# Annex
+
+## I GIZ's Gender Architecture
+
+### Management Board
+- Puts the gender strategy into effect and sets the corporate-policy orientation for its implementation; the Managing Director responsible for the Human Resources Department is responsible for equal opportunity policies within the company
+- Appoints a Gender Ambassador among the members of the Strategy Committee
+
+### Strategy Committee
+- Advises the Management Board on fundamental corporate-policy issues with regard to gender equality
+- Annual debate concerning gender
+
+### Management Committee
+- Steers and ensures the implementation of the Management Board's corporate-policy instructions on gender in operational business
+- Advises and makes decisions on topic related and procedural issues
+
+### Gender Ambassador
+- Actively promotes GIZ's Gender Strategy and gender equality as such and acts as an interface between specialist and executive management levels
+- Acts as line manager for the GIZ Gender Commissioner and arranges for the Strategy Committee's annual debate on gender
+
+### GIZ Gender Commissioner
+- Coordinator and GIZ spokesperson for gender
+- See Annex II for detailed ToR
+
+### Human Resources Department
+- Draws up strategies on equal opportunities and gender equality within the company
+- Plans and implements measures to promote equal opportunities and gender equality within the company
+- Supports managers in implementing the GIZ Internal Plan for Equal Opportunities
+
+11
+---
+# GIZ Gender Strategy
+
+## Sectoral division – Gender
+
+- Ensures excellence in sectoral expertise on gender as well as the ability to deliver sector-specific services
+- Takes responsibility for setting up and further developing specialised gender know-how and sectoral/methodological knowledge management
+- Promotes the integration of gender into commission management and develops interdisciplinary service packages in cooperation with other departments
+
+## Safeguards+Gender Desk
+
+- The Safeguards+Gender Desk of the Internal Customer Services Division (PIC) of the Sectoral Department provides advice on gender-related procedural questions in connection with the Safeguards+Gender Management System
+- Checks the plausibility of preliminary (Safeguards+Gender checklist, possibly gender scan, provisional gender analysis) and in-depth gender analysis
+
+## Gender coordination group
+
+- Consists of the Gender Ambassador, the GIZ Gender Commissioner, the gender focal points of the departments and corporate units including the GIZ representative offices in Brussels and Berlin, the Equal Opportunity Commissioner at company level, one representative of the responsible sectoral division and the sector programme
+- Supports and monitors decentralised implementation of the gender strategy within the organisational units
+  - Compiles the results achieved in the different organisational units and identifies joint areas of action on that basis
+  - Supports the GIZ Gender Commissioner in compiling the report on implementing the gender strategy and documents to be submitted to the Strategy Committee
+  - Appoints members for working groups established by the GIZ Gender Commissioner
+
+## Gender network
+
+- Consists of all Head Office and field structure gender focal points
+- Communicates the gender strategy inside the company
+- Encourages the discussion of innovative topics, issues and challenges that are important from a corporate policy stance
+- Ensures a supra-regional and cross-sectoral exchange on digital networking formats and at least one Gender Network Meeting each year
+---
+| Gender reloaded: Vision needs Attitude – Attitude meets Action
+
+### Thematic forum on gender
+
+- Consists of planning officers and gender focal points from the Sectoral Department (FMB) and the Sector and Global Programmes Department (GloBe)
+- Is coordinated by the two departmental gender focal points of FMB and GloBe
+- Draws up joint sectoral documents on specific themes
+- Contributes to knowledge management on gender in commission management processes and in sectors
+- Organises joint events with internal and external discussion partners
+
+### Gender focal points
+
+- The basis for effective work by the gender officers in the various organisational and work units is a set of custom-tailored ToRs that are adopted by the individual management teams.
+- Gender focal points at departmental and corporate unit level actively support the GIZ Gender Commissioner in performing his/her duties
+- In general, all gender officers:
+  - advise and support managers in implementing the gender strategy in their organisational unit or work unit
+  - contribute to monitoring and annual reporting and to company-wide gender-specific initiatives and events
+  - participate in networks and in formats for sharing information and knowledge management
+
+### Equal Opportunity Commissioner
+
+- Promote and monitor the implementation of the GIZ employer/staff council agreement on equal opportunities and compliance with the principles of the German General Equal Treatment Act (AGG) with regard to protection against gender-based discrimination and sexual harassment within the company.
+- The Equal Opportunity Commissioner at corporate level is the employer's contact for all overarching issues related to equal opportunities and gender equality within the company and measures that relate to or affect the company as a whole, a number of units within the company, or cross-departmental workforce groups
+
+13
+---
+GIZ Gender Strategy |
+
+## II ToR for the GIZ Gender Commissioner
+Role: Coordinator and GIZ spokesperson for gender
+
+### (1) Implementation of the gender strategy throughout the company
+- Coordinating the steps involved in implementing the strategy
+- Elaborating the annual report on implementation of the strategy and coordinating company-wide monitoring
+- Initiating and chairing specialised working groups; steering those working groups or delegating this steering role in consultation with the gender coordination group
+- Coordinating the Gender Week and Gender Competitions
+- Elaborating and coordinating the external image and communication of the Gender Strategy
+- Systematising the progress on implementing the strategy achieved by the various departments, corporate units and other organisational units within the company and communicating this information to managerial level and the gender networks
+
+### (2) Sectoral advice
+- Advisory services to the Gender Ambassador, the Management Committee and the Strategy Committee
+- Delivering advisory services on gender mainstreaming throughout the company
+- Providing impetus for further conceptual development work and the consideration of gender in corporate-strategy documents and processes
+- Positioning the issue of gender within GIZ by providing specialist inputs
+- Ensuring a regular strategic exchange with the Equal Opportunity Commissioner
+- Supporting the gender focal points of the departments and corporate units in networking inside and outside the company
+- Underpinning knowledge management at company level in cooperation with, and with the active support of the Sectoral Department (FMB), the operational departments and the Client Liaison and Business Development Department (AGE)
+
+### (3) Administration
+- Coordinating and supporting the gender coordination group
+- Coordinating and supporting the gender network
+- Producing and following up on the annual plan
+- Accepting responsibility for the pertinent budget
+
+14
+---
+Gender reloaded: Vision needs Attitude – Attitude meets Action
+
+### III ToR of the gender focal points in the departments and corporate units
+
+Role: To coordinate the implementation of the gender strategy within the organisational units and to act as representatives of these organisational units
+
+#### (1) Implementation of the gender strategy within the organisational units
+- Facilitating and supporting the implementation of organisational unit-specific action documents and measures, concepts and/or strategies
+- Advising managerial groups on the implementation of the gender strategy
+- Supporting managerial groups in reviewing implementation needs
+- Coordinating and supporting the gender focal points of their respective organisational units with a special focus on the field structure
+- Supporting the organisation of the Gender Week and Gender Competitions and other company-wide initiatives and events on the subject of gender
+
+#### (2) Sectoral advice
+- Advising the members of organisational unit-specific gender networks within Germany and abroad on mainstreaming the institutionalisation of gender
+- Securing the flow of information between the field structure and Head Office and among the various Head Office organisational units
+- Facilitating identification and placing of gender focal points or experts and communicating specialist expertise
+- Where applicable, advising on the integration of gender aspects into the terms of reference for appraisal missions, etc.
+- Attending ZUK (approval of the draft strategy) and ZAK (approval of the offer design) meetings within the pertinent department
+- Identifying issues and putting these on the agenda
+
+#### (3) Implementation of the gender strategy throughout the company
+- Representing the respective organisational unit within the gender coordination group and attending meetings regularly
+- Representing the respective organisational unit in consultation and coordination processes
+- Supporting the GIZ Gender Commissioner in her/his monitoring work
+
+15
+---
+GIZ Gender Strategy |
+
+## IV Glossary
+
+| Term | Definition |
+|------|------------|
+| Diversity, diversity management | Diversity considers the following aspects in particular: gender, sexual orientation, ethnic and cultural background, religion and beliefs, disability, and age. Diversity management as a tool of modern corporate management has the objective of making constructive and gainful use of diversity within the workforce. Diversity management has also taken on a compliance dimension with the enactment of a range of anti-discrimination law, such as the German General Act on Equal Treatment (AGG). |
+| Empowerment | Empowerment means giving an individual power or transferring responsibility to them. The term is often used in the context of economic empowerment or women's empowerment. Empowerment includes strategies and measures intended to increase the degree of autonomy and self-determination that individuals and societies have over their lives and to enable them to (re-)assert their interests independently, responsibly and autonomously. In line with the 1995 Beijing Declaration and Platform for Action, the empowerment approach, among other things, particularly aims to use awareness-raising at all levels of society to strengthen women as legal entities and to broaden their agency. |
+| Gender | Gender refers to an individual's social gender as opposed to their sex, which is biologically determined. It includes socially constructed gender roles and relationships, perceptions and expectations. These factors are contextual, dynamic and open to change. They are reflected in such areas as social standards, legislation, traditions, religion and so on. |
+| Gender competence | Gender competence includes the ability to reflect critically on gender relationships and social gender attributions (e.g. gender stereotypes) by using knowledge of how such attributions arise and their social impact. Gender competence also includes the ability to use and apply this knowledge, for example in project planning and implementation and in the design of new approaches and strategies. |
+
+16
+---
+| Gender reloaded: Vision needs Attitude – Attitude meets Action
+
+## Gender-sensitive design of all procedures, gender-sensitive approach
+
+A gender-sensitive approach takes account of gender-specific inequalities and gender discrimination and of the diverse interests, needs and potentials of different genders within a specific context. It recognises and identifies existing gender-specific differences, problems and inequalities and integrates them into strategies and measures. The objective is to ensure that no unintended negative impact results from these strategies and measures and that individuals are able to participate in and benefit from (development cooperation) measures irrespective of their gender.
+
+## Gender identity
+
+Gender identity is the gender with which an individual identifies. Gender diversity is based on feelings of belonging to a particular gender and gender identity, for example as a woman, a man, transgender or intersex or other local and indigenous self-identities, such as Hijra in India and Pakistan.
+
+## Equal opportunities within the company
+
+Equal opportunities within the company aim to secure equality at work for women and men. Within GIZ, such equality is based primarily on the company/staff council agreement on gender equality and on the gender strategy.
+
+![A logo with the text "GLEICHSTELLUNG IN DER GIZ" and an image of three overlapping silhouettes in red, white, and blue.]
+
+## Safeguards+Gender
+
+GIZ's binding Safeguards+Gender Management System is there to check and ensure that projects and programmes systematically identify potential for promoting gender equality and women's rights as well as unintended negative impacts or potential external risks with regard to gender equality at an early stage, and actively address them during the entire project cycle.
+
+![A circular logo with the text "SAFEGUARDS+GENDER" surrounded by colorful segments.]
+
+17
+---
+GIZ Gender Strategy |
+
+| Term | Definition |
+|------|------------|
+| Sexual harassment | Sexual harassment is a form of gender-based violence encompassing acts of unwanted physical, verbal or non-verbal conduct of a sexual nature, which have a purpose and/or effect of violating the victim's dignity and creating an intimidating, hostile, degrading, humiliating or offensive environment. Individuals of any gender may be both the perpetrator and the target of sexual harassment. The latter includes among others: unwanted physical proximity, touching or assault of a sexual nature; gestures and other forms of non-verbal communication with a sexual nature; sexually-oriented comments on individuals and/or their body, behaviour, sex life or sexual identity; sexually demeaning language and comments designed to undermine dignity, such as jokes with sexual content; requests to carry out sexual acts; sharing or displaying pornographic or sexist images and stalking with a sexual basis. Sexual harassment is particularly severe when a person's hierarchical dependence is exploited in a training context or at the workplace and when personal or professional disadvantages are threatened or advantages are promised. |
+| Sexual orientation | An individual's sexual orientation is the gender to which a person feels drawn emotionally, physically and/or sexually. Distinctions are made between lesbian, gay, heterosexual, bisexual and asexual orientations. The abbreviation LGBTIQ stands for lesbian, gay, bisexual, transgender, intersex & queer. |
+| Transformation of gender relationships, gender-transformative approach | Gender-transformative approaches aim to change gender-specific role attributions, unequal power relationships, structures and social norms, and rules that lead to gender-specific disadvantages, discrimination and/or marginalisation so as to achieve (greater) gender justice. A gender-transformative approach therefore not only focuses on the manifestations and symptoms of gender inequality (such as lack of access) but also tackles its root causes, such as sociocultural norms, discriminatory legal provisions and social systems. |
+
+18
+---
+| Gender reloaded: Vision needs Attitude – Attitude meets Action
+
+| Wide-ranging understanding | Gender diversity, gender inclusivity, an inclusive under- |
+| of gender; gender diversity | standing of gender, and a non-binary understanding of |
+|                             | gender all mean inclusion of all genders, not just male |
+|                             | and female. However, in most cases a binary understand- |
+|                             | ing of gender (male and female) forms the basis for leg- |
+|                             | islation and dominant social orders. Self-determination of |
+|                             | gender identity is a basic human right under internation- |
+|                             | al law. |
+
+19
+---
+Deutsche Gesellschaft für  
+Internationale Zusammenarbeit (GIZ) GmbH
+
+Registered offices  
+Bonn and Eschborn, Germany
+
+Friedrich-Ebert-Allee 36 + 40    Dag-Hammarskjöld-Weg 1 - 5  
+53113 Bonn, Germany             65760 Eschborn, Germany  
+T +49 228 44 60-0               T +49 61 96 79-0  
+F +49 228 44 60-17 66           F +49 61 96 79-11 15
+
+E [email protected]  
+I www.giz.de

rag.py

@@ -0,0 +1,229 @@
+"This file contains the implementation of the RAG pipeline."
+
+from pathlib import Path
+
+from haystack import Pipeline
+from haystack.components.builders import PromptBuilder
+from haystack.components.converters import MarkdownToDocument
+from haystack.components.embedders import (
+    SentenceTransformersDocumentEmbedder,
+    SentenceTransformersTextEmbedder,
+)
+from haystack.components.generators import HuggingFaceAPIGenerator
+from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter
+from haystack.components.retrievers import InMemoryEmbeddingRetriever
+from haystack.components.writers import DocumentWriter
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.utils import Secret
+
+# Define the paths to the document and the model for embedding the documents and the user query
+DOCUMENT_PATH = Path("gender_document.md")
+EMBEDDING_MODEL = "all-MiniLM-L6-v2"
+
+
+def process_document(document_store: InMemoryDocumentStore) -> Pipeline:
+    """This function processes the document and stores it in the document store.
+    It contains of the following components:
+    - MarkdownToDocument: Converts the markdown file to a document (https://docs.haystack.deepset.ai/docs/markdowntodocument)
+    - DocumentCleaner: Cleans the document (https://docs.haystack.deepset.ai/docs/documentcleaner)
+    - DocumentSplitter: Splits the document into chunks (https://docs.haystack.deepset.ai/docs/documentsplitter)
+    - DocumentWriter: Writes the document to the document store (https://docs.haystack.deepset.ai/docs/documentwriter)
+    - SentenceTransformersDocumentEmbedder: Embeds the documents, more precisely the chunks (https://docs.haystack.deepset.ai/docs/sentencetransformersdocumentembedder)
+
+
+    Parameters
+    ----------
+    document_store : InMemoryDocumentStore
+        The document store where the processed document should be stored.
+
+    Returns
+    -------
+    Pipeline
+        The pipeline containing the components to parse, clean, split, embed and write the document to the document store.
+        To run the pipeline, you can use the `pipeline.run()` method. If a component needs input or arguments, you can pass them as a dictionary to the `run()` method.
+        For example: `pipeline.run({"converter": {"sources": [DOCUMENT_PATH]}})`.
+    """
+
+    # initialize the pipeline
+    pipeline = Pipeline()
+
+    # add the components to the pipeline. If you want to add more components, you can do it here.
+    # If you want to the settings of the components, you can do it here.
+    # MarkdownToDocument
+    pipeline.add_component("converter", MarkdownToDocument())
+
+    # DocumentCleaner
+    pipeline.add_component("cleaner", DocumentCleaner())
+
+    # DocumentSplitter
+    pipeline.add_component(
+        "splitter",
+        DocumentSplitter(
+            split_by="word", split_length=300, respect_sentence_boundary=True
+        ),
+    )
+
+    # DocumentWriter
+    pipeline.add_component("writer", DocumentWriter(document_store=document_store))
+
+    # SentenceTransformersDocumentEmbedder
+    pipeline.add_component(
+        "embedder",
+        SentenceTransformersDocumentEmbedder(
+            EMBEDDING_MODEL,
+        ),
+    )
+
+    # connect the components
+    pipeline.connect("converter", "cleaner")
+    pipeline.connect("cleaner", "splitter")
+    pipeline.connect("splitter", "embedder")
+    pipeline.connect("embedder", "writer")
+    return pipeline
+
+
+def load_document_store(document_store_settings: dict) -> InMemoryDocumentStore:
+    """This function loads the document store with the given settings.
+
+    Parameters
+    ----------
+    document_store_settings : dict
+        The settings for the document store. The settings are passed as a dictionary.
+        You can find the available settings here: https://docs.haystack.deepset.ai/docs/inmemorydocumentstore
+
+    Returns
+    -------
+    InMemoryDocumentStore
+        _description_
+    """
+    document_store = InMemoryDocumentStore(**document_store_settings)
+    return document_store
+
+
+def get_query_pipeline(
+    document_store: InMemoryDocumentStore, generator: HuggingFaceAPIGenerator
+) -> Pipeline:
+    """
+    This function creates a query pipeline that contains the following components:
+    - SentenceTransformersTextEmbedder: Embeds the user query (https://docs.haystack.deepset.ai/docs/sentencetransformerstextembedder)
+    - InMemoryEmbeddingRetriever: Retrieves the most similar documents to the user query (https://docs.haystack.deepset.ai/docs/inmemoryembeddingretriever)
+    - PromptBuilder: Builds the prompt for the generator (https://docs.haystack.deepset.ai/docs/promptbuilder)
+    - HuggingFaceAPIGenerator: Generates the answer to the user query (https://docs.haystack.deepset.ai/docs/huggingfaceapigenerator)
+
+    Parameters
+    ----------
+    document_store : InMemoryDocumentStore
+        The document store where the documents are stored.
+    llm_provider : HuggingFaceAPIGenerator
+        The llm_provider that generates the answer to the user query.
+
+    Returns
+    -------
+    Pipeline
+        The query pipeline containing the components to embed the user query, retrieve the most similar documents, build the prompt and generate the answer.
+    """
+
+    # initialize the query pipeline
+    query_pipeline = Pipeline()
+
+    # add the components to the query pipeline
+    # SentenceTransformersTextEmbedder
+    query_pipeline.add_component(
+        "text_embedder", SentenceTransformersTextEmbedder(EMBEDDING_MODEL)
+    )
+
+    # InMemoryEmbeddingRetriever
+    query_pipeline.add_component(
+        "retriever", InMemoryEmbeddingRetriever(document_store=document_store, top_k=10)
+    )
+
+    # template for the PromptBuilder
+    template = """
+    Given the following information, answer the question. If the information is insufficient, answer with "Answer is not possible". Please do not provide any additional information and ask clarifying questions.
+
+    Context: 
+    {% for document in documents %}
+        {{ document.content }}
+    {% endfor %}
+
+    Question: {{ query }}?
+    """
+
+    # PromptBuilder
+    query_pipeline.add_component("prompt_builder", PromptBuilder(template=template))
+
+    # HuggingFaceAPIGenerator
+    query_pipeline.add_component("llm", generator)
+
+    # connect the components
+    query_pipeline.connect("text_embedder.embedding", "retriever.query_embedding")
+    query_pipeline.connect("retriever", "prompt_builder.documents")
+    query_pipeline.connect("prompt_builder", "llm")
+    return query_pipeline
+
+
+def init_generator() -> HuggingFaceAPIGenerator:
+    """This function initializes the HuggingFaceAPIGenerator with the given settings.
+    You can find the available models here: https://huggingface.co/models?inference=warm&pipeline_tag=text-generation&sort=trending
+    Please note that you need to provide a valid token to use the HuggingFaceAPIGenerator.
+    For testing purposes, you can hardcode the token in the script.
+    For deployment on Hugging Face Spaces, please safe the token as a secret (Settings -> Secrets) and load it with `Secret.from_env_var("your_token_name")`.
+
+    Returns
+    -------
+    HuggingFaceAPIGenerator
+        _description_
+    """
+
+    # initialize the HuggingFaceAPIGenerator
+    llm_provider = HuggingFaceAPIGenerator(
+        api_type="serverless_inference_api",
+        api_params={"model": "HuggingFaceH4/zephyr-7b-beta", "stop": ["Question"]},
+        token=Secret.from_token(""),
+    )
+    return llm_provider
+
+
+def rag_pipeline() -> Pipeline:
+    """This function wraps the whole RAG pipeline.
+    It loads the document store, processes the document, initializes the generator and
+    creates the query pipeline.
+
+    Returns
+    -------
+    Pipeline
+        The RAG pipeline containing the components to process the document and generate
+        the answer to the user query. It is enough to import and load this function for the chat application.
+        You can run the pipeline with the `pipeline.run()` method.
+        If a component needs input or arguments, you can pass them as a dictionary to the `run()` method.
+        For example:
+        result = rag.run(
+            {"prompt_builder": {"query": prompt}, "text_embedder": {"text": prompt}},
+        )
+        For debugging purposes, you can include the outputs for example from the retriever
+        result = rag.run(
+            {"prompt_builder": {"query": prompt}, "text_embedder": {"text": prompt}},
+            include_outputs_from=["retriever", "llm"],
+        )
+    """
+    # define document_store_settings
+    document_store_settings = {"embedding_similarity_function": "cosine"}
+
+    # load the document store
+    document_store = load_document_store(document_store_settings)
+
+    # process the document and write it to the document store
+    document_pipeline = process_document(document_store=document_store)
+
+    # run the document pipeline
+    document_pipeline.run({"converter": {"sources": [DOCUMENT_PATH]}})
+
+    # initialize the generator
+    llm_provider = init_generator()
+
+    # create the query pipeline
+    query_pipeline = get_query_pipeline(
+        document_store=document_store, generator=llm_provider
+    )
+
+    return query_pipeline