Duplicate from GIZ/SDSN-demo

Co-authored-by: Prashant Singh <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+appStore/img/giz_sdsn.jpg filter=lfs diff=lfs merge=lfs -text
+appStore/img/paris.png filter=lfs diff=lfs merge=lfs -text
+appStore/img/pic1.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: SDSN Demo
+emoji: 📈
+colorFrom: purple
+colorTo: blue
+sdk: streamlit
+sdk_version: 1.10.0
+app_file: app.py
+pinned: false
+duplicated_from: GIZ/SDSN-demo
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,18 @@
+import appStore.keyword_search as keyword_search
+import appStore.sdg_analysis as sdg_analysis
+import appStore.coherence as coherence
+import appStore.info as info
+from appStore.multiapp import MultiApp
+import streamlit as st
+
+st.set_page_config(page_title = 'Climate Policy Intelligence', 
+                   initial_sidebar_state='expanded', layout="wide") 
+
+app = MultiApp()
+
+app.add_app("About","house", info.app)
+app.add_app("Search","search", keyword_search.app)
+app.add_app("SDG Analysis","gear",sdg_analysis.app)
+app.add_app("NDC Comparison","exclude", coherence.app)
+
+app.run()

appStore/__init__.py

@@ -0,0 +1 @@
+# creating appstore package

appStore/coherence.py

@@ -0,0 +1,156 @@
+# set path
+import glob, os, sys; 
+sys.path.append('../utils')
+
+import streamlit as st
+import ast
+import logging
+from utils.ndc_explorer import countrySpecificCCA, countrySpecificCCM
+from utils.checkconfig import getconfig
+from utils.semantic_search import runSemanticPreprocessingPipeline,process_semantic_output
+from utils.semantic_search import semanticSearchPipeline, runSemanticPipeline
+from st_aggrid import AgGrid
+from st_aggrid.shared import ColumnsAutoSizeMode
+
+# Reading data and Declaring necessary variables
+with open('docStore/ndcs/countryList.txt') as dfile:
+    countryList = dfile.read()
+countryList = ast.literal_eval(countryList)
+countrynames = list(countryList.keys())
+    
+with open('docStore/ndcs/cca.txt', encoding='utf-8', errors='ignore') as dfile:
+    cca_sent = dfile.read()
+cca_sent = ast.literal_eval(cca_sent)
+            
+with open('docStore/ndcs/ccm.txt', encoding='utf-8', errors='ignore') as dfile:
+    ccm_sent = dfile.read()
+ccm_sent = ast.literal_eval(ccm_sent)
+
+config = getconfig('paramconfig.cfg')
+split_by = config.get('coherence','SPLIT_BY')
+split_length = int(config.get('coherence','SPLIT_LENGTH'))
+split_overlap = int(config.get('coherence','SPLIT_OVERLAP'))
+split_respect_sentence_boundary = bool(int(config.get('coherence',
+                                    'RESPECT_SENTENCE_BOUNDARY')))
+remove_punc = bool(int(config.get('coherence','REMOVE_PUNC')))
+embedding_model = config.get('coherence','RETRIEVER')
+embedding_model_format = config.get('coherence','RETRIEVER_FORMAT')
+embedding_layer = int(config.get('coherence','RETRIEVER_EMB_LAYER'))
+embedding_dim  = int(config.get('coherence','EMBEDDING_DIM'))
+max_seq_len = int(config.get('coherence','MAX_SEQ_LENGTH')) 
+retriever_top_k = int(config.get('coherence','RETRIEVER_TOP_K'))
+
+
+
+def app():
+
+    #### APP INFO #####
+    with st.container():
+        st.markdown("<h1 style='text-align: center;  \
+                      color: black;'> NDC Comparison</h1>", 
+                      unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+    with st.expander("ℹ️ - About this app", expanded=False):
+
+        st.write(
+            """     
+            The *NDC Comparison* application provides easy evaluation of 
+            coherence between a given policy document and a country’s (Intended)\
+            Nationally Determined Contribution (INDCs/NDCs) using open-source \
+            data from the German Institute of Development and Sustainability’s \
+            (IDOS) [NDC Explorer](https://klimalog.idos-research.de/ndc/#NDCExplorer/worldMap?NewAndUpdatedNDC??income???catIncome).\
+            """)
+        st.write("")
+        st.write(""" User can select a country context via the drop-down menu \
+            on the left-hand side of the application. Subsequently, the user is \
+            given the opportunity to manually upload another policy document \
+            from the same national context or to select a pre-loaded example \
+            document. Thereafter, the user can choose between two categories \
+            to compare coherence between the documents: climate change adaptation \
+            and climate change mitigation. Based on the selected information, \
+            the application identifies relevant paragraphs in the uploaded \
+            document and assigns them to the respective indicator from the NDC \
+            Explorer. Currently, the NDC Explorer has 20 indicators under \
+            climate change mitigation (e.g., fossil fuel production, REDD+) and \
+            22 indicators under climate change adaptation (e.g., sea level rise,\
+            investment needs). The assignment of the paragraph to a corresponding\
+            indicator is based on vector similarities in which top 3 results
+            if found are shown to the user. """)
+        st.write("")
+        st.write("")
+        st.markdown("Some runtime metrics tested with cpu: Intel(R) Xeon(R) CPU @ 2.20GHz, memory: 13GB")
+        col1,col2= st.columns(2)
+        with col1:
+            st.caption("OCR File processing")
+            # st.markdown('<div style="text-align: center;">50 sec</div>', unsafe_allow_html=True)
+            st.write("50 sec")
+           
+        with col2:
+            st.caption("NDC comparison on 200 paragraphs(~ 35 pages)")
+            # st.markdown('<div style="text-align: center;">12 sec</div>', unsafe_allow_html=True)
+            st.write("140 sec")
+    
+    with st.sidebar:
+
+        option = st.selectbox('Select Country', (countrynames))
+        countryCode = countryList[option]
+        st.markdown("---")
+
+        genre = st.radio( "Select Category",('Climate Change Adaptation', 
+                                            'Climate Change Mitigation'))
+        st.markdown("---")
+    
+    with st.container():
+        if st.button("Compare with NDC"):
+            sent_cca = countrySpecificCCA(cca_sent,1,countryCode)
+            sent_ccm = countrySpecificCCM(ccm_sent,1,countryCode)
+
+            if 'filepath' in st.session_state:
+                allDocuments = runSemanticPreprocessingPipeline(
+                        file_path= st.session_state['filepath'],
+                        file_name  = st.session_state['filename'],
+                        split_by=split_by,
+                        split_length= split_length,
+                        split_overlap=split_overlap,
+                        remove_punc= remove_punc,
+                split_respect_sentence_boundary=split_respect_sentence_boundary)
+                # genre = st.radio( "Select Category",('Climate Change Adaptation', 'Climate Change Mitigation'))
+                if genre == 'Climate Change Adaptation':
+                    sent_dict = sent_cca
+                else:
+                    sent_dict = sent_ccm
+                sent_labels = []
+                for key,sent in sent_dict.items():
+                            sent_labels.append(sent)
+                if len(allDocuments['documents']) > 100:
+                    warning_msg = ": This might take sometime, please sit back and relax."
+                else:
+                    warning_msg = ""
+                logging.info("starting Coherence analysis, \
+                    country selected {}".format(option))
+                with st.spinner("Performing Coherence Analysis for {} \
+                    under {} category{}".format(option,genre,warning_msg)):
+                    semanticsearch_pipeline, doc_store = semanticSearchPipeline(documents = allDocuments['documents'],
+                            embedding_model= embedding_model, 
+                            embedding_layer= embedding_layer,
+                            embedding_model_format= embedding_model_format,
+                            retriever_top_k= retriever_top_k,
+                            embedding_dim=embedding_dim,
+                            max_seq_len=max_seq_len, useQueryCheck=False)
+                    raw_output = runSemanticPipeline(pipeline=semanticsearch_pipeline,queries=sent_labels)
+                    results_df = process_semantic_output(raw_output)
+                    results_df = results_df.drop(['answer','answer_offset',
+                                'context_offset','context','reader_score','id'],
+                                axis = 1)
+                    
+                    for i,key in enumerate(list(sent_dict.keys())):
+                        st.subheader("Relevant paragraphs for topic: {}".format(key))
+                        df = results_df[results_df['query']==sent_dict[key]].reset_index(drop=True)
+                        for j in range(3):
+                            st.write('Result {}.'.format(j+1))
+                            st.write(df.loc[j]['content']+'\n')
+                    
+            else:
+                st.info("🤔 No document found, please try to upload it at the sidebar!")
+                logging.warning("Terminated as no document provided")

appStore/info.py

@@ -0,0 +1,72 @@
+import streamlit as st
+
+def app():
+    
+    
+    with open('style.css') as f:
+        st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
+    
+    st.markdown("<h2 style='text-align: center;  \
+                      color: black;'> Policy Action Tracker</h2>", 
+                      unsafe_allow_html=True)
+
+    
+    st.markdown("<div style='text-align: center; \
+                    color: grey;'>The Policy Action Tracker is an open-source\
+                         digital tool which aims to assist policy analysts and \
+                          other users in extracting and filtering relevant \
+                            information from policy documents.</div>",
+                        unsafe_allow_html=True)
+    footer = """
+           <div class="footer-custom">
+               Guidance & Feedback - <a href="https://www.linkedin.com/in/maren-bernlöhr-149891222" target="_blank">Maren Bernlöhr</a> |
+               <a href="https://www.linkedin.com/in/manuelkuhm" target="_blank">Manuel Kuhm</a> |
+               Developer - <a href="https://www.linkedin.com/in/erik-lehmann-giz/" target="_blank">Erik Lehmann</a>  |   
+               <a href="https://www.linkedin.com/in/jonas-nothnagel-bb42b114b/" target="_blank">Jonas Nothnagel</a>   |
+               <a href="https://www.linkedin.com/in/prashantpsingh/" target="_blank">Prashant Singh</a> |
+               
+           </div>
+       """
+    st.markdown(footer, unsafe_allow_html=True)
+
+    c1, c2, c3 =  st.columns([8,1,12])
+    with c1:
+        st.image("docStore/img/ndc.png")
+    with c3:
+        st.markdown('<div style="text-align: justify;">The manual extraction \
+        of relevant information from text documents is a \
+    time-consuming task for any policy analyst. As the amount and length of \
+    public policy documents in relation to sustainable development (such as \
+    National Development Plans and Nationally Determined Contributions) \
+    continuously increases, a major challenge for policy action tracking – the \
+    evaluation of stated goals and targets and their actual implementation on \
+    the ground – arises. Luckily, Artificial Intelligence (AI) and Natural \
+    Language Processing (NLP) methods can help in shortening and easing this \
+    task for policy analysts.</div><br>',
+    unsafe_allow_html=True)
+
+    intro = """
+    <div style="text-align: justify;">
+
+    For this purpose, the United Nations Sustainable Development Solutions \
+    Network (SDSN) and the Deutsche Gesellschaft für Internationale \
+    Zusammenarbeit (GIZ) GmbH are collaborated in the development \
+    of this AI-powered open-source web application that helps find and extract \
+    relevant information from public policy documents faster to facilitate \
+    evidence-based decision-making processes in sustainable development and beyond.  
+
+    This tool allows policy analysts and other users the possibility to rapidly \
+    search for relevant information/paragraphs in the document according to the \
+    user’s interest, classify the document’s content according to the Sustainable \
+    Development Goals (SDGs), and compare climate-related policy documents and NDCs \
+    across countries using open data from the German Institute of Development and \
+    Sustainability’s (IDOS) NDC Explorer. 
+    To understand the application's functionalities and learn more about ß
+    the project, see the attached concept note. We hope you like our application 😊
+
+
+    </div>
+    <br>
+    """
+    st.markdown(intro, unsafe_allow_html=True)
+    # st.image("docStore/img/paris.png")

appStore/keyword_search.py

@@ -0,0 +1,176 @@
+# set path
+import glob, os, sys; 
+sys.path.append('../utils')
+
+import streamlit as st
+import json
+import logging
+from utils.lexical_search import runLexicalPreprocessingPipeline, lexical_search
+from utils.semantic_search import runSemanticPreprocessingPipeline, semantic_keywordsearch
+from utils.checkconfig import getconfig
+from utils.streamlitcheck import checkbox_without_preselect
+
+# Declare all the necessary variables
+config = getconfig('paramconfig.cfg')
+split_by = config.get('semantic_search','SPLIT_BY')
+split_length = int(config.get('semantic_search','SPLIT_LENGTH'))
+split_overlap = int(config.get('semantic_search','SPLIT_OVERLAP'))
+split_respect_sentence_boundary = bool(int(config.get('semantic_search',
+                                    'RESPECT_SENTENCE_BOUNDARY')))
+remove_punc = bool(int(config.get('semantic_search','REMOVE_PUNC')))
+embedding_model = config.get('semantic_search','RETRIEVER')
+embedding_model_format = config.get('semantic_search','RETRIEVER_FORMAT')
+embedding_layer = int(config.get('semantic_search','RETRIEVER_EMB_LAYER'))
+embedding_dim  = int(config.get('semantic_search','EMBEDDING_DIM'))
+max_seq_len = int(config.get('semantic_search','MAX_SEQ_LENGTH')) 
+retriever_top_k = int(config.get('semantic_search','RETRIEVER_TOP_K'))
+reader_model = config.get('semantic_search','READER')
+reader_top_k = int(config.get('semantic_search','RETRIEVER_TOP_K'))
+top_k_per_candidate = int(config.get('semantic_search','READER_TOP_K_PER_CANDIDATE')) 
+lexical_split_by= config.get('lexical_search','SPLIT_BY')
+lexical_split_length=int(config.get('lexical_search','SPLIT_LENGTH'))
+lexical_split_overlap = int(config.get('lexical_search','SPLIT_OVERLAP'))
+lexical_remove_punc = bool(int(config.get('lexical_search','REMOVE_PUNC')))
+lexical_top_k=int(config.get('lexical_search','TOP_K'))
+
+def app():
+
+    with st.container():
+        st.markdown("<h1 style='text-align: center;  \
+                      color: black;'> Search</h1>", 
+                      unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=False):
+
+        st.write(
+            """     
+            The *Search* app is an interface \
+            for doing contextual and keyword searches in \
+            policy documents.  \
+            """)
+        st.write("")
+        st.write(""" The application allows its user to perform a search\
+             based on two options: a lexical search([TFIDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf))\
+              and semantic search. [bi-encoder](https://www.sbert.net/examples/applications/retrieve_rerank/README.html)\
+             The lexical search only \
+            displays paragraphs in the document with exact matching results, \
+            the semantic search shows paragraphs with meaningful connections \
+            (e.g., synonyms) based on the search context. Both \
+            methods employ a probabilistic retrieval framework in its identification\
+            of relevant paragraphs. By defualt the search is performed using \
+            'Semantic Search', and to find 'Exact/Lexical Matches' please tick the \
+            checkbox provided which will by-pass semantic search. Furthermore,\
+            the application allows the user to search for pre-defined keywords \
+            from different thematic buckets present in sidebar.""")
+        st.write("")
+        st.write(""" The Exact Matches gives back top {} findings, and Semantic
+        search provides with top {} answers.""".format(lexical_top_k, retriever_top_k))
+        st.write("")
+        st.write("")
+        st.markdown("Some runtime metrics tested with cpu: Intel(R) Xeon(R) CPU @ 2.20GHz, memory: 13GB")
+        col1,col2,col3= st.columns([2,4,4])
+        with col1:
+            st.caption("OCR File processing")
+            # st.markdown('<div style="text-align: center;">50 sec</div>', unsafe_allow_html=True)
+            st.write("50 sec")
+           
+        with col2:
+            st.caption("Lexical Search on 200 paragraphs(~ 35 pages)")
+            # st.markdown('<div style="text-align: center;">12 sec</div>', unsafe_allow_html=True)
+            st.write("15 sec")
+           
+        with col3:
+            st.caption("Semantic search on 200 paragraphs(~ 35 pages)")
+            # st.markdown('<div style="text-align: center;">120 sec</div>', unsafe_allow_html=True)
+            st.write("120 sec(including emebedding creation)")
+ 
+    with st.sidebar:
+        with open('docStore/sample/keywordexample.json','r') as json_file:
+            keywordexample = json.load(json_file)
+        
+        # genre = st.radio("Select Keyword Category", list(keywordexample.keys()))
+        st.caption("Select Keyword Category")
+        genre = checkbox_without_preselect(list(keywordexample.keys()))
+        if genre:
+            keywordList = keywordexample[genre]
+        else:
+            keywordList = None
+        
+        st.markdown("---")
+    
+    with st.container():
+        type_hinting = "Please enter here your question and we \
+                        will look for an answer in the document\
+                        OR enter the keyword you are looking \
+                        for and we will look for similar\
+                        context in the document.\
+                        You can also explore predefined sets of keywords from sidebar. "
+        if keywordList is not None:
+        #     queryList = st.text_input("You selected the {} category we \
+        #                 will look for these keywords in document".format(genre)
+        #                             value="{}".format(keywordList))
+            queryList = st.text_input(type_hinting,
+                                        value = "{}".format(keywordList))
+        else:
+             queryList = st.text_input(type_hinting,
+                                       placeholder="Enter keyword/query here")
+
+        searchtype = st.checkbox("Show only Exact Matches")
+        if st.button("Find them"):
+
+            if queryList == "":
+                st.info("🤔 No keyword provided, if you dont have any, \
+                                please try example sets from sidebar!")
+                logging.warning("Terminated as no keyword provided")
+            else:
+                if 'filepath' in st.session_state:
+                      
+                    if searchtype:
+                        all_documents = runLexicalPreprocessingPipeline(
+                                    file_name=st.session_state['filename'],
+                                    file_path=st.session_state['filepath'],
+                                    split_by=lexical_split_by,
+                                    split_length=lexical_split_length,
+                                    split_overlap=lexical_split_overlap,
+                                    remove_punc=lexical_remove_punc)
+                        logging.info("performing lexical search")
+                        with st.spinner("Performing Exact matching search \
+                                        (Lexical search) for you"):
+                            lexical_search(query=queryList,
+                        documents = all_documents['documents'],
+                                top_k = lexical_top_k )
+                    else:
+                        all_documents = runSemanticPreprocessingPipeline(
+                                            file_path= st.session_state['filepath'],
+                                            file_name  = st.session_state['filename'],
+                                            split_by=split_by,
+                                            split_length= split_length,
+                                            split_overlap=split_overlap,
+                                            remove_punc= remove_punc,
+                        split_respect_sentence_boundary=split_respect_sentence_boundary)
+                        if len(all_documents['documents']) > 100:
+                            warning_msg = ": This might take sometime, please sit back and relax."
+                        else:
+                            warning_msg = ""
+
+                        logging.info("starting semantic search")
+                        with st.spinner("Performing Similar/Contextual search{}".format(warning_msg)):
+                            semantic_keywordsearch(query = queryList, 
+                            documents = all_documents['documents'],
+                            embedding_model=embedding_model, 
+                            embedding_layer=embedding_layer,
+                            embedding_model_format=embedding_model_format,
+                            reader_model=reader_model,reader_top_k=reader_top_k,
+                            retriever_top_k=retriever_top_k, embedding_dim=embedding_dim,
+                            max_seq_len=max_seq_len,
+                            top_k_per_candidate = top_k_per_candidate)
+
+                else:
+                    st.info("🤔 No document found, please try to upload it at the sidebar!")
+                    logging.warning("Terminated as no document provided")
+        
+
+
+                    

appStore/multiapp.py

@@ -0,0 +1,70 @@
+"""Frameworks for running multiple Streamlit applications as a single app.
+"""
+import streamlit as st
+from PIL import Image
+from streamlit_option_menu import option_menu
+from utils.uploadAndExample import add_upload
+
+class MultiApp:
+    """Framework for combining multiple streamlit applications.
+    Usage:
+        def foo():
+            st.title("Hello Foo")
+        def bar():
+            st.title("Hello Bar")
+        app = MultiApp()
+        app.add_app("Foo", foo)
+        app.add_app("Bar", bar)
+        app.run()
+    It is also possible keep each application in a separate file.
+        import foo
+        import bar
+        app = MultiApp()
+        app.add_app("Foo", foo.app)
+        app.add_app("Bar", bar.app)
+        app.run()
+    """
+    def __init__(self):
+        self.apps = []
+
+    def add_app(self,title,icon, func):
+        """Adds a new application.
+        Parameters
+        ----------
+        func:
+            the python function to render this app.
+        title:
+            title of the app. Appears in the dropdown in the sidebar.
+        """
+        self.apps.append({
+            "title": title,
+            "icon": icon,
+            "function": func
+        })
+
+    def run(self):
+        
+        st.sidebar.write(format_func=lambda app: app['title'])
+        image = Image.open('docStore/img/sdsn.png')
+        st.sidebar.image(image, width =200)
+       
+        with st.sidebar:
+            selected = option_menu(None, [page["title"] for page in self.apps],
+                                   icons=[page["icon"] for page in self.apps],
+                                   menu_icon="cast", default_index=0)
+            st.markdown("---")          
+        
+        
+        for index, item in enumerate(self.apps):
+            if item["title"] == selected:
+                self.apps[index]["function"]()
+                break
+                
+   
+        choice = st.sidebar.radio(label = 'Select the Document',
+                            help = 'You can upload the document \
+                            or else you can try a example document', 
+                            options = ('Upload Document', 'Try Example'), 
+                            horizontal = True)
+        add_upload(choice)
+       

appStore/sdg_analysis.py

@@ -0,0 +1,179 @@
+# set path
+import glob, os, sys; 
+sys.path.append('../utils')
+
+#import needed libraries
+import seaborn as sns
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import streamlit as st
+from st_aggrid import AgGrid
+from st_aggrid.shared import ColumnsAutoSizeMode
+from utils.sdg_classifier import sdg_classification
+from utils.sdg_classifier import runSDGPreprocessingPipeline, load_sdgClassifier
+from utils.keyword_extraction import textrank
+import logging
+logger = logging.getLogger(__name__)
+from utils.checkconfig import getconfig
+
+
+# Declare all the necessary variables
+config = getconfig('paramconfig.cfg')
+model_name = config.get('sdg','MODEL')
+split_by = config.get('sdg','SPLIT_BY')
+split_length = int(config.get('sdg','SPLIT_LENGTH'))
+split_overlap = int(config.get('sdg','SPLIT_OVERLAP'))
+remove_punc = bool(int(config.get('sdg','REMOVE_PUNC')))
+split_respect_sentence_boundary = bool(int(config.get('sdg','RESPECT_SENTENCE_BOUNDARY')))
+threshold = float(config.get('sdg','THRESHOLD'))
+top_n = int(config.get('sdg','TOP_KEY'))
+
+
+def app():
+
+    #### APP INFO #####
+    with st.container():
+        st.markdown("<h1 style='text-align: center; color: black;'> SDG Classification and Keyphrase Extraction </h1>", unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=False):
+
+        st.write(
+            """     
+            The *SDG Analysis* app is an easy-to-use interface built \
+                in Streamlit for analyzing policy documents with respect to SDG \
+                 Classification for the paragraphs/texts in the document and \
+                extracting the keyphrase per SDG label - developed by GIZ Data \
+                 and the Sustainable Development Solution Network. \n
+            """)
+        st.write("""**Document Processing:** The Uploaded/Selected document is \
+            automatically cleaned and split into paragraphs with a maximum \
+            length of 120 words using a Haystack preprocessing pipeline. The \
+            length of 120 is an empirical value which should reflect the length \
+            of a “context” and should limit the paragraph length deviation. \
+            However, since we want to respect the sentence boundary the limit \
+            can breach and hence this limit of 120 is tentative.  \n
+            """)
+        st.write("""**SDG cLassification:** The application assigns paragraphs \
+            to 16 of the 17 United Nations Sustainable Development Goals (SDGs).\
+            SDG 17 “Partnerships for the Goals” is excluded from the analysis due \
+            to its broad nature which could potentially inflate the results. \
+            Each paragraph is assigned to one SDG only. Again, the results are \
+            displayed in a summary table including the number of the SDG, a \
+            relevancy score highlighted through a green color shading, and the \
+            respective text of the analyzed paragraph. Additionally, a pie \
+            chart with a blue color shading is displayed which illustrates the \
+            three most prominent SDGs in the document. The SDG classification \
+            uses open-source training [data](https://zenodo.org/record/5550238#.Y25ICHbMJPY) \
+            from [OSDG.ai](https://osdg.ai/) which is a global \
+            partnerships and growing community of researchers and institutions \
+            interested in the classification of research according to the \
+            Sustainable Development Goals. The summary table only displays \
+            paragraphs with a calculated relevancy score above 85%.  \n""")
+
+        st.write("""**Keyphrase Extraction:** The application extracts 15 \
+            keyphrases from the document, for each SDG label and displays the \
+            results in a summary table. The keyphrases are extracted using \
+            using [Textrank](https://github.com/summanlp/textrank)\
+            which is an easy-to-use computational less expensive \
+            model leveraging combination of TFIDF and Graph networks.
+            """)
+        st.write("")
+        st.write("")
+        st.markdown("Some runtime metrics tested with cpu: Intel(R) Xeon(R) CPU @ 2.20GHz, memory: 13GB")
+        col1,col2,col3,col4 = st.columns([2,2,4,4])
+        with col1:
+            st.caption("Loading Time Classifier")
+            # st.markdown('<div style="text-align: center;">12 sec</div>', unsafe_allow_html=True)
+            st.write("12 sec")
+        with col2:
+            st.caption("OCR File processing")
+            # st.markdown('<div style="text-align: center;">50 sec</div>', unsafe_allow_html=True)
+            st.write("50 sec")
+        with col3:
+            st.caption("SDG Classification of 200 paragraphs(~ 35 pages)")
+            # st.markdown('<div style="text-align: center;">120 sec</div>', unsafe_allow_html=True)
+            st.write("120 sec")
+        with col4:
+            st.caption("Keyword extraction for 200 paragraphs(~ 35 pages)")
+            # st.markdown('<div style="text-align: center;">3 sec</div>', unsafe_allow_html=True)
+            st.write("3 sec")
+
+        
+
+    
+    ### Main app code ###
+    with st.container():
+        if st.button("RUN SDG Analysis"):
+                   
+            if 'filepath' in st.session_state:
+                file_name = st.session_state['filename']
+                file_path = st.session_state['filepath']
+                classifier = load_sdgClassifier(classifier_name=model_name)
+                st.session_state['sdg_classifier'] = classifier
+                all_documents = runSDGPreprocessingPipeline(file_name= file_name,
+                                        file_path= file_path, split_by= split_by,
+                                        split_length= split_length,
+                split_respect_sentence_boundary= split_respect_sentence_boundary,
+                split_overlap= split_overlap, remove_punc= remove_punc)
+
+                if len(all_documents['documents']) > 100:
+                    warning_msg = ": This might take sometime, please sit back and relax."
+                else:
+                    warning_msg = ""
+
+                with st.spinner("Running SDG Classification{}".format(warning_msg)):
+
+                    df, x = sdg_classification(haystack_doc=all_documents['documents'],
+                                                threshold= threshold)
+                    df = df.drop(['Relevancy'], axis = 1)
+                    sdg_labels = x.SDG.unique()
+                    textrank_keyword_list = []
+                    for label in sdg_labels:
+                        sdgdata = " ".join(df[df.SDG == label].text.to_list())
+                        textranklist_ = textrank(textdata=sdgdata, words= top_n)
+                        if len(textranklist_) > 0:
+                            textrank_keyword_list.append({'SDG':label, 'TextRank Keywords':",".join(textranklist_)})
+                    textrank_keywords_df = pd.DataFrame(textrank_keyword_list)
+
+
+                    plt.rcParams['font.size'] = 25
+                    colors = plt.get_cmap('Blues')(np.linspace(0.2, 0.7, len(x)))
+                    # plot
+                    fig, ax = plt.subplots()
+                    ax.pie(x['count'], colors=colors, radius=2, center=(4, 4),
+                        wedgeprops={"linewidth": 1, "edgecolor": "white"},
+                        textprops={'fontsize': 14}, 
+                        frame=False,labels =list(x.SDG_Num),
+                        labeldistance=1.2)
+                    # fig.savefig('temp.png', bbox_inches='tight',dpi= 100)
+                    
+
+                    st.markdown("#### Anything related to SDGs? ####")
+
+                    c4, c5, c6 = st.columns([1,2,2])
+
+                    with c5:
+                        st.pyplot(fig)
+                    with c6:
+                        labeldf = x['SDG_name'].values.tolist()
+                        labeldf = "<br>".join(labeldf)
+                        st.markdown(labeldf, unsafe_allow_html=True)
+                    st.write("")
+                    st.markdown("###### What keywords are present under SDG classified text? ######")
+
+                    AgGrid(textrank_keywords_df, reload_data = False, 
+                            update_mode="value_changed",
+                    columns_auto_size_mode = ColumnsAutoSizeMode.FIT_CONTENTS)
+                    st.write("")
+                    st.markdown("###### Top few SDG Classified paragraph/text results ######")
+
+                    AgGrid(df, reload_data = False, update_mode="value_changed",
+                    columns_auto_size_mode = ColumnsAutoSizeMode.FIT_CONTENTS)
+            else:
+                st.info("🤔 No document found, please try to upload it at the sidebar!")
+                logging.warning("Terminated as no document provided")
+
+

docStore/ndcs/cca.txt

@@ -0,0 +1,81 @@
+{"climate_risks_droughts": {"category": "climate change adaptation","id": {0:"(I)NDC not submitted or not yet included in analysis",
+                                                                            1: "Droughts are not climate risks concerns",
+								            2:  "Droughts are among the five climate risks concerns"}},
+"climate_risks_extreme_weather": {"category": "climate change adaptation", "id": {0:"(I)NDC not submitted or not yet included in analysis",
+                                                                            1: "Extreme Weathers are not climate risks concerns",
+								            2:  "Extreme Weathers are among the five climate risks concerns"}},
+"climate_risks_floods": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+									 1: "Floods are not climate risks concerns",
+								         2:  "Floods are among the five climate risks concerns"}},
+"climate_risks_temp_increase": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+									 1: "Temperature increase are not climate risks concerns",
+								         2:  "Temperature increase are among the five climate risks concerns"}},
+"climate_risks_sea_level_rise": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+									 1: "Sea level rise is not a climate risks concerns",
+								         2:  "Sea level rise is among the five climate risks concerns"}},
+
+"priority_sectors_agriculture": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+										1: "Agricultural sector is not that important in the context of adaptation ambitions",
+										2: "In the context of adaptation ambitions Agricultural sector is very important for the country",
+										3: "Agriculture sector plays an importance for the country, and therefore in the adaptation ambitions Agriculture sector has special actions and aims"}},
+
+"priority_sectors_ecosystems": {"category": "climate change adaptation","id": {0 :"(I)NDC not submitted or not yet included in analysis",
+ 									       1 :"Biodiversity and preservation of Ecosystems is not that important in the context of adaptation ambitions",
+									       2: "In the context of adaptation ambitions Biodiversity and preservation of Ecosystems is very important for the country",
+									       3: "Biodiversity and Ecosystems plays an importance for the country, and therefore in the adaptation ambitions Biodiversity and Ecosystems has special actions and aims"}},
+"priority_sectors_forestry": {"category": "climate change adaptation", "id": {0: "(I)NDC not submitted or not yet included in analysis",
+									      1: "Forestry sector is not that important in the context of adaptation ambitions",
+									      2: "In the context of adaptation ambitions Forestry sector is very important for the country",
+									      3: "Forestry sector plays an importance for the country, and therefore in the adaptation ambitions Agriculture sector has special actions and aims"}},
+"priority_sectors_health": {"category": "climate change adaptation","id": { 0: "(I)NDC not submitted or not yet included in analysis",
+									    1: "Health sector is not that important in the context of adaptation ambitions",
+									    2: "In the context of adaptation ambitions Health sector is very important for the country",
+									    3: "Health sector plays an importance for the country, and therefore in the adaptation ambitions Agriculture sector has special actions and aims"}},
+
+"priority_sectors_water": {"category": "climate change adaptation","id": { 0 : "(I)NDC not submitted or not yet included in analysis",
+ 									   1: "Water sector is not that important in the context of adaptation ambitions",
+									   2: "In the context of adaptation ambitions Water sector is very important for the country",
+									   3: "Water sector plays an importance for the country, and therefore in the adaptation ambitions Agriculture sector has special actions and aims"}},
+
+"vulnerability_agriculture": {"category": "climate change adaptation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+									     1: "Agriculture is a not a vulnerable sector",
+   									     2: "Agriculture is a vulnerable sector"}},
+"vulnerability_coastal_zones": {"category": "climate change adaptation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+									       1: "Coastal Zone is a not a vulnerable sector",
+   									       2: "Coastal Zone is a vulnerable sector"}},
+"vulnerability_ecosystems": {"category": "climate change adaptation", "id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+									     1: "Biodiversity and Ecosystems is a not a vulnerable sector",
+   									     2: "Biodiversity and Ecosystems is a vulnerable sector"}},
+ "vulnerability_health": {"category": "climate change adaptation","id": {0:"(I)NDC not submitted or not yet included in analysis",
+									 1: "Health is a not a vulnerable sector",
+   									 2: "Health is a vulnerable sector"}},
+ "vulnerability_water": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+    									1: "Water is a not a vulnerable sector",
+   									2: "Water is a vulnerable sector"}},
+
+"costs_of_adaptation": {"category": "climate change adaptation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+								       1: "The partial cost of adaptation is tentatively around few million dollars",
+								       2: " The cost of adaptation will be 0-1 billion US$ until 2030",
+								       3: " The cost of adaptation will be 1-5 billion US$ until 2030",
+								       4: " The cost of adaptation will be 5-10 billion US$ until 2030",
+								       5: " The cost of adaptation will be 10-20 billion US$ until 2030",
+								       6:  "The cost of adaptation will be more than 20 billion US$ until 2030"}},
+"costs_of_future_climate_related_hazards": {"category": "climate change adaptation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+											   1: "The future losses from climate change will be huge",
+   											   2: "The climate hazards cause significant loss to economy and life, and the cost of Future losses could go around few million dollars"}},
+
+"costs_of_recent_climate_related_hazards": {"category": "climate change adaptation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+											   1: "No losses indicated",
+   									                   2: "In the recent climate hazards there has been significant Economic losses.",
+  										           3: "In the recent climate hazards the impact on human life has been significant",
+											   4: "In the recent climate hazards the impact on human life has been significant and the economic loss amounts to 5.3"}},
+"quantified_adaptation_targets": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+										 1:"No quantitative adaptation target",
+										 2: "In this sector we aim to achieve the adaptation targets of 5.6 in coming few years",
+										 3: "In this sector we aim to achieve the adaptation targets of 5.6 in coming few years",
+										 4: "In this sector we aim to achieve the adaptation targets of 5.6 in coming few years"}},
+
+"slow_onset_others": {"category": "climate change adaptation","id": {0: "(I)NDC not submitted or not yet included in analysis",
+							             1:"Apart from sea level rise and temperature increase, no other specific slow onset process",
+							             2: "There are other slow onset processes additional to sea level rise and temperature increase like loss of biodiversity, desertification, glacier retreat, salinisation or ocean acidification"}},
+}

docStore/ndcs/ccm.txt

@@ -0,0 +1,86 @@
+{"agriculture": {"category": "climate change mitigation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+							        1: "Agriculture sector is not considered for climate change mitigation",
+								2: "Agriculture sector contribution in greenhouse gases emission is significant and therefore is part of climate change mitigation",
+								3: "Agriculture sector contribution in greenhouse gases emission is significant. Given the importance of agriculture sector for economy and and its adverse contribution in greenhouse gas emissions it is a Focus area for climate change mitigation and needs to be prioritised"}},
+
+"energy_efficiency": {"category": "climate change mitigation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+							        1: "Energy Efficiency is not considered for climate change mitigation",
+								2: "Energy sector contribution in greenhouse gases emission is significant and therefore Energy Efficiency is part of climate change mitigation",
+								3: "Energy sector contribution in greenhouse gases emission is significant. Given the importance of the energy sector for economy and its adverse contribution to greenhouse gas emissions, energy efficiency is a Focus area for climate change mitigation and needs to be prioritised. The quantified renewable energy targets like for example in solar, geothermal, wind power are provided."}},
+
+"fossil_fuel_production": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+									  1:"There is no recorded FFP (2016)",
+									  2: "Fossil fuel Production is important for economy",
+									  3:"Fossil fuel Production is important to provide for the basic requirements of the people in the country",
+									  4:"The country's dependence on Fossil fuel production to meet energy and other requirements cannot be ignored, however the climate change impact due to the same cannot be ignored. The plans and actions as part of climate change mitigation includes measures to address production (e.g. subsidy removal; taxes); cleaner production",
+								          5: "Fossil fuel Production is important to provide for the basic requirements of the people in the country.The country's dependence on Fossil fuel production to meet energy and other requirements cannot be ignored, however the climate change impact due to same cannot be ignored. The plans and actions as part of climate change mitigation includes measures to address production (e.g. subsidy removal; taxes); cleaner production"}},
+"fossil_fuel_subsidiaries": {"category": "climate change mitigation","id":{0: "(I)NDC not submitted or not yet included in analysis",
+									   1:"fossil Fuel subsidiaries are not considered",
+									   2:"the alternates/subsidiaries to fossil Fuel need to be considered to meet the mitigations ambitions",
+									   3:"The fossil fuel contribution towards greenhouse gas emissions is very high and therefore there is a need to find the alternatives/substitutes for the same. The replacement of fossil fuels with alternates is a priority focus area in the mitigation actions to meet mitigation ambitions."}},
+
+"land_use_and_forestry": {"category": "climate change mitigation", "id":{0:"(I)NDC not submitted or not yet included in analysis",
+  									 1:"land use and forestry are not considered",
+									 2:"the land use and forestry contribute to greenhouse gas emissions and need to be considered to meet the mitigations ambitions",
+									 3:"The land use and forestry contribution towards greenhouse gas emissions is significant and therefore there is need to quantify the mitigation potential land use and forestry."}},
+"land_use_change": {"category": "climate change mitigation", "id": {0:"(I)NDC not submitted or not yet included in analysis",
+								    1: "land use change Not mentioned",
+							            2: "land use change is being considered, but there are no mitigation targets",
+								    3: "land use change is being considered as part of mitigation targets",
+								    4: "land use change can play an important role in mitigation efforts. As part of mitigation plan there are quantified targets for land use change.",
+								    5: "land use change can play an important role in mitigation efforts. As part of mitigation plan there are quantified targets for land use change."}},
+
+"renewable_energy": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+								     1:"renewable energy is not considered",
+							             2:"Renewable energy are direct measure to reduce the greenhouse gas emissions and need to be considered to meet the mitigations ambitions",
+								     3:"Renewable energy are direct measure to reduce the greenhouse gas emissions and therefore there is need to quantify the mitigation potential in terms of renewable energy targets and specific sub-sectors of action (e.g. solar, geothermal, wind power)"}},
+
+"temp_target": {"category": "climate change mitigation", "id": { 0: "(I)NDC not submitted or not yet included in analysis",
+ 								 1:"Not mentioning global effort to limit global temperature increase to 2 degree celsius or 1.5 degree C",
+								 2:"there is urgent need to limit global temperature increase to 2 degree celsius",						
+								 3:"there is urgent need to limit global temperature increase to 1.5 degree C",
+							         4:"there is urgent need to limit global temperature increase to 2 degree celsius",
+ 								 5:"there is urgent need to limit global temperature increase to 1.5 degree C"}},
+"waste": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+							 1:"Waste as a topic is not mentioned",
+ 							 2:"Waste reduction or management can play important role in mitigation plan and ambitions",
+							 3:"Waste reduction or management can play an important role in sustainable development and hence is a focus area in mitigation plan and ambitions"}},
+"transport": {"category": "climate change mitigation","id": {0:"(I)NDC not submitted or not yet included in analysis",
+							     1:"Transport is not considered",
+							     2:"Transport contribute to greenhouse gas emissions and need to be considered to meet the mitigations ambitions",
+							     3:"transport sector contribution towards greenhouse gas emissions is significant and therefore there is need to focus/prioritise the transport sector to meet the mitigation potential"}},
+
+"reducing_non_co2_gases": {"category": "climate change mitigation","id": {0:"(I)NDC not submitted or not yet included in analysis",
+									  1:"Reduction of non CO2 gases not indicated",
+									  2:"Efforts should be made in reduction of NOn CO2 gases too."}},
+
+
+"base_year": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+							     1: "No base year",
+     							     2: "the base year or reference point for measurement of emissions is year 19XX"}},
+
+"carbon_capture_and_storage": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+									      1: "carbon capture and storage not indicated",
+									      2:"With Technology advancement the mitigation efforts can also in form of carbon capture and storage.",
+									      3: "With technological advancement the mitigation efforts can also be in form of carbon capture and storage. This should be  a focus area and more options need to be explored to do carbon capture and storage."}},
+
+"costs_of_ccm": {"category": "climate change mitigation","id":{ 0: "(I)NDC not submitted or not yet included in analysis",
+								1: "(partial) costs not indicated",
+								2: " the mitigation actions and efforts will cost 0-1 billion US$ until 2030",
+								3:"the mitigation actions and efforts will cost 1-5 billion US$ until 2030",
+								4:"the mitigation actions and efforts will cost5-10 billion US$ until 2030",
+								5: "the mitigation actions and efforts will cost 10-20 billion US$ until 2030",
+								6:"the mitigation actions and efforts will cost will be more than 20 billion US$ until 2030"}},
+
+"market_mechanisms": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+								     1: "International market mechanisms not mentioned",
+ 								     2:"One good mechanism to deal with greenhouse gas emissions is to explore International market mechanisms",
+								     3: "International market mechanisms are not a good way of dealing with mitigation ambitions and therefore should not be considered. Greenhouse gas emissions cannot be part of tradable commodity.",
+								     4: "Carbon emissions of greenhouse gas emissions are now a tradable commodity and these can provide a good source for funds and achieving mitigation ambitions. Therefore it is important to exploreInternational market mechanisms. It is important that such means should be explored and there will be plan of actions soon to include these in meeting mitigations target",
+								     5: "Carbon emissions of greenhouse gas emissions are now a tradable commodity and these can provide a good source for funds and achieving mitigation ambitions. Therefore it is important to exploreInternational market mechanisms. It is important that such means should be explored and there will be plan of actions soon to include these in meeting mitigations target"}},
+
+"redd": {"category": "climate change mitigation","id":{ 0:"(I)NDC not submitted or not yet included in analysis",
+							1: "REDD+ not mentioned",
+							2: "Reducing Emissions of Deforestation and Forest Degradation/REDD+",
+							3: "Reducing Emissions of Deforestation and Forest Degradation/REDD+"}},
+}

docStore/ndcs/countryList.txt

@@ -0,0 +1,170 @@
+{'Afghanistan': 'AFG',
+ 'Albania': 'ALB',
+ 'Algeria': 'DZA',
+ 'Andorra': 'AND',
+ 'Angola': 'AGO',
+ 'Antigua and Barbuda': 'ATG',
+ 'Argentina': 'ARG',
+ 'Armenia': 'ARM',
+ 'Australia': 'AUS',
+ 'Azerbaijan': 'AZE',
+ 'Bahamas': 'BHS',
+ 'Bahrain': 'BHR',
+ 'Bangladesh': 'BGD',
+ 'Barbados': 'BRB',
+ 'Belarus': 'BLR',
+ 'Belize': 'BLZ',
+ 'Benin': 'BEN',
+ 'Bhutan': 'BTN',
+ 'Bolivia': 'BOL',
+ 'Bosnia and Herzegovina': 'BIH',
+ 'Botswana': 'BWA',
+ 'Brazil ': 'BRA',
+ 'Brunei Darussalam': 'BRN',
+ 'Burkina Faso': 'BFA',
+ 'Burundi ': 'BDI',
+ 'Cabo Verde': 'CPV',
+ 'Cambodia': 'KHM',
+ 'Cameroon': 'CMR',
+ 'Canada': 'CAN',
+ 'Central African Republic': 'CAF',
+ 'Chad': 'TCD',
+ 'Chile': 'CHL',
+ 'China': 'CHN',
+ 'Colombia': 'COL',
+ 'Comoros': 'COM',
+ 'Congo': 'COG',
+ 'Cook Islands': 'COK',
+ 'Costa Rica': 'CRI',
+ 'Cote dIvoire': 'CIV',
+ 'Cuba': 'CUB',
+ "Democratic People's Republic of Korea": 'PRK',
+ 'Democratic Republic of Congo': 'COD',
+ 'Djibouti': 'DJI',
+ 'Dominica': 'DMA',
+ 'Dominican Republic': 'DOM',
+ 'Ecuador': 'ECU',
+ 'Egypt': 'EGY',
+ 'El Salvador': 'SLV',
+ 'Equatorial Guinea': 'GNQ',
+ 'Eritrea': 'ERI',
+ 'Ethiopia': 'ETH',
+ 'European Union': 'EU',
+ 'Fiji': 'FJI',
+ 'Gabon': 'GAB',
+ 'Gambia': 'GMB',
+ 'Georgia': 'GEO',
+ 'Ghana': 'GHA',
+ 'Grenada': 'GRD',
+ 'Guatemala': 'GTM',
+ 'Guinea': 'GIN',
+ 'Guinea Bissau': 'GNB',
+ 'Guyana': 'GUY',
+ 'Haiti': 'HTI',
+ 'Honduras': 'HND',
+ 'Iceland': 'ISL',
+ 'India': 'IND',
+ 'Indonesia': 'IDN',
+ 'Iran': 'IRN',
+ 'Iraq': 'IRQ',
+ 'Israel': 'ISR',
+ 'Jamaica': 'JAM',
+ 'Japan': 'JPN',
+ 'Jordan': 'JOR',
+ 'Kazakhstan': 'KAZ',
+ 'Kenya': 'KEN',
+ 'Kingdom of Eswatini': 'SWZ',
+ 'Kiribati': 'KIR',
+ 'Kuwait': 'KWT',
+ 'Kyrgyzstan': 'KGZ',
+ 'Lao Peoples Democratic Republic': 'LAO',
+ 'Lebanon': 'LBN',
+ 'Lesotho': 'LSO',
+ 'Liberia': 'LBR',
+ 'Libya': 'LBY',
+ 'Liechtenstein': 'LIE',
+ 'Madagascar': 'MDG',
+ 'Malawi': 'MWI',
+ 'Malaysia': 'MYS',
+ 'Maldives': 'MDV',
+ 'Mali': 'MLI',
+ 'Marshall Islands': 'MHL',
+ 'Mauritania': 'MRT',
+ 'Mauritius': 'MUS',
+ 'Mexico': 'MEX',
+ 'Micronesia': 'FSM',
+ 'Monaco': 'MCO',
+ 'Mongolia': 'MNG',
+ 'Montenegro': 'MNE',
+ 'Morocco': 'MAR',
+ 'Mozambique': 'MOZ',
+ 'Myanmar': 'MMR',
+ 'Namibia': 'NAM',
+ 'Nauru': 'NRU',
+ 'Nepal': 'NPL',
+ 'New Zealand': 'NZL',
+ 'Nicaragua': 'NIC',
+ 'Niger': 'NER',
+ 'Nigeria': 'NGA',
+ 'Niue': 'NIU',
+ 'Norway': 'NOR',
+ 'Oman': 'OMN',
+ 'Pakistan': 'PAK',
+ 'Palau ': 'PLW',
+ 'Palestine': 'PSE',
+ 'Panama': 'PAN',
+ 'Papua New Guinea': 'PNG',
+ 'Paraguay': 'PRY',
+ 'Peru': 'PER',
+ 'Philippines': 'PHL',
+ 'Qatar': 'QAT',
+ 'Republic of Moldova': 'MDA',
+ 'Republic of North Macedonia': 'MKD',
+ 'Russian Federation': 'RUS',
+ 'Rwanda': 'RWA',
+ 'Saint Kitts and Nevis': 'KNA',
+ 'Saint Lucia': 'LCA',
+ 'Saint Vincent and the Grenadines': 'VCT',
+ 'Samoa': 'WSM',
+ 'San Marino': 'SMR',
+ 'Sao Tome and Principe': 'STP',
+ 'Saudi Arabia': 'SAU',
+ 'Senegal': 'SEN',
+ 'Serbia': 'SRB',
+ 'Seychelles': 'SYC',
+ 'Sierra Leone': 'SLE',
+ 'Singapore': 'SGP',
+ 'Solomon Islands': 'SLB',
+ 'Somalia': 'SOM',
+ 'South Africa': 'ZAF',
+ 'South Korea': 'KOR',
+ 'South Sudan': 'SSD',
+ 'Sri Lanka': 'LKA',
+ 'Sudan': 'SDN',
+ 'Suriname': 'SUR',
+ 'Switzerland': 'CHE',
+ 'Syria': 'SYR',
+ 'Tajikistan': 'TJK',
+ 'Thailand': 'THA',
+ 'Timor Leste': 'TLS',
+ 'Togo': 'TGO',
+ 'Tonga': 'TON',
+ 'Trinidad and Tobago': 'TTO',
+ 'Tunisia': 'TUN',
+ 'Turkey': 'TUR',
+ 'Turkmenistan': 'TKM',
+ 'Tuvalu': 'TUV',
+ 'Uganda': 'UGA',
+ 'Ukraine': 'UKR',
+ 'United Arab Emirates': 'ARE',
+ 'United Kingdom': 'GBR',
+ 'United Republic of Tanzania': 'TZA',
+ 'United States of America': 'USA',
+ 'Uruguay': 'URY',
+ 'Uzbekistan': 'UZB',
+ 'Vanuatu': 'VUT',
+ 'Venezuela ': 'VEN',
+ 'Vietnam': 'VNM',
+ 'Yemen': 'YEM',
+ 'Zambia': 'ZMB',
+ 'Zimbabwe': 'ZWE'}

docStore/sample/Ethiopia_s_2021_10 Year Development Plan.txt

@@ -0,0 +1,737 @@
+Ethiopia 2030: The Pathway to Prosperity
+Ten Years Perspective Development Plan (2021 � 2030)
+1. Baselines and Assumptions
+2. Strategic pillars
+3. Departures
+4. Macroeconomic goals
+5. Implications of the COVID-19 pandemic and necessary mitigation measures
+6. Potentials/capabilities
+7. Focus areas
+7.1. Productive sectors
+7.2. Services sector
+7.3. Enabling sectors
+8. Balanced and competitive development (nationally, regionally and locally)
+9. Monitoring and Evaluation
+Content
+1. Baselines and Assumptions
+Poverty Reduction (%)
+Key performances of previous years
+45.5 44.2
+38.7
+29.6
+23.5
+19
+0
+5
+10
+15
+20
+25
+30
+35
+40
+45
+50
+1994 2000 2005 2011 2016 2020
+Percent
+Year
+Proportion of people living below poverty line
+10.5
+8.8
+10.1
+7.7
+9
+5.19-6.20
+0 2 4 6 8 10 12
+GTP I: 2011-2015
+GTP II: 2015/16
+GTP II: 2016/17
+GTP II: 2017/18
+GTP II: 2018/19
+GTP II: 2019/20 (projection, with
+COVID-19)
+GDP growth rate (%)
+1. Baselines and Assumptions
+Share of economic sectors in GDP (%) Merchandise export as % of GDP
+8.66
+7.33
+6.57
+5.93
+4.91
+3.86 3.56 3.37
+2.77
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+Percent
+Year
+46.9
+45
+43.5
+41.4
+39.5
+37.1 35.9
+34.5
+32.8
+13.4
+15
+17.3
+18.8
+21
+23.5
+25.7 26.9 27.8
+4.7 4.8 5 5.3 5.6 6.1 6.9 6.8 6.8
+7.1
+8.6
+10.7 12
+14.2
+16.2
+17.8 19.1 20.1
+39.8 40.1 39.2 39.8 39.4 38.4 38.6 39.4
+0
+5
+10
+15
+20
+25
+30
+35
+40
+45
+50
+2010/11 2011/12 2012/13 2013/14 2014/15 2015/16 2016/17 2017/18 2018/19
+Percent
+Agriculture Industry Manufacturing Construction Services
+1. Baselines and Assumptions
+Labour force participation (2013)
+73%
+7%
+20%
+Agriculture
+Industry
+Services
+7%
+22%
+71%
+Agriculture
+Industry
+Services
+Urban labour force participation (2013)
+1. Baselines and Assumptions
+High and increasing Unemployment Rate
+� Urban unemployment rate = 19.1% in 2018
+� Youth unemployment rate = 25.3 %
+? Male = 18.6%
+? Female 30.9 %
+� Rural unemployment rate = 2% in 2013
+� Declining per capita rural land creating
+disguised unemployment
+402,869
+471,535
+Male Female Total Male Female Total
+2014 2018
+15-19 yr. 20-24 yr. 25-29 yr. Linear (20-24 yr.)
+Number of unemployed people in urban areas
+1. Baselines and Assumptions
+Challenges
+1. Macroeconomic imbalances
+?Sustained high inflation
+?High and rising unemployment especially
+in urban areas
+?High and rising debt burden
+?Chronic foreign currency shortage
+?Sluggish (though encouraging) rate of
+structural change
+2. Vulnerability to shocks (COVID-19, Climate
+changes, Desert Locust infestation, etc)
+3. Poor quality and high inequity in
+infrastructure projects
+4. Poor quality services in health and
+education
+� High repetition and dropout rates from school
+1. Baselines and Assumptions
+� Poor quality of growth and slow
+structural change
+� Excessive aid and loan
+dependence for financing
+infrastructural and construction
+investments
+� Limited success in expanding
+manufacturing and modern
+agriculture which have high job
+creation potentials
+� Weak institutional capacity as
+the main culprit of all failures
+? Provision of quality services
+(electricity, water, telephone,
+internet)
+? Creation of enough jobs and
+improved living standards
+? Generation of reliable foreign
+exchange revenue and debtsustainable
+national economic
+capacity
+? Completion of development
+projects and investment plans
+under public-private
+partnerships
+� Low reward for merit, productivity and effort
+while low disincentive for laziness, wastefulness
+and corruption
+� Slow institutional change and transformation in:
+? Government policies
+? Investor attitude
+? Youth behaviour
+? Role of the intellectuals
+� The need for sustained increase in production
+and productivity
+� The need to set a common national vision to
+achieve major successes with consensus and
+popular legitimacy
+Major areas of failure in the economy
+1. Baselines and Assumptions
+� Poor quality of growth and slow
+structural change
+� Excessive aid and loan
+dependence for financing
+infrastructural and construction
+investments
+� Limited success in expanding
+manufacturing and modern
+agriculture which have high job
+creation potentials
+� Weak institutional capacity as
+the main culprit of all failures
+? Provision of quality services
+(electricity, water, telephone,
+internet)
+? Creation of enough jobs and
+improved living standards
+? Generation of reliable foreign
+exchange revenue and debtsustainable
+national economic
+capacity
+? Completion of development
+projects and investment plans
+under public-private
+partnerships
+� Low reward for merit, productivity and effort
+while low disincentive for laziness, wastefulness
+and corruption
+� Slow institutional change and transformation in:
+? Government policies
+? Investor attitude
+? Youth behaviour
+? Role of the intellectuals
+� The need for sustained increase in production
+and productivity
+� The need to set a common national vision to
+achieve major successes with consensus and
+popular legitimacy
+Major areas of failure in the economy
+2. Departures
+1. Emphasis on quality of economic growth
+2. Participation and coordination of sectors in the planning process
+3. Sectoral linkages and multi-sectoral development focus
+4. Preparation of national development corridors based on development potentials
+5. Focus on solving institutional bottlenecks
+6. The ongoing home grown economic reform programme as a sprinting board
+7. Emphasis on resilience building, innovation and entrepreneurship
+3. Strategic pillars
+1. Ensure quality growth
+2. Improve productivity and competitiveness
+3. Undertake institutional transformation
+4. Ensure private sector's leadership in the economy
+5. Ensure equitable participation of women and children
+6. Build climate resilient green economy
+3. Strategic pillars
+� Increasing export revenues and substituting imports by
+reducing production costs
+� Availing quality and massive infrastructure
+? Linking infrastructural development with development corridors
+� Producing required human resources with quality
+� Producing enough and quality human resources
+� Prioritizing innovative production systems
+� Linking incentives with export revenue and job creation
+performances
+� Modernizing and enhancing the logistic system
+� Creating technological competences needed for longterm
+growth
+� The economic growth should ensure:
+? Participation of all citizens and equitable utilization of the
+growth proceeds
+? Improved standard of living of every citizen
+? Reduced poverty in all indicators
+? Reduced inflation and unemployment
+� The economic growth should lead to increased
+aggregate supply
+� Focus on modern agriculture, manufacturing and
+mining
+� Emphasis on exploiting the sources of growth through
+structural change
+1.Ensuring quality economic growth 2. Raising production and productivity
+3. Strategic pillars
+� Build democratic and judicial institutions that ensure elite bargain,
+national consensus, common vision and government legitimacy
+� Build private sector and competition friendly bureaucracy
+� Coordinate with parents, the society and teachers to make
+educational institutions centers of excellence and virtuous citizens
+� Coordinate with parents as well as social and religious leaders to
+encourage religious institutions and their teachings contribute
+towards poverty reduction efforts
+� Prepare policies, strategies and legal frameworks for achieving
+prosperity
+� Increased focus on innovation and research
+� Creating strong social security system
+3. Institutional Transformation 4. Private sector's leadership in the economy
+� Create conducive investment climate and incentivize
+domestic investors in key sectors
+� Build strong and market-led public-private partnerships in
+order to ensure the establishment of inclusive and
+pragmatic market economy
+� Enhance access and quality of infrastructure to attract
+quality foreign direct investment
+� Identify new sources of growth, empower and stimulate
+the private sector, and supplement the private sector in
+strategic areas
+� Emphasis for public-private partnership on problem
+solving innovations and research activities
+3. Strategic pillars
+� Ensure gender equity in economic and social
+sectors
+? Participation of women at all levels of education
+? Asset ownership of women
+� Ensure fair participation of women and youth in
+leadership and decision making positions
+� Create awareness among citizens about the role of
+women and youth in the country�s overall
+development
+� Increase basin development efforts to fight land
+degradation and to reduce pollutions
+� Improve productivity and reduce GHG emissions
+� Increase forest protection and development
+� Increase production of electricity from renewable
+sources for domestic use and for export
+� Focus on modern and energy saving technologies
+5. Equitable participation of women and children 6. Climate resilient green economy
+4. Macroeconomic Goals
+Assumptions
+? Requirement to significantly reduce
+poverty
+? Available national potentials
+? Potential for investment in the economy
+? Existing potentials in each sector
+? Low productivity that needs to be
+improved
+� Make Ethiopia a middle income
+economy by 2022
+� Raise per capita income to USD 1,115
+in 2022
+? Threshold for middle-income is USD 1,026
+? Plus human development index and
+economic vulnerability index
+� Raise per capita income to USD 2,220
+by 2030
+Sectoral growth Targets (2021-2030)
+Assured middle- income potential
+10.2%
+Average
+Growth
+Target
+Percentage of population below poverty line
+4. Macroeconomic Goals
+Structural change
+Financing Gaps
+Reduce urban unemployment to less than 9%
+?1.36 million new jobs need to be
+created per annum
+Sectoral composition of GDP Labour force participation
+Economic
+Sectors
+Performance Target
+2011 2015 2018/19 2030
+Agriculture 45 39.7 32.8 22.0
+Industry 15.1 21.2 27.6 35.9
+Manufacturing 4.7 5.5 6.8 17.2
+Services 39.9 39 39.4 42.1
+5. Implications of the COVID-19 pandemic and necessary mitigation measures
+� GDP growth for 2019/20 fiscal year is projected to be lower than its target of 9.0% by between 2.81
+and 3.80 percentage points (equivalent to 58.3 - 78.8 billion birr) due to COVID-19 pandemic
+� If the current scenario continues, next year�s GDP growth could decline by 2.8 percentage points
+� Returning the economy to its high growth trajectory requires focusing on sectors with high
+productivity and job creation potentials
+� Public investment should focus on empowering the private sector
+� Promoting both domestic and foreign investments with the right set of incentives (merit based)
+� Modernizing production systems and improving uptake of technology
+� Conducting demand analysis for export commodities to remedy for the declining trend in exports
+and foreign exchange earnings.
+6. Potentials
+� Endowment of various natural resources contributing to the growth potential
+� Huge unutilized arable land creates great potential for the success of the plan
+� Endowment of gemstones, ornamental, energy, metals, and metallic minerals
+� Gold, coal, iron ore, potash, tantalum, marble, petroleum and other natural resources
+Natural
+Resources
+� Large youth population and potential for demographic dividend
+� Cumulative capacity in education and health
+� Positive attitude and noble culture of reaching agreement among citizens
+Human
+capital
+6. Potentials
+Built physical and material capitals
+?Transport and communication
+? Irrigation infrastructures for modern agriculture
+?Industrial Parks
+?Mega energy infrastructures
+Physical
+capital
+Unexploited
+growth
+potentials
+� Utilizing the tourism potential through modernization
+� Using the mining subsector as a source of input as well as a competitive industry in its
+own right
+6. Potentials
+� Solving supply side bottlenecks to satisfy the existing demand
+� Improving international acceptance and reliable partnerships
+? The �medemer�/synergy philosophy
+? The ongoing political reform measures
+? The Homegrown Economic Reform programme
+� Increased finance from partners and multilateral institutions
+? Increased availability of foreign exchange
+? Reduced debt stress for the short to medium term
+? Increased potential for development
+Increased
+demand as
+potential
+Political Capital
+Continental
+and regional
+integrations
+� Regional and continental economic integration agreements
+� International and continental free trade agreements
+6. Potentials
+Low
+technology as
+a potential
+� Undeniably low status of technological development
+� International mobility and spillover effect of technology
+� Potential for development and catching up by filling the technological gaps
+� Doubling crop productivity from the current 24-36 quintals per hectare will result
+in 7% increase in crop production
+� Raise the production efficiency of manufacturing from the current 50% to 80%
+7. Focus Areas
+7.1. Productive sectors: agriculture, manufacturing, mining
+7.2. Service sector: tourism
+7.3. Enabling sectors: energy, transport, sustainable finance,
+innovation and technology, urban development, irrigation,
+human capital development
+7.1. Productive sectors
+Agriculture Objectives
+1. Free agriculture from rain dependence
+2. Agricultural mechanization services
+3. Contract farming, cluster approach and
+land consolidation
+4. Livestock, animal feed and animal health
+5. Horticulture (irrigation and urban farming)
+6. Private sector participation
+7. Institutional implementation capacity
+8. Climate resilient sustainable agricultural
+development
+1. Improve income and livelihood options for farming and pastoral
+communities through increased productivity and competitiveness
+2. Modernize agriculture and ensure national food and nutrition security
+3. Raise export of agricultural output and substitute imports
+4. Make agriculture a viable and profitable enterprise through value addition
+5. Create rural employment opportunities
+6. Enhance livestock health access and quality
+7. Preserve animal genetic resources and increase pastoral research
+8. Improve the development of animal feed and access to markets
+9. Develop livestock specific extension package for each livestock type
+Focus Areas
+7.1. Productive sector
+Manufacturing Industry
+Objectives
+1. Production of quality and competitive food, textile, housing and
+pharmaceutical products for export and domestic markets
+2. Production and productivity of existing manufacturing industries
+3. Utilization of locally available inputs
+4. Value chains, linkages and interdependencies
+5. Linkages between large scale metallurgical and engineering,
+chemical and pharmaceutical industries with other industries
+6. Job creation, cluster approaches and expanding small and medium
+scale manufacturing
+7. Private sector participation and partnership
+1. Establish basis for domestic industrialization
+2. Value addition through enhanced inter-sectoral
+linkages
+3. Enhance productivity through private sector
+leadership and supportive role of the
+government
+? Create job opportunities for the youth leaving
+agriculture and concentrating in urban areas
+? Make exportable commodities internationally
+competitive
+? Ensure structural change
+Focus areas
+7.1. Productive sectors
+Mining
+Objectives
+� Foreign exchange earning and
+domestic revenues
+� Increased investment in mining
+� Participation of manufacturing
+industries that add value
+� Job creation
+� Add value for improved contribution of the subsector
+� Increase inter-sectoral linkages to raise raw material inputs to other
+sectors
+� Make mining a competent subsector and induce structural change
+� Increase human resource and technological capabilities through
+research and trainings
+� Raise foreign exchange revenue from mining through increased
+exploration and production
+� Improve traditional mining production and marketing systems
+� Improve the country�s geological information
+Focus areas
+7.2. Service sector
+Tourism
+Objectives
+� Identification and developing destinations
+� Infrastructure
+� Competitiveness
+?improve existing destinations
+?develop new destinations
+? diversify service and raise quality
+� Market linkages, branding, and promotion
+� Technology, research and development
+� Preservation, maintenance and proper
+utilization of heritage resources
+� Expand job opportunities
+� Raise incomes
+� Build information management
+systems
+� Increase implementation capacity
+Focus areas
+7.3. Enabling sectors
+Urban development
+Objectives
+? Prioritize productive sectors in job creation and enterprise
+development plans
+? Rapid development and equity goals in land provision system
+? Participation of indigenous people in land redevelopment and
+expansion
+? Urban land registration and cadaster system, modern
+property valuation
+? Greenery and public spaces as well as waste disposal and
+management in urban planning and implementation
+? Housing development and financing options to reduce
+housing shortages
+? Integrated infrastructure and services provision
+? Role of private sector in infrastructure development and
+service provision
+� Expand micro and small-scale
+enterprises to reduce urban
+unemployment
+� Develop and avail urban land based on
+demand, equity and cost effectiveness
+� Make quality housing accessible both in
+rural and urban areas
+� Develop quality and integrated
+infrastructure as well as service
+provision in towns
+� Improve financial management and
+resource utilization in urban areas
+Focus areas
+7.3. Enabling sectors
+Innovation and Technology
+Objectives
+? Access to innovation and
+technological information
+? Developing a digital economy
+? Productivity enhancement and
+competitiveness
+? Build a digital economy
+? Develop national scientific research and technological
+capabilities
+? Support problem solving research and development of
+technologies necessary for raising production,
+productivity and service provision
+? Create jobs and capital that are based on technology
+? Develop technological and data security protection
+systems
+Focus areas
+7.3. Enabling sectors
+Sustainable finance
+Objectives
+� Access to modern finance and saving culture in rural
+areas
+� Support to the private sector and corporations to
+reinvest profits in productive sectors
+� Role of private financial institutions in manufacturing
+and agriculture
+� Digital revenue collection system
+� Tax equity (contraband, tax evasion, and bringing the
+underground economy to the tax system)
+� Domestic and foreign strategic partnerships
+� Transform financing from short term to long-term,
+sustainable and quality sources
+� Ensure financing quality based on sectoral prioritization
+and reduction of wastage
+� Increase the number of domestic saving institutions both
+in rural and urban areas
+� Support domestic finance with foreign exchange capacity
+and foreign direct investment
+� Modernize domestic revenue collection system
+� Raise voluntary tax payment attitude
+� Bring the informal sector to the formal tax system
+Focus areas
+7.3. Enabling sectors
+Transport
+Objectives
+� Access to infrastructure
+� Implementation capacity
+� Participation of the private sector and the general
+public
+� Financing capacity
+� Ensure equitable access to transport infrastructure and
+services
+� Improve transport safety
+� Make logistics services fast and reliable
+� Build transport infrastructure and service that is
+resilient to climate change
+Focus areas
+7.3. Enabling sectors
+Energy
+Objectives
+? Equity in access to electricity services
+? Energy access and quality
+? Alternative sources of energy
+? Reliability of electricity infrastructure
+? Investment and income in energy subsector
+� Ensure equitable access to transport
+infrastructure and services
+� Improve transport safety
+� Make logistics services fast and reliable
+� Build transport infrastructure and service that is
+resilient to climate change
+Focus areas
+7.3. Enabling sectors
+Irrigation
+Objectives
+? Medium and large scale irrigation infrastructure
+? Job creation
+? Share of government expenditure and alternative
+financing options
+? Institutional capacity and human resource
+development
+? Improve agricultural output and productivity
+? Reduce government spending and enhance
+institutional capacity and human resources
+development
+? Ensure the inclusion of all genders and
+disabled citizens
+? Develop alternative financing options for
+irrigation development
+Focus areas
+7.3. Enabling sectors
+Human capital development
+Objectives
+� Make education and training inclusive and equitable by
+harmonizing the system with ability, need and capacity
+� Develop capacity of educational institutions (teacher capacity,
+inputs and technology)
+� Establish education and training quality assurance system
+� Avail free and compulsory education for pre-primary to junior
+secondary levels and free education at the senior secondary levels
+equitably
+� Ensure the relevance of education and training system and
+synchronize education policy with economic and social
+development needs
+� Make the education and training policy compatible with the
+nation�s contemporary capacities as well as global and regional
+market opportunities
+� Enhance commitment, capability and responsibility of citizens
+? Ensure equitable and quality health services
+? Raise average life expectancy
+? Achieve universal health coverage through
+proactive and prevention health system
+? Curtail preventable maternal and child deaths
+? Reduce incidences of contagious and noncontagious
+related diseases and deaths
+? Build capacity for health tourism through
+increased treatment capabilities
+? Create a healthy society that is free from
+addictions and use technology for supporting
+knowledge led economic development
+Focus areas
+8 Nationally, regionally and locally balanced and competitive development
+1. Lack of synchronization of investment with
+resource potentials and development needs
+2. Poor alignment of federal, regional and
+district level investment plans with the
+national development goals and envisioned
+settlement patterns
+3. Poor regional coordination due to low
+consideration for trans-regional and
+spatial issues in development plans of
+regional states
+4. Inter-regional and intra-regional
+disparities in infrastructural development
+and access to services
+Challenges
+8. Nationally, regionally and locally balanced and competitive development
+1. Ensure that the investment flow and
+infrastructural development plans fairly go hand in
+hand with resource potential and development
+needs
+?Developing underutilized natural resources
+?Equitable distribution and access to
+infrastructure
+?Sustainable environmental protection
+2. Ensure the inclusion of pastoral and agro-pastoral
+areas in the development
+?Focused infrastructural development in pastoral
+areas such as education and health sector input
+provision as well as governance
+?Market linkages with other areas and the central
+markets
+?Improve rural finance (credit and insurance) to
+encourage fattening, milk processing, leather
+production and irrigation agriculture
+Focus areas
+9. Monitoring and Evaluation
+10 Years Perspective
+Plan KPIs
+Federal Implementing
+Institutions
+Planning and
+Development Commission
+Generate Data (Census,
+Sample and administrative
+data)
+Annual Reports
+Dialogue forums
+(Civic Organizations, professional
+associations, development partners,
+intellectuals)
+Central Statistical Agency
+Database
+National
+Information Portal
+National Statistics
+Development Strategic
+plan
+Evaluation Reports
+Prime Minister�s Office
+House of People�s
+Representatives
+Thank you!

docStore/sample/files.json

@@ -0,0 +1,3 @@
+{"Ethiopia: 10 Year Development Plan":"docStore/sample/Ethiopia_s_2021_10 Year Development Plan.txt",
+"South Africa:Low Emission strategy":"docStore/sample/South Africa_s Low Emission Development Strategy.txt"
+ }

docStore/sample/keywordexample.json

@@ -0,0 +1,8 @@
+{
+"Climate":"Climate,Adaptation,Mitigation,Decarbonization,Carbon neutrality,Net zero Emissions",
+"Food":"Food security,Nutrition,Diets,Food loss",
+"Implementation":"Implementation,transformation,reform,integration,strategy,policy",
+"Nature":"Nature,Nature-based solutions,Biodiversity,Degradation",
+"Social":"Indigenous,Local community(ies),Rural livelihoods,Minority",
+"Gender":"gender, women empowernment, women economic power, gender bias"
+}

packages.txt

@@ -0,0 +1,4 @@
+poppler-utils
+xpdf
+tesseract-ocr 
+libtesseract-dev

paramconfig.cfg

@@ -0,0 +1,47 @@
+[lexical_search]
+TOP_K = 20
+SPLIT_BY = word
+SPLIT_LENGTH = 120
+SPLIT_OVERLAP = 0
+REMOVE_PUNC = 0
+
+[semantic_search]
+RETRIEVER_TOP_K = 10
+MAX_SEQ_LENGTH = 384
+RETRIEVER = all-mpnet-base-v2
+RETRIEVER_FORMAT = sentence_transformers
+EMBEDDING_DIM = 768
+RETRIEVER_EMB_LAYER = -1
+READER = deepset/tinyroberta-squad2
+READER_TOP_K = 10
+READER_TOP_K_PER_CANDIDATE = 1
+SPLIT_BY = word
+SPLIT_LENGTH = 120
+SPLIT_OVERLAP = 10
+RESPECT_SENTENCE_BOUNDARY = 1
+REMOVE_PUNC = 0
+
+[sdg]
+THRESHOLD = 0.85
+MODEL = jonas/bert-base-uncased-finetuned-sdg
+SPLIT_BY = word
+REMOVE_PUNC = 0
+SPLIT_LENGTH = 120
+SPLIT_OVERLAP = 10
+RESPECT_SENTENCE_BOUNDARY = 1
+TOP_KEY = 15
+
+
+[coherence]
+RETRIEVER_TOP_K = 10
+MAX_SEQ_LENGTH = 512
+RETRIEVER = msmarco-distilbert-dot-v5
+RETRIEVER_FORMAT = sentence_transformers
+RETRIEVER_EMB_LAYER = -1
+EMBEDDING_DIM = 768
+THRESHOLD = 0.55
+SPLIT_BY = word
+SPLIT_LENGTH = 120
+SPLIT_OVERLAP = 10
+RESPECT_SENTENCE_BOUNDARY = 1
+REMOVE_PUNC = 0

requirements.txt

@@ -0,0 +1,19 @@
+farm-haystack == 1.10
+farm-haystack[ocr]==1.10.0
+spacy==3.2.0
+https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0.tar.gz#egg=en_core_web_sm
+matplotlib==3.5.1
+nltk==3.7
+numpy==1.22.1
+pandas==1.4.0
+pdfplumber==0.6.2
+Pillow==9.1.1
+seaborn==0.11.2
+transformers==4.21.2
+st-annotated-text==3.0.0
+markdown==3.4.1
+summa==1.2.0
+altair==4.0
+streamlit-aggrid
+python-docx
+streamlit_option_menu

style.css

@@ -0,0 +1,180 @@
+
+.row-widget.stTextInput > div:first-of-type {
+    background: #fff;
+    display: flex;
+    border: 1px solid #dfe1e5;
+    box-shadow: none;
+    border-radius: 24px;
+    height: 50px;
+    width: auto;
+    margin: 10px auto 30px;
+}
+
+.row-widget.stTextInput > div:first-of-type:hover,
+.row-widget.stTextInput > div:first-of-type:focus {
+    box-shadow: 1px 1px 2px 1px rgba(0, 0, 0, 0.2);
+}
+
+.row-widget.stTextInput .st-bq {
+    background-color: #fff;
+}
+
+.row-widget.stTextInput > label {
+    color: #b3b3b3;
+}
+
+.row-widget.stButton > button {
+    border-radius: 24px;
+    background-color: #B6C9B1;
+    color: #fff;
+    border: none;
+    padding: 6px 20px;
+    float: right;
+    background-image: none;
+}
+
+.row-widget.stButton > button:hover {
+    box-shadow: 1px 1px 2px 1px rgba(0, 0, 0, 0.2);
+}
+
+.row-widget.stButton > button:focus {
+    border: none;
+    color: #fff;
+}
+
+.footer-custom {
+    position: fixed;
+    bottom: 0;
+    width: 100%;
+    color: var(--text-color);
+    max-width: 698px;
+    font-size: 14px;
+    height: 50px;
+    padding: 10px 0;
+    z-index: 50;
+}
+
+.main {
+    padding: 20px;
+}
+
+footer {
+    display: none !important;
+}
+
+.footer-custom a {
+    color: var(--text-color);
+}
+
+#wikipedia-assistant {
+    font-size: 36px;
+}
+
+.generated-answer p {
+    font-size: 16px;
+    font-weight: bold;
+}
+
+.react-json-view {
+    margin: 40px 0 80px;
+}
+
+.tooltip {
+    text-align: center;
+    line-height: 20px;
+    display: table-caption;
+    font-size: 10px;
+    border-radius: 50%;
+    height: 20px;
+    width: 20px;
+    position: relative;
+    cursor: pointer;
+    color:#000;
+}
+
+.tooltip .tooltiptext {
+    visibility: hidden;
+    width: 280px;
+    text-align: center;
+    border-radius: 6px;
+    padding: 10px;
+    position: absolute;
+    z-index: 1;
+    top: 25px;
+    left: 50%;
+    margin-left: -140px;
+    font-size: 14px;
+    background-color: #fff;
+    border: 1px solid #ccc;
+    box-shadow: 0px 0px 3px 1px rgba(0, 0, 0, 0.16);
+    color: #000;
+}
+
+.tooltip:hover .tooltiptext {
+    visibility: visible;
+}
+
+.sentence-wrapper {
+    border-left: 4px solid #ffc423;
+    padding-left: 20px;
+    margin-bottom: 40px;
+}
+
+#context {
+    padding: 2rem 0 1rem;
+}
+
+hr {
+    margin: 2em 0 1em;
+}
+
+  
+.technical-details-info {
+    margin-bottom: 100px;
+}
+
+.loader-wrapper {
+    display: flex;
+    align-items: center;
+    background-color: rgba(250, 202, 43, 0.2);
+    padding: 15px 20px;
+    border-radius: 6px;
+}
+
+.loader-wrapper p {
+    margin-bottom: 0;
+    margin-left: 20px;
+}
+
+.loader {
+    width: 30px;
+    height: 30px;
+    border: dotted 5px #868686;
+    border-radius: 100%;
+    animation: spin 1s linear infinite;
+}
+
+.loader-note {
+    font-size: 14px;
+    color: #b3b3b3;
+    margin-left: 5px;
+}
+
+@keyframes spin {
+  0% { 
+    transform: rotate(0deg) scale(0.8); 
+    border-top-color: transparent;
+    border-right-color: transparent;
+  }
+  50% { transform: rotate(180deg) scale(1.2);   
+    border-color: #949494; 
+    border-top-color: transparent;
+    border-right-color: transparent;
+  }
+  100% { transform: rotate(360deg) scale(0.8); 
+    border-color: #bbbbbb;     
+    border-top-color: transparent;
+    border-right-color: transparent;
+  }
+}
+

utils/__init__.py

@@ -0,0 +1 @@
+# adding for package implementation

utils/checkconfig.py

@@ -0,0 +1,15 @@
+import configparser
+import logging
+
+def getconfig(configfile_path:str):
+    """
+    configfile_path: file path of .cfg file
+    """
+
+    config = configparser.ConfigParser()
+
+    try:
+        config.read_file(open(configfile_path))
+        return config
+    except:
+        logging.warning("config file not found")

utils/keyword_extraction.py

@@ -0,0 +1,140 @@
+import pandas as pd
+# from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+# import nltk
+# nltk.download('stopwords')
+# from nltk.corpus import stopwords
+import pickle
+from typing import List, Text
+import logging
+from summa import keywords
+
+try:
+    import streamlit as st    
+except ImportError:
+    logging.info("Streamlit not installed")
+
+
+def sort_coo(coo_matrix):
+    """
+    It takes Coordinate format scipy sparse matrix and extracts info from same.\
+    1. https://kavita-ganesan.com/python-keyword-extraction/#.Y2-TFHbMJPb
+    """
+    tuples = zip(coo_matrix.col, coo_matrix.data)
+    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)
+
+def extract_topn_from_vector(feature_names, sorted_items, top_n=10):
+    """get the feature names and tf-idf score of top n items
+    
+    Params
+    ---------
+    feature_names: list of words from vectorizer
+    sorted_items: tuple returned by sort_coo function defined in  \
+    keyword_extraction.py
+    topn: topn words to be extracted using tfidf
+
+    Return
+    ----------
+    results: top extracted keywords
+
+    """
+    
+    #use only topn items from vector
+    sorted_items = sorted_items[:top_n]
+    score_vals = []
+    feature_vals = []
+    
+    # word index and corresponding tf-idf score
+    for idx, score in sorted_items:
+        
+        #keep track of feature name and its corresponding score
+        score_vals.append(round(score, 3))
+        feature_vals.append(feature_names[idx])
+
+    results= {}
+    for idx in range(len(feature_vals)):
+        results[feature_vals[idx]]=score_vals[idx]
+    
+    return results
+
+
+def tfidf_keyword(textdata:str, vectorizer, tfidfmodel, top_n):
+    """
+    TFIDF based keywords extraction
+    
+    Params
+    ---------
+    vectorizer: trained cont vectorizer model
+    tfidfmodel: TFIDF Tranformer model
+    top_n: Top N keywords to be extracted
+    textdata: text data to which needs keyword extraction
+
+    Return
+    ----------
+    keywords: top extracted keywords
+
+    """
+    features = vectorizer.get_feature_names_out()
+    tf_idf_vector=tfidfmodel.transform(vectorizer.transform(textdata))
+    sorted_items=sort_coo(tf_idf_vector.tocoo())
+    results=extract_topn_from_vector(features,sorted_items,top_n)
+    keywords = [keyword for keyword in results]
+    return keywords
+
+def keyword_extraction(sdg:int,sdgdata:List[Text], top_n:int=10):
+    """
+    TFIDF based keywords extraction
+    
+    Params
+    ---------
+    sdg: which sdg tfidf model to be used
+    sdgdata: text data to which needs keyword extraction
+
+
+    Return
+    ----------
+    keywords: top extracted keywords
+
+    """
+    model_path = "docStore/sdg{}/".format(sdg)
+    vectorizer = pickle.load(open(model_path+'vectorizer.pkl', 'rb'))
+    tfidfmodel = pickle.load(open(model_path+'tfidfmodel.pkl', 'rb'))
+    features = vectorizer.get_feature_names_out()
+    tf_idf_vector=tfidfmodel.transform(vectorizer.transform(sdgdata))
+    sorted_items=sort_coo(tf_idf_vector.tocoo())
+    top_n = top_n
+    results=extract_topn_from_vector(features,sorted_items,top_n)
+    keywords = [keyword for keyword in results]
+    return keywords
+
+@st.cache(allow_output_mutation=True)
+def textrank(textdata:Text, ratio:float = 0.1, words:int = 0)->List[str]:
+    """
+    wrappper function to perform textrank, uses either ratio or wordcount to
+    extract top keywords limited by words or ratio.
+    1. https://github.com/summanlp/textrank/blob/master/summa/keywords.py
+
+    Params
+    --------
+    textdata: text data to perform the textrank.
+    ratio: float to limit the number of keywords as proportion of total token \
+        in textdata
+    words: number of keywords to be extracted. Takes priority over ratio if \
+        Non zero. Howevr incase the pagerank returns lesser keywords than \
+        compared to fix value then ratio is used.
+    
+    Return
+    --------
+    results: extracted keywords
+    """
+    if words == 0:
+        logging.info("Textrank using defulat ratio value = 0.1, as no words limit given")
+        results = keywords.keywords(textdata, ratio= ratio).split("\n")
+    else:
+        try:
+            results = keywords.keywords(textdata, words= words).split("\n")
+        except:
+            results = keywords.keywords(textdata, ratio = ratio).split("\n")
+
+    return results
+
+

utils/lexical_search.py

@@ -0,0 +1,251 @@
+from haystack.nodes import TfidfRetriever
+from haystack.document_stores import InMemoryDocumentStore
+import spacy
+import re
+from spacy.matcher import Matcher
+from markdown import markdown
+from annotated_text import annotation
+from haystack.schema import Document
+from typing import List, Text, Tuple
+from typing_extensions import Literal
+from utils.preprocessing import processingpipeline
+from utils.streamlitcheck import check_streamlit
+import logging
+try:
+    from termcolor import colored
+except:
+    pass
+
+try:
+    import streamlit as st    
+except ImportError:
+    logging.info("Streamlit not installed")
+
+
+def runLexicalPreprocessingPipeline(file_name:str,file_path:str,
+                        split_by: Literal["sentence", "word"] = 'word', 
+                        split_length:int = 80, split_overlap:int = 0, 
+                        remove_punc:bool = False,)->List[Document]:
+    """
+    creates the pipeline and runs the preprocessing pipeline, 
+    the params for pipeline are fetched from paramconfig. As lexical doesnt gets
+    affected by overlap, threfore split_overlap = 0 in default paramconfig and 
+    split_by = word.
+
+    Params
+    ------------
+
+    file_name: filename, in case of streamlit application use 
+    st.session_state['filename']
+    file_path: filepath, in case of streamlit application use 
+    st.session_state['filepath']
+    split_by: document splitting strategy either as word or sentence
+    split_length: when synthetically creating the paragrpahs from document,
+                    it defines the length of paragraph.
+    split_overlap: Number of words or sentences that overlap when creating
+        the paragraphs. This is done as one sentence or 'some words' make sense
+        when  read in together with others. Therefore the overlap is used.
+    splititng of text.
+    removePunc: to remove all Punctuation including ',' and '.' or not
+
+    Return
+    --------------
+    List[Document]: When preprocessing pipeline is run, the output dictionary 
+    has four objects. For the lexicaal search using TFIDFRetriever we 
+    need to use the List of Haystack Document, which can be fetched by 
+    key = 'documents' on output.
+
+    """
+    
+    lexical_processing_pipeline = processingpipeline()
+
+
+    output_lexical_pre = lexical_processing_pipeline.run(file_paths = file_path, 
+                            params= {"FileConverter": {"file_path": file_path, \
+                                        "file_name": file_name}, 
+                                        "UdfPreProcessor": {"remove_punc": remove_punc, \
+                                            "split_by": split_by, \
+                                            "split_length":split_length,\
+                                            "split_overlap": split_overlap}})
+
+    return output_lexical_pre
+
+
+def tokenize_lexical_query(query:str)-> List[str]:
+    """
+    Removes the stop words from query and returns the list of important keywords
+    in query. For the lexical search the relevent paragraphs in document are 
+    retreived using TfIDFretreiver from Haystack. However to highlight these 
+    keywords we need the tokenized form of query.
+
+    Params
+    --------
+    query: string which represents either list of keywords user is looking for 
+            or a query in form of Question.
+    
+    Return
+    -----------
+    token_list: list of important keywords in the query.
+     
+    """
+    nlp = spacy.load("en_core_web_sm")    
+    token_list = [token.text.lower() for token in nlp(query) 
+                  if not (token.is_stop or token.is_punct)]
+    return token_list
+
+def runSpacyMatcher(token_list:List[str], document:Text
+                    )->Tuple[List[List[int]],spacy.tokens.doc.Doc]:
+    """
+    Using the spacy in backend finds the keywords in the document using the 
+    Matcher class from spacy. We can alternatively use the regex, but spacy
+    finds all keywords in serialized manner which helps in annotation of answers.
+
+    Params
+    -------
+    token_list: this is token list which tokenize_lexical_query function returns
+    document: text in which we need to find the tokens
+
+    Return
+    --------
+    matches: List of [start_index, end_index] in the spacydoc(at word level not 
+    character) for the keywords in token list.
+
+    spacydoc: the keyword index in the spacydoc are at word level and not character,
+    therefore to allow the annotator to work seamlessly we return the spacydoc.
+
+    """
+    nlp = spacy.load("en_core_web_sm")
+    spacydoc = nlp(document)
+    matcher = Matcher(nlp.vocab)
+    token_pattern = [[{"LOWER":token}] for token in token_list]
+    matcher.add(",".join(token_list), token_pattern)
+    spacymatches = matcher(spacydoc)
+
+    # getting start and end index in spacydoc so that annotator can work seamlessly
+    matches = []
+    for match_id, start, end in spacymatches:
+        matches = matches + [[start, end]]
+    
+    return matches, spacydoc
+
+def runRegexMatcher(token_list:List[str], document:Text):
+    """
+    Using the regex in backend finds the keywords in the document.
+
+    Params
+    -------
+    token_list: this is token list which tokenize_lexical_query function returns
+
+    document: text in which we need to find the tokens
+
+    Return
+    --------
+    matches: List of [start_index, end_index] in the document for the keywords 
+    in token list at character level.
+
+    document: the keyword index returned by regex are at character level,
+    therefore to allow the annotator to work seamlessly we return the text back.
+
+    """
+    matches = []
+    for token in token_list:
+        matches = (matches + 
+                  [[val.start(), val.start() + 
+                  len(token)] for val in re.finditer(token, document)])
+    
+    return matches, document
+
+def spacyAnnotator(matches: List[List[int]], document:spacy.tokens.doc.Doc):
+    """
+    This is spacy Annotator and needs spacy.doc
+    Annotates the text in the document defined by list of [start index, end index]
+    Example: "How are you today", if document type is text, matches = [[0,3]]
+    will give answer = "How", however in case we used the spacy matcher then the
+    matches = [[0,3]] will give answer = "How are you". However if spacy is used
+    to find "How" then the matches = [[0,1]] for the string defined above.
+
+    Params
+    -----------
+    matches: As mentioned its list of list. Example [[0,1],[10,13]]
+    document: document which needs to be indexed.
+
+
+    Return
+    --------
+    will send the output to either app front end using streamlit or 
+    write directly to output screen.
+
+    """
+    start = 0
+    annotated_text = ""
+    for match in matches:
+        start_idx = match[0]
+        end_idx = match[1]
+
+        if check_streamlit():
+            annotated_text = (annotated_text + document[start:start_idx].text 
+                            + str(annotation(body=document[start_idx:end_idx].text,
+                            label="ANSWER", background="#964448", color='#ffffff')))
+        else:
+            annotated_text = (annotated_text + document[start:start_idx].text
+                            + colored(document[start_idx:end_idx].text,
+                          "green", attrs = ['bold']))
+
+
+        start = end_idx
+    
+    annotated_text = annotated_text + document[end_idx:].text
+
+
+    if check_streamlit():
+
+        st.write(
+                markdown(annotated_text),
+                unsafe_allow_html=True,
+            )
+    else:
+        print(annotated_text)
+
+def lexical_search(query:Text, documents:List[Document],top_k:int):
+    """
+    Performs the Lexical search on the List of haystack documents which is 
+    returned by preprocessing Pipeline.
+
+    Params
+    -------
+    query: Keywords that need to be searche in documents.
+    documents: List of Haystack documents returned by preprocessing pipeline.
+    top_k: Number of Top results to be fetched.
+    
+    """
+
+    document_store = InMemoryDocumentStore()
+    document_store.write_documents(documents)
+    
+    # Haystack Retriever works with document stores only.
+    retriever = TfidfRetriever(document_store)
+    results = retriever.retrieve(query=query, top_k = top_k)          
+    query_tokens = tokenize_lexical_query(query)
+    flag = True
+    for count, result in enumerate(results):
+        matches, doc = runSpacyMatcher(query_tokens,result.content)
+
+        if len(matches) != 0:
+            if flag:
+                flag = False
+                if check_streamlit():
+                    st.markdown("##### Top few lexical search (TFIDF) hits #####")
+                else:
+                    print("Top few lexical search (TFIDF) hits")
+            
+            if check_streamlit():
+                st.write("Result {}".format(count+1))
+            else:
+                print("Results {}".format(count +1))
+            spacyAnnotator(matches, doc)
+
+    if flag:
+        if check_streamlit():
+            st.info("🤔 No relevant result found. Please try another keyword.")
+        else:
+            print("No relevant result found. Please try another keyword.")   

utils/ndc_explorer.py

@@ -0,0 +1,90 @@
+
+import urllib.request
+import json
+
+link = "https://klimalog.die-gdi.de/ndc/open-data/dataset.json"
+def get_document(country_code: str):
+    """
+    read the country NDC data from 
+    https://klimalog.die-gdi.de/ndc/open-data/dataset.json 
+    using the country code.
+    
+    Params
+    -------
+    country_code:"""
+    with urllib.request.urlopen(link) as urlfile:
+        data =  json.loads(urlfile.read())
+    categoriesData = {}
+    categoriesData['categories']= data['categories']
+    categoriesData['subcategories']= data['subcategories']
+    keys_sub = categoriesData['subcategories'].keys()
+    documentType= 'NDCs'
+    if documentType in data.keys():
+        if country_code in data[documentType].keys():
+            get_dict = {}
+            for key, value in data[documentType][country_code].items():
+                if key not in ['country_name','region_id', 'region_name']:
+                    get_dict[key] = value['classification']
+                else:
+                    get_dict[key] = value
+        else:
+            return None
+    else:
+        return None
+
+    country = {}
+    for key in categoriesData['categories']:
+        country[key]= {}
+    for key,value in categoriesData['subcategories'].items():
+        country[value['category']][key] = get_dict[key]
+    
+    return country
+        
+            
+def countrySpecificCCA(cca_sent:dict, threshold:int, countryCode:str):
+    """
+    based on the countrycode, reads the country data from
+    https://klimalog.die-gdi.de/ndc/open-data/dataset.json
+    using get_documents from utils.ndc_explorer.py
+    then based on thereshold value filters the Climate Change Adaptation
+    targets assigned by NDC explorer team to that country. Using the sentences
+    create by Data services team of GIZ for each target level, tries to find the
+    relevant passages from the document by doing the semantic search.
+
+    Params
+    -------
+    cca_sent: dictionary with key as 'target labels' and manufactured sentences 
+    reflecting the target level. Please see the docStore/ndcs/cca.txt
+
+    threshold: NDC target have many categoriees ranging from [0-5], with 0 
+    refelcting most relaxed attitude and 5 being most aggrisive towards Climate 
+    change. We select the threshold value beyond which we need to focus on.
+
+    countryCode: standard country code to allow us to fetch the country specific
+    data.
+
+    """
+    temp = {}
+    doc = get_document(countryCode)
+    for key,value in cca_sent.items():
+        id_ = doc['climate change adaptation'][key]['id']
+        if id_ >threshold:
+            temp[key] = value['id'][id_]
+    return temp
+
+                
+def countrySpecificCCM(ccm_sent, threshold, countryCode):
+    """
+    see the documentation of countrySpecificCCA. This is same instead of 
+    this gets the data pertaining to Adaptation
+    
+    """
+
+    temp = {}
+    doc = get_document(countryCode)
+    for key,value in ccm_sent.items():
+        id_ = doc['climate change mitigation'][key]['id']
+        if id_ >threshold:
+            temp[key] = value['id'][id_]
+    
+    return temp

utils/preprocessing.py

@@ -0,0 +1,260 @@
+from haystack.nodes.base import BaseComponent
+from haystack.schema import Document
+from haystack.nodes import PDFToTextOCRConverter, PDFToTextConverter
+from haystack.nodes import TextConverter, DocxToTextConverter, PreProcessor
+from typing import Callable, Dict, List, Optional, Text, Tuple, Union
+from typing_extensions import Literal
+import pandas as pd
+import logging
+import re
+import string
+from haystack.pipelines import Pipeline
+
+def useOCR(file_path: str)-> Text:
+    """
+    Converts image pdfs into text, Using the Farm-haystack[OCR]
+    
+    Params
+    ----------
+    file_path: file_path of uploade file, returned by add_upload function in 
+    uploadAndExample.py
+    
+    Returns the text file as string.
+    """
+
+    
+    converter = PDFToTextOCRConverter(remove_numeric_tables=True, 
+                                      valid_languages=["eng"])
+    docs = converter.convert(file_path=file_path, meta=None)
+    return docs[0].content
+
+
+
+
+class FileConverter(BaseComponent):
+    """
+    Wrapper class to convert uploaded document into text by calling appropriate 
+    Converter class, will use internally haystack PDFToTextOCR in case of image 
+    pdf. Cannot use the FileClassifier from haystack as its doesnt has any 
+    label/output class for image.
+
+    1. https://haystack.deepset.ai/pipeline_nodes/custom-nodes
+    2. https://docs.haystack.deepset.ai/docs/file_converters
+    3. https://github.com/deepset-ai/haystack/tree/main/haystack/nodes/file_converter
+    4. https://docs.haystack.deepset.ai/reference/file-converters-api
+
+
+    """
+
+    outgoing_edges = 1
+
+    def run(self, file_name: str , file_path: str, encoding: Optional[str]=None,
+            id_hash_keys: Optional[List[str]] = None,
+            ) -> Tuple[dict,str]:
+        """ this is required method to invoke the component in 
+            the pipeline implementation. 
+            
+        Params
+        ----------
+        file_name: name of file
+        file_path: file_path of uploade file, returned by add_upload function in 
+                    uploadAndExample.py
+        
+        See the links provided in Class docstring/description to see other params
+        
+        Return
+        ---------
+        output: dictionary, with key as identifier and value could be anything 
+                we need to return. In this case its the List of Hasyatck Document
+        
+        output_1: As there is only one outgoing edge, we pass 'output_1' string
+        """
+        try:
+            if file_name.endswith('.pdf'):
+                converter = PDFToTextConverter(remove_numeric_tables=True)
+            if file_name.endswith('.txt'):
+                converter = TextConverter(remove_numeric_tables=True)
+            if file_name.endswith('.docx'):
+                converter = DocxToTextConverter() 
+        except Exception as e:
+            logging.error(e)
+            return 
+
+
+
+        documents = []
+
+        document = converter.convert(
+                      file_path=file_path, meta=None, 
+                      encoding=encoding, id_hash_keys=id_hash_keys
+                      )[0]
+
+        text = document.content
+
+        # if file is image pdf then it will have {'content': "\x0c\x0c\x0c\x0c"}
+        # subsitute this substring with '',and check if content is empty string
+
+        text = re.sub(r'\x0c', '', text)
+        documents.append(Document(content=text, 
+                              meta={"name": file_name}, 
+                              id_hash_keys=id_hash_keys))
+
+        
+        # check if text is empty and apply pdfOCR converter.
+        for i in documents:
+            if i.content == "":
+                logging.info("Using OCR")
+                i.content = useOCR(file_path)
+        
+        logging.info('file conversion succesful')
+        output = {'documents': documents}
+        return output, 'output_1'
+
+    def run_batch():
+        """
+        we dont have requirement to process the multiple files in one go
+        therefore nothing here, however to use the custom node we need to have
+        this method for the class.
+        """
+        
+        return
+
+
+def basic(s:str, remove_punc:bool = False):
+
+    """
+    Performs basic cleaning of text.
+
+    Params
+    ----------
+    s: string to be processed
+    removePunc: to remove all Punctuation including ',' and '.' or not
+    
+    Returns: processed string: see comments in the source code for more info
+    """
+    
+    # Remove URLs
+    s = re.sub(r'^https?:\/\/.*[\r\n]*', ' ', s, flags=re.MULTILINE)
+    s = re.sub(r"http\S+", " ", s)
+
+    # Remove new line characters
+    s = re.sub('\n', ' ', s) 
+
+    # Remove punctuations
+    if remove_punc == True:
+      translator = str.maketrans(' ', ' ', string.punctuation) 
+      s = s.translate(translator)
+    # Remove distracting single quotes and dotted pattern
+    s = re.sub("\'", " ", s)
+    s = s.replace("..","") 
+    
+    return s.strip()
+
+
+class UdfPreProcessor(BaseComponent):
+    """
+    class to preprocess the document returned by FileConverter. It will check
+    for splitting strategy and splits the document by word or sentences and then
+    synthetically create the paragraphs. 
+
+    1. https://docs.haystack.deepset.ai/docs/preprocessor
+    2. https://docs.haystack.deepset.ai/reference/preprocessor-api
+    3. https://github.com/deepset-ai/haystack/tree/main/haystack/nodes/preprocessor
+
+    """
+    outgoing_edges = 1
+
+    def run(self, documents:List[Document], remove_punc:bool=False, 
+            split_by: Literal["sentence", "word"] = 'sentence',
+            split_length:int = 2, split_respect_sentence_boundary:bool = False,
+            split_overlap:int = 0):
+
+        """ this is required method to invoke the component in 
+        the pipeline implementation. 
+            
+        Params
+        ----------
+        documents: documents from the output dictionary returned by Fileconverter
+        remove_punc: to remove all Punctuation including ',' and '.' or not
+        split_by: document splitting strategy either as word or sentence
+        split_length: when synthetically creating the paragrpahs from document,
+                      it defines the length of paragraph.
+        split_respect_sentence_boundary: Used when using 'word' strategy for 
+        splititng of text.
+        split_overlap: Number of words or sentences that overlap when creating
+        the paragraphs. This is done as one sentence or 'some words' make sense
+        when  read in together with others. Therefore the overlap is used.
+        
+        Return
+        ---------
+        output: dictionary, with key as identifier and value could be anything 
+                we need to return. In this case the output will contain 4 objects
+                the paragraphs text list as List, Haystack document, Dataframe and 
+                one raw text file.
+        
+        output_1: As there is only one outgoing edge, we pass 'output_1' string
+      
+        """
+        
+        if split_by == 'sentence':
+            split_respect_sentence_boundary = False
+    
+        else:
+            split_respect_sentence_boundary = split_respect_sentence_boundary
+      
+        preprocessor = PreProcessor(
+            clean_empty_lines=True,
+            clean_whitespace=True,
+            clean_header_footer=True,
+            split_by=split_by,
+            split_length=split_length,
+            split_respect_sentence_boundary= split_respect_sentence_boundary,
+            split_overlap=split_overlap,
+
+            # will add page number only in case of PDF not for text/docx file.
+            add_page_number=True
+            )
+        
+        for i in documents:
+            # # basic cleaning before passing it to preprocessor.
+            # i = basic(i)
+            docs_processed = preprocessor.process([i])
+            for item in docs_processed:
+                item.content = basic(item.content, remove_punc= remove_punc)
+
+        df = pd.DataFrame(docs_processed)
+        all_text = " ".join(df.content.to_list())
+        para_list = df.content.to_list()
+        logging.info('document split into {} paragraphs'.format(len(para_list)))
+        output = {'documents': docs_processed,
+                  'dataframe': df,
+                  'text': all_text,
+                  'paraList': para_list
+                 }
+        return output, "output_1"
+    def run_batch():
+        """
+            we dont have requirement to process the multiple files in one go
+            therefore nothing here, however to use the custom node we need to have
+            this method for the class.
+        """
+        return
+
+def processingpipeline():
+    """
+    Returns the preprocessing pipeline. Will use FileConverter and UdfPreProcesor 
+    from utils.preprocessing
+
+    """
+
+    preprocessing_pipeline = Pipeline()
+    file_converter = FileConverter()
+    custom_preprocessor = UdfPreProcessor()
+
+    preprocessing_pipeline.add_node(component=file_converter, 
+                                    name="FileConverter", inputs=["File"])
+    preprocessing_pipeline.add_node(component = custom_preprocessor, 
+                            name ='UdfPreProcessor', inputs=["FileConverter"])
+
+    return preprocessing_pipeline
+

utils/sdg_classifier.py

@@ -0,0 +1,177 @@
+from haystack.nodes import TransformersDocumentClassifier
+from haystack.schema import Document
+from typing import List, Tuple
+from typing_extensions import Literal
+import logging
+import pandas as pd
+from pandas import DataFrame, Series
+from utils.checkconfig import getconfig
+from utils.streamlitcheck import check_streamlit
+from utils.preprocessing import processingpipeline
+try:
+    import streamlit as st
+except ImportError:
+    logging.info("Streamlit not installed")
+
+## Labels dictionary ###
+_lab_dict = {0: 'no_cat',
+            1:'SDG 1 - No poverty',
+            2:'SDG 2 - Zero hunger',
+            3:'SDG 3 - Good health and well-being',
+            4:'SDG 4 - Quality education',
+            5:'SDG 5 - Gender equality',
+            6:'SDG 6 - Clean water and sanitation',
+            7:'SDG 7 - Affordable and clean energy',
+            8:'SDG 8 - Decent work and economic growth', 
+            9:'SDG 9 - Industry, Innovation and Infrastructure',
+            10:'SDG 10 - Reduced inequality',
+            11:'SDG 11 - Sustainable cities and communities',
+            12:'SDG 12 - Responsible consumption and production',
+            13:'SDG 13 - Climate action',
+            14:'SDG 14 - Life below water',
+            15:'SDG 15 - Life on land',
+            16:'SDG 16 - Peace, justice and strong institutions',
+            17:'SDG 17 - Partnership for the goals',}
+
+@st.cache(allow_output_mutation=True)
+def load_sdgClassifier(config_file:str = None, classifier_name:str = None):
+    """
+    loads the document classifier using haystack, where the name/path of model
+    in HF-hub as string is used to fetch the model object.Either configfile or 
+    model should be passed.
+    1. https://docs.haystack.deepset.ai/reference/document-classifier-api
+    2. https://docs.haystack.deepset.ai/docs/document_classifier
+
+    Params
+    --------
+    config_file: config file path from which to read the model name
+    classifier_name: if modelname is passed, it takes a priority if not \
+    found then will look for configfile, else raise error.
+
+
+    Return: document classifier model
+    """
+    if not classifier_name:
+        if not config_file:
+            logging.warning("Pass either model name or config file")
+            return
+        else:
+            config = getconfig(config_file)
+            classifier_name = config.get('sdg','MODEL')
+    
+    logging.info("Loading classifier")    
+    doc_classifier = TransformersDocumentClassifier(
+                        model_name_or_path=classifier_name,
+                        task="text-classification")
+
+    return doc_classifier
+        
+
+@st.cache(allow_output_mutation=True)
+def sdg_classification(haystack_doc:List[Document],
+                        threshold:float = 0.8, 
+                        classifier_model:TransformersDocumentClassifier= None
+                        )->Tuple[DataFrame,Series]:
+    """
+    Text-Classification on the list of texts provided. Classifier provides the 
+    most appropriate label for each text. these labels are in terms of if text 
+    belongs to which particular Sustainable Devleopment Goal (SDG).
+
+    Params
+    ---------
+    haystack_doc: List of haystack Documents. The output of Preprocessing Pipeline 
+    contains the list of paragraphs in different format,here the list of 
+    Haystack Documents is used.
+    threshold: threshold value for the model to keep the results from classifier
+    classifiermodel: you can pass the classifier model directly,which takes priority
+    however if not then looks for model in streamlit session.
+    In case of streamlit avoid passing the model directly.
+
+
+    Returns
+    ----------
+    df: Dataframe with two columns['SDG:int', 'text']
+    x: Series object with the unique SDG covered in the document uploaded and 
+    the number of times it is covered/discussed/count_of_paragraphs. 
+
+    """
+    logging.info("Working on SDG Classification")
+    if not classifier_model:
+        if check_streamlit():
+            classifier_model = st.session_state['sdg_classifier']
+        else:
+            logging.warning("No streamlit envinornment found, Pass the classifier")
+            return
+    
+    results = classifier_model.predict(haystack_doc)
+
+
+    labels_= [(l.meta['classification']['label'],
+            l.meta['classification']['score'],l.content,) for l in results]
+
+    df = DataFrame(labels_, columns=["SDG","Relevancy","text"])
+    
+    df = df.sort_values(by="Relevancy", ascending=False).reset_index(drop=True)  
+    df.index += 1
+    df =df[df['Relevancy']>threshold]
+
+    # creating the dataframe for value counts of SDG, along with 'title' of SDGs
+    x = df['SDG'].value_counts()
+    x = x.rename('count')
+    x = x.rename_axis('SDG').reset_index()
+    x["SDG"] = pd.to_numeric(x["SDG"])
+    x = x.sort_values(by=['count'], ascending=False)
+    x['SDG_name'] = x['SDG'].apply(lambda x: _lab_dict[x])
+    x['SDG_Num'] = x['SDG'].apply(lambda x: "SDG "+str(x))
+
+    df['SDG'] = pd.to_numeric(df['SDG'])
+    df = df.sort_values('SDG')
+    
+    return df, x
+
+def runSDGPreprocessingPipeline(file_name:str, file_path:str, 
+            split_by: Literal["sentence", "word"] = 'sentence',
+            split_length:int = 2, split_respect_sentence_boundary:bool = False,
+            split_overlap:int = 0,remove_punc:bool = False)->List[Document]:
+    """
+    creates the pipeline and runs the preprocessing pipeline, 
+    the params for pipeline are fetched from paramconfig
+
+    Params
+    ------------
+
+    file_name: filename, in case of streamlit application use 
+    st.session_state['filename']
+    file_path: filepath, in case of streamlit application use st.session_state['filepath']
+    split_by: document splitting strategy either as word or sentence
+    split_length: when synthetically creating the paragrpahs from document,
+                    it defines the length of paragraph.
+    split_respect_sentence_boundary: Used when using 'word' strategy for 
+    splititng of text.
+    split_overlap: Number of words or sentences that overlap when creating
+        the paragraphs. This is done as one sentence or 'some words' make sense
+        when  read in together with others. Therefore the overlap is used.
+    remove_punc: to remove all Punctuation including ',' and '.' or not
+
+
+    Return
+    --------------
+    List[Document]: When preprocessing pipeline is run, the output dictionary 
+    has four objects. For the Haysatck implementation of SDG classification we, 
+    need to use the List of Haystack Document, which can be fetched by 
+    key = 'documents' on output.
+
+    """
+
+    sdg_processing_pipeline = processingpipeline()
+
+    output_sdg_pre = sdg_processing_pipeline.run(file_paths = file_path, 
+                            params= {"FileConverter": {"file_path": file_path, \
+                                        "file_name": file_name}, 
+                                     "UdfPreProcessor": {"remove_punc": remove_punc, \
+                                            "split_by": split_by, \
+                                            "split_length":split_length,\
+                                            "split_overlap": split_overlap, \
+        "split_respect_sentence_boundary":split_respect_sentence_boundary}})
+    
+    return output_sdg_pre

utils/semantic_search.py

@@ -0,0 +1,582 @@
+from haystack.nodes import TransformersQueryClassifier, Docs2Answers
+from haystack.nodes import EmbeddingRetriever, FARMReader
+from haystack.nodes.base import BaseComponent
+from haystack.document_stores import InMemoryDocumentStore
+from markdown import markdown
+from annotated_text import annotation
+from haystack.schema import Document
+from typing import List, Text, Union
+from typing_extensions import Literal
+from utils.preprocessing import processingpipeline
+from utils.streamlitcheck import check_streamlit
+from haystack.pipelines import Pipeline
+import pandas as pd
+import logging
+try:
+    from termcolor import colored
+except:
+    pass
+try:
+    import streamlit as st    
+except ImportError:
+    logging.info("Streamlit not installed")
+
+
+@st.cache(allow_output_mutation=True)
+def loadQueryClassifier():
+    """
+    retuns the haystack query classifier model
+    model = shahrukhx01/bert-mini-finetune-question-detection
+    
+    """
+    query_classifier = TransformersQueryClassifier(model_name_or_path=
+                            "shahrukhx01/bert-mini-finetune-question-detection")
+    return query_classifier
+
+class QueryCheck(BaseComponent):
+    """
+    Uses Query Classifier from Haystack, process the query based on query type.
+    Ability to determine the statements is not so good, therefore the chances 
+    statement also get modified. Ex: "List water related issues" will be 
+    identified by the model as keywords, and therefore it be processed as "what 
+    are the 'list all water related issues' related issues and discussions?". 
+    This is one shortcoming but is igonred for now, as semantic search will not 
+    get affected a lot, by this. If you want to pass keywords list and want to 
+    do batch processing use. run_batch. Example: if you want to find relevant 
+    passages for water, food security, poverty then querylist = ["water", "food 
+    security","poverty"] and then execute QueryCheck.run_batch(queries = querylist)
+
+    1. https://docs.haystack.deepset.ai/docs/query_classifier
+
+    """
+
+    outgoing_edges = 1
+
+    def run(self, query:str):
+        """
+        mandatory method to use the custom node. Determines the query type, if 
+        if the query is of type keyword/statement will modify it to make it more
+        useful for sentence transoformers.
+
+        Params
+        --------
+        query: query/statement/keywords in form of string
+
+        Return
+        ------
+        output: dictionary, with key as identifier and value could be anything 
+                we need to return. In this case the output contain key = 'query'.
+        
+        output_1: As there is only one outgoing edge, we pass 'output_1' string
+        
+        """
+        query_classifier = loadQueryClassifier()
+        result = query_classifier.run(query=query)
+
+        if result[1] == "output_1":
+            output = {"query":query,
+                       "query_type": 'question/statement'}
+        else:
+            output = {"query": "what are the {} related issues and \
+                        discussions?".format(query),
+                      "query_type": 'statements/keyword'}
+        logging.info(output)
+        return output, "output_1"
+    
+    def run_batch(self, queries:List[str]):
+        """
+        running multiple queries in one go, howeevr need the queries to be passed
+        as list of string. Example: if you want to find relevant passages for
+        water, food security, poverty then querylist = ["water", "food security",
+        "poverty"] and then execute QueryCheck.run_batch(queries = querylist)
+
+        Params
+        --------
+        queries: queries/statements/keywords in form of string encapsulated 
+                within List
+
+        Return
+        ------
+        output: dictionary, with key as identifier and value could be anything 
+                we need to return. In this case the output contain key = 'queries'.
+        
+        output_1: As there is only one outgoing edge, we pass 'output_1' string
+        """
+        query_classifier = loadQueryClassifier()
+        query_list = []
+        for query in queries:
+            result = query_classifier.run(query=query)
+            if result[1] == "output_1":
+                query_list.append(query)
+            else:
+                query_list.append("what are the {} related issues and \
+                    discussions?".format(query))
+        output = {'queries':query_list}
+        logging.info(output)
+        return output, "output_1"
+
+
+@st.cache(allow_output_mutation=True)
+def runSemanticPreprocessingPipeline(file_path:str, file_name:str, 
+                split_by: Literal["sentence", "word"] = 'sentence',
+                split_length:int = 2, split_overlap:int = 0,
+                split_respect_sentence_boundary:bool = False,
+                remove_punc:bool = False)->List[Document]:
+    """
+    creates the pipeline and runs the preprocessing pipeline.
+
+    Params
+    ------------
+
+    file_name: filename, in case of streamlit application use 
+            st.session_state['filename']
+    file_path: filepath, in case of streamlit application use 
+            st.session_state['filepath']
+    split_by: document splitting strategy either as word or sentence
+    split_length: when synthetically creating the paragrpahs from document,
+            it defines the length of paragraph.
+    split_overlap: Number of words or sentences that overlap when creating the 
+            paragraphs. This is done as one sentence or 'some words' make sense
+            when  read in together with others. Therefore the overlap is used.
+    split_respect_sentence_boundary: Used when using 'word' strategy for 
+            splititng of text.
+    remove_punc: to remove all Punctuation including ',' and '.' or not
+
+    Return
+    --------------
+    List[Document]: When preprocessing pipeline is run, the output dictionary 
+        has four objects. For the Haysatck implementation of semantic search we, 
+        need to use the List of Haystack Document, which can be fetched by 
+        key = 'documents' on output.
+
+    """
+
+    semantic_processing_pipeline = processingpipeline()
+
+    output_semantic_pre = semantic_processing_pipeline.run(file_paths = file_path, 
+                            params= {"FileConverter": {"file_path": file_path, \
+                                        "file_name": file_name}, 
+                                "UdfPreProcessor": {"remove_punc": remove_punc, \
+                                            "split_by": split_by, \
+                                            "split_length":split_length,\
+                                            "split_overlap": split_overlap,
+        "split_respect_sentence_boundary":split_respect_sentence_boundary}})
+
+    return output_semantic_pre
+
+
+@st.cache(hash_funcs={"builtins.SwigPyObject": lambda _: None},
+                                        allow_output_mutation=True)
+def loadRetriever(embedding_model:Text=None, embedding_model_format:Text = None, 
+                 embedding_layer:int = None,  retriever_top_k:int = 10, 
+                 max_seq_len:int=512, document_store:InMemoryDocumentStore=None):
+    """
+    Returns the Retriever model based on params provided.
+    1. https://docs.haystack.deepset.ai/docs/retriever#embedding-retrieval-recommended
+    2. https://www.sbert.net/examples/applications/semantic-search/README.html
+    3. https://github.com/deepset-ai/haystack/blob/main/haystack/nodes/retriever/dense.py
+
+    
+    Params
+    ---------
+    embedding_model: Name of the model to be used for embedding. Check the links
+            provided in documentation
+    embedding_model_format: check the github link of Haystack provided in 
+            documentation embedding_layer: check the github link of Haystack 
+            provided in documentation retriever_top_k: Number of Top results to
+            be returned by 
+    retriever max_seq_len: everymodel has max seq len it can handle, check in 
+            model card. Needed to hanlde the edge cases.
+    document_store: InMemoryDocumentStore, write haystack Document list to 
+            DocumentStore and pass the same to function call. Can be done using 
+            createDocumentStore from utils.
+    
+    Return
+    -------
+    retriever: embedding model
+    """
+    logging.info("loading retriever")
+    if document_store is None:
+        logging.warning("Retriever initialization requires the DocumentStore")
+        return
+    
+    retriever = EmbeddingRetriever(
+                embedding_model=embedding_model,top_k = retriever_top_k,
+                document_store = document_store,
+                emb_extraction_layer=embedding_layer, scale_score =True,
+                model_format=embedding_model_format, use_gpu = True, 
+                max_seq_len = max_seq_len )
+    if check_streamlit:
+        st.session_state['retriever'] = retriever
+    return retriever
+
+@st.cache(hash_funcs={"builtins.SwigPyObject": lambda _: None},
+                    allow_output_mutation=True)
+def createDocumentStore(documents:List[Document], similarity:str = 'dot_product', 
+                        embedding_dim:int = 768):
+    """
+    Creates the InMemory Document Store from haystack list of Documents.
+    It is  mandatory component for Retriever to work in Haystack frame work.
+    
+    Params
+    -------
+    documents: List of haystack document. If using the preprocessing pipeline, 
+            can be fetched key = 'documents; on output of preprocessing pipeline.
+    similarity: scoring function, can be either 'cosine' or 'dot_product'
+    embedding_dim: Document store has default value of embedding size = 768, and
+            update_embeddings method of Docstore cannot infer the embedding size of 
+            retiever automatically, therefore set this value as per the model card.
+    
+    Return
+    -------
+    document_store: InMemory Document Store object type.
+    
+    """
+    document_store = InMemoryDocumentStore(similarity = similarity, 
+                                        embedding_dim = embedding_dim )
+    document_store.write_documents(documents)
+    
+    return document_store
+
+
+@st.cache(hash_funcs={"builtins.SwigPyObject": lambda _: None},
+                                        allow_output_mutation=True)
+def semanticSearchPipeline(documents:List[Document], embedding_model:Text =  None, 
+                embedding_model_format:Text = None,embedding_layer:int = None,
+                embedding_dim:int = 768,retriever_top_k:int = 10,
+                reader_model:str =  None, reader_top_k:int = 10,
+                max_seq_len:int =512,useQueryCheck = True,
+                top_k_per_candidate:int = 1):
+    """
+    creates the semantic search pipeline and document Store object from the
+    list of haystack documents. The top_k for the Reader and Retirever are kept  
+    same, so that all the results returned by Retriever are used, however the 
+    context is extracted by Reader for each retrieved result. The querycheck is 
+    added as node to process the query. This pipeline is suited for keyword search,
+    and to some extent extractive QA purpose. The purpose of Reader is strictly to
+    highlight the context for retrieved result and not for QA, however as stated
+    it can work for QA too in limited sense.
+    There are 4 variants of pipeline it can return
+    1.QueryCheck > Retriever > Reader
+    2.Retriever > Reader
+    3.QueryCheck > Retriever > Docs2Answers : If reader is None, 
+    then Doc2answer is used to keep the output of pipeline structurally same.
+    4.Retriever > Docs2Answers 
+
+    Links
+
+    1. https://docs.haystack.deepset.ai/docs/retriever#embedding-retrieval-recommended
+    2. https://www.sbert.net/examples/applications/semantic-search/README.html
+    3. https://github.com/deepset-ai/haystack/blob/main/haystack/nodes/retriever/dense.py
+    4. https://docs.haystack.deepset.ai/docs/reader
+
+
+    Params
+    ----------
+    documents: list of Haystack Documents, returned by preprocessig pipeline.
+    embedding_model: Name of the model to be used for embedding. Check the links
+            provided in documentation
+    embedding_model_format: check the github link of Haystack provided in 
+            documentation
+    embedding_layer: check the github link of Haystack provided in documentation
+    embedding_dim: Document store has default value of embedding size = 768, and
+            update_embeddings method of Docstore cannot infer the embedding size of 
+            retiever automatically, therefore set this value as per the model card.
+    retriever_top_k: Number of Top results to be returned by retriever
+    reader_model: Name of the model to be used for Reader node in hasyatck 
+            Pipeline. Check the links provided in documentation
+    reader_top_k: Reader will use retrieved results to further find better matches.
+            As purpose here is to use reader to extract context, the value is
+            same as retriever_top_k.
+    max_seq_len:everymodel has max seq len it can handle, check in model card. 
+            Needed to hanlde the edge cases
+    useQueryCheck: Whether to use the querycheck which modifies the query or not.
+    top_k_per_candidate:How many answers to extract for each candidate doc 
+            that is coming from the retriever
+
+    Return
+    ---------
+    semanticsearch_pipeline: Haystack Pipeline object, with all the necessary 
+            nodes [QueryCheck, Retriever, Reader/Docs2Answer]. If reader is None, 
+            then Doc2answer is used to keep the output of pipeline structurally 
+            same.
+
+    document_store: As retriever can work only with Haystack Document Store, the
+            list of document returned by preprocessing pipeline are fed into to
+            get InMemmoryDocumentStore object type, with retriever updating the 
+            embeddings of each paragraph in document store.
+
+    """
+    document_store = createDocumentStore(documents=documents, 
+                                    embedding_dim=embedding_dim)                  
+    retriever = loadRetriever(embedding_model = embedding_model,
+                    embedding_model_format=embedding_model_format,
+                    embedding_layer=embedding_layer,  
+                    retriever_top_k= retriever_top_k, 
+                    document_store = document_store,
+                    max_seq_len=max_seq_len)           
+    document_store.update_embeddings(retriever)
+    semantic_search_pipeline = Pipeline()
+    if useQueryCheck and reader_model:
+        querycheck = QueryCheck()
+        reader = FARMReader(model_name_or_path=reader_model,
+                    top_k = reader_top_k, use_gpu=True,
+                    top_k_per_candidate = top_k_per_candidate)
+        semantic_search_pipeline.add_node(component = querycheck, 
+                    name = "QueryCheck",inputs = ["Query"])
+        semantic_search_pipeline.add_node(component = retriever, 
+                    name = "EmbeddingRetriever",inputs = ["QueryCheck.output_1"])
+        semantic_search_pipeline.add_node(component = reader, name = "FARMReader",
+                                        inputs= ["EmbeddingRetriever"])
+
+    elif reader_model :
+        reader = FARMReader(model_name_or_path=reader_model,
+                    top_k = reader_top_k, use_gpu=True,
+                    top_k_per_candidate = top_k_per_candidate)
+        semantic_search_pipeline.add_node(component = retriever, 
+                    name = "EmbeddingRetriever",inputs = ["Query"])
+        semantic_search_pipeline.add_node(component = reader,
+                    name = "FARMReader",inputs= ["EmbeddingRetriever"])
+    elif useQueryCheck and not reader_model:
+        querycheck = QueryCheck()
+        docs2answers = Docs2Answers() 
+        semantic_search_pipeline.add_node(component = querycheck,
+                        name = "QueryCheck",inputs = ["Query"])
+        semantic_search_pipeline.add_node(component = retriever,
+                        name = "EmbeddingRetriever",inputs = ["QueryCheck.output_1"])
+        semantic_search_pipeline.add_node(component = docs2answers,
+                        name = "Docs2Answers",inputs= ["EmbeddingRetriever"])
+    elif not useQueryCheck and not reader_model:
+        docs2answers = Docs2Answers()
+        semantic_search_pipeline.add_node(component = retriever, 
+                        name = "EmbeddingRetriever",inputs = ["Query"])
+        semantic_search_pipeline.add_node(component = docs2answers, 
+                        name = "Docs2Answers",inputs= ["EmbeddingRetriever"])            
+        
+    logging.info(semantic_search_pipeline.components)
+    return semantic_search_pipeline, document_store
+
+def runSemanticPipeline(pipeline:Pipeline, queries:Union[list,str])->dict:
+    """
+    will use the haystack run or run_batch based on if single query is passed 
+    as string or multiple queries as List[str]
+    
+    Params
+    -------
+    pipeline: haystack pipeline, this is same as returned by semanticSearchPipeline
+            from utils.semanticsearch
+
+    queries: Either a single query or list of queries.
+
+    Return
+    -------
+    results: Dict containing answers and documents as key and their respective 
+            values
+
+    """
+
+    if type(queries) == list:
+        results = pipeline.run_batch(queries=queries)
+    elif type(queries) == str:
+        results = pipeline.run(query=queries)
+    else:
+        logging.info("Please check the input type for the queries")
+        return
+
+    return results
+
+def process_query_output(results:dict)->pd.DataFrame:
+    """
+    Returns the dataframe with necessary information like including
+    ['query','answer','answer_offset','context_offset','context','content',
+    'reader_score','retriever_score','id',]. This is designed for output given 
+    by semantic search pipeline with single query and final node as reader.
+    The output of pipeline having Docs2Answers as final node or multiple queries
+    need to be handled separately. In these other cases, use process_semantic_output
+    from utils.semantic_search which uses this function internally to make one
+    combined dataframe.
+    
+    Params
+    ---------
+    results: this dictionary should have key,values with 
+            keys = [query,answers,documents], however answers is optional.
+            in case of [Doc2Answers as final node], process_semantic_output 
+            doesnt return answers thereby setting all values contained in 
+            answers to 'None'
+        
+    Return
+    --------
+    df: dataframe with all the columns mentioned in function description.
+    
+    """
+    query_text = results['query']
+    if 'answers' in results.keys():
+        answer_dict = {}
+
+        for answer in results['answers']:
+            answer_dict[answer.document_id] = answer.to_dict()
+    else:
+        answer_dict = {}
+    docs = results['documents']
+    df = pd.DataFrame(columns=['query','answer','answer_offset','context_offset',
+                            'context','content','reader_score','retriever_score',
+                            'id'])
+    for doc in docs:
+        row_list = {}
+        row_list['query'] = query_text
+        row_list['retriever_score'] = doc.score
+        row_list['id'] = doc.id
+        row_list['content'] = doc.content
+        if doc.id in answer_dict.keys():
+            row_list['answer'] = answer_dict[doc.id]['answer']
+            row_list['context'] = answer_dict[doc.id]['context']
+            row_list['reader_score'] = answer_dict[doc.id]['score']
+            answer_offset = answer_dict[doc.id]['offsets_in_document'][0]
+            row_list['answer_offset'] = [answer_offset['start'],answer_offset['end']]
+            start_idx = doc.content.find(row_list['context'])
+            end_idx = start_idx + len(row_list['context'])
+            row_list['context_offset'] = [start_idx, end_idx]
+        else:
+            row_list['answer'] = None
+            row_list['context'] = None
+            row_list['reader_score'] = None
+            row_list['answer_offset'] = None
+            row_list['context_offset'] = None
+        df_dictionary = pd.DataFrame([row_list])
+        df = pd.concat([df, df_dictionary], ignore_index=True)
+    
+    return df
+
+def process_semantic_output(results):
+    """
+    Returns the dataframe with necessary information like including
+    ['query','answer','answer_offset','context_offset','context','content',
+    'reader_score','retriever_score','id',]. Distingushes if its single query or
+    multi queries by reading the pipeline output dictionary keys.
+    Uses the process_query_output to get the dataframe for each query and create
+    one concataneted dataframe. In case of Docs2Answers as final node, deletes 
+    the answers part. See documentations of process_query_output.
+    
+    Params
+    ---------
+    results: raw output of runSemanticPipeline. 
+        
+    Return
+    --------
+    df: dataframe with all the columns mentioned in function description.
+    
+    """
+    output = {}
+    if 'query' in results.keys():
+        output['query'] = results['query']
+        output['documents'] = results['documents']
+        if results['node_id'] == 'Docs2Answers':
+            pass
+        else:
+            output['answers'] = results['answers']
+        df = process_query_output(output)
+        return df
+    if 'queries' in results.keys():
+        df = pd.DataFrame(columns=['query','answer','answer_offset',
+                                   'context_offset','context','content',
+                                   'reader_score','retriever_score','id'])
+        for query,answers,documents in zip(results['queries'],
+                    results['answers'],results['documents']):
+            output = {}
+            output['query'] = query
+            output['documents'] = documents
+            if results['node_id'] == 'Docs2Answers':
+                    pass
+            else:
+                output['answers'] = answers
+            
+            temp = process_query_output(output)
+            df = pd.concat([df, temp], ignore_index=True)
+
+            
+    return df
+
+def semanticsearchAnnotator(matches:List[List[int]], document:Text):
+    """
+    Annotates the text in the document defined by list of [start index, end index]
+    Example: "How are you today", if document type is text, matches = [[0,3]]
+    will give answer = "How", however in case we used the spacy matcher then the
+    matches = [[0,3]] will give answer = "How are you". However if spacy is used
+    to find "How" then the matches = [[0,1]] for the string defined above.
+
+    """
+    start = 0
+    annotated_text = ""
+    for match in matches:
+        start_idx = match[0]
+        end_idx = match[1]
+        if check_streamlit():
+            annotated_text = (annotated_text + document[start:start_idx]
+                            + str(annotation(body=document[start_idx:end_idx],
+                            label="Context", background="#964448", color='#ffffff')))
+        else:
+            annotated_text = (annotated_text + document[start:start_idx]
+                            + colored(document[start_idx:end_idx],
+                          "green", attrs = ['bold']))
+        start = end_idx
+    
+    annotated_text = annotated_text + document[end_idx:]
+
+    if check_streamlit():
+
+        st.write(
+                markdown(annotated_text),
+                unsafe_allow_html=True,
+            )
+    else:
+        print(annotated_text)
+    
+
+def semantic_keywordsearch(query:Text,documents:List[Document],
+                embedding_model:Text, 
+                embedding_model_format:Text, 
+                embedding_layer:int,  reader_model:str,
+                retriever_top_k:int = 10, reader_top_k:int = 10,
+                return_results:bool = False, embedding_dim:int = 768,
+                max_seq_len:int = 512,top_k_per_candidate:int =1,
+                sort_by:Literal["retriever", "reader"] = 'retriever'):
+    """
+    Performs the Semantic search on the List of haystack documents which is 
+    returned by preprocessing Pipeline.
+
+    Params
+    -------
+    query: Keywords that need to be searche in documents.
+    documents: List fo Haystack documents returned by preprocessing pipeline.
+    
+    """
+    semanticsearch_pipeline, doc_store = semanticSearchPipeline(documents = documents,
+                        embedding_model= embedding_model, 
+                        embedding_layer= embedding_layer,
+                        embedding_model_format= embedding_model_format,
+                        reader_model= reader_model, retriever_top_k= retriever_top_k,
+                        reader_top_k= reader_top_k, embedding_dim=embedding_dim,
+                        max_seq_len=max_seq_len,
+                        top_k_per_candidate=top_k_per_candidate)
+
+    raw_output = runSemanticPipeline(semanticsearch_pipeline,query)
+    results_df = process_semantic_output(raw_output)
+    if sort_by == 'retriever':
+        results_df = results_df.sort_values(by=['retriever_score'], ascending=False)
+    else:
+        results_df = results_df.sort_values(by=['reader_score'], ascending=False)
+
+    if return_results:
+        return results_df
+    else:
+        if check_streamlit:
+            st.markdown("##### Top few semantic search results #####")
+        else:
+            print("Top few semantic search results")
+        for i in range(len(results_df)):
+            if check_streamlit:
+                st.write("Result {}".format(i+1))
+            else:
+                print("Result {}".format(i+1))
+            semanticsearchAnnotator([results_df.loc[i]['context_offset']],
+                        results_df.loc[i]['content'] )

utils/streamlitcheck.py

@@ -0,0 +1,42 @@
+import logging
+try:
+    import streamlit as st
+except ImportError:
+    logging.info("Streamlit not installed")
+
+
+def check_streamlit():
+    """
+    Function to check whether python code is run within streamlit
+
+    Returns
+    -------
+    use_streamlit : boolean
+        True if code is run within streamlit, else False
+    """
+    try:
+        from streamlit.scriptrunner.script_run_context import get_script_run_ctx
+        if not get_script_run_ctx():
+            use_streamlit = False
+        else:
+            use_streamlit = True
+    except ModuleNotFoundError:
+        use_streamlit = False
+    return use_streamlit
+
+def disable_other_checkboxes(*other_checkboxes_keys):
+    for checkbox_key in other_checkboxes_keys:
+        st.session_state[checkbox_key] = False
+
+def checkbox_without_preselect(keylist):
+    dict_ = {}
+    for i,key_val in enumerate(keylist):
+        dict_[i] = st.checkbox(key_val,key = key_val,
+        on_change = disable_other_checkboxes,
+        args=tuple(list(filter(lambda x: x!= key_val, keylist))),)
+    
+    for key,val in dict_.items():
+        if val == True:
+            return keylist[int(key)]
+    
+    return None

utils/uploadAndExample.py

@@ -0,0 +1,33 @@
+import streamlit as st
+import tempfile
+import json
+
+def add_upload(choice):
+    """
+    Provdies the user with choice to either 'Upload Document' or 'Try Example'.
+    Based on user choice runs streamlit processes and save the path and name of
+    the 'file' to streamlit session_state which then can be fetched later.
+
+    """
+    
+    if choice == 'Upload Document':
+        uploaded_file = st.sidebar.file_uploader('Upload the File',
+                            type=['pdf', 'docx', 'txt'])
+        if uploaded_file is not None:
+            with tempfile.NamedTemporaryFile(mode="wb", delete = False) as temp:
+                bytes_data = uploaded_file.getvalue()
+                temp.write(bytes_data)
+                st.session_state['filename'] = uploaded_file.name
+                st.session_state['filepath'] = temp.name
+
+                
+    else:
+        # listing the options
+        with open('docStore/sample/files.json','r') as json_file:
+            files = json.load(json_file)
+
+        option = st.sidebar.selectbox('Select the example document',
+                              list(files.keys()))
+        file_name = file_path  = files[option]
+        st.session_state['filename'] = file_name
+        st.session_state['filepath'] = file_path

ver0.1 scripts/cleaning.py

@@ -0,0 +1,168 @@
+import logging
+import pandas as pd
+import numpy as np
+import string  
+import nltk 
+import spacy
+import en_core_web_sm
+import re
+import streamlit as st
+
+from haystack.nodes import PreProcessor
+
+'''basic cleaning - suitable for transformer models'''
+def basic(s,SDG = False):
+    """
+    :param s: string to be processed
+    :return: processed string: see comments in the source code for more info
+    """
+    # Text Lowercase
+    #s = s.lower() 
+    # Remove punctuation
+    #translator = str.maketrans(' ', ' ', string.punctuation) 
+    #s = s.translate(translator)
+    # Remove URLs
+    s = re.sub(r'^https?:\/\/.*[\r\n]*', ' ', s, flags=re.MULTILINE)
+    s = re.sub(r"http\S+", " ", s)
+    if SDG == True:
+        s = s.lower()
+        translator = str.maketrans(' ', ' ', string.punctuation)  
+        s = s.translate(translator)
+        s = re.sub('\n', ' ', s)
+        s = re.sub("\'", " ", s)
+        s = re.sub(r'\d+', ' ', s)
+        s = re.sub(r'\W+', ' ', s) 
+
+    # Remove new line characters
+    #s = re.sub('\n', ' ', s) 
+  
+    # Remove distracting single quotes
+    #s = re.sub("\'", " ", s) 
+    # Remove all remaining numbers and non alphanumeric characters
+    #s = re.sub(r'\d+', ' ', s) 
+    #s = re.sub(r'\W+', ' ', s)
+
+    # define custom words to replace:
+    #s = re.sub(r'strengthenedstakeholder', 'strengthened stakeholder', s)
+    
+    return s.strip()
+
+
+def preprocessingForSDG(document):
+
+    """
+    takes in haystack document object and splits it into paragraphs and applies simple cleaning.
+
+    Returns cleaned list of haystack document objects. One paragraph per object. Also returns pandas df and 
+    list that contains all text joined together.
+    """    
+
+    preprocessor = PreProcessor(
+        clean_empty_lines=True,
+        clean_whitespace=True,
+        clean_header_footer=True,
+        split_by="word",
+        split_length=120,
+        split_respect_sentence_boundary=False,
+        #split_overlap=1
+    )
+    for i in document:
+        docs_processed = preprocessor.process([i])
+        for item in docs_processed:
+            item.content = basic(item.content, SDG = True)
+
+    with st.spinner("👑 document being splitted into paragraphs"):
+        logging.info("document has been splitted to {} paragraphs".format(len(docs_processed)))
+    
+    # create dataframe of text and list of all text
+    df = pd.DataFrame(docs_processed)
+    all_text = " ".join(df.content.to_list())
+    par_list = df.content.to_list()
+
+    return docs_processed, df, all_text, par_list
+
+def preprocessing(document):
+
+    """
+    takes in haystack document object and splits it into paragraphs and applies simple cleaning.
+
+    Returns cleaned list of haystack document objects. One paragraph per object. Also returns pandas df and 
+    list that contains all text joined together.
+    """    
+
+    preprocessor = PreProcessor(
+        clean_empty_lines=True,
+        clean_whitespace=True,
+        clean_header_footer=True,
+        split_by="sentence",
+        split_length=3,
+        split_respect_sentence_boundary=False,
+        split_overlap=1
+    )
+    for i in document:
+        docs_processed = preprocessor.process([i])
+        for item in docs_processed:
+            item.content = basic(item.content)
+
+    with st.spinner("👑 document being splitted into paragraphs"):
+        logging.info("document has been splitted to {} paragraphs".format(len(docs_processed)))
+    
+    # create dataframe of text and list of all text
+    df = pd.DataFrame(docs_processed)
+    all_text = " ".join(df.content.to_list())
+    par_list = df.content.to_list()
+
+    return docs_processed, df, all_text, par_list
+
+'''processing with spacy - suitable for models such as tf-idf, word2vec'''
+def spacy_clean(alpha:str, use_nlp:bool = True) -> str:
+
+    """
+
+    Clean and tokenise a string using Spacy. Keeps only alphabetic characters, removes stopwords and
+
+    filters out all but proper nouns, nounts, verbs and adjectives.
+
+    Parameters
+    ----------
+    alpha : str
+
+            The input string.
+
+    use_nlp : bool, default False
+
+            Indicates whether Spacy needs to use NLP. Enable this when using this function on its own.
+
+            Should be set to False if used inside nlp.pipeline   
+
+     Returns
+    -------
+    ' '.join(beta) : a concatenated list of lemmatised tokens, i.e. a processed string
+
+    Notes
+    -----
+    Fails if alpha is an NA value. Performance decreases as len(alpha) gets large.
+    Use together with nlp.pipeline for batch processing.
+
+    """
+
+    nlp = spacy.load("en_core_web_sm", disable=["parser", "ner", "textcat"])
+
+    if use_nlp:
+
+        alpha = nlp(alpha)
+
+        
+
+    beta = []
+
+    for tok in alpha:
+
+        if all([tok.is_alpha, not tok.is_stop, tok.pos_ in ['PROPN', 'NOUN', 'VERB', 'ADJ']]):
+
+            beta.append(tok.lemma_)
+
+            
+    text = ' '.join(beta)
+    text = text.lower()
+    return text

ver0.1 scripts/coherence.py

@@ -0,0 +1,267 @@
+# set path
+import glob, os, sys; sys.path.append('../udfPreprocess')
+
+#import helper
+import udfPreprocess.docPreprocessing as pre
+import udfPreprocess.cleaning as clean
+
+#import needed libraries
+import seaborn as sns
+from pandas import DataFrame
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+from sklearn.metrics.pairwise import cosine_similarity
+# from keybert import KeyBERT
+from transformers import pipeline
+import matplotlib.pyplot as plt
+import numpy as np
+import streamlit as st
+import pandas as pd 
+from rank_bm25 import BM25Okapi
+from sklearn.feature_extraction import _stop_words
+import string
+from tqdm.autonotebook import tqdm
+import numpy as np
+import urllib.request
+import ast
+import tempfile
+import sqlite3
+import json
+import urllib.request
+import ast
+import docx
+from docx.shared import Inches
+from docx.shared import Pt
+from docx.enum.style import WD_STYLE_TYPE 
+
+def app():
+    # Sidebar
+    st.sidebar.title('Check Coherence')
+    st.sidebar.write(' ')
+    with open('ndcs/countryList.txt') as dfile:
+        countryList = dfile.read()
+
+    countryList = ast.literal_eval(countryList)
+    countrynames = list(countryList.keys())
+    
+    option = st.sidebar.selectbox('Select Country', (countrynames))
+    countryCode = countryList[option]
+
+
+    with st.container():
+        st.markdown("<h1 style='text-align: center; color: black;'> Check Coherence of Policy Document with NDCs</h1>", unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=True):
+
+        st.write(
+            """     
+            The *Check Coherence* app is an easy-to-use interface built in Streamlit for doing analysis of policy document and finding the coherence between NDCs/New-Updated NDCs- developed by GIZ Data and the Sustainable Development Solution Network.
+            """
+        )
+
+        st.markdown("")
+
+    st.markdown("")
+    st.markdown("##  📌 Step One: Upload document of the country selected ")
+    
+    with st.container():
+            docs = None
+            # asking user for either upload or select existing doc
+            choice = st.radio(label = 'Select the Document',
+                              help = 'You can upload the document \
+                              or else you can try a example document.', 
+                              options = ('Upload Document', 'Try Example'), 
+                              horizontal = True)
+
+            if choice == 'Upload Document':
+              uploaded_file = st.file_uploader('Upload the File', type=['pdf', 'docx', 'txt'])
+              if uploaded_file is not None:
+                with tempfile.NamedTemporaryFile(mode="wb") as temp:
+                    bytes_data = uploaded_file.getvalue()
+                    temp.write(bytes_data)
+
+                    st.write("Uploaded Filename: ", uploaded_file.name)
+                    file_name =  uploaded_file.name
+                    file_path = temp.name
+                    docs = pre.load_document(file_path, file_name)
+                    haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+
+            else:
+              # listing the options
+              option = st.selectbox('Select the example document',
+                                    ('South Africa:Low Emission strategy', 
+                                    'Ethiopia: 10 Year Development Plan'))
+              if option is 'South Africa:Low Emission strategy':
+                file_name = file_path  = 'sample/South Africa_s Low Emission Development Strategy.txt'
+                countryCode = countryList['South Africa']
+                st.write("Selected document:", file_name.split('/')[1])
+                # with open('sample/South Africa_s Low Emission Development Strategy.txt') as dfile:
+                # file = open('sample/South Africa_s Low Emission Development Strategy.txt', 'wb')
+              else:
+                # with open('sample/Ethiopia_s_2021_10 Year Development Plan.txt') as dfile:
+                file_name = file_path =  'sample/Ethiopia_s_2021_10 Year Development Plan.txt'
+                countryCode = countryList['Ethiopia']
+                st.write("Selected document:", file_name.split('/')[1])
+              
+              if option is not None:
+                docs = pre.load_document(file_path,file_name)
+                haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+
+            with open('ndcs/cca.txt', encoding='utf-8', errors='ignore') as dfile:
+                cca_sent = dfile.read()
+
+            cca_sent = ast.literal_eval(cca_sent)
+            
+            with open('ndcs/ccm.txt', encoding='utf-8', errors='ignore') as dfile:
+                ccm_sent = dfile.read()
+
+            ccm_sent = ast.literal_eval(ccm_sent)
+            
+            with open('ndcs/countryList.txt') as dfile:
+                countryList = dfile.read()
+
+            countryList = ast.literal_eval(countryList)
+            
+            def get_document(countryCode: str):
+                link = "https://klimalog.die-gdi.de/ndc/open-data/dataset.json"  
+                with urllib.request.urlopen(link) as urlfile:
+                    data =  json.loads(urlfile.read())
+                categoriesData = {}
+                categoriesData['categories']= data['categories']
+                categoriesData['subcategories']= data['subcategories']
+                keys_sub = categoriesData['subcategories'].keys()
+                documentType= 'NDCs'
+                if documentType in data.keys():
+                    if countryCode in data[documentType].keys():
+                        get_dict = {}
+                        for key, value in data[documentType][countryCode].items():
+                            if key not in ['country_name','region_id', 'region_name']:
+                                get_dict[key] = value['classification']
+                            else:
+                                get_dict[key] = value
+                    else:
+                        return None
+                else:
+                    return None
+
+                country = {}
+                for key in categoriesData['categories']:
+                    country[key]= {}
+                for key,value in categoriesData['subcategories'].items():
+                    country[value['category']][key] = get_dict[key]
+                
+                return country
+        
+        #   country_ndc = get_document('NDCs', countryList[option])
+            
+            def countrySpecificCCA(cca_sent, threshold, countryCode):
+                temp = {}
+                doc = get_document(countryCode)
+                for key,value in cca_sent.items():
+                    id_ = doc['climate change adaptation'][key]['id']
+                    if id_ >threshold:
+                        temp[key] = value['id'][id_]
+                return temp
+            
+                
+            def countrySpecificCCM(ccm_sent, threshold, countryCode):
+                temp = {}
+                doc = get_document(countryCode)
+                for key,value in ccm_sent.items():
+                    id_ = doc['climate change mitigation'][key]['id']
+                    if id_ >threshold:
+                        temp[key] = value['id'][id_]
+                
+                return temp
+
+        
+        
+            if docs is not None:
+                    sent_cca = countrySpecificCCA(cca_sent,1,countryCode)
+                    sent_ccm = countrySpecificCCM(ccm_sent,1,countryCode)
+                    #st.write(sent_ccm)
+                    @st.cache(allow_output_mutation=True)
+                    def load_sentenceTransformer(name):
+                        return SentenceTransformer(name)
+                    model = load_sentenceTransformer('all-MiniLM-L6-v2')
+          
+                    document_embeddings = model.encode(paraList, show_progress_bar=True)
+                
+                    genre = st.radio( "Select Category",('Climate Change Adaptation', 'Climate Change Mitigation'))
+                    if genre == 'Climate Change Adaptation':
+                        sent_dict = sent_cca
+                        sent_labels = []
+                        for key,sent in sent_dict.items():
+                            sent_labels.append(sent)
+                        label_embeddings = model.encode(sent_labels, show_progress_bar=True)
+                        similarity_high_threshold = 0.55
+                        similarity_matrix = cosine_similarity(label_embeddings, document_embeddings)
+                        label_indices, paragraph_indices = np.where(similarity_matrix>similarity_high_threshold)
+
+                        positive_indices = list(zip(label_indices.tolist(), paragraph_indices.tolist()))
+                        
+                        
+                    else:
+                        sent_dict = sent_ccm
+                        sent_labels = []
+                        for key,sent in sent_dict.items():
+                            sent_labels.append(sent)
+                        label_embeddings = model.encode(sent_labels, show_progress_bar=True)
+                        similarity_high_threshold = 0.55
+                        similarity_matrix = cosine_similarity(label_embeddings, document_embeddings)
+                        label_indices, paragraph_indices = np.where(similarity_matrix>similarity_high_threshold)
+
+                        positive_indices = list(zip(label_indices.tolist(), paragraph_indices.tolist()))
+                        
+            
+                #    sent_labels = []
+                #   for key,sent in sent_dict.items():
+                  #      sent_labels.append(sent)
+                    
+            
+                  # label_embeddings = model.encode(sent_labels, show_progress_bar=True)
+            
+                    #similarity_high_threshold = 0.55
+                  # similarity_matrix = cosine_similarity(label_embeddings, document_embeddings)
+                    #label_indices, paragraph_indices = np.where(similarity_matrix>similarity_high_threshold)
+
+                    #positive_indices = list(zip(label_indices.tolist(), paragraph_indices.tolist()))
+                    document = docx.Document()
+                    document.add_heading('Document name:{}'.format(file_name), 2)
+                    section = document.sections[0]
+
+                      # Calling the footer
+                    footer = section.footer
+                    
+                    # Calling the paragraph already present in
+                    # the footer section
+                    footer_para = footer.paragraphs[0]
+                    
+                    font_styles = document.styles
+                    font_charstyle = font_styles.add_style('CommentsStyle', WD_STYLE_TYPE.CHARACTER)
+                    font_object = font_charstyle.font
+                    font_object.size = Pt(7)
+                    # Adding the centered zoned footer
+                    footer_para.add_run('''\tPowered by GIZ Data and the Sustainable Development Solution Network hosted at Hugging-Face spaces:                        https://huggingface.co/spaces/ppsingh/streamlit_dev''', style='CommentsStyle')
+                    
+                    document.add_paragraph("Country Code for which NDC is carried out {}".format(countryCode))
+                    
+                    for _label_idx, _paragraph_idx in positive_indices:
+                        st.write("This paragraph: \n")
+                        document.add_paragraph("This paragraph: \n")
+                        st.write(paraList[_paragraph_idx])
+                        st.write(f"Is relevant to: \n {list(sent_dict.keys())[_label_idx]}")
+                        document.add_paragraph(f"Is relevant to: \n {list(sent_dict.keys())[_label_idx]}")
+                        st.write('-'*10)
+                        document.add_paragraph('-'*10)
+                    
+                    document.save('demo.docx')
+                    with open("demo.docx", "rb") as file:
+                                btn = st.download_button(
+                                label="Download file",
+                                data=file,
+                                file_name="demo.docx",
+                                mime="txt/docx"
+                                  )  
+            

ver0.1 scripts/docPreprocessing.py

@@ -0,0 +1,75 @@
+from typing import Callable, Dict, List, Optional
+
+from pathlib import Path
+import re
+import logging
+import string 
+import streamlit as st
+logger = logging.getLogger(__name__)
+
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+from haystack.utils import convert_files_to_docs, fetch_archive_from_http
+from haystack.nodes.file_converter import BaseConverter, DocxToTextConverter 
+from haystack.nodes.file_converter import PDFToTextConverter, TextConverter
+from haystack.schema import Document
+import pdfplumber
+
+import pandas as pd
+
+import tempfile
+import sqlite3
+
+
+
+def load_document(
+    file_path: str,
+    file_name,
+    encoding: Optional[str] = None,
+    id_hash_keys: Optional[List[str]] = None,
+) -> List[Document]:
+    
+    """
+    takes docx, txt and pdf files as input and \
+    extracts text as well as the filename as metadata. \
+    Since haystack does not take care of all pdf files, \
+    pdfplumber is attached to the pipeline in case the pdf \ 
+    extraction fails via Haystack.
+
+    Returns a list of type haystack.schema.Document
+    """
+
+    if file_name.endswith('.pdf'):
+        converter = PDFToTextConverter(remove_numeric_tables=True)
+    if file_name.endswith('.txt'):
+        converter = TextConverter()
+    if file_name.endswith('.docx'):
+        converter = DocxToTextConverter()
+
+
+    documents = []
+    logger.info("Converting {}".format(file_name))
+    # PDFToTextConverter, TextConverter, and DocxToTextConverter 
+    # return a list containing a single Document
+    document = converter.convert(
+                file_path=file_path, meta=None, 
+                encoding=encoding, id_hash_keys=id_hash_keys
+                )[0]
+    text = document.content
+    documents.append(Document(content=text, 
+                              meta={"name": file_name}, 
+                              id_hash_keys=id_hash_keys))
+    
+    '''check if text is empty and apply different pdf processor. \
+    This can happen whith certain pdf types.'''
+    for i in documents: 
+        if i.content == "":
+            with st.spinner("using pdfplumber"):
+                text = []
+                with pdfplumber.open(file_path) as pdf:
+                    for page in pdf.pages:
+                        text.append(page.extract_text())
+                i.content = ' '.join([page for page in text])
+    
+    return documents

ver0.1 scripts/keyword_search.py

@@ -0,0 +1,169 @@
+# set path
+import glob, os, sys
+from udfPreprocess.search import semantic_search
+sys.path.append('../udfPreprocess')
+
+#import helper
+import udfPreprocess.docPreprocessing as pre
+import udfPreprocess.cleaning as clean
+from udfPreprocess.search import bm25_tokenizer, bm25TokenizeDoc, lexical_search
+#import needed libraries
+import seaborn as sns
+from pandas import DataFrame
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+# from keybert import KeyBERT
+from transformers import pipeline
+import matplotlib.pyplot as plt
+import numpy as np
+import streamlit as st
+import pandas as pd 
+from rank_bm25 import BM25Okapi
+from sklearn.feature_extraction import _stop_words
+import string
+from tqdm.autonotebook import tqdm
+import numpy as np
+import docx
+from docx.shared import Inches
+from docx.shared import Pt
+from docx.enum.style import WD_STYLE_TYPE 
+import logging
+logger = logging.getLogger(__name__)
+import tempfile
+import sqlite3
+import json
+import configparser
+
+
+def app():
+
+    with st.container():
+        st.markdown("<h1 style='text-align: center;  \
+                      color: black;'> Search</h1>", 
+                      unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=False):
+
+        st.write(
+            """     
+            The *Keyword Search* app is an easy-to-use interface \ 
+            built in Streamlit for doing keyword search in \
+            policy document - developed by GIZ Data and the \
+            Sustainable Development Solution Network.
+            """)
+
+        st.markdown("")
+  
+    
+                  
+    with st.sidebar:
+        with open('sample/keywordexample.json','r') as json_file:
+            keywordexample = json.load(json_file)
+        
+        genre = st.radio("Select Keyword Category", list(keywordexample.keys()))
+        if genre == 'Food':
+            keywordList = keywordexample['Food']
+        elif genre == 'Climate':
+            keywordList = keywordexample['Climate']
+        elif genre == 'Social':
+            keywordList = keywordexample['Social']
+        elif genre == 'Nature':
+            keywordList = keywordexample['Nature']
+        elif genre == 'Implementation':
+            keywordList = keywordexample['Implementation']
+        else:
+            keywordList = None
+        
+        searchtype = st.selectbox("Do you want to find exact macthes or similar meaning/context", ['Exact Matches', 'Similar context/meaning'])
+
+        
+    with st.container():
+        if keywordList is not None:
+            queryList = st.text_input("You selcted the {} category we will look for these keywords in document".format(genre),
+                                    value="{}".format(keywordList))
+        else:
+            queryList = st.text_input("Please enter here your question and we will look \
+                                     for an answer in the document OR enter the keyword you \
+                                     are looking for and we will \
+                                     we will look for similar context \
+                                     in the document.",
+                                    placeholder="Enter keyword here")
+
+        if st.button("Find them"):
+
+            if queryList == "":
+                st.info("🤔 No keyword provided, if you dont have any, please try example sets from sidebar!")
+                logging.warning("Terminated as no keyword provided")
+            else:
+                
+                if 'docs' in st.session_state:
+                    docs = st.session_state['docs']
+                    paraList = st.session_state['paraList']
+                   
+                    if searchtype == 'Exact Matches':
+                        queryList = list(queryList.split(","))
+                        logging.info("performing lexical search")
+                        tokenized_corpus = bm25TokenizeDoc(paraList)
+                        # st.write(len(tokenized_corpus))
+                        document_bm25 = BM25Okapi(tokenized_corpus)
+
+                        with st.spinner("Performing Exact matching search (Lexical search) for you"):
+                            st.markdown("##### Top few lexical search (BM25) hits #####")
+
+                            for keyword in queryList:
+
+                                bm25_hits = lexical_search(keyword,document_bm25)
+                              
+                                
+                                counter = 0
+                                for hit in bm25_hits:
+                                    if hit['score'] > 0.00:
+                                        counter += 1
+                                        if counter == 1:
+                                            st.markdown("###### Results for keyword: **{}** ######".format(keyword))
+                                        # st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+                                        st.write("\t {}: {}\t".format(counter, paraList[hit['corpus_id']].replace("\n", " ")))
+                                        
+                                
+                                if counter == 0:
+                                    st.write("No results found for '**{}**' ".format(keyword))
+                                
+                                st.markdown("---")
+                    else:
+                        logging.info("starting semantic search")
+                        with st.spinner("Performing Similar/Contextual search"):
+                            query = "Find {} related issues ?".format(queryList)
+                            config = configparser.ConfigParser()
+                            config.read_file(open('udfPreprocess/paramconfig.cfg'))
+                            threshold = float(config.get('semantic_search','THRESHOLD'))
+                            # st.write(query)                          
+                            semantic_hits = semantic_search(query,paraList)
+                            st.markdown("##### Few Semantic search hits for {} related topics #####".format(queryList))
+
+                            for i,queryhit in enumerate(semantic_hits):
+
+                                # st.markdown("###### Results for query: **{}** ######".format(queryList[i]))
+                                counter = 0
+                                for hit in queryhit:
+                                    counter += 1
+                                    
+                                                
+                                    if hit['score'] > threshold:
+                                    # st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+                                        st.write("\t {}: \t {}".format(counter, paraList[hit['corpus_id']].replace("\n", " ")))
+
+                                    # document.add_paragraph("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+                                st.markdown("---")
+                            # st.write(semantic_hits)
+
+                          
+
+
+                else:
+                    st.info("🤔 No document found, please try to upload it at the sidebar!")
+                    logging.warning("Terminated as no keyword provided")
+                    
+     
+                
+    

ver0.1 scripts/sdg.py

@@ -0,0 +1,57 @@
+import glob, os, sys; 
+sys.path.append('../udfPreprocess')
+
+#import helper
+import udfPreprocess.docPreprocessing as pre
+import udfPreprocess.cleaning as clean
+
+#import needed libraries
+import seaborn as sns
+from pandas import DataFrame
+from keybert import KeyBERT
+from transformers import pipeline
+import matplotlib.pyplot as plt
+import numpy as np
+import streamlit as st
+import pandas as pd
+import docx
+from docx.shared import Inches
+from docx.shared import Pt
+from docx.enum.style import WD_STYLE_TYPE 
+
+import tempfile
+import sqlite3
+import logging
+logger = logging.getLogger(__name__)
+import configparser
+
+@st.cache(allow_output_mutation=True)
+def load_sdgClassifier():
+    classifier = pipeline("text-classification", model= "jonas/sdg_classifier_osdg")
+    logging.info("Loading classifier")
+    return classifier
+
+def sdg_classification(par_list):
+    logging.info("running SDG classifiication")
+    config = configparser.ConfigParser()
+    config.read_file(open('udfPreprocess/paramconfig.cfg'))
+    threshold = float(config.get('sdg','THRESHOLD'))
+
+
+    classifier = load_sdgClassifier()
+    labels = classifier(par_list)
+    
+    labels_= [(l['label'],l['score']) for l in labels]
+    # df2 = DataFrame(labels_, columns=["SDG", "Relevancy"])
+    df2 = DataFrame(labels_, columns=["SDG", "Relevancy"])
+
+    df2['text'] = par_list      
+    df2 = df2.sort_values(by="Relevancy", ascending=False).reset_index(drop=True)  
+    df2.index += 1
+    df2 =df2[df2['Relevancy']>threshold]
+    x = df2['SDG'].value_counts()
+    df3 = df2.copy()
+    df3= df3.drop(['Relevancy'], axis = 1)
+    
+
+    return df3, x

ver0.1 scripts/sdg_analysis.py

@@ -0,0 +1,160 @@
+# set path
+import glob, os, sys; 
+sys.path.append('../udfPreprocess')
+
+#import helper
+
+
+#import needed libraries
+import seaborn as sns
+import matplotlib.pyplot as plt
+import numpy as np
+import streamlit as st
+import docx
+from docx.shared import Inches
+from docx.shared import Pt
+from docx.enum.style import WD_STYLE_TYPE
+from udfPreprocess.sdg_classifier import sdg_classification
+from udfPreprocess.sdg_classifier import runSDGPreprocessingPipeline
+import configparser
+import tempfile
+import sqlite3
+import logging
+logger = logging.getLogger(__name__)
+
+
+
+def app():
+
+    with st.container():
+        st.markdown("<h1 style='text-align: center; color: black;'> SDSN x GIZ Policy Action Tracking v0.1</h1>", unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=False):
+
+        st.write(
+            """     
+            The *Analyse Policy Document* app is an easy-to-use interface built in Streamlit for analyzing policy documents with respect to SDG Classification for the paragraphs/texts in the document - developed by GIZ Data and the Sustainable Development Solution Network. \n
+            """)
+        st.markdown("")
+
+
+    with st.container():
+        
+
+            
+        if 'filepath' in st.session_state:
+            paraList = runSDGPreprocessingPipeline()
+            with st.spinner("Running SDG"):
+
+                df, x = sdg_classification(paraList)
+
+
+                # classifier = load_sdgClassifier()
+
+                # labels = classifier(par_list)
+                # labels_= [(l['label'],l['score']) for l in labels]
+                # df2 = DataFrame(labels_, columns=["SDG", "Relevancy"])
+                # df2['text'] = par_list      
+                # df2 = df2.sort_values(by="Relevancy", ascending=False).reset_index(drop=True)  
+                # df2.index += 1
+                # df2 =df2[df2['Relevancy']>.85]
+                # x = df2['SDG'].value_counts()
+                # df3 = df2.copy()
+
+                plt.rcParams['font.size'] = 25
+                colors = plt.get_cmap('Blues')(np.linspace(0.2, 0.7, len(x)))
+                # plot
+                fig, ax = plt.subplots()
+                ax.pie(x, colors=colors, radius=2, center=(4, 4),
+                    wedgeprops={"linewidth": 1, "edgecolor": "white"}, frame=False,labels =list(x.index))
+                # fig.savefig('temp.png', bbox_inches='tight',dpi= 100)
+                st.markdown("#### Anything related to SDGs? ####")
+
+                # st.markdown("#### 🎈 Anything related to SDGs? ####")
+
+                c4, c5, c6 = st.columns([2, 2, 2])
+
+                # Add styling
+                cmGreen = sns.light_palette("green", as_cmap=True)
+                cmRed = sns.light_palette("red", as_cmap=True)
+                # df2 = df2.style.background_gradient(
+                #     cmap=cmGreen,
+                #     subset=[
+                #         "Relevancy",
+                #     ],
+                # )
+
+                # format_dictionary = {
+                #     "Relevancy": "{:.1%}",
+                # }
+
+                # df2 = df2.format(format_dictionary)
+
+                with c5:
+                    st.pyplot(fig)
+                    
+                c7, c8, c9 = st.columns([1, 10, 1])
+                with c8:
+                    st.table(df)
+
+
+#     1. Keyword heatmap \n
+ #               2. SDG Classification for the paragraphs/texts in the document
+ #       
+    
+    # with st.container():
+    #     if 'docs' in st.session_state:
+    #         docs = st.session_state['docs']
+    #         docs_processed, df, all_text, par_list = clean.preprocessingForSDG(docs)
+    #         # paraList = st.session_state['paraList']
+    #         logging.info("keybert")
+    #         with st.spinner("Running Key bert"):
+
+    #             kw_model = load_keyBert()
+
+    #             keywords = kw_model.extract_keywords(
+    #             all_text,
+    #             keyphrase_ngram_range=(1, 3),
+    #             use_mmr=True,
+    #             stop_words="english",
+    #             top_n=10,
+    #             diversity=0.7,
+    #             )
+
+    #             st.markdown("## 🎈 What is my document about?")
+            
+    #             df = (
+    #                 DataFrame(keywords, columns=["Keyword/Keyphrase", "Relevancy"])
+    #                 .sort_values(by="Relevancy", ascending=False)
+    #                 .reset_index(drop=True)
+    #             )
+    #             df1 = (
+    #                 DataFrame(keywords, columns=["Keyword/Keyphrase", "Relevancy"])
+    #                 .sort_values(by="Relevancy", ascending=False)
+    #                 .reset_index(drop=True)
+    #             )
+    #             df.index += 1
+
+    #             # Add styling
+    #             cmGreen = sns.light_palette("green", as_cmap=True)
+    #             cmRed = sns.light_palette("red", as_cmap=True)
+    #             df = df.style.background_gradient(
+    #                 cmap=cmGreen,
+    #                 subset=[
+    #                     "Relevancy",
+    #                 ],
+    #             )
+
+    #             c1, c2, c3 = st.columns([1, 3, 1])
+
+    #             format_dictionary = {
+    #                 "Relevancy": "{:.1%}",
+    #             }
+
+    #             df = df.format(format_dictionary)
+
+    #             with c2:
+    #  
+    #               st.table(df)

ver0.1 scripts/search.py

@@ -0,0 +1,141 @@
+import glob, os, sys; sys.path.append('../utils')
+
+#import needed libraries
+import seaborn as sns
+from pandas import DataFrame
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+# from keybert import KeyBERT
+from transformers import pipeline
+import matplotlib.pyplot as plt
+import numpy as np
+import streamlit as st
+import pandas as pd 
+from rank_bm25 import BM25Okapi
+from sklearn.feature_extraction import _stop_words
+import string
+from tqdm.autonotebook import tqdm
+import numpy as np
+import docx
+from docx.shared import Inches
+from docx.shared import Pt
+from docx.enum.style import WD_STYLE_TYPE 
+import logging
+logger = logging.getLogger(__name__)
+import tempfile
+import sqlite3
+import configparser
+
+### These are lexcial search related functions #####
+
+def bm25_tokenizer(text):
+    tokenized_doc = []
+    for token in text.lower().split():
+        token = token.strip(string.punctuation)
+
+        if len(token) > 0 and token not in _stop_words.ENGLISH_STOP_WORDS:
+            tokenized_doc.append(token)
+    return tokenized_doc
+    
+def bm25TokenizeDoc(paraList):
+    tokenized_corpus = []
+    ##########Commenting this for now########### will incorporate paragrpah splitting later.
+    # for passage in tqdm(paraList):
+        # if len(passage.split()) >256:
+        #     # st.write("Splitting")
+        #     temp  = " ".join(passage.split()[:256])
+        #     tokenized_corpus.append(bm25_tokenizer(temp))
+        #     temp  = " ".join(passage.split()[256:])
+        #     tokenized_corpus.append(bm25_tokenizer(temp))
+        # else:
+        #     tokenized_corpus.append(bm25_tokenizer(passage))
+    ######################################################################################33333
+    for passage in tqdm(paraList):
+        tokenized_corpus.append(bm25_tokenizer(passage))    
+
+    return tokenized_corpus
+
+def lexical_search(keyword, document_bm25):
+    config = configparser.ConfigParser()
+    config.read_file(open('udfPreprocess/paramconfig.cfg'))
+    top_k = int(config.get('lexical_search','TOP_K'))
+    bm25_scores = document_bm25.get_scores(bm25_tokenizer(keyword))
+    top_n = np.argpartition(bm25_scores, -top_k)[-top_k:]
+    bm25_hits = [{'corpus_id': idx, 'score': bm25_scores[idx]} for idx in top_n]
+    bm25_hits = sorted(bm25_hits, key=lambda x: x['score'], reverse=True)
+    return bm25_hits
+
+@st.cache(allow_output_mutation=True)
+def load_sentenceTransformer(name):
+    return SentenceTransformer(name)
+
+
+def semantic_search(keywordlist,paraList):
+    
+    ##### Sematic Search #####
+    #query = "Does document contain {} issues ?".format(keyword)
+    config = configparser.ConfigParser()
+    config.read_file(open('udfPreprocess/paramconfig.cfg'))
+    model_name = config.get('semantic_search','MODEL_NAME')
+
+    bi_encoder = load_sentenceTransformer(model_name)
+    bi_encoder.max_seq_length = int(config.get('semantic_search','MAX_SEQ_LENGTH'))     #Truncate long passages to 256 tokens
+    top_k = int(config.get('semantic_search','TOP_K'))  
+    document_embeddings = bi_encoder.encode(paraList, convert_to_tensor=True, show_progress_bar=False)
+    question_embedding = bi_encoder.encode(keywordlist, convert_to_tensor=True)
+
+    hits = util.semantic_search(question_embedding, document_embeddings, top_k=top_k)
+    
+    return hits
+
+def show_results(keywordList):
+            document = docx.Document()
+            # document.add_heading('Document name:{}'.format(file_name), 2)
+            section = document.sections[0]
+
+           # Calling the footer
+            footer = section.footer
+        
+            # Calling the paragraph already present in
+        # the footer section
+            footer_para = footer.paragraphs[0]
+        
+            font_styles = document.styles
+            font_charstyle = font_styles.add_style('CommentsStyle', WD_STYLE_TYPE.CHARACTER)
+            font_object = font_charstyle.font
+            font_object.size = Pt(7)
+        # Adding the centered zoned footer
+            footer_para.add_run('''\tPowered by GIZ Data and the Sustainable Development Solution Network hosted at Hugging-Face spaces: https://huggingface.co/spaces/ppsingh/streamlit_dev''', style='CommentsStyle')
+            document.add_heading('Your Seacrhed for {}'.format(keywordList), level=1)
+            for keyword in keywordList:
+          
+              st.write("Results for Query: {}".format(keyword))
+              para = document.add_paragraph().add_run("Results for Query: {}".format(keyword))
+              para.font.size = Pt(12)
+              bm25_hits, hits = search(keyword)     
+
+              st.markdown("""
+                      We will provide with 2 kind of results. The 'lexical search' and the semantic search. 
+                      """)  
+              # In the semantic search part we provide two kind of results one with only Retriever (Bi-Encoder) and other the ReRanker (Cross Encoder)           
+              st.markdown("Top few lexical search (BM25) hits")
+              document.add_paragraph("Top few lexical search (BM25) hits")
+
+              for hit in bm25_hits[0:5]:
+                  if hit['score'] > 0.00:   
+                      st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+                      document.add_paragraph("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+          
+        
+        
+        #   st.table(bm25_hits[0:3])
+          
+              st.markdown("\n-------------------------\n")
+              st.markdown("Top few Bi-Encoder Retrieval hits")
+              document.add_paragraph("\n-------------------------\n")
+              document.add_paragraph("Top few Bi-Encoder Retrieval hits")
+
+              hits = sorted(hits, key=lambda x: x['score'], reverse=True)
+              for hit in hits[0:5]:
+                #  if hit['score'] > 0.45:
+                  st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+                  document.add_paragraph("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))

ver0.1 scripts/uploadAndExample.py

@@ -0,0 +1,52 @@
+import streamlit as st
+import tempfile
+import udfPreprocess.docPreprocessing as pre
+import udfPreprocess.cleaning as clean
+
+def add_upload(choice):
+
+    
+    if choice == 'Upload Document':
+          uploaded_file = st.sidebar.file_uploader('Upload the File', type=['pdf', 'docx', 'txt'])
+          if uploaded_file is not None:
+            with tempfile.NamedTemporaryFile(mode="wb") as temp:
+                bytes_data = uploaded_file.getvalue()
+                temp.write(bytes_data)
+                st.session_state['filename'] = uploaded_file.name
+                # st.write("Uploaded Filename: ", uploaded_file.name)
+                file_name =  uploaded_file.name
+                file_path = temp.name
+                # docs = pre.load_document(file_path, file_name)
+                # haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+                st.session_state['filename'] = file_name
+                # st.session_state['paraList'] = paraList
+                st.session_state['filepath'] = file_path
+
+                
+
+    else:
+          # listing the options
+          option = st.sidebar.selectbox('Select the example document',
+                                ('South Africa:Low Emission strategy', 
+                                'Ethiopia: 10 Year Development Plan'))
+          if option is 'South Africa:Low Emission strategy':
+            file_name = file_path  = 'sample/South Africa_s Low Emission Development Strategy.txt'
+            st.session_state['filename'] = file_name
+            st.sesion_state['filepath'] = file_path
+            # st.write("Selected document:", file_name.split('/')[1])
+            # with open('sample/South Africa_s Low Emission Development Strategy.txt') as dfile:
+            # file = open('sample/South Africa_s Low Emission Development Strategy.txt', 'wb')
+          else:
+            # with open('sample/Ethiopia_s_2021_10 Year Development Plan.txt') as dfile:
+            file_name = file_path =  'sample/Ethiopia_s_2021_10 Year Development Plan.txt'
+            st.session_state['filename'] = file_name
+            st.session_state['filepath'] = file_path
+            # st.write("Selected document:", file_name.split('/')[1])
+          
+          # if option is not None:
+          #   docs = pre.load_document(file_path,file_name)
+          #   haystackDoc, dataframeDoc, textData, paraList = clean.preprocessing(docs)
+          #   st.session_state['docs'] = docs
+          #   st.session_state['paraList'] = paraList
+          
+