BERT_rationale_benchmark/utils.py
def load_documents_from_file(data_dir: str, docids: Set[str] = None) -> Dict[str, List[List[str]]]:
    """Loads a subset of available documents from 'docs.jsonl' file on disk.

    Each document is assumed to be serialized as newline ('\n') separated sentences.
    Each sentence is assumed to be space (' ') joined tokens.
    """
    docs_file = os.path.join(data_dir, 'docs.jsonl')
    documents = load_jsonl(docs_file)
    documents = {doc['docid']: doc['document'] for doc in documents}
    # res = dict()
    # if docids is None:
    #     docids = sorted(list(documents.keys()))
    # else:
    #     docids = sorted(set(str(d) for d in docids))
    # for d in docids:
    #     lines = documents[d].split('\n')
    #     tokenized = [line.strip().split(' ') for line in lines]
    #     res[d] = tokenized
    return documents
Loads a subset of available documents from 'docs.jsonl' file on disk.

    Each document is assumed to be serialized as newline ('\n') separated sentences.
    Each sentence is assumed to be space (' ') joined tokens.
    
Loads a subset of available documents from 'docs.jsonl' file on disk.
Each document is assumed to be serialized as newline ('\n') separated sentences.
Each sentence is assumed to be space (' ') joined tokens.
Loads a subset of available documents from ' docs . jsonl ' file on disk . Each document is assumed to be serialized as newline ( ' \ n ' ) separated sentences . Each sentence is assumed to be space ( ' ' ) joined tokens .
def load_documents_from_file(data_dir: str, docids: Set[str] = None) -> Dict[str, List[List[str]]]:
    docs_file = os.path.join(data_dir, 'docs.jsonl')
    documents = load_jsonl(docs_file)
    documents = {doc['docid']: doc['document'] for doc in documents}
    return documents
def load_documents_from_file ( data_dir : str , docids : Set [ str ] = None ) -> Dict [ str , List [ List [ str ] ] ] : docs_file = os . path . join ( data_dir , 'docs.jsonl' ) documents = load_jsonl ( docs_file ) documents = { doc [ 'docid' ] : doc [ 'document' ] for doc in documents } return documents
Loads a subset of available documents from 'docs.jsonl' file on disk.
Loads a subset of available documents from ' docs . jsonl ' file on disk .
"""Loads a subset of available documents from 'docs.jsonl' file on disk.

    Each document is assumed to be serialized as newline ('\n') separated sentences.
    Each sentence is assumed to be space (' ') joined tokens.
    """ # res = dict() # if docids is None: #     docids = sorted(list(documents.keys())) # else: #     docids = sorted(set(str(d) for d in docids)) # for d in docids: #     lines = documents[d].split('\n') #     tokenized = [line.strip().split(' ') for line in lines] #     res[d] = tokenized
data_dir
str
docids
Set[str]