BERT_rationale_benchmark/utils.py
def load_documents(data_dir: str, docids: Set[str] = None) -> Dict[str, List[List[str]]]:
    """Loads a subset of available documents from disk.

    Each document is assumed to be serialized as newline ('\n') separated sentences.
    Each sentence is assumed to be space (' ') joined tokens.
    """
    if os.path.exists(os.path.join(data_dir, 'docs.jsonl')):
        assert not os.path.exists(os.path.join(data_dir, 'docs'))
        return load_documents_from_file(data_dir, docids)

    docs_dir = os.path.join(data_dir, 'docs')
    res = dict()
    if docids is None:
        docids = sorted(os.listdir(docs_dir))
    else:
        docids = sorted(set(str(d) for d in docids))
    for d in docids:
        with open(os.path.join(docs_dir, d), 'r') as inf:
            res[d] = inf.read()
    return res
Loads a subset of available documents from disk.

    Each document is assumed to be serialized as newline ('\n') separated sentences.
    Each sentence is assumed to be space (' ') joined tokens.
    
Loads a subset of available documents from disk.
Each document is assumed to be serialized as newline ('\n') separated sentences.
Each sentence is assumed to be space (' ') joined tokens.
Loads a subset of available documents from disk . Each document is assumed to be serialized as newline ( ' \ n ' ) separated sentences . Each sentence is assumed to be space ( ' ' ) joined tokens .
def load_documents(data_dir: str, docids: Set[str] = None) -> Dict[str, List[List[str]]]:
    if os.path.exists(os.path.join(data_dir, 'docs.jsonl')):
        assert not os.path.exists(os.path.join(data_dir, 'docs'))
        return load_documents_from_file(data_dir, docids)
    docs_dir = os.path.join(data_dir, 'docs')
    res = dict()
    if docids is None:
        docids = sorted(os.listdir(docs_dir))
    else:
        docids = sorted(set(str(d) for d in docids))
    for d in docids:
        with open(os.path.join(docs_dir, d), 'r') as inf:
            res[d] = inf.read()
    return res
def load_documents ( data_dir : str , docids : Set [ str ] = None ) -> Dict [ str , List [ List [ str ] ] ] : if os . path . exists ( os . path . join ( data_dir , 'docs.jsonl' ) ) : assert not os . path . exists ( os . path . join ( data_dir , 'docs' ) ) return load_documents_from_file ( data_dir , docids ) docs_dir = os . path . join ( data_dir , 'docs' ) res = dict ( ) if docids is None : docids = sorted ( os . listdir ( docs_dir ) ) else : docids = sorted ( set ( str ( d ) for d in docids ) ) for d in docids : with open ( os . path . join ( docs_dir , d ) , 'r' ) as inf : res [ d ] = inf . read ( ) return res
Loads a subset of available documents from disk.
Loads a subset of available documents from disk .
"""Loads a subset of available documents from disk.

    Each document is assumed to be serialized as newline ('\n') separated sentences.
    Each sentence is assumed to be space (' ') joined tokens.
    """
data_dir
str
docids
Set[str]