pipelines/ner_wikiner/scripts/partition.py
def main(data_loc: Path, out_dir: Path, fraction: float = 0.2, seed: int = 0):
    """Partition the data into train/test/dev split."""
    random.seed(0)
    lines = (
        bz2.open(str(data_loc), mode="rt", encoding="utf8").read().strip().split("\n")
    )
    lines = [line for line in lines if line.strip()]
    random.shuffle(lines)
    dev_size = int(fraction * len(lines))
    test_size = int(fraction * len(lines))
    train_size = (len(lines) - dev_size) - test_size
    if out_dir.exists():
        shutil.rmtree(out_dir)

    out_dir.mkdir(parents=True)
    with (out_dir / "train.iob").open("w", encoding="utf8") as file_:
        file_.write("\n".join(lines[:train_size]))
    with (out_dir / "dev.iob").open("w", encoding="utf8") as file_:
        file_.write("\n".join(lines[train_size : train_size + dev_size]))
    with (out_dir / "test.iob").open("w", encoding="utf8") as file_:
        file_.write("\n".join(lines[train_size + dev_size :]))
Partition the data into train/test/dev split.
Partition the data into train/test/dev split.
Partition the data into train / test / dev split .
def main(data_loc: Path, out_dir: Path, fraction: float = 0.2, seed: int = 0):
    random.seed(0)
    lines = (
        bz2.open(str(data_loc), mode="rt", encoding="utf8").read().strip().split("\n")
    )
    lines = [line for line in lines if line.strip()]
    random.shuffle(lines)
    dev_size = int(fraction * len(lines))
    test_size = int(fraction * len(lines))
    train_size = (len(lines) - dev_size) - test_size
    if out_dir.exists():
        shutil.rmtree(out_dir)
    out_dir.mkdir(parents=True)
    with (out_dir / "train.iob").open("w", encoding="utf8") as file_:
        file_.write("\n".join(lines[:train_size]))
    with (out_dir / "dev.iob").open("w", encoding="utf8") as file_:
        file_.write("\n".join(lines[train_size : train_size + dev_size]))
    with (out_dir / "test.iob").open("w", encoding="utf8") as file_:
        file_.write("\n".join(lines[train_size + dev_size :]))
def main ( data_loc : Path , out_dir : Path , fraction : float = 0.2 , seed : int = 0 ) : random . seed ( 0 ) lines = ( bz2 . open ( str ( data_loc ) , mode = "rt" , encoding = "utf8" ) . read ( ) . strip ( ) . split ( "\n" ) ) lines = [ line for line in lines if line . strip ( ) ] random . shuffle ( lines ) dev_size = int ( fraction * len ( lines ) ) test_size = int ( fraction * len ( lines ) ) train_size = ( len ( lines ) - dev_size ) - test_size if out_dir . exists ( ) : shutil . rmtree ( out_dir ) out_dir . mkdir ( parents = True ) with ( out_dir / "train.iob" ) . open ( "w" , encoding = "utf8" ) as file_ : file_ . write ( "\n" . join ( lines [ : train_size ] ) ) with ( out_dir / "dev.iob" ) . open ( "w" , encoding = "utf8" ) as file_ : file_ . write ( "\n" . join ( lines [ train_size : train_size + dev_size ] ) ) with ( out_dir / "test.iob" ) . open ( "w" , encoding = "utf8" ) as file_ : file_ . write ( "\n" . join ( lines [ train_size + dev_size : ] ) )
Partition the data into train/test/dev split.
Partition the data into train / test / dev split .
"""Partition the data into train/test/dev split."""
data_loc
Path
out_dir
Path
fraction
float
seed
int