Python: Convert a text file multi-level JSON
Question: I am writing a script in `python` to go recursively through each file, create
a JSON object from the file that looks like this:

    
    
    target_id   length  eff_length  est_counts  tpm
    ENST00000619216.1   68  33.8839 2.83333 4.64528
    ENST00000473358.1   712 428.88  0   0
    ENST00000469289.1   535 306.32  0   0
    ENST00000607096.1   138 69.943  0   0
    ENST00000417324.1   1187    844.464 0   0
    ENST00000461467.1   590 342.551 3.44007 0.557892
    ENST00000335137.3   918 588.421 0   0
    ENST00000466430.5   2748    2405.46 75.1098 1.73463
    ENST00000495576.1   1319    976.464 11.1999 0.637186
    

This is my script:

    
    
    import glob
    import os
    import json
    
    # define datasets
    # Dataset name
    datasets = ['pnoc']
    
    # open file in append mode
    f = open('mydict','a')
    
    # define a new object
    data={}
    
    # traverse through folders of datasets
    for d in datasets:
        samples = glob.glob(d + "/data"  + "/*.tsv")
        for s in samples:
            # get the SampleName without extension and path
            fname = os.path.splitext(os.path.basename(s))[0]
    
            # split the basename to get sample name and norm method
            sname, keyword, norm = fname.partition('.')
    
            # determing Normalization method based on filename
            if norm == "abundance":
                norm = "kallisto"
            elif norm == "rsem_genes.results":
                norm = "rsem_genes"
            else:
                norm = "rsem_isoforms"
    
            # read each file
            with open(s) as samp:
                next(samp)
                for line in samp:
                    sp = line.split('\t')
                    data.setdefault(sname,[]).append({"ID": sp[0],"Expression": sp[4]})
                    json.dump(data, f)
    f.close()
    

I want a JSON object on the following lines:

    
    
    # 20000 Sample names, 3 Normalization methods and 60000 IDs in each file.
    DatasetName1 {
        SampleName1 {
            Type {
                Normalization1 {
                    { ID1: value, Expression: value },
                    { ID2: value, Expression: value },
                    ...
                    { ID60000: value, Expression: value }
                },
                Normalization2 {
                    { ID1: value, Expression: value },
                    { ID2: value, Expression: value },
                    ...
                    { ID60000: value, Expression: value }
                },
                Normalization3 {
                    { ID1: value, Expression: value },
                    { ID2: value, Expression: value },
                    ...
                    { ID60000: value, Expression: value }
                }
            }   
        },
        SampleName2 {
            Type {
                Normalization1 {
                    { ID1: value, Expression: value },
                    { ID2: value, Expression: value },
                    ...
                    { ID60000: value, Expression: value }
                },
                Normalization2 {
                    { ID1: value, Expression: value },
                    { ID2: value, Expression: value },
                    ...
                    { ID60000: value, Expression: value }
                },
                Normalization3 {
                    { ID1: value, Expression: value },
                    { ID2: value, Expression: value },
                    ...
                    { ID60000: value, Expression: value }
                }
            }   
        },
        ...
        SampleName20000{
            Type {
                Normalization1 {
                    { ID1: value, Expression: value },
                    { ID2: value, Expression: value },
                    ...
                    { ID60000: value, Expression: value }
                },
                Normalization2 {
                    { ID1: value, Expression: value },
                    { ID2: value, Expression: value },
                    ...
                    { ID60000: value, Expression: value }
                },
                Normalization3 {
                    { ID1: value, Expression: value },
                    { ID2: value, Expression: value },
                    ...
                    { ID60000: value, Expression: value }
                }
            }
        }
    }
    

So my question is - When converting a text file to JSON, how do I set the
levels in my JSON output?

Thanks!


Answer: First, instead of setting the default value over and over, you should make use
of
[`defaultdict`](https://docs.python.org/2/library/collections.html#collections.defaultdict).

Secondly, I think your proposed structure is off, and you should be using some
arrays within (JSON-like structure):

    
    
    {
        DatasetName1: {
            SampleName1: {
                Type: {
                    Normalization1: [
                        { ID1: value, Expression: value },
                        { ID2: value, Expression: value },
                        ...
                        { ID60000: value, Expression: value }
                    ],
                    Normalization2: [
                        { ID1: value, Expression: value },
                        { ID2: value, Expression: value },
                        ...
                        { ID60000: value, Expression: value }
                    ],
                    Normalization3: [
                        { ID1: value, Expression: value },
                        { ID2: value, Expression: value },
                        ...
                        { ID60000: value, Expression: value }
                    ]
                }
            },
            SampleName2: {
                Type: {
                    Normalization1: [
                        { ID1: value, Expression: value },
                        { ID2: value, Expression: value },
                        ...
                        { ID60000: value, Expression: value }
                    ],
                    Normalization2: [
                        { ID1: value, Expression: value },
                        { ID2: value, Expression: value },
                        ...
                        { ID60000: value, Expression: value }
                    ],
                    Normalization3: [
                        { ID1: value, Expression: value },
                        { ID2: value, Expression: value },
                        ...
                        { ID60000: value, Expression: value }
                    ]
                }
            },
            ...
            SampleName20000: {
                Type: {
                    Normalization1: [
                        { ID1: value, Expression: value },
                        { ID2: value, Expression: value },
                        ...
                        { ID60000: value, Expression: value }
                    ],
                    Normalization2: [
                        { ID1: value, Expression: value },
                        { ID2: value, Expression: value },
                        ...
                        { ID60000: value, Expression: value }
                    ],
                    Normalization3: [
                        { ID1: value, Expression: value },
                        { ID2: value, Expression: value },
                        ...
                        { ID60000: value, Expression: value }
                    ]
                }
            }
        }, {
        DatasetName2: {
            ...
        }, ...
    }
    

So your resulting code (untested) should look like this (long as your norm
method logic is correct):

    
    
    from glob import glob
    from os import path
    from json import dump
    from collections import defaultdict
    
    # define datasets, and result dict
    datasets, results = ['pnoc'], defaultdict(dict)
    
    # open file in append mode
    with open('mydict','a') as f:
        # traverse through folders of datasets
        for d in datasets:
            for s in glob(d + "/data"  + "/*.tsv"):
                sample = {"Type": defaultdict(list)}
    
                # get the basename without extension and path
                fname = path.splitext(path.basename(s))[0]
    
                # split the basename to get sample name and norm method
                sname, keyword, norm = fname.partition('.')
    
                # determing norm method based on filename
                if norm == "abundance":
                    norm = "kallisto"
                elif norm == "rsem_genes.results":
                    norm = "rsem_genes"
                else:
                    norm = "rsem_isoforms"
    
                # read each file
                with open(s) as samp:
                    next(samp)              # Skip first line of file
    
                    # Loop through each line and extract the ID and TPM
                    for (id, _, __, ___, tpm) in (line.split('\t') for line in samp):
                        # Add this line to the list for respective normalization method
                        sample['Type'][norm].append({"ID": id, "Expression": float(tpm)})
                # Add sample to dataset
                results[d][sname] = sample
        dump(results, f)
    

This will save the result in a JSON format.

