We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
merge_json behavior when no labels.
@kamo-naoyuki, I've found that there is a case when we don't have any label output in some utterances for the switchboard evaluation data (`rt03`). But current `data2json.sh` cannot allow such cases 
https://github.com/espnet/espnet/blob/master/utils/merge_scp2json.py#L210-L214
Can you relax this?
This check is good for the training data, but it excludes such utterances in the evaluation set, and makes the evaluation different from other benchmarks.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `utils/merge_scp2json.py`
Content:
```
1 #!/usr/bin/env python
2 # encoding: utf-8
3 
4 from __future__ import print_function
5 from __future__ import unicode_literals
6 
7 import argparse
8 import codecs
9 from io import open
10 import json
11 import logging
12 import sys
13 
14 from espnet.utils.cli_utils import get_commandline_args
15 
16 PY2 = sys.version_info[0] == 2
17 sys.stdin = codecs.getreader('utf-8')(sys.stdin if PY2 else sys.stdin.buffer)
18 sys.stdout = codecs.getwriter('utf-8')(
19     sys.stdout if PY2 else sys.stdout.buffer)
20 
21 
22 # Special types:
23 def shape(x):
24     """Change str to List[int]
25 
26     >>> shape('3,5')
27     [3, 5]
28     >>> shape(' [3, 5] ')
29     [3, 5]
30 
31     """
32 
33     # x: ' [3, 5] ' -> '3, 5'
34     x = x.strip()
35     if x[0] == '[':
36         x = x[1:]
37     if x[-1] == ']':
38         x = x[:-1]
39 
40     return list(map(int, x.split(',')))
41 
42 
43 if __name__ == '__main__':
44     parser = argparse.ArgumentParser(
45         description='Given each file paths with such format as '
46                     '<key>:<file>:<type>. type> can be omitted and the default '
47                     'is "str". e.g. {} '
48                     '--input-scps feat:data/feats.scp shape:data/utt2feat_shape:shape '
49                     '--input-scps feat:data/feats2.scp shape:data/utt2feat2_shape:shape '
50                     '--output-scps text:data/text shape:data/utt2text_shape:shape '
51                     '--scps utt2spk:data/utt2spk'.format(sys.argv[0]),
52         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
53     parser.add_argument('--input-scps', type=str, nargs='*', action='append',
54                         default=[], help='Json files for the inputs')
55     parser.add_argument('--output-scps', type=str, nargs='*', action='append',
56                         default=[], help='Json files for the outputs')
57     parser.add_argument('--scps', type=str, nargs='+', default=[],
58                         help='The json files except for the input and outputs')
59     parser.add_argument('--verbose', '-V', default=1, type=int,
60                         help='Verbose option')
61     parser.add_argument('--out', '-O', type=str,
62                         help='The output filename. '
63                              'If omitted, then output to sys.stdout')
64 
65     args = parser.parse_args()
66     args.scps = [args.scps]
67 
68     # logging info
69     logfmt = "%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s"
70     if args.verbose > 0:
71         logging.basicConfig(level=logging.INFO, format=logfmt)
72     else:
73         logging.basicConfig(level=logging.WARN, format=logfmt)
74     logging.info(get_commandline_args())
75 
76     # List[List[Tuple[str, str, Callable[[str], Any], str, str]]]
77     input_infos = []
78     output_infos = []
79     infos = []
80     for lis_list, key_scps_list in [(input_infos, args.input_scps),
81                                     (output_infos, args.output_scps),
82                                     (infos, args.scps)]:
83         for key_scps in key_scps_list:
84             lis = []
85             for key_scp in key_scps:
86                 sps = key_scp.split(':')
87                 if len(sps) == 2:
88                     key, scp = sps
89                     type_func = None
90                     type_func_str = 'none'
91                 elif len(sps) == 3:
92                     key, scp, type_func_str = sps
93                     fail = False
94 
95                     try:
96                         # type_func: Callable[[str], Any]
97                         # e.g. type_func_str = "int" -> type_func = int
98                         type_func = eval(type_func_str)
99                     except Exception:
100                         raise RuntimeError(
101                             'Unknown type: {}'.format(type_func_str))
102 
103                     if not callable(type_func):
104                         raise RuntimeError(
105                             'Unknown type: {}'.format(type_func_str))
106 
107                 else:
108                     raise RuntimeError(
109                         'Format <key>:<filepath> '
110                         'or <key>:<filepath>:<type>  '
111                         'e.g. feat:data/feat.scp '
112                         'or shape:data/feat.scp:shape: {}'.format(key_scp))
113 
114                 for item in lis:
115                     if key == item[0]:
116                         raise RuntimeError('The key "{}" is duplicated: {} {}'
117                                            .format(key, item[3], key_scp))
118 
119                 lis.append((key, scp, type_func, key_scp, type_func_str))
120             lis_list.append(lis)
121 
122     # Open  scp files
123     input_fscps = [[open(i[1], 'r', encoding='utf-8')
124                     for i in il] for il in input_infos]
125     output_fscps = [[open(i[1], 'r', encoding='utf-8') for i in il]
126                     for il in output_infos]
127     fscps = [[open(i[1], 'r', encoding='utf-8') for i in il] for il in infos]
128 
129     # Note(kamo): What is done here?
130     # The final goal is creating a JSON file such as.
131     # {
132     #     "utts": {
133     #         "sample_id1": {(omitted)},
134     #         "sample_id2": {(omitted)},
135     #          ....
136     #     }
137     # }
138     #
139     # To reduce memory usage, reading the input text files for each lines
140     # and writing JSON elements per samples.
141     if args.out is None:
142         out = sys.stdout
143     else:
144         out = open(args.out, 'w', encoding='utf-8')
145     out.write('{\n    "utts": {\n')
146     nutt = 0
147     while True:
148         nutt += 1
149         # List[List[str]]
150         input_lines = [[f.readline() for f in fl] for fl in input_fscps]
151         output_lines = [[f.readline() for f in fl] for fl in output_fscps]
152         lines = [[f.readline() for f in fl] for fl in fscps]
153 
154         # Get the first line
155         concat = sum(input_lines + output_lines + lines, [])
156         if len(concat) == 0:
157             break
158         first = concat[0]
159 
160         # Sanity check: Must be sorted by the first column and have same keys
161         count = 0
162         for ls_list in (input_lines, output_lines, lines):
163             for ls in ls_list:
164                 for line in ls:
165                     if line == ''or first == '':
166                         if line != first:
167                             concat = sum(
168                                 input_infos + output_infos + infos, [])
169                             raise RuntimeError(
170                                 'The number of lines mismatch '
171                                 'between: "{}" and "{}"'
172                                 .format(concat[0][1], concat[count][1]))
173 
174                     elif line.split()[0] != first.split()[0]:
175                         concat = sum(input_infos + output_infos + infos, [])
176                         raise RuntimeError(
177                             'The keys are mismatch at {}th line '
178                             'between "{}" and "{}":\n>>> {}\n>>> {}'
179                             .format(nutt, concat[0][1], concat[count][1],
180                                     first.rstrip(), line.rstrip()))
181                     count += 1
182 
183         # The end of file
184         if first == '':
185             if nutt != 1:
186                 out.write('\n')
187             break
188         if nutt != 1:
189             out.write(',\n')
190 
191         entry = {}
192         for inout, _lines, _infos in [('input', input_lines, input_infos),
193                                       ('output', output_lines, output_infos),
194                                       ('other', lines, infos)]:
195 
196             lis = []
197             for idx, (line_list, info_list) \
198                     in enumerate(zip(_lines, _infos), 1):
199                 if inout == 'input':
200                     d = {'name': 'input{}'.format(idx)}
201                 elif inout == 'output':
202                     d = {'name': 'target{}'.format(idx)}
203                 else:
204                     d = {}
205 
206                 # info_list: List[Tuple[str, str, Callable]]
207                 # line_list: List[str]
208                 for line, info in zip(line_list, info_list):
209                     sps = line.split(None, 1)
210                     if len(sps) < 2:
211                         raise RuntimeError(
212                             'Format error {}th line in {}: '
213                             ' Expecting "<key> <value>":\n>>> {}'
214                             .format(nutt, info[1], line))
215                     uttid, value = sps
216                     key = info[0]
217                     type_func = info[2]
218                     value = value.rstrip()
219 
220                     if type_func is not None:
221                         try:
222                             # type_func: Callable[[str], Any]
223                             value = type_func(value)
224                         except Exception:
225                             logging.error('"{}" is an invalid function '
226                                           'for the {} th line in {}: \n>>> {}'
227                                           .format(info[4], nutt, info[1], line))
228                             raise
229 
230                     d[key] = value
231                 lis.append(d)
232 
233             if inout != 'other':
234                 entry[inout] = lis
235             else:
236                 # If key == 'other'. only has the first item
237                 entry.update(lis[0])
238 
239         entry = json.dumps(entry, indent=4, ensure_ascii=False,
240                            sort_keys=True, separators=(',', ': '))
241         # Add indent
242         indent = '    ' * 2
243         entry = ('\n' + indent).join(entry.split('\n'))
244 
245         uttid = first.split()[0]
246         out.write('        "{}": {}'.format(uttid, entry))
247 
248     out.write('    }\n}\n')
249 
250     logging.info('{} entries in {}'.format(nutt, out.name))
251
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/utils/merge_scp2json.py b/utils/merge_scp2json.py
--- a/utils/merge_scp2json.py
+++ b/utils/merge_scp2json.py
@@ -6,6 +6,7 @@
 
 import argparse
 import codecs
+from distutils.util import strtobool
 from io import open
 import json
 import logging
@@ -58,6 +59,9 @@
                         help='The json files except for the input and outputs')
     parser.add_argument('--verbose', '-V', default=1, type=int,
                         help='Verbose option')
+    parser.add_argument('--allow-one-column', type=strtobool, default=False,
+                        help='Allow one column in input scp files. '
+                             'In this case, the value will be empty string.')
     parser.add_argument('--out', '-O', type=str,
                         help='The output filename. '
                              'If omitted, then output to sys.stdout')
@@ -162,7 +166,7 @@
         for ls_list in (input_lines, output_lines, lines):
             for ls in ls_list:
                 for line in ls:
-                    if line == ''or first == '':
+                    if line == '' or first == '':
                         if line != first:
                             concat = sum(
                                 input_infos + output_infos + infos, [])
@@ -208,11 +212,16 @@
                 for line, info in zip(line_list, info_list):
                     sps = line.split(None, 1)
                     if len(sps) < 2:
-                        raise RuntimeError(
-                            'Format error {}th line in {}: '
-                            ' Expecting "<key> <value>":\n>>> {}'
-                            .format(nutt, info[1], line))
-                    uttid, value = sps
+                        if not args.allow_one_column:
+                            raise RuntimeError(
+                                'Format error {}th line in {}: '
+                                ' Expecting "<key> <value>":\n>>> {}'
+                                .format(nutt, info[1], line))
+                        uttid = sps[0]
+                        value = ''
+                    else:
+                        uttid, value = sps
+
                     key = info[0]
                     type_func = info[2]
                     value = value.rstrip()

{"golden_diff": "diff --git a/utils/merge_scp2json.py b/utils/merge_scp2json.py\n--- a/utils/merge_scp2json.py\n+++ b/utils/merge_scp2json.py\n@@ -6,6 +6,7 @@\n \n import argparse\n import codecs\n+from distutils.util import strtobool\n from io import open\n import json\n import logging\n@@ -58,6 +59,9 @@\n                         help='The json files except for the input and outputs')\n     parser.add_argument('--verbose', '-V', default=1, type=int,\n                         help='Verbose option')\n+    parser.add_argument('--allow-one-column', type=strtobool, default=False,\n+                        help='Allow one column in input scp files. '\n+                             'In this case, the value will be empty string.')\n     parser.add_argument('--out', '-O', type=str,\n                         help='The output filename. '\n                              'If omitted, then output to sys.stdout')\n@@ -162,7 +166,7 @@\n         for ls_list in (input_lines, output_lines, lines):\n             for ls in ls_list:\n                 for line in ls:\n-                    if line == ''or first == '':\n+                    if line == '' or first == '':\n                         if line != first:\n                             concat = sum(\n                                 input_infos + output_infos + infos, [])\n@@ -208,11 +212,16 @@\n                 for line, info in zip(line_list, info_list):\n                     sps = line.split(None, 1)\n                     if len(sps) < 2:\n-                        raise RuntimeError(\n-                            'Format error {}th line in {}: '\n-                            ' Expecting \"<key> <value>\":\\n>>> {}'\n-                            .format(nutt, info[1], line))\n-                    uttid, value = sps\n+                        if not args.allow_one_column:\n+                            raise RuntimeError(\n+                                'Format error {}th line in {}: '\n+                                ' Expecting \"<key> <value>\":\\n>>> {}'\n+                                .format(nutt, info[1], line))\n+                        uttid = sps[0]\n+                        value = ''\n+                    else:\n+                        uttid, value = sps\n+\n                     key = info[0]\n                     type_func = info[2]\n                     value = value.rstrip()\n", "issue": "merge_json behavior when no labels.\n@kamo-naoyuki, I've found that there is a case when we don't have any label output in some utterances for the switchboard evaluation data (`rt03`). But current `data2json.sh` cannot allow such cases \r\nhttps://github.com/espnet/espnet/blob/master/utils/merge_scp2json.py#L210-L214\r\nCan you relax this?\r\nThis check is good for the training data, but it excludes such utterances in the evaluation set, and makes the evaluation different from other benchmarks.\n", "before_files": [{"content": "#!/usr/bin/env python\n# encoding: utf-8\n\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport argparse\nimport codecs\nfrom io import open\nimport json\nimport logging\nimport sys\n\nfrom espnet.utils.cli_utils import get_commandline_args\n\nPY2 = sys.version_info[0] == 2\nsys.stdin = codecs.getreader('utf-8')(sys.stdin if PY2 else sys.stdin.buffer)\nsys.stdout = codecs.getwriter('utf-8')(\n    sys.stdout if PY2 else sys.stdout.buffer)\n\n\n# Special types:\ndef shape(x):\n    \"\"\"Change str to List[int]\n\n    >>> shape('3,5')\n    [3, 5]\n    >>> shape(' [3, 5] ')\n    [3, 5]\n\n    \"\"\"\n\n    # x: ' [3, 5] ' -> '3, 5'\n    x = x.strip()\n    if x[0] == '[':\n        x = x[1:]\n    if x[-1] == ']':\n        x = x[:-1]\n\n    return list(map(int, x.split(',')))\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(\n        description='Given each file paths with such format as '\n                    '<key>:<file>:<type>. type> can be omitted and the default '\n                    'is \"str\". e.g. {} '\n                    '--input-scps feat:data/feats.scp shape:data/utt2feat_shape:shape '\n                    '--input-scps feat:data/feats2.scp shape:data/utt2feat2_shape:shape '\n                    '--output-scps text:data/text shape:data/utt2text_shape:shape '\n                    '--scps utt2spk:data/utt2spk'.format(sys.argv[0]),\n        formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n    parser.add_argument('--input-scps', type=str, nargs='*', action='append',\n                        default=[], help='Json files for the inputs')\n    parser.add_argument('--output-scps', type=str, nargs='*', action='append',\n                        default=[], help='Json files for the outputs')\n    parser.add_argument('--scps', type=str, nargs='+', default=[],\n                        help='The json files except for the input and outputs')\n    parser.add_argument('--verbose', '-V', default=1, type=int,\n                        help='Verbose option')\n    parser.add_argument('--out', '-O', type=str,\n                        help='The output filename. '\n                             'If omitted, then output to sys.stdout')\n\n    args = parser.parse_args()\n    args.scps = [args.scps]\n\n    # logging info\n    logfmt = \"%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s\"\n    if args.verbose > 0:\n        logging.basicConfig(level=logging.INFO, format=logfmt)\n    else:\n        logging.basicConfig(level=logging.WARN, format=logfmt)\n    logging.info(get_commandline_args())\n\n    # List[List[Tuple[str, str, Callable[[str], Any], str, str]]]\n    input_infos = []\n    output_infos = []\n    infos = []\n    for lis_list, key_scps_list in [(input_infos, args.input_scps),\n                                    (output_infos, args.output_scps),\n                                    (infos, args.scps)]:\n        for key_scps in key_scps_list:\n            lis = []\n            for key_scp in key_scps:\n                sps = key_scp.split(':')\n                if len(sps) == 2:\n                    key, scp = sps\n                    type_func = None\n                    type_func_str = 'none'\n                elif len(sps) == 3:\n                    key, scp, type_func_str = sps\n                    fail = False\n\n                    try:\n                        # type_func: Callable[[str], Any]\n                        # e.g. type_func_str = \"int\" -> type_func = int\n                        type_func = eval(type_func_str)\n                    except Exception:\n                        raise RuntimeError(\n                            'Unknown type: {}'.format(type_func_str))\n\n                    if not callable(type_func):\n                        raise RuntimeError(\n                            'Unknown type: {}'.format(type_func_str))\n\n                else:\n                    raise RuntimeError(\n                        'Format <key>:<filepath> '\n                        'or <key>:<filepath>:<type>  '\n                        'e.g. feat:data/feat.scp '\n                        'or shape:data/feat.scp:shape: {}'.format(key_scp))\n\n                for item in lis:\n                    if key == item[0]:\n                        raise RuntimeError('The key \"{}\" is duplicated: {} {}'\n                                           .format(key, item[3], key_scp))\n\n                lis.append((key, scp, type_func, key_scp, type_func_str))\n            lis_list.append(lis)\n\n    # Open  scp files\n    input_fscps = [[open(i[1], 'r', encoding='utf-8')\n                    for i in il] for il in input_infos]\n    output_fscps = [[open(i[1], 'r', encoding='utf-8') for i in il]\n                    for il in output_infos]\n    fscps = [[open(i[1], 'r', encoding='utf-8') for i in il] for il in infos]\n\n    # Note(kamo): What is done here?\n    # The final goal is creating a JSON file such as.\n    # {\n    #     \"utts\": {\n    #         \"sample_id1\": {(omitted)},\n    #         \"sample_id2\": {(omitted)},\n    #          ....\n    #     }\n    # }\n    #\n    # To reduce memory usage, reading the input text files for each lines\n    # and writing JSON elements per samples.\n    if args.out is None:\n        out = sys.stdout\n    else:\n        out = open(args.out, 'w', encoding='utf-8')\n    out.write('{\\n    \"utts\": {\\n')\n    nutt = 0\n    while True:\n        nutt += 1\n        # List[List[str]]\n        input_lines = [[f.readline() for f in fl] for fl in input_fscps]\n        output_lines = [[f.readline() for f in fl] for fl in output_fscps]\n        lines = [[f.readline() for f in fl] for fl in fscps]\n\n        # Get the first line\n        concat = sum(input_lines + output_lines + lines, [])\n        if len(concat) == 0:\n            break\n        first = concat[0]\n\n        # Sanity check: Must be sorted by the first column and have same keys\n        count = 0\n        for ls_list in (input_lines, output_lines, lines):\n            for ls in ls_list:\n                for line in ls:\n                    if line == ''or first == '':\n                        if line != first:\n                            concat = sum(\n                                input_infos + output_infos + infos, [])\n                            raise RuntimeError(\n                                'The number of lines mismatch '\n                                'between: \"{}\" and \"{}\"'\n                                .format(concat[0][1], concat[count][1]))\n\n                    elif line.split()[0] != first.split()[0]:\n                        concat = sum(input_infos + output_infos + infos, [])\n                        raise RuntimeError(\n                            'The keys are mismatch at {}th line '\n                            'between \"{}\" and \"{}\":\\n>>> {}\\n>>> {}'\n                            .format(nutt, concat[0][1], concat[count][1],\n                                    first.rstrip(), line.rstrip()))\n                    count += 1\n\n        # The end of file\n        if first == '':\n            if nutt != 1:\n                out.write('\\n')\n            break\n        if nutt != 1:\n            out.write(',\\n')\n\n        entry = {}\n        for inout, _lines, _infos in [('input', input_lines, input_infos),\n                                      ('output', output_lines, output_infos),\n                                      ('other', lines, infos)]:\n\n            lis = []\n            for idx, (line_list, info_list) \\\n                    in enumerate(zip(_lines, _infos), 1):\n                if inout == 'input':\n                    d = {'name': 'input{}'.format(idx)}\n                elif inout == 'output':\n                    d = {'name': 'target{}'.format(idx)}\n                else:\n                    d = {}\n\n                # info_list: List[Tuple[str, str, Callable]]\n                # line_list: List[str]\n                for line, info in zip(line_list, info_list):\n                    sps = line.split(None, 1)\n                    if len(sps) < 2:\n                        raise RuntimeError(\n                            'Format error {}th line in {}: '\n                            ' Expecting \"<key> <value>\":\\n>>> {}'\n                            .format(nutt, info[1], line))\n                    uttid, value = sps\n                    key = info[0]\n                    type_func = info[2]\n                    value = value.rstrip()\n\n                    if type_func is not None:\n                        try:\n                            # type_func: Callable[[str], Any]\n                            value = type_func(value)\n                        except Exception:\n                            logging.error('\"{}\" is an invalid function '\n                                          'for the {} th line in {}: \\n>>> {}'\n                                          .format(info[4], nutt, info[1], line))\n                            raise\n\n                    d[key] = value\n                lis.append(d)\n\n            if inout != 'other':\n                entry[inout] = lis\n            else:\n                # If key == 'other'. only has the first item\n                entry.update(lis[0])\n\n        entry = json.dumps(entry, indent=4, ensure_ascii=False,\n                           sort_keys=True, separators=(',', ': '))\n        # Add indent\n        indent = '    ' * 2\n        entry = ('\\n' + indent).join(entry.split('\\n'))\n\n        uttid = first.split()[0]\n        out.write('        \"{}\": {}'.format(uttid, entry))\n\n    out.write('    }\\n}\\n')\n\n    logging.info('{} entries in {}'.format(nutt, out.name))\n", "path": "utils/merge_scp2json.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# encoding: utf-8\n\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport argparse\nimport codecs\nfrom distutils.util import strtobool\nfrom io import open\nimport json\nimport logging\nimport sys\n\nfrom espnet.utils.cli_utils import get_commandline_args\n\nPY2 = sys.version_info[0] == 2\nsys.stdin = codecs.getreader('utf-8')(sys.stdin if PY2 else sys.stdin.buffer)\nsys.stdout = codecs.getwriter('utf-8')(\n    sys.stdout if PY2 else sys.stdout.buffer)\n\n\n# Special types:\ndef shape(x):\n    \"\"\"Change str to List[int]\n\n    >>> shape('3,5')\n    [3, 5]\n    >>> shape(' [3, 5] ')\n    [3, 5]\n\n    \"\"\"\n\n    # x: ' [3, 5] ' -> '3, 5'\n    x = x.strip()\n    if x[0] == '[':\n        x = x[1:]\n    if x[-1] == ']':\n        x = x[:-1]\n\n    return list(map(int, x.split(',')))\n\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(\n        description='Given each file paths with such format as '\n                    '<key>:<file>:<type>. type> can be omitted and the default '\n                    'is \"str\". e.g. {} '\n                    '--input-scps feat:data/feats.scp shape:data/utt2feat_shape:shape '\n                    '--input-scps feat:data/feats2.scp shape:data/utt2feat2_shape:shape '\n                    '--output-scps text:data/text shape:data/utt2text_shape:shape '\n                    '--scps utt2spk:data/utt2spk'.format(sys.argv[0]),\n        formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n    parser.add_argument('--input-scps', type=str, nargs='*', action='append',\n                        default=[], help='Json files for the inputs')\n    parser.add_argument('--output-scps', type=str, nargs='*', action='append',\n                        default=[], help='Json files for the outputs')\n    parser.add_argument('--scps', type=str, nargs='+', default=[],\n                        help='The json files except for the input and outputs')\n    parser.add_argument('--verbose', '-V', default=1, type=int,\n                        help='Verbose option')\n    parser.add_argument('--allow-one-column', type=strtobool, default=False,\n                        help='Allow one column in input scp files. '\n                             'In this case, the value will be empty string.')\n    parser.add_argument('--out', '-O', type=str,\n                        help='The output filename. '\n                             'If omitted, then output to sys.stdout')\n\n    args = parser.parse_args()\n    args.scps = [args.scps]\n\n    # logging info\n    logfmt = \"%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s\"\n    if args.verbose > 0:\n        logging.basicConfig(level=logging.INFO, format=logfmt)\n    else:\n        logging.basicConfig(level=logging.WARN, format=logfmt)\n    logging.info(get_commandline_args())\n\n    # List[List[Tuple[str, str, Callable[[str], Any], str, str]]]\n    input_infos = []\n    output_infos = []\n    infos = []\n    for lis_list, key_scps_list in [(input_infos, args.input_scps),\n                                    (output_infos, args.output_scps),\n                                    (infos, args.scps)]:\n        for key_scps in key_scps_list:\n            lis = []\n            for key_scp in key_scps:\n                sps = key_scp.split(':')\n                if len(sps) == 2:\n                    key, scp = sps\n                    type_func = None\n                    type_func_str = 'none'\n                elif len(sps) == 3:\n                    key, scp, type_func_str = sps\n                    fail = False\n\n                    try:\n                        # type_func: Callable[[str], Any]\n                        # e.g. type_func_str = \"int\" -> type_func = int\n                        type_func = eval(type_func_str)\n                    except Exception:\n                        raise RuntimeError(\n                            'Unknown type: {}'.format(type_func_str))\n\n                    if not callable(type_func):\n                        raise RuntimeError(\n                            'Unknown type: {}'.format(type_func_str))\n\n                else:\n                    raise RuntimeError(\n                        'Format <key>:<filepath> '\n                        'or <key>:<filepath>:<type>  '\n                        'e.g. feat:data/feat.scp '\n                        'or shape:data/feat.scp:shape: {}'.format(key_scp))\n\n                for item in lis:\n                    if key == item[0]:\n                        raise RuntimeError('The key \"{}\" is duplicated: {} {}'\n                                           .format(key, item[3], key_scp))\n\n                lis.append((key, scp, type_func, key_scp, type_func_str))\n            lis_list.append(lis)\n\n    # Open  scp files\n    input_fscps = [[open(i[1], 'r', encoding='utf-8')\n                    for i in il] for il in input_infos]\n    output_fscps = [[open(i[1], 'r', encoding='utf-8') for i in il]\n                    for il in output_infos]\n    fscps = [[open(i[1], 'r', encoding='utf-8') for i in il] for il in infos]\n\n    # Note(kamo): What is done here?\n    # The final goal is creating a JSON file such as.\n    # {\n    #     \"utts\": {\n    #         \"sample_id1\": {(omitted)},\n    #         \"sample_id2\": {(omitted)},\n    #          ....\n    #     }\n    # }\n    #\n    # To reduce memory usage, reading the input text files for each lines\n    # and writing JSON elements per samples.\n    if args.out is None:\n        out = sys.stdout\n    else:\n        out = open(args.out, 'w', encoding='utf-8')\n    out.write('{\\n    \"utts\": {\\n')\n    nutt = 0\n    while True:\n        nutt += 1\n        # List[List[str]]\n        input_lines = [[f.readline() for f in fl] for fl in input_fscps]\n        output_lines = [[f.readline() for f in fl] for fl in output_fscps]\n        lines = [[f.readline() for f in fl] for fl in fscps]\n\n        # Get the first line\n        concat = sum(input_lines + output_lines + lines, [])\n        if len(concat) == 0:\n            break\n        first = concat[0]\n\n        # Sanity check: Must be sorted by the first column and have same keys\n        count = 0\n        for ls_list in (input_lines, output_lines, lines):\n            for ls in ls_list:\n                for line in ls:\n                    if line == '' or first == '':\n                        if line != first:\n                            concat = sum(\n                                input_infos + output_infos + infos, [])\n                            raise RuntimeError(\n                                'The number of lines mismatch '\n                                'between: \"{}\" and \"{}\"'\n                                .format(concat[0][1], concat[count][1]))\n\n                    elif line.split()[0] != first.split()[0]:\n                        concat = sum(input_infos + output_infos + infos, [])\n                        raise RuntimeError(\n                            'The keys are mismatch at {}th line '\n                            'between \"{}\" and \"{}\":\\n>>> {}\\n>>> {}'\n                            .format(nutt, concat[0][1], concat[count][1],\n                                    first.rstrip(), line.rstrip()))\n                    count += 1\n\n        # The end of file\n        if first == '':\n            if nutt != 1:\n                out.write('\\n')\n            break\n        if nutt != 1:\n            out.write(',\\n')\n\n        entry = {}\n        for inout, _lines, _infos in [('input', input_lines, input_infos),\n                                      ('output', output_lines, output_infos),\n                                      ('other', lines, infos)]:\n\n            lis = []\n            for idx, (line_list, info_list) \\\n                    in enumerate(zip(_lines, _infos), 1):\n                if inout == 'input':\n                    d = {'name': 'input{}'.format(idx)}\n                elif inout == 'output':\n                    d = {'name': 'target{}'.format(idx)}\n                else:\n                    d = {}\n\n                # info_list: List[Tuple[str, str, Callable]]\n                # line_list: List[str]\n                for line, info in zip(line_list, info_list):\n                    sps = line.split(None, 1)\n                    if len(sps) < 2:\n                        if not args.allow_one_column:\n                            raise RuntimeError(\n                                'Format error {}th line in {}: '\n                                ' Expecting \"<key> <value>\":\\n>>> {}'\n                                .format(nutt, info[1], line))\n                        uttid = sps[0]\n                        value = ''\n                    else:\n                        uttid, value = sps\n\n                    key = info[0]\n                    type_func = info[2]\n                    value = value.rstrip()\n\n                    if type_func is not None:\n                        try:\n                            # type_func: Callable[[str], Any]\n                            value = type_func(value)\n                        except Exception:\n                            logging.error('\"{}\" is an invalid function '\n                                          'for the {} th line in {}: \\n>>> {}'\n                                          .format(info[4], nutt, info[1], line))\n                            raise\n\n                    d[key] = value\n                lis.append(d)\n\n            if inout != 'other':\n                entry[inout] = lis\n            else:\n                # If key == 'other'. only has the first item\n                entry.update(lis[0])\n\n        entry = json.dumps(entry, indent=4, ensure_ascii=False,\n                           sort_keys=True, separators=(',', ': '))\n        # Add indent\n        indent = '    ' * 2\n        entry = ('\\n' + indent).join(entry.split('\\n'))\n\n        uttid = first.split()[0]\n        out.write('        \"{}\": {}'.format(uttid, entry))\n\n    out.write('    }\\n}\\n')\n\n    logging.info('{} entries in {}'.format(nutt, out.name))\n", "path": "utils/merge_scp2json.py"}]}