We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
'Database is Locked' error when channel importing is running
We get errors in `jonboiser`'s `available-channels` branch. It's not reliably replicating (!), but the general observations are:

- It happens during `startremotechannelimport` or `diskchannelimport`
- The actual error is raised when another API or session call happens

My sense here is that it's caused by the long content import and annotation phase of channel importing.

Possible solutions:
- shorten length of transactions
- do stuff outside of transactions, before committing them

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `kolibri/content/utils/annotation.py`
Content:
```
1 import logging as logger
2 import os
3 
4 from django.conf import settings
5 from kolibri.content.apps import KolibriContentConfig
6 from kolibri.content.models import ChannelMetadata, ContentNode, File, LocalFile
7 from le_utils.constants import content_kinds
8 from sqlalchemy import and_, exists, func, select
9 
10 from .channels import get_channel_ids_for_content_database_dir
11 from .paths import get_content_file_name, get_content_storage_file_path
12 from .sqlalchemybridge import Bridge
13 
14 logging = logger.getLogger(__name__)
15 
16 CONTENT_APP_NAME = KolibriContentConfig.label
17 
18 CHUNKSIZE = 10000
19 
20 def update_channel_metadata():
21     """
22     If we are potentially moving from a version of Kolibri that did not import its content data,
23     scan through the settings.CONTENT_DATABASE_DIR folder for all channel content databases,
24     and pull the data from each database if we have not already imported it.
25     """
26     from .channel_import import import_channel_from_local_db
27     channel_ids = get_channel_ids_for_content_database_dir(settings.CONTENT_DATABASE_DIR)
28     imported = False
29     for channel_id in channel_ids:
30         if not ChannelMetadata.objects.filter(id=channel_id).exists():
31             import_channel_from_local_db(channel_id)
32             imported = True
33     if imported:
34         set_availability()
35 
36 
37 def set_leaf_node_availability_from_local_file_availability():
38     bridge = Bridge(app_name=CONTENT_APP_NAME)
39 
40     ContentNodeTable = bridge.get_table(ContentNode)
41     FileTable = bridge.get_table(File)
42     LocalFileTable = bridge.get_table(LocalFile)
43 
44     connection = bridge.get_connection()
45 
46     file_statement = select([LocalFileTable.c.available]).where(
47         FileTable.c.local_file_id == LocalFileTable.c.id,
48     ).limit(1)
49 
50     logging.info('Setting availability of File objects based on LocalFile availability')
51 
52     connection.execute(FileTable.update().values(available=file_statement).execution_options(autocommit=True))
53 
54     contentnode_statement = select([FileTable.c.contentnode_id]).where(
55         and_(
56             FileTable.c.available == True,  # noqa
57             FileTable.c.supplementary == False
58         )
59     ).where(ContentNodeTable.c.id == FileTable.c.contentnode_id)
60 
61     logging.info('Setting availability of non-topic ContentNode objects based on File availability')
62 
63     connection.execute(ContentNodeTable.update().where(
64         ContentNodeTable.c.kind != content_kinds.TOPIC).values(available=exists(contentnode_statement)).execution_options(autocommit=True))
65 
66     bridge.end()
67 
68 def mark_local_files_as_available(checksums):
69     """
70     Shortcut method to update database if we are sure that the files are available.
71     Can be used after successful downloads to flag availability without having to do expensive disk reads.
72     """
73     bridge = Bridge(app_name=CONTENT_APP_NAME)
74 
75     LocalFileClass = bridge.get_class(LocalFile)
76 
77     logging.info('Setting availability of {number} LocalFile objects based on passed in checksums'.format(number=len(checksums)))
78 
79     for i in range(0, len(checksums), CHUNKSIZE):
80         bridge.session.bulk_update_mappings(LocalFileClass, ({
81             'id': checksum,
82             'available': True
83         } for checksum in checksums[i:i+CHUNKSIZE]))
84         bridge.session.flush()
85 
86     bridge.session.commit()
87 
88     bridge.end()
89 
90 def set_local_file_availability_from_disk(checksums=None):
91     bridge = Bridge(app_name=CONTENT_APP_NAME)
92 
93     LocalFileClass = bridge.get_class(LocalFile)
94 
95     if checksums is None:
96         logging.info('Setting availability of LocalFile objects based on disk availability')
97         files = bridge.session.query(LocalFileClass).all()
98     elif type(checksums) == list:
99         logging.info('Setting availability of {number} LocalFile objects based on disk availability'.format(number=len(checksums)))
100         files = bridge.session.query(LocalFileClass).filter(LocalFileClass.id.in_(checksums)).all()
101     else:
102         logging.info('Setting availability of LocalFile object with checksum {checksum} based on disk availability'.format(checksum=checksums))
103         files = [bridge.session.query(LocalFileClass).get(checksums)]
104 
105     checksums_to_update = [
106         file.id for file in files if os.path.exists(get_content_storage_file_path(get_content_file_name(file)))
107     ]
108 
109     bridge.end()
110 
111     mark_local_files_as_available(checksums_to_update)
112 
113 def recurse_availability_up_tree():
114     bridge = Bridge(app_name=CONTENT_APP_NAME)
115 
116     ContentNodeClass = bridge.get_class(ContentNode)
117 
118     ContentNodeTable = bridge.get_table(ContentNode)
119 
120     connection = bridge.get_connection()
121 
122     node_depth = bridge.session.query(func.max(ContentNodeClass.level)).scalar()
123 
124     logging.info('Setting availability of ContentNode objects with children for {levels} levels'.format(levels=node_depth))
125 
126     child = ContentNodeTable.alias()
127 
128     # Go from the deepest level to the shallowest
129     for level in range(node_depth, 0, -1):
130 
131         available_nodes = select([child.c.available]).where(
132             and_(
133                 child.c.available == True,  # noqa
134                 child.c.level == level,
135             )
136         ).where(ContentNodeTable.c.id == child.c.parent_id)
137 
138         logging.info('Setting availability of ContentNode objects with children for level {level}'.format(level=level))
139         # Only modify topic availability here
140         connection.execute(ContentNodeTable.update().where(
141             ContentNodeTable.c.level == level - 1).where(
142             ContentNodeTable.c.kind == content_kinds.TOPIC).values(available=exists(available_nodes)).execution_options(autocommit=True))
143 
144     bridge.end()
145 
146 def set_availability(checksums=None):
147     if checksums is None:
148         set_local_file_availability_from_disk()
149     else:
150         mark_local_files_as_available(checksums)
151 
152     set_leaf_node_availability_from_local_file_availability()
153     recurse_availability_up_tree()
154
```

Path: `kolibri/content/management/commands/importcontent.py`
Content:
```
1 import os
2 
3 from django.conf import settings
4 from django.core.management.base import CommandError
5 from django.db.models import Sum
6 from kolibri.tasks.management.commands.base import AsyncCommand
7 from requests.exceptions import HTTPError
8 
9 from ...models import LocalFile
10 from ...utils import annotation, paths, transfer
11 
12 # constants to specify the transfer method to be used
13 DOWNLOAD_METHOD = "download"
14 COPY_METHOD = "copy"
15 
16 
17 class Command(AsyncCommand):
18 
19     def add_arguments(self, parser):
20         # let's save the parser in case we need to print a help statement
21         self._parser = parser
22 
23         # we want two groups of arguments. One group is when the
24         # 'importcontent disk' command is given, where we'll expect a file
25         # directory to be given. Another is the 'importcontent network'
26         # command to be given, where we'll expect a channel.
27 
28         # However, some optional arguments apply to both groups. Add them here!
29         node_ids_help_text = """
30         Specify one or more node IDs to import. Only the files associated to those node IDs will be imported.
31 
32         e.g.
33 
34         kolibri manage importcontent --node_ids <id1>,<id2>, [<ids>,...] {network, disk} <channel id>
35         """
36         parser.add_argument(
37             "--node_ids", "-n",
38             # Split the comma separated string we get, into a list of strings
39             type=lambda x: x.split(","),
40             default=[],
41             required=False,
42             dest="node_ids",
43             help=node_ids_help_text,
44         )
45 
46         exclude_node_ids_help_text = """
47         Specify one or more node IDs to exclude. Files associated to those node IDs will be not be imported.
48 
49         e.g.
50 
51         kolibri manage importcontent --exclude_node_ids <id1>,<id2>, [<ids>,...] {network, disk} <channel id>
52         """
53         parser.add_argument(
54             "--exclude_node_ids",
55             # Split the comma separated string we get, into a list of string
56             type=lambda x: x.split(","),
57             default=[],
58             required=False,
59             dest="exclude_node_ids",
60             help=exclude_node_ids_help_text
61         )
62 
63         # to implement these two groups of commands and their corresponding
64         # arguments, we'll need argparse.subparsers.
65         subparsers = parser.add_subparsers(dest='command', help="The following subcommands are available.")
66 
67         # the network command has a channel id required positional argument,
68         # and some optional content_id arguments.
69 
70         # TODO: implement a --content-domain parameter, for optionally
71         # specifying the domain for the curation server.
72 
73         # Note: cmd should be the management command instance, as though the
74         # interface for adding arguments is argparse, Django overrides the
75         # parser object with its own thing, hence why we need to add cmd. See
76         # http://stackoverflow.com/questions/36706220/is-it-possible-to-create-subparsers-in-a-django-management-command
77         network_subparser = subparsers.add_parser(
78             name='network',
79             cmd=self,
80             help="Download the given channel through the network.",
81         )
82         network_subparser.add_argument('channel_id', type=str)
83 
84         default_studio_url = settings.CENTRAL_CONTENT_DOWNLOAD_BASE_URL
85         network_subparser.add_argument(
86             "--baseurl",
87             type=str,
88             default=default_studio_url,
89             dest="baseurl",
90         )
91 
92         disk_subparser = subparsers.add_parser(
93             name='disk',
94             cmd=self,
95             help='Copy the content from the given folder.'
96         )
97         disk_subparser.add_argument('channel_id', type=str)
98         disk_subparser.add_argument('directory', type=str)
99 
100     def download_content(self, channel_id, node_ids=None, exclude_node_ids=None, baseurl=None):
101         self._transfer(DOWNLOAD_METHOD, channel_id, node_ids=node_ids, exclude_node_ids=exclude_node_ids, baseurl=baseurl)
102 
103     def copy_content(self, channel_id, path, node_ids=None, exclude_node_ids=None):
104         self._transfer(COPY_METHOD, channel_id, path=path, node_ids=node_ids, exclude_node_ids=exclude_node_ids)
105 
106     def _transfer(self, method, channel_id, path=None, node_ids=None, exclude_node_ids=None, baseurl=None):  # noqa: max-complexity=16
107 
108         files_to_download = LocalFile.objects.filter(files__contentnode__channel_id=channel_id, available=False)
109 
110         if node_ids:
111             files_to_download = files_to_download.filter(files__contentnode__in=node_ids)
112 
113         if exclude_node_ids:
114             files_to_download = files_to_download.exclude(files__contentnode__in=exclude_node_ids)
115 
116         # Make sure the files are unique, to avoid duplicating downloads
117         files_to_download = files_to_download.distinct()
118 
119         total_bytes_to_transfer = files_to_download.aggregate(Sum('file_size'))['file_size__sum'] or 0
120 
121         downloaded_files = []
122         file_checksums_to_annotate = []
123 
124         with self.start_progress(total=total_bytes_to_transfer) as overall_progress_update:
125 
126             for f in files_to_download:
127 
128                 if self.is_cancelled():
129                     break
130 
131                 filename = f.get_filename()
132                 dest = paths.get_content_storage_file_path(filename)
133 
134                 # if the file already exists, add its size to our overall progress, and skip
135                 if os.path.isfile(dest) and os.path.getsize(dest) == f.file_size:
136                     overall_progress_update(f.file_size)
137                     file_checksums_to_annotate.append(f.id)
138                     continue
139 
140                 # determine where we're downloading/copying from, and create appropriate transfer object
141                 if method == DOWNLOAD_METHOD:
142                     url = paths.get_content_storage_remote_url(filename, baseurl=baseurl)
143                     filetransfer = transfer.FileDownload(url, dest)
144                 elif method == COPY_METHOD:
145                     srcpath = paths.get_content_storage_file_path(filename, datafolder=path)
146                     filetransfer = transfer.FileCopy(srcpath, dest)
147 
148                 try:
149 
150                     with filetransfer:
151 
152                         with self.start_progress(total=filetransfer.total_size) as file_dl_progress_update:
153 
154                             for chunk in filetransfer:
155                                 if self.is_cancelled():
156                                     filetransfer.cancel()
157                                     break
158                                 length = len(chunk)
159                                 overall_progress_update(length)
160                                 file_dl_progress_update(length)
161                             else:
162                                 # If the for loop didn't break, add this to downloaded files.
163                                 downloaded_files.append(dest)
164 
165                     file_checksums_to_annotate.append(f.id)
166 
167                 except HTTPError:
168                     overall_progress_update(f.file_size)
169 
170             if self.is_cancelled():
171                 # Cancelled, clean up any already downloading files.
172                 for dest in downloaded_files:
173                     os.remove(dest)
174                 self.cancel()
175             else:
176                 annotation.set_availability(file_checksums_to_annotate)
177 
178     def handle_async(self, *args, **options):
179         if options['command'] == 'network':
180             self.download_content(options["channel_id"],
181                                   node_ids=options["node_ids"],
182                                   exclude_node_ids=options['exclude_node_ids'],
183                                   baseurl=options["baseurl"])
184         elif options['command'] == 'disk':
185             self.copy_content(options["channel_id"],
186                               options["directory"],
187                               node_ids=options["node_ids"],
188                               exclude_node_ids=options["exclude_node_ids"])
189         else:
190             self._parser.print_help()
191             raise CommandError("Please give a valid subcommand. You gave: {}".format(options["command"]))
192
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/kolibri/content/management/commands/importcontent.py b/kolibri/content/management/commands/importcontent.py
--- a/kolibri/content/management/commands/importcontent.py
+++ b/kolibri/content/management/commands/importcontent.py
@@ -173,7 +173,7 @@
                     os.remove(dest)
                 self.cancel()
             else:
-                annotation.set_availability(file_checksums_to_annotate)
+                annotation.set_availability(channel_id, file_checksums_to_annotate)
 
     def handle_async(self, *args, **options):
         if options['command'] == 'network':
diff --git a/kolibri/content/utils/annotation.py b/kolibri/content/utils/annotation.py
--- a/kolibri/content/utils/annotation.py
+++ b/kolibri/content/utils/annotation.py
@@ -110,7 +110,7 @@
 
     mark_local_files_as_available(checksums_to_update)
 
-def recurse_availability_up_tree():
+def recurse_availability_up_tree(channel_id):
     bridge = Bridge(app_name=CONTENT_APP_NAME)
 
     ContentNodeClass = bridge.get_class(ContentNode)
@@ -138,16 +138,18 @@
         logging.info('Setting availability of ContentNode objects with children for level {level}'.format(level=level))
         # Only modify topic availability here
         connection.execute(ContentNodeTable.update().where(
-            ContentNodeTable.c.level == level - 1).where(
-            ContentNodeTable.c.kind == content_kinds.TOPIC).values(available=exists(available_nodes)).execution_options(autocommit=True))
+            and_(
+                ContentNodeTable.c.level == level - 1,
+                ContentNodeTable.c.channel_id == channel_id,
+                ContentNodeTable.c.kind == content_kinds.TOPIC)).values(available=exists(available_nodes)).execution_options(autocommit=True))
 
     bridge.end()
 
-def set_availability(checksums=None):
+def set_availability(channel_id, checksums=None):
     if checksums is None:
         set_local_file_availability_from_disk()
     else:
         mark_local_files_as_available(checksums)
 
     set_leaf_node_availability_from_local_file_availability()
-    recurse_availability_up_tree()
+    recurse_availability_up_tree(channel_id)

{"golden_diff": "diff --git a/kolibri/content/management/commands/importcontent.py b/kolibri/content/management/commands/importcontent.py\n--- a/kolibri/content/management/commands/importcontent.py\n+++ b/kolibri/content/management/commands/importcontent.py\n@@ -173,7 +173,7 @@\n                     os.remove(dest)\n                 self.cancel()\n             else:\n-                annotation.set_availability(file_checksums_to_annotate)\n+                annotation.set_availability(channel_id, file_checksums_to_annotate)\n \n     def handle_async(self, *args, **options):\n         if options['command'] == 'network':\ndiff --git a/kolibri/content/utils/annotation.py b/kolibri/content/utils/annotation.py\n--- a/kolibri/content/utils/annotation.py\n+++ b/kolibri/content/utils/annotation.py\n@@ -110,7 +110,7 @@\n \n     mark_local_files_as_available(checksums_to_update)\n \n-def recurse_availability_up_tree():\n+def recurse_availability_up_tree(channel_id):\n     bridge = Bridge(app_name=CONTENT_APP_NAME)\n \n     ContentNodeClass = bridge.get_class(ContentNode)\n@@ -138,16 +138,18 @@\n         logging.info('Setting availability of ContentNode objects with children for level {level}'.format(level=level))\n         # Only modify topic availability here\n         connection.execute(ContentNodeTable.update().where(\n-            ContentNodeTable.c.level == level - 1).where(\n-            ContentNodeTable.c.kind == content_kinds.TOPIC).values(available=exists(available_nodes)).execution_options(autocommit=True))\n+            and_(\n+                ContentNodeTable.c.level == level - 1,\n+                ContentNodeTable.c.channel_id == channel_id,\n+                ContentNodeTable.c.kind == content_kinds.TOPIC)).values(available=exists(available_nodes)).execution_options(autocommit=True))\n \n     bridge.end()\n \n-def set_availability(checksums=None):\n+def set_availability(channel_id, checksums=None):\n     if checksums is None:\n         set_local_file_availability_from_disk()\n     else:\n         mark_local_files_as_available(checksums)\n \n     set_leaf_node_availability_from_local_file_availability()\n-    recurse_availability_up_tree()\n+    recurse_availability_up_tree(channel_id)\n", "issue": "'Database is Locked' error when channel importing is running\nWe get errors in `jonboiser`'s `available-channels` branch. It's not reliably replicating (!), but the general observations are:\r\n\r\n- It happens during `startremotechannelimport` or `diskchannelimport`\r\n- The actual error is raised when another API or session call happens\r\n\r\nMy sense here is that it's caused by the long content import and annotation phase of channel importing.\r\n\r\nPossible solutions:\r\n- shorten length of transactions\r\n- do stuff outside of transactions, before committing them\n", "before_files": [{"content": "import logging as logger\nimport os\n\nfrom django.conf import settings\nfrom kolibri.content.apps import KolibriContentConfig\nfrom kolibri.content.models import ChannelMetadata, ContentNode, File, LocalFile\nfrom le_utils.constants import content_kinds\nfrom sqlalchemy import and_, exists, func, select\n\nfrom .channels import get_channel_ids_for_content_database_dir\nfrom .paths import get_content_file_name, get_content_storage_file_path\nfrom .sqlalchemybridge import Bridge\n\nlogging = logger.getLogger(__name__)\n\nCONTENT_APP_NAME = KolibriContentConfig.label\n\nCHUNKSIZE = 10000\n\ndef update_channel_metadata():\n    \"\"\"\n    If we are potentially moving from a version of Kolibri that did not import its content data,\n    scan through the settings.CONTENT_DATABASE_DIR folder for all channel content databases,\n    and pull the data from each database if we have not already imported it.\n    \"\"\"\n    from .channel_import import import_channel_from_local_db\n    channel_ids = get_channel_ids_for_content_database_dir(settings.CONTENT_DATABASE_DIR)\n    imported = False\n    for channel_id in channel_ids:\n        if not ChannelMetadata.objects.filter(id=channel_id).exists():\n            import_channel_from_local_db(channel_id)\n            imported = True\n    if imported:\n        set_availability()\n\n\ndef set_leaf_node_availability_from_local_file_availability():\n    bridge = Bridge(app_name=CONTENT_APP_NAME)\n\n    ContentNodeTable = bridge.get_table(ContentNode)\n    FileTable = bridge.get_table(File)\n    LocalFileTable = bridge.get_table(LocalFile)\n\n    connection = bridge.get_connection()\n\n    file_statement = select([LocalFileTable.c.available]).where(\n        FileTable.c.local_file_id == LocalFileTable.c.id,\n    ).limit(1)\n\n    logging.info('Setting availability of File objects based on LocalFile availability')\n\n    connection.execute(FileTable.update().values(available=file_statement).execution_options(autocommit=True))\n\n    contentnode_statement = select([FileTable.c.contentnode_id]).where(\n        and_(\n            FileTable.c.available == True,  # noqa\n            FileTable.c.supplementary == False\n        )\n    ).where(ContentNodeTable.c.id == FileTable.c.contentnode_id)\n\n    logging.info('Setting availability of non-topic ContentNode objects based on File availability')\n\n    connection.execute(ContentNodeTable.update().where(\n        ContentNodeTable.c.kind != content_kinds.TOPIC).values(available=exists(contentnode_statement)).execution_options(autocommit=True))\n\n    bridge.end()\n\ndef mark_local_files_as_available(checksums):\n    \"\"\"\n    Shortcut method to update database if we are sure that the files are available.\n    Can be used after successful downloads to flag availability without having to do expensive disk reads.\n    \"\"\"\n    bridge = Bridge(app_name=CONTENT_APP_NAME)\n\n    LocalFileClass = bridge.get_class(LocalFile)\n\n    logging.info('Setting availability of {number} LocalFile objects based on passed in checksums'.format(number=len(checksums)))\n\n    for i in range(0, len(checksums), CHUNKSIZE):\n        bridge.session.bulk_update_mappings(LocalFileClass, ({\n            'id': checksum,\n            'available': True\n        } for checksum in checksums[i:i+CHUNKSIZE]))\n        bridge.session.flush()\n\n    bridge.session.commit()\n\n    bridge.end()\n\ndef set_local_file_availability_from_disk(checksums=None):\n    bridge = Bridge(app_name=CONTENT_APP_NAME)\n\n    LocalFileClass = bridge.get_class(LocalFile)\n\n    if checksums is None:\n        logging.info('Setting availability of LocalFile objects based on disk availability')\n        files = bridge.session.query(LocalFileClass).all()\n    elif type(checksums) == list:\n        logging.info('Setting availability of {number} LocalFile objects based on disk availability'.format(number=len(checksums)))\n        files = bridge.session.query(LocalFileClass).filter(LocalFileClass.id.in_(checksums)).all()\n    else:\n        logging.info('Setting availability of LocalFile object with checksum {checksum} based on disk availability'.format(checksum=checksums))\n        files = [bridge.session.query(LocalFileClass).get(checksums)]\n\n    checksums_to_update = [\n        file.id for file in files if os.path.exists(get_content_storage_file_path(get_content_file_name(file)))\n    ]\n\n    bridge.end()\n\n    mark_local_files_as_available(checksums_to_update)\n\ndef recurse_availability_up_tree():\n    bridge = Bridge(app_name=CONTENT_APP_NAME)\n\n    ContentNodeClass = bridge.get_class(ContentNode)\n\n    ContentNodeTable = bridge.get_table(ContentNode)\n\n    connection = bridge.get_connection()\n\n    node_depth = bridge.session.query(func.max(ContentNodeClass.level)).scalar()\n\n    logging.info('Setting availability of ContentNode objects with children for {levels} levels'.format(levels=node_depth))\n\n    child = ContentNodeTable.alias()\n\n    # Go from the deepest level to the shallowest\n    for level in range(node_depth, 0, -1):\n\n        available_nodes = select([child.c.available]).where(\n            and_(\n                child.c.available == True,  # noqa\n                child.c.level == level,\n            )\n        ).where(ContentNodeTable.c.id == child.c.parent_id)\n\n        logging.info('Setting availability of ContentNode objects with children for level {level}'.format(level=level))\n        # Only modify topic availability here\n        connection.execute(ContentNodeTable.update().where(\n            ContentNodeTable.c.level == level - 1).where(\n            ContentNodeTable.c.kind == content_kinds.TOPIC).values(available=exists(available_nodes)).execution_options(autocommit=True))\n\n    bridge.end()\n\ndef set_availability(checksums=None):\n    if checksums is None:\n        set_local_file_availability_from_disk()\n    else:\n        mark_local_files_as_available(checksums)\n\n    set_leaf_node_availability_from_local_file_availability()\n    recurse_availability_up_tree()\n", "path": "kolibri/content/utils/annotation.py"}, {"content": "import os\n\nfrom django.conf import settings\nfrom django.core.management.base import CommandError\nfrom django.db.models import Sum\nfrom kolibri.tasks.management.commands.base import AsyncCommand\nfrom requests.exceptions import HTTPError\n\nfrom ...models import LocalFile\nfrom ...utils import annotation, paths, transfer\n\n# constants to specify the transfer method to be used\nDOWNLOAD_METHOD = \"download\"\nCOPY_METHOD = \"copy\"\n\n\nclass Command(AsyncCommand):\n\n    def add_arguments(self, parser):\n        # let's save the parser in case we need to print a help statement\n        self._parser = parser\n\n        # we want two groups of arguments. One group is when the\n        # 'importcontent disk' command is given, where we'll expect a file\n        # directory to be given. Another is the 'importcontent network'\n        # command to be given, where we'll expect a channel.\n\n        # However, some optional arguments apply to both groups. Add them here!\n        node_ids_help_text = \"\"\"\n        Specify one or more node IDs to import. Only the files associated to those node IDs will be imported.\n\n        e.g.\n\n        kolibri manage importcontent --node_ids <id1>,<id2>, [<ids>,...] {network, disk} <channel id>\n        \"\"\"\n        parser.add_argument(\n            \"--node_ids\", \"-n\",\n            # Split the comma separated string we get, into a list of strings\n            type=lambda x: x.split(\",\"),\n            default=[],\n            required=False,\n            dest=\"node_ids\",\n            help=node_ids_help_text,\n        )\n\n        exclude_node_ids_help_text = \"\"\"\n        Specify one or more node IDs to exclude. Files associated to those node IDs will be not be imported.\n\n        e.g.\n\n        kolibri manage importcontent --exclude_node_ids <id1>,<id2>, [<ids>,...] {network, disk} <channel id>\n        \"\"\"\n        parser.add_argument(\n            \"--exclude_node_ids\",\n            # Split the comma separated string we get, into a list of string\n            type=lambda x: x.split(\",\"),\n            default=[],\n            required=False,\n            dest=\"exclude_node_ids\",\n            help=exclude_node_ids_help_text\n        )\n\n        # to implement these two groups of commands and their corresponding\n        # arguments, we'll need argparse.subparsers.\n        subparsers = parser.add_subparsers(dest='command', help=\"The following subcommands are available.\")\n\n        # the network command has a channel id required positional argument,\n        # and some optional content_id arguments.\n\n        # TODO: implement a --content-domain parameter, for optionally\n        # specifying the domain for the curation server.\n\n        # Note: cmd should be the management command instance, as though the\n        # interface for adding arguments is argparse, Django overrides the\n        # parser object with its own thing, hence why we need to add cmd. See\n        # http://stackoverflow.com/questions/36706220/is-it-possible-to-create-subparsers-in-a-django-management-command\n        network_subparser = subparsers.add_parser(\n            name='network',\n            cmd=self,\n            help=\"Download the given channel through the network.\",\n        )\n        network_subparser.add_argument('channel_id', type=str)\n\n        default_studio_url = settings.CENTRAL_CONTENT_DOWNLOAD_BASE_URL\n        network_subparser.add_argument(\n            \"--baseurl\",\n            type=str,\n            default=default_studio_url,\n            dest=\"baseurl\",\n        )\n\n        disk_subparser = subparsers.add_parser(\n            name='disk',\n            cmd=self,\n            help='Copy the content from the given folder.'\n        )\n        disk_subparser.add_argument('channel_id', type=str)\n        disk_subparser.add_argument('directory', type=str)\n\n    def download_content(self, channel_id, node_ids=None, exclude_node_ids=None, baseurl=None):\n        self._transfer(DOWNLOAD_METHOD, channel_id, node_ids=node_ids, exclude_node_ids=exclude_node_ids, baseurl=baseurl)\n\n    def copy_content(self, channel_id, path, node_ids=None, exclude_node_ids=None):\n        self._transfer(COPY_METHOD, channel_id, path=path, node_ids=node_ids, exclude_node_ids=exclude_node_ids)\n\n    def _transfer(self, method, channel_id, path=None, node_ids=None, exclude_node_ids=None, baseurl=None):  # noqa: max-complexity=16\n\n        files_to_download = LocalFile.objects.filter(files__contentnode__channel_id=channel_id, available=False)\n\n        if node_ids:\n            files_to_download = files_to_download.filter(files__contentnode__in=node_ids)\n\n        if exclude_node_ids:\n            files_to_download = files_to_download.exclude(files__contentnode__in=exclude_node_ids)\n\n        # Make sure the files are unique, to avoid duplicating downloads\n        files_to_download = files_to_download.distinct()\n\n        total_bytes_to_transfer = files_to_download.aggregate(Sum('file_size'))['file_size__sum'] or 0\n\n        downloaded_files = []\n        file_checksums_to_annotate = []\n\n        with self.start_progress(total=total_bytes_to_transfer) as overall_progress_update:\n\n            for f in files_to_download:\n\n                if self.is_cancelled():\n                    break\n\n                filename = f.get_filename()\n                dest = paths.get_content_storage_file_path(filename)\n\n                # if the file already exists, add its size to our overall progress, and skip\n                if os.path.isfile(dest) and os.path.getsize(dest) == f.file_size:\n                    overall_progress_update(f.file_size)\n                    file_checksums_to_annotate.append(f.id)\n                    continue\n\n                # determine where we're downloading/copying from, and create appropriate transfer object\n                if method == DOWNLOAD_METHOD:\n                    url = paths.get_content_storage_remote_url(filename, baseurl=baseurl)\n                    filetransfer = transfer.FileDownload(url, dest)\n                elif method == COPY_METHOD:\n                    srcpath = paths.get_content_storage_file_path(filename, datafolder=path)\n                    filetransfer = transfer.FileCopy(srcpath, dest)\n\n                try:\n\n                    with filetransfer:\n\n                        with self.start_progress(total=filetransfer.total_size) as file_dl_progress_update:\n\n                            for chunk in filetransfer:\n                                if self.is_cancelled():\n                                    filetransfer.cancel()\n                                    break\n                                length = len(chunk)\n                                overall_progress_update(length)\n                                file_dl_progress_update(length)\n                            else:\n                                # If the for loop didn't break, add this to downloaded files.\n                                downloaded_files.append(dest)\n\n                    file_checksums_to_annotate.append(f.id)\n\n                except HTTPError:\n                    overall_progress_update(f.file_size)\n\n            if self.is_cancelled():\n                # Cancelled, clean up any already downloading files.\n                for dest in downloaded_files:\n                    os.remove(dest)\n                self.cancel()\n            else:\n                annotation.set_availability(file_checksums_to_annotate)\n\n    def handle_async(self, *args, **options):\n        if options['command'] == 'network':\n            self.download_content(options[\"channel_id\"],\n                                  node_ids=options[\"node_ids\"],\n                                  exclude_node_ids=options['exclude_node_ids'],\n                                  baseurl=options[\"baseurl\"])\n        elif options['command'] == 'disk':\n            self.copy_content(options[\"channel_id\"],\n                              options[\"directory\"],\n                              node_ids=options[\"node_ids\"],\n                              exclude_node_ids=options[\"exclude_node_ids\"])\n        else:\n            self._parser.print_help()\n            raise CommandError(\"Please give a valid subcommand. You gave: {}\".format(options[\"command\"]))\n", "path": "kolibri/content/management/commands/importcontent.py"}], "after_files": [{"content": "import logging as logger\nimport os\n\nfrom django.conf import settings\nfrom kolibri.content.apps import KolibriContentConfig\nfrom kolibri.content.models import ChannelMetadata, ContentNode, File, LocalFile\nfrom le_utils.constants import content_kinds\nfrom sqlalchemy import and_, exists, func, select\n\nfrom .channels import get_channel_ids_for_content_database_dir\nfrom .paths import get_content_file_name, get_content_storage_file_path\nfrom .sqlalchemybridge import Bridge\n\nlogging = logger.getLogger(__name__)\n\nCONTENT_APP_NAME = KolibriContentConfig.label\n\nCHUNKSIZE = 10000\n\ndef update_channel_metadata():\n    \"\"\"\n    If we are potentially moving from a version of Kolibri that did not import its content data,\n    scan through the settings.CONTENT_DATABASE_DIR folder for all channel content databases,\n    and pull the data from each database if we have not already imported it.\n    \"\"\"\n    from .channel_import import import_channel_from_local_db\n    channel_ids = get_channel_ids_for_content_database_dir(settings.CONTENT_DATABASE_DIR)\n    imported = False\n    for channel_id in channel_ids:\n        if not ChannelMetadata.objects.filter(id=channel_id).exists():\n            import_channel_from_local_db(channel_id)\n            imported = True\n    if imported:\n        set_availability()\n\n\ndef set_leaf_node_availability_from_local_file_availability():\n    bridge = Bridge(app_name=CONTENT_APP_NAME)\n\n    ContentNodeTable = bridge.get_table(ContentNode)\n    FileTable = bridge.get_table(File)\n    LocalFileTable = bridge.get_table(LocalFile)\n\n    connection = bridge.get_connection()\n\n    file_statement = select([LocalFileTable.c.available]).where(\n        FileTable.c.local_file_id == LocalFileTable.c.id,\n    ).limit(1)\n\n    logging.info('Setting availability of File objects based on LocalFile availability')\n\n    connection.execute(FileTable.update().values(available=file_statement).execution_options(autocommit=True))\n\n    contentnode_statement = select([FileTable.c.contentnode_id]).where(\n        and_(\n            FileTable.c.available == True,  # noqa\n            FileTable.c.supplementary == False\n        )\n    ).where(ContentNodeTable.c.id == FileTable.c.contentnode_id)\n\n    logging.info('Setting availability of non-topic ContentNode objects based on File availability')\n\n    connection.execute(ContentNodeTable.update().where(\n        ContentNodeTable.c.kind != content_kinds.TOPIC).values(available=exists(contentnode_statement)).execution_options(autocommit=True))\n\n    bridge.end()\n\ndef mark_local_files_as_available(checksums):\n    \"\"\"\n    Shortcut method to update database if we are sure that the files are available.\n    Can be used after successful downloads to flag availability without having to do expensive disk reads.\n    \"\"\"\n    bridge = Bridge(app_name=CONTENT_APP_NAME)\n\n    LocalFileClass = bridge.get_class(LocalFile)\n\n    logging.info('Setting availability of {number} LocalFile objects based on passed in checksums'.format(number=len(checksums)))\n\n    for i in range(0, len(checksums), CHUNKSIZE):\n        bridge.session.bulk_update_mappings(LocalFileClass, ({\n            'id': checksum,\n            'available': True\n        } for checksum in checksums[i:i+CHUNKSIZE]))\n        bridge.session.flush()\n\n    bridge.session.commit()\n\n    bridge.end()\n\ndef set_local_file_availability_from_disk(checksums=None):\n    bridge = Bridge(app_name=CONTENT_APP_NAME)\n\n    LocalFileClass = bridge.get_class(LocalFile)\n\n    if checksums is None:\n        logging.info('Setting availability of LocalFile objects based on disk availability')\n        files = bridge.session.query(LocalFileClass).all()\n    elif type(checksums) == list:\n        logging.info('Setting availability of {number} LocalFile objects based on disk availability'.format(number=len(checksums)))\n        files = bridge.session.query(LocalFileClass).filter(LocalFileClass.id.in_(checksums)).all()\n    else:\n        logging.info('Setting availability of LocalFile object with checksum {checksum} based on disk availability'.format(checksum=checksums))\n        files = [bridge.session.query(LocalFileClass).get(checksums)]\n\n    checksums_to_update = [\n        file.id for file in files if os.path.exists(get_content_storage_file_path(get_content_file_name(file)))\n    ]\n\n    bridge.end()\n\n    mark_local_files_as_available(checksums_to_update)\n\ndef recurse_availability_up_tree(channel_id):\n    bridge = Bridge(app_name=CONTENT_APP_NAME)\n\n    ContentNodeClass = bridge.get_class(ContentNode)\n\n    ContentNodeTable = bridge.get_table(ContentNode)\n\n    connection = bridge.get_connection()\n\n    node_depth = bridge.session.query(func.max(ContentNodeClass.level)).scalar()\n\n    logging.info('Setting availability of ContentNode objects with children for {levels} levels'.format(levels=node_depth))\n\n    child = ContentNodeTable.alias()\n\n    # Go from the deepest level to the shallowest\n    for level in range(node_depth, 0, -1):\n\n        available_nodes = select([child.c.available]).where(\n            and_(\n                child.c.available == True,  # noqa\n                child.c.level == level,\n            )\n        ).where(ContentNodeTable.c.id == child.c.parent_id)\n\n        logging.info('Setting availability of ContentNode objects with children for level {level}'.format(level=level))\n        # Only modify topic availability here\n        connection.execute(ContentNodeTable.update().where(\n            and_(\n                ContentNodeTable.c.level == level - 1,\n                ContentNodeTable.c.channel_id == channel_id,\n                ContentNodeTable.c.kind == content_kinds.TOPIC)).values(available=exists(available_nodes)).execution_options(autocommit=True))\n\n    bridge.end()\n\ndef set_availability(channel_id, checksums=None):\n    if checksums is None:\n        set_local_file_availability_from_disk()\n    else:\n        mark_local_files_as_available(checksums)\n\n    set_leaf_node_availability_from_local_file_availability()\n    recurse_availability_up_tree(channel_id)\n", "path": "kolibri/content/utils/annotation.py"}, {"content": "import os\n\nfrom django.conf import settings\nfrom django.core.management.base import CommandError\nfrom django.db.models import Sum\nfrom kolibri.tasks.management.commands.base import AsyncCommand\nfrom requests.exceptions import HTTPError\n\nfrom ...models import LocalFile\nfrom ...utils import annotation, paths, transfer\n\n# constants to specify the transfer method to be used\nDOWNLOAD_METHOD = \"download\"\nCOPY_METHOD = \"copy\"\n\n\nclass Command(AsyncCommand):\n\n    def add_arguments(self, parser):\n        # let's save the parser in case we need to print a help statement\n        self._parser = parser\n\n        # we want two groups of arguments. One group is when the\n        # 'importcontent disk' command is given, where we'll expect a file\n        # directory to be given. Another is the 'importcontent network'\n        # command to be given, where we'll expect a channel.\n\n        # However, some optional arguments apply to both groups. Add them here!\n        node_ids_help_text = \"\"\"\n        Specify one or more node IDs to import. Only the files associated to those node IDs will be imported.\n\n        e.g.\n\n        kolibri manage importcontent --node_ids <id1>,<id2>, [<ids>,...] {network, disk} <channel id>\n        \"\"\"\n        parser.add_argument(\n            \"--node_ids\", \"-n\",\n            # Split the comma separated string we get, into a list of strings\n            type=lambda x: x.split(\",\"),\n            default=[],\n            required=False,\n            dest=\"node_ids\",\n            help=node_ids_help_text,\n        )\n\n        exclude_node_ids_help_text = \"\"\"\n        Specify one or more node IDs to exclude. Files associated to those node IDs will be not be imported.\n\n        e.g.\n\n        kolibri manage importcontent --exclude_node_ids <id1>,<id2>, [<ids>,...] {network, disk} <channel id>\n        \"\"\"\n        parser.add_argument(\n            \"--exclude_node_ids\",\n            # Split the comma separated string we get, into a list of string\n            type=lambda x: x.split(\",\"),\n            default=[],\n            required=False,\n            dest=\"exclude_node_ids\",\n            help=exclude_node_ids_help_text\n        )\n\n        # to implement these two groups of commands and their corresponding\n        # arguments, we'll need argparse.subparsers.\n        subparsers = parser.add_subparsers(dest='command', help=\"The following subcommands are available.\")\n\n        # the network command has a channel id required positional argument,\n        # and some optional content_id arguments.\n\n        # TODO: implement a --content-domain parameter, for optionally\n        # specifying the domain for the curation server.\n\n        # Note: cmd should be the management command instance, as though the\n        # interface for adding arguments is argparse, Django overrides the\n        # parser object with its own thing, hence why we need to add cmd. See\n        # http://stackoverflow.com/questions/36706220/is-it-possible-to-create-subparsers-in-a-django-management-command\n        network_subparser = subparsers.add_parser(\n            name='network',\n            cmd=self,\n            help=\"Download the given channel through the network.\",\n        )\n        network_subparser.add_argument('channel_id', type=str)\n\n        default_studio_url = settings.CENTRAL_CONTENT_DOWNLOAD_BASE_URL\n        network_subparser.add_argument(\n            \"--baseurl\",\n            type=str,\n            default=default_studio_url,\n            dest=\"baseurl\",\n        )\n\n        disk_subparser = subparsers.add_parser(\n            name='disk',\n            cmd=self,\n            help='Copy the content from the given folder.'\n        )\n        disk_subparser.add_argument('channel_id', type=str)\n        disk_subparser.add_argument('directory', type=str)\n\n    def download_content(self, channel_id, node_ids=None, exclude_node_ids=None, baseurl=None):\n        self._transfer(DOWNLOAD_METHOD, channel_id, node_ids=node_ids, exclude_node_ids=exclude_node_ids, baseurl=baseurl)\n\n    def copy_content(self, channel_id, path, node_ids=None, exclude_node_ids=None):\n        self._transfer(COPY_METHOD, channel_id, path=path, node_ids=node_ids, exclude_node_ids=exclude_node_ids)\n\n    def _transfer(self, method, channel_id, path=None, node_ids=None, exclude_node_ids=None, baseurl=None):  # noqa: max-complexity=16\n\n        files_to_download = LocalFile.objects.filter(files__contentnode__channel_id=channel_id, available=False)\n\n        if node_ids:\n            files_to_download = files_to_download.filter(files__contentnode__in=node_ids)\n\n        if exclude_node_ids:\n            files_to_download = files_to_download.exclude(files__contentnode__in=exclude_node_ids)\n\n        # Make sure the files are unique, to avoid duplicating downloads\n        files_to_download = files_to_download.distinct()\n\n        total_bytes_to_transfer = files_to_download.aggregate(Sum('file_size'))['file_size__sum'] or 0\n\n        downloaded_files = []\n        file_checksums_to_annotate = []\n\n        with self.start_progress(total=total_bytes_to_transfer) as overall_progress_update:\n\n            for f in files_to_download:\n\n                if self.is_cancelled():\n                    break\n\n                filename = f.get_filename()\n                dest = paths.get_content_storage_file_path(filename)\n\n                # if the file already exists, add its size to our overall progress, and skip\n                if os.path.isfile(dest) and os.path.getsize(dest) == f.file_size:\n                    overall_progress_update(f.file_size)\n                    file_checksums_to_annotate.append(f.id)\n                    continue\n\n                # determine where we're downloading/copying from, and create appropriate transfer object\n                if method == DOWNLOAD_METHOD:\n                    url = paths.get_content_storage_remote_url(filename, baseurl=baseurl)\n                    filetransfer = transfer.FileDownload(url, dest)\n                elif method == COPY_METHOD:\n                    srcpath = paths.get_content_storage_file_path(filename, datafolder=path)\n                    filetransfer = transfer.FileCopy(srcpath, dest)\n\n                try:\n\n                    with filetransfer:\n\n                        with self.start_progress(total=filetransfer.total_size) as file_dl_progress_update:\n\n                            for chunk in filetransfer:\n                                if self.is_cancelled():\n                                    filetransfer.cancel()\n                                    break\n                                length = len(chunk)\n                                overall_progress_update(length)\n                                file_dl_progress_update(length)\n                            else:\n                                # If the for loop didn't break, add this to downloaded files.\n                                downloaded_files.append(dest)\n\n                    file_checksums_to_annotate.append(f.id)\n\n                except HTTPError:\n                    overall_progress_update(f.file_size)\n\n            if self.is_cancelled():\n                # Cancelled, clean up any already downloading files.\n                for dest in downloaded_files:\n                    os.remove(dest)\n                self.cancel()\n            else:\n                annotation.set_availability(channel_id, file_checksums_to_annotate)\n\n    def handle_async(self, *args, **options):\n        if options['command'] == 'network':\n            self.download_content(options[\"channel_id\"],\n                                  node_ids=options[\"node_ids\"],\n                                  exclude_node_ids=options['exclude_node_ids'],\n                                  baseurl=options[\"baseurl\"])\n        elif options['command'] == 'disk':\n            self.copy_content(options[\"channel_id\"],\n                              options[\"directory\"],\n                              node_ids=options[\"node_ids\"],\n                              exclude_node_ids=options[\"exclude_node_ids\"])\n        else:\n            self._parser.print_help()\n            raise CommandError(\"Please give a valid subcommand. You gave: {}\".format(options[\"command\"]))\n", "path": "kolibri/content/management/commands/importcontent.py"}]}