12-09-2024

LICENSE.md

@@ -0,0 +1,24 @@
+# Creative Commons Attribution-NonCommercial 4.0 International (CC BY-NC 4.0)
+
+## Permissions
+- **Share** — copy and redistribute the material in any medium or format
+- **Adapt** — remix, transform, and build upon the material
+
+The licensor cannot revoke these freedoms as long as you follow the license terms.
+
+## Limitations
+- **No Commercial Use** — You may not use the material for commercial purposes.
+- **No Additional Restrictions** — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.
+- **No Endorsement** — Nothing in this license constitutes or may be construed as permission to assert or imply that you are, or that your use of the material is, connected with, or sponsored, endorsed, or granted official status by, the licensor or others designated to receive attribution.
+
+## Conditions
+- **Attribution** — You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.
+- **Indicate Changes** — If you modify the material, you must indicate that changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.
+- **Include License** — You must include the text of, or the URI or hyperlink to, this license in any copies or substantial portions of the material.
+
+## Notices
+You do not have to comply with the license for elements of the material in the public domain or where your use is permitted by an applicable exception or limitation.
+
+No warranties are given. The license may not give you all of the permissions necessary for your intended use. For example, other rights such as publicity, privacy, or moral rights may limit how you use the material.
+
+For more details, see the full license text [here](https://creativecommons.org/licenses/by-nc/4.0/legalcode).

backend/api/queue.py

@@ -19,7 +19,7 @@ env = environ.Env()
 
 
 @csrf_exempt
-@ratelimit(key='ip', rate='10/m')
+@ratelimit(key='ip', rate='30/m')
 def request_chapter(request):
     try:
         if request.method != "POST": return HttpResponseBadRequest('Allowed POST request only!', status=400)

backend/api/web_scrape.py

@@ -1,6 +1,6 @@
 
 import json, environ, requests, os, subprocess
-import asyncio, uuid, shutil
+import asyncio, uuid, shutil, time
 
 from django.http import HttpResponse, JsonResponse, HttpResponseBadRequest, StreamingHttpResponse
 from django_ratelimit.decorators import ratelimit
@@ -9,10 +9,10 @@ from asgiref.sync import sync_to_async
 
 from backend.module import web_scrap
 from backend.module.utils import manage_image
-from backend.models.model_cache import RequestCache
 from core.settings import BASE_DIR
 from backend.module.utils import cloudflare_turnstile
-from backend.models.model_1 import WebscrapeGetCoverCache
+from backend.models.model_1 import WebScrapeGetCoverCache, WebScrapeGetCache
+from backend.models.model_cache import RequestWebScrapeGetCoverCache, RequestWebScrapeGetCache
 
 from backend.module.utils import directory_info, date_utils
 
@@ -23,7 +23,7 @@ STORAGE_DIR = os.path.join(BASE_DIR,"storage")
 if not os.path.exists(STORAGE_DIR): os.makedirs(STORAGE_DIR)
 
 @csrf_exempt
-@ratelimit(key='ip', rate='20/m')
+@ratelimit(key='ip', rate='30/m')
 def get_list(request):
     if request.method != "POST": return HttpResponseBadRequest('Allowed POST request only!', status=400)
     token = request.META.get('HTTP_X_CLOUDFLARE_TURNSTILE_TOKEN')
@@ -34,11 +34,6 @@ def get_list(request):
     page = payload.get("page")
     source = payload.get("source")
     
-    
-    
-    
-    
-    
     if search.get("text"): DATA = web_scrap.source_control[source].search.scrap(search=search,page=page)
     else: DATA = web_scrap.source_control["colamanga"].get_list.scrap(page=page)
 
@@ -47,7 +42,7 @@ def get_list(request):
 
 
 @csrf_exempt
-@ratelimit(key='ip', rate='20/m')
+@ratelimit(key='ip', rate='30/m')
 def get(request):
     if request.method != "POST": return HttpResponseBadRequest('Allowed POST request only!', status=400)
     token = request.META.get('HTTP_X_CLOUDFLARE_TURNSTILE_TOKEN')
@@ -56,12 +51,57 @@ def get(request):
     payload = json.loads(request.body)
     id = payload.get("id")
     source = payload.get("source")
-    
+
+    file_path = ""
+    file_name = ""
+    chunk_size = 8192
+
     try:
-        DATA = web_scrap.source_control[source].get.scrap(id=id)
-        return JsonResponse({"data":DATA}) 
-    except Exception as e:
+        query_result = WebScrapeGetCache.objects.filter(source=source,comic_id=id).first()
+        if (
+            query_result 
+            and query_result.datetime >= date_utils.utc_time().add(-5,'hour').get()
+            and os.path.exists(query_result.file_path)
+        ): 
+            file_path = query_result.file_path
+            file_name = os.path.basename(file_path)
+
+        else:
+            request_query = RequestWebScrapeGetCache.objects.filter(source=source,comic_id=id).first()
+            if not request_query:
+                RequestWebScrapeGetCache(
+                    source=source,
+                    comic_id=id,
+                ).save()
+            
+            timeout = date_utils.utc_time().add(30,'second').get()
+            while True:
+                if date_utils.utc_time().get() >= timeout: return HttpResponseBadRequest('Request timeout!', status=408)
+                count = RequestWebScrapeGetCache.objects.filter(source=source,comic_id=id).count()
+                if count: time.sleep(1)
+                else: break
+            query_result = WebScrapeGetCache.objects.filter(source=source,comic_id=id).first()
+            
+            if (query_result):
+                file_path = query_result.file_path
+                if not os.path.exists(file_path): return HttpResponseBadRequest('Worker is done but item not found.!', status=404)
+                file_name = os.path.basename(file_path)
+            else:
+                return HttpResponseBadRequest('Worker is done but item not found.!', status=404)
+            
+            
+        
+        def file_iterator():
+            with open(file_path, 'r') as f:
+                while chunk := f.read(chunk_size):
+                    yield chunk
 
+        response = StreamingHttpResponse(file_iterator(), content_type='application/json')
+        response['Content-Length'] = os.path.getsize(file_path)
+        response['Content-Disposition'] = f'attachment; filename="{file_name}"'
+        return response
+    except Exception as e:
+        print(e)
         return HttpResponseBadRequest(str(e), status=500)
 
 
@@ -73,10 +113,9 @@ def get_cover(request,source,id,cover_id):
     file_path = ""
     file_name = ""
     chunk_size = 8192
-    MAX_COVER_STORAGE_SIZE = 10 * 1024 * 1024 * 1024
 
     try:
-        query_result = WebscrapeGetCoverCache.objects.filter(source=source,comic_id=id,cover_id=cover_id).first()
+        query_result = WebScrapeGetCoverCache.objects.filter(source=source,comic_id=id,cover_id=cover_id).first()
         if (
             query_result 
             and os.path.exists(query_result.file_path)
@@ -86,37 +125,28 @@ def get_cover(request,source,id,cover_id):
             file_name = os.path.basename(file_path)
 
         else:
-            if not os.path.exists(os.path.join(STORAGE_DIR,"covers")): os.makedirs(os.path.join(STORAGE_DIR,"covers"))
+            request_query = RequestWebScrapeGetCoverCache.objects.filter(source=source,comic_id=id,cover_id=cover_id).first()
+            if not request_query:
+                RequestWebScrapeGetCoverCache(
+                    source=source,
+                    comic_id=id,
+                    cover_id=cover_id
+                ).save()
             
+            timeout = date_utils.utc_time().add(30,'second').get()
             while True:
-                storage_size = directory_info.GetDirectorySize(directory=os.path.join(STORAGE_DIR,"covers"),max_threads=5)
-                if (storage_size >= MAX_COVER_STORAGE_SIZE):
-                    query_result = WebscrapeGetCoverCache.objects.order_by("datetime").first()
-                    if (query_result):
-                        file_path = query_result.file_path
-                        if os.path.exists(file_path): shutil.rmtree(file_path)
-                        WebscrapeGetCoverCache.objects.filter(file_path=query_result.file_path).delete()
-                    else: 
-                        shutil.rmtree(os.path.join(STORAGE_DIR,"covers"))
-                        break
+                if date_utils.utc_time().get() >= timeout: return HttpResponseBadRequest('Request timeout!', status=408)
+                count = RequestWebScrapeGetCoverCache.objects.filter(source=source,comic_id=id,cover_id=cover_id).count()
+                if count: time.sleep(1)
                 else: break
-                print(storage_size)
-            
-            DATA = web_scrap.source_control[source].get_cover.scrap(id=id,cover_id=cover_id)
-            if not DATA: HttpResponseBadRequest('Image Not found!', status=404)
-            
-            file_path = os.path.join(STORAGE_DIR,"covers",f'{source}-{id}-{cover_id}.png')
-            file_name = os.path.basename(file_path)
-            
-            with open(file_path, "wb") as f: f.write(DATA)
-            
-            WebscrapeGetCoverCache(
-                file_path=file_path,
-                source=source,
-                comic_id=id,
-                cover_id=cover_id,
-            ).save()
+            query_result = WebScrapeGetCoverCache.objects.filter(source=source,comic_id=id,cover_id=cover_id).first()
             
+            if (query_result):
+                file_path = query_result.file_path
+                if not os.path.exists(file_path): return HttpResponseBadRequest('Worker is done but item not found.!', status=404)
+                file_name = os.path.basename(file_path)
+            else:
+                return HttpResponseBadRequest('Worker is done but item not found.!', status=404)
             
         
         def file_iterator():
@@ -124,12 +154,12 @@ def get_cover(request,source,id,cover_id):
                 while chunk := f.read(chunk_size):
                     yield chunk
 
-        response = StreamingHttpResponse(file_iterator())
-        response['Content-Type'] = 'application/octet-stream'
+        response = StreamingHttpResponse(file_iterator(), content_type='image/png')
         response['Content-Length'] = os.path.getsize(file_path)
         response['Content-Disposition'] = f'attachment; filename="{file_name}"'
         return response
     except Exception as e:
+        print(e)
         return HttpResponseBadRequest(str(e), status=500)
 
     

backend/invoke_worker/__init__.py

@@ -5,5 +5,7 @@ if len(sys.argv) > 1 and not (sys.argv[1] in ['migrate', "makemigrations", "clea
     from backend.invoke_worker import (
         session,
         chapter_queue,
+        webscrape_get_cover,
+        webscrape_get,
     )
 

backend/invoke_worker/chapter_queue.py

@@ -73,8 +73,10 @@ class Job(Thread):
                     if (stored):
                         SocketRequestChapterQueueCache.objects.filter(id=query_result.id).delete()
                         connections['cache'].close()
+                        connections['DB1'].close()
                     else:
                         connections['cache'].close()
+                        connections['DB1'].close()
                         
                         if (options.get("colorize") or options.get("translate").get("state")):
                             script = []
@@ -169,9 +171,11 @@ class Job(Thread):
                                         }
                                     })
                                 connections['cache'].close()
+                                connections['DB1'].close()
                                 
                             else: 
                                 connections['cache'].close()
+                                connections['DB1'].close()
                                 raise Exception("#1 Dowload chapter error!")
                         else:
                             input_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"original")
@@ -226,10 +230,13 @@ class Job(Thread):
                                     })
                             else: 
                                 connections['cache'].close()
+                                connections['DB1'].close()
                                 raise Exception("#2 Dowload chapter error!")
                         connections['cache'].close()
+                        connections['DB1'].close()
                 else:
                     connections['cache'].close()
+                    connections['DB1'].close()
                     sleep(5)
             except Exception as e: 
                 print("[Error] Chapter Queue Socket:", e) 
@@ -256,6 +263,7 @@ class Job(Thread):
                 
                 
                 connections['cache'].close()
+                connections['DB1'].close()
                 sleep(10)
 
 thread = Job()
@@ -303,6 +311,7 @@ class UpdateSocketQueue(Thread):
                 print(e)
                 
             connections['cache'].close()
+            connections['DB1'].close()
             sleep(10)
 thread = UpdateSocketQueue()
 thread.daemon = True

backend/invoke_worker/webscrape_get.py

@@ -0,0 +1,116 @@
+
+import json, environ, requests, os, subprocess
+import asyncio, uuid, shutil, sys
+
+from django.http import HttpResponse, JsonResponse, HttpResponseBadRequest, StreamingHttpResponse
+from django_ratelimit.decorators import ratelimit
+from django.views.decorators.csrf import csrf_exempt
+from asgiref.sync import sync_to_async
+from django.db import connections
+
+from backend.module import web_scrap
+from backend.module.utils import manage_image
+from backend.models.model_cache import RequestWebScrapeGetCache
+from core.settings import BASE_DIR
+from backend.module.utils import cloudflare_turnstile
+from backend.models.model_1 import WebScrapeGetCache
+
+from backend.module.utils import directory_info, date_utils
+
+from django_thread import Thread
+from time import sleep
+
+MAX_GET_STORAGE_SIZE  = 10 * 1024 * 1024 * 1024 # GB
+
+STORAGE_DIR = os.path.join(BASE_DIR,"storage")
+if not os.path.exists(STORAGE_DIR): os.makedirs(STORAGE_DIR)
+
+class Get(Thread):
+    def run(self):
+        while True: 
+            connections['cache'].close()
+            connections['DB1'].close()
+            is_request = False
+            
+            request_query = RequestWebScrapeGetCache.objects.order_by("datetime").first()
+            try:
+                if request_query:
+                    query_result = WebScrapeGetCache.objects.filter(source=request_query.source,comic_id=request_query.comic_id).first()
+                    if (
+                        query_result
+                        and os.path.exists(query_result.file_path)
+                        and query_result.datetime >= date_utils.utc_time().add(-5,'hour').get()
+                        
+                    ): 
+                        RequestWebScrapeGetCache.objects.filter(
+                            source=request_query.source,
+                            comic_id=request_query.comic_id,
+                        ).delete()
+                        is_request = False
+                    elif query_result: 
+                        WebScrapeGetCache.objects.filter(
+                            file_path=query_result.file_path
+                        ).delete()
+                        is_request = True
+                    else: is_request = True
+                else: is_request = False
+                
+                if is_request:
+                    if not os.path.exists(os.path.join(STORAGE_DIR,"get",request_query.source)): os.makedirs(os.path.join(STORAGE_DIR,"get",request_query.source))
+                    
+                    connections['cache'].close()
+                    connections['DB1'].close()
+                    
+                    while True:
+                        storage_size = directory_info.GetDirectorySize(directory=os.path.join(STORAGE_DIR,"get"),max_threads=5)
+                        if (storage_size >= MAX_GET_STORAGE_SIZE ):
+                            request_query = WebScrapeGetCache.objects.order_by("datetime").first()
+                            if (request_query):
+                                file_path = request_query.file_path
+                                if os.path.exists(file_path): os.remove(file_path)
+                                WebScrapeGetCache.objects.filter(file_path=request_query.file_path).delete()
+                            else: 
+                                shutil.rmtree(os.path.join(STORAGE_DIR,"get"))
+                                break
+                        else: break
+                    
+                    
+
+                    DATA = web_scrap.source_control[request_query.source].get.scrap(id=request_query.comic_id)
+                    if not DATA: raise Exception('Image Not found!')
+                    
+                    file_path = os.path.join(STORAGE_DIR,"get",request_query.source,f'{request_query.comic_id}.json')
+                    
+                    with open(file_path, "w") as f: json.dump(DATA, f, indent=None, separators=(',', ':'))
+                    
+                    connections['cache'].close()
+                    connections['DB1'].close()
+                    WebScrapeGetCache(
+                        file_path=file_path,
+                        source=request_query.source,
+                        comic_id=request_query.comic_id,
+                    ).save()
+                    RequestWebScrapeGetCache.objects.filter(
+                        source=request_query.source,
+                        comic_id=request_query.comic_id,
+                    ).delete()
+                else:
+                    sleep(5)
+            except Exception as e:
+                connections['cache'].close()
+                connections['DB1'].close()
+                RequestWebScrapeGetCache.objects.filter(
+                    source=request_query.source,
+                    comic_id=request_query.comic_id,
+                ).delete()
+                print("[Error 'Webscrape-Get' Worker]: ") 
+                exc_type, exc_obj, exc_tb = sys.exc_info()
+                line_number = exc_tb.tb_lineno
+                print(f"Error on line {line_number}: {e}")
+                sleep(5)
+
+thread = Get()
+thread.daemon = True
+thread.start()
+
+

backend/invoke_worker/webscrape_get_cover.py

@@ -0,0 +1,121 @@
+
+import json, environ, requests, os, subprocess
+import asyncio, uuid, shutil, sys
+
+from django.http import HttpResponse, JsonResponse, HttpResponseBadRequest, StreamingHttpResponse
+from django_ratelimit.decorators import ratelimit
+from django.views.decorators.csrf import csrf_exempt
+from asgiref.sync import sync_to_async
+from django.db import connections
+
+from backend.module import web_scrap
+from backend.module.utils import manage_image
+from backend.models.model_cache import RequestWebScrapeGetCoverCache
+from core.settings import BASE_DIR
+from backend.module.utils import cloudflare_turnstile
+from backend.models.model_1 import WebScrapeGetCoverCache
+
+from backend.module.utils import directory_info, date_utils
+
+from django_thread import Thread
+from time import sleep
+
+MAX_COVER_STORAGE_SIZE = 20 * 1024 * 1024 * 1024 # GB
+
+STORAGE_DIR = os.path.join(BASE_DIR,"storage")
+if not os.path.exists(STORAGE_DIR): os.makedirs(STORAGE_DIR)
+
+class GetCover(Thread):
+    def run(self):
+        while True: 
+            connections['cache'].close()
+            connections['DB1'].close()
+            is_request = False
+            
+            request_query = RequestWebScrapeGetCoverCache.objects.order_by("datetime").first()
+            try:
+                
+                
+                if request_query:
+                    query_result = WebScrapeGetCoverCache.objects.filter(source=request_query.source,comic_id=request_query.comic_id,cover_id=request_query.cover_id).first()
+                    if (
+                        query_result
+                        and os.path.exists(query_result.file_path)
+                        and query_result.datetime >= date_utils.utc_time().add(-5,'hour').get()
+                    ): 
+                        RequestWebScrapeGetCoverCache.objects.filter(
+                            source=request_query.source,
+                            comic_id=request_query.comic_id,
+                            cover_id=request_query.cover_id
+                        ).delete()
+                        is_request = False
+                    elif query_result: 
+                        WebScrapeGetCoverCache.objects.filter(
+                            file_path=query_result.file_path
+                        ).delete()
+                        is_request = True
+                    else: is_request = True
+                else: is_request = False
+                
+                if is_request:
+                    if not os.path.exists(os.path.join(STORAGE_DIR,"covers",request_query.source)): os.makedirs(os.path.join(STORAGE_DIR,"covers",request_query.source))
+                    
+                    connections['cache'].close()
+                    connections['DB1'].close()
+                    
+                    while True:
+                        storage_size = directory_info.GetDirectorySize(directory=os.path.join(STORAGE_DIR,"covers"),max_threads=5)
+                        if (storage_size >= MAX_COVER_STORAGE_SIZE):
+                            request_query = WebScrapeGetCoverCache.objects.order_by("datetime").first()
+                            if (request_query):
+                                file_path = request_query.file_path
+                                if os.path.exists(file_path): os.remove(file_path)
+                                WebScrapeGetCoverCache.objects.filter(file_path=request_query.file_path).delete()
+                            else: 
+                                shutil.rmtree(os.path.join(STORAGE_DIR,"covers"))
+                                break
+                        else: break
+                    
+                    
+
+                    DATA = web_scrap.source_control[request_query.source].get_cover.scrap(id=request_query.comic_id,cover_id=request_query.cover_id)
+                    if not DATA: raise Exception('Image Not found!')
+                    
+                    file_path = os.path.join(STORAGE_DIR,"covers",request_query.source,f'{request_query.comic_id}-{request_query.cover_id}.png')
+                    
+                    with open(file_path, "wb") as f: f.write(DATA)
+                    
+                    connections['cache'].close()
+                    connections['DB1'].close()
+                    WebScrapeGetCoverCache(
+                        file_path=file_path,
+                        source=request_query.source,
+                        comic_id=request_query.comic_id,
+                        cover_id=request_query.cover_id,
+                    ).save()
+                    RequestWebScrapeGetCoverCache.objects.filter(
+                        source=request_query.source,
+                        comic_id=request_query.comic_id,
+                        cover_id=request_query.cover_id
+                    ).delete()
+                else:
+                    sleep(5)
+            except Exception as e:
+                connections['cache'].close()
+                connections['DB1'].close()
+                RequestWebScrapeGetCoverCache.objects.filter(
+                    source=request_query.source,
+                    comic_id=request_query.comic_id,
+                    cover_id=request_query.cover_id
+                ).delete()
+                print("[Error 'Webscrape-Get-Cover' Worker]: ") 
+                exc_type, exc_obj, exc_tb = sys.exc_info()
+                line_number = exc_tb.tb_lineno
+                print(f"Error on line {line_number}: {e}")
+                sleep(5)
+
+thread = GetCover()
+thread.daemon = True
+thread.start()
+
+

backend/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.1.1 on 2024-12-01 11:29
+# Generated by Django 5.1.1 on 2024-12-07 07:22
 
 import backend.models.model_1
 import backend.models.model_cache
@@ -37,9 +37,12 @@ class Migration(migrations.Migration):
             ],
         ),
         migrations.CreateModel(
-            name='RequestCache',
+            name='RequestWebScrapeGetCoverCache',
             fields=[
-                ('client', models.UUIDField(primary_key=True, serialize=False)),
+                ('id', models.UUIDField(primary_key=True, serialize=False)),
+                ('source', models.TextField()),
+                ('comic_id', models.TextField()),
+                ('cover_id', models.TextField()),
                 ('datetime', models.DateTimeField(default=backend.models.model_cache.get_current_utc_time)),
             ],
         ),
@@ -57,4 +60,14 @@ class Migration(migrations.Migration):
                 ('datetime', models.DateTimeField(default=backend.models.model_cache.get_current_utc_time)),
             ],
         ),
+        migrations.CreateModel(
+            name='WebScrapeGetCoverCache',
+            fields=[
+                ('file_path', models.TextField(primary_key=True, serialize=False)),
+                ('source', models.TextField()),
+                ('comic_id', models.TextField()),
+                ('cover_id', models.TextField()),
+                ('datetime', models.DateTimeField(default=backend.models.model_1.get_current_utc_time)),
+            ],
+        ),
     ]

backend/migrations/0002_alter_requestwebscrapegetcovercache_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.1.1 on 2024-12-07 07:58
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('backend', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='requestwebscrapegetcovercache',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
+        ),
+    ]

backend/migrations/0003_requestwebscrapegetcache.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1.1 on 2024-12-07 15:52
+
+import backend.models.model_cache
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('backend', '0002_alter_requestwebscrapegetcovercache_id'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RequestWebScrapeGetCache',
+            fields=[
+                ('page', models.IntegerField(primary_key=True, serialize=False)),
+                ('source', models.TextField()),
+                ('comic_id', models.TextField()),
+                ('datetime', models.DateTimeField(default=backend.models.model_cache.get_current_utc_time)),
+            ],
+        ),
+    ]

backend/migrations/0004_remove_requestwebscrapegetcache_page_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1.1 on 2024-12-07 15:53
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('backend', '0003_requestwebscrapegetcache'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='requestwebscrapegetcache',
+            name='page',
+        ),
+        migrations.AddField(
+            model_name='requestwebscrapegetcache',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
+        ),
+    ]

backend/migrations/{0002_webscrapegetcovercache.py → 0005_webscrapegetcache.py}

@@ -1,4 +1,4 @@
-# Generated by Django 5.1.1 on 2024-12-01 11:38
+# Generated by Django 5.1.1 on 2024-12-07 16:01
 
 import backend.models.model_1
 from django.db import migrations, models
@@ -7,17 +7,16 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('backend', '0001_initial'),
+        ('backend', '0004_remove_requestwebscrapegetcache_page_and_more'),
     ]
 
     operations = [
         migrations.CreateModel(
-            name='WebscrapeGetCoverCache',
+            name='WebScrapeGetCache',
             fields=[
                 ('file_path', models.TextField(primary_key=True, serialize=False)),
                 ('source', models.TextField()),
                 ('comic_id', models.TextField()),
-                ('cover_id', models.TextField()),
                 ('datetime', models.DateTimeField(default=backend.models.model_1.get_current_utc_time)),
             ],
         ),

backend/models/model_1.py

@@ -20,9 +20,19 @@ class ComicStorageCache(models.Model):
     target_lang = models.TextField()
     datetime = models.DateTimeField(default=get_current_utc_time)
     
-class WebscrapeGetCoverCache(models.Model):
+
+class WebScrapeGetCache(models.Model):
+    file_path = models.TextField(primary_key=True)
+    source = models.TextField()
+    comic_id = models.TextField()
+    datetime = models.DateTimeField(default=get_current_utc_time)
+    
+    
+class WebScrapeGetCoverCache(models.Model):
     file_path = models.TextField(primary_key=True)
     source = models.TextField()
     comic_id = models.TextField()
     cover_id = models.TextField()
     datetime = models.DateTimeField(default=get_current_utc_time)
+    
+

backend/models/model_cache.py

@@ -4,8 +4,17 @@ import uuid
 
 def get_current_utc_time(): return date_utils.utc_time().get()
 
-class RequestCache(models.Model):
-    client = models.UUIDField(primary_key=True)
+class RequestWebScrapeGetCoverCache(models.Model):
+    id = models.UUIDField(primary_key=True, default = uuid.uuid4)
+    source = models.TextField()
+    comic_id = models.TextField()
+    cover_id = models.TextField()
+    datetime = models.DateTimeField(default=get_current_utc_time)
+    
+class RequestWebScrapeGetCache(models.Model):
+    id = models.UUIDField(primary_key=True, default = uuid.uuid4)
+    source = models.TextField()
+    comic_id = models.TextField()
     datetime = models.DateTimeField(default=get_current_utc_time)
 
 class CloudflareTurnStileCache(models.Model):

core/middleware.py

@@ -36,8 +36,11 @@ class SequentialRequestMiddleware:
         request_type = request.scope.get("type")
         request_path = request.path
         print(request_path)
-        if request_type == "http":
-            
+        if (
+            request_type == "http"
+            and request_path in ["/api/web_scrap/get_list/"]
+        ):
+            print("IT THREAD")
             with TimeoutContext(30) as executor:
                 self.__Lock.acquire()
                 try:

core/routers.py

@@ -7,7 +7,6 @@ CACHE_MODELS = [cls.__name__.lower() for name, cls in inspect.getmembers(model_c
 DB1_MODELS = [cls.__name__.lower() for name, cls in inspect.getmembers(model_1, inspect.isclass) if cls.__module__ == model_1.__name__]
 DB2_MODELS = [cls.__name__.lower() for name, cls in inspect.getmembers(model_2, inspect.isclass) if cls.__module__ == model_2.__name__]
 
-
 class Router:
     def db_for_read(self, model, **hints):
         if model._meta.model_name in CACHE_MODELS:
@@ -26,7 +25,7 @@ class Router:
         elif model._meta.model_name in DB2_MODELS:
             return 'DB2'
         return 'default'
-    
+
     def allow_relation(self, obj1, obj2, **hints):
         return True
 

frontend/app/bookmark/components/comic_component.tsx

@@ -49,7 +49,6 @@ const ComicComponent = ({item, SELECTED_BOOKMARK, SET_SELECTED_BOOKMARK}:any) =>
             setIsLoading(true)
             setStyles(__styles(themeTypeContext,Dimensions))
             const stored_bookmark = await Storage.get("bookmark") || []
-            console.log(stored_bookmark)
             cover.current  = await CoverStorage.get(`${item.source}-${item.id}`) || ""
             setIsLoading(false)
         })()
@@ -60,7 +59,12 @@ const ComicComponent = ({item, SELECTED_BOOKMARK, SET_SELECTED_BOOKMARK}:any) =>
         };
     },[]))
 
-    return (<>{styles && !isLoading && <>
+    return (<>{styles && !isLoading && <View
+        style={{
+            marginHorizontal: Math.max((Dimensions.width+Dimensions.height)/2*0.015,8),
+            marginVertical: 16,
+        }}
+    >
         <TouchableRipple 
             rippleColor={Theme[themeTypeContext].ripple_color_outlined}
             onPress={()=>{router.navigate(`/view/${item.source}/${item.id}/?mode=local`)}}
@@ -76,7 +80,7 @@ const ComicComponent = ({item, SELECTED_BOOKMARK, SET_SELECTED_BOOKMARK}:any) =>
                 <Text style={styles.item_title}>{item.info.title}</Text>
             </>
         </TouchableRipple>
-    </>}</>)
+    </View>}</>)
     
 
 }

frontend/app/bookmark/components/widgets/bookmark.tsx

@@ -81,13 +81,20 @@ const BookmarkWidget: React.FC<BookmarkWidgetProps> = ({
                                         gap:18,
                                     }}
                                 >
-                                    <Text
+                                    <View
                                         style={{
-                                            color:"white",
-                                            fontFamily:"roboto-medium",
-                                            fontSize:(Dimensions.width+Dimensions.height)/2*0.025
+                                            flex:1,
                                         }}
-                                    >{item.label}</Text>
+                                    >
+                                        <Text
+                                            style={{
+                                                color:"white",
+                                                fontFamily:"roboto-medium",
+                                                fontSize:(Dimensions.width+Dimensions.height)/2*0.025,
+                                                width:"100%"
+                                            }}
+                                        >{item.label}</Text>
+                                    </View>
                                     <View
                                         style={{
                                             width:"auto",

frontend/app/bookmark/index.tsx

@@ -51,23 +51,50 @@ const Index = ({}:any) => {
 
     const [COMIC_DATA, SET_COMIC_DATA] = useState<any>([])
 
-    useEffect(() => {
+    const [reRender, setReRender] = useState(false)
+    const max_column = useRef(0)
+
+
+    useFocusEffect(useCallback(() => {
+    
+        if (!reRender) return
+        else setReRender(false)
+        
+    },[reRender]))
+
+    useFocusEffect(useCallback(() => {
+        const item_box = Math.max(((Dimensions.width+Dimensions.height)/2)*0.225,100) + (Math.max((Dimensions.width+Dimensions.height)/2*0.015,8))*2;
+
+        const number_of_column = Math.floor((Dimensions.width-(((Dimensions.width+Dimensions.height)/2)*0.035)-32)/item_box)
+        if (max_column.current !== number_of_column) {
+            max_column.current = number_of_column
+            setReRender(true)
+        }
+    }, [Dimensions]))
+
+
+    // First load + Refresh ->
+    useFocusEffect(useCallback(() => {
+        (async ()=>{
+            if (!onRefresh) return
+            else setOnRefresh(false)
+        })()
+    },[onRefresh]))
+
+    useFocusEffect(useCallback(() => {
         (async ()=>{
             if (!SELECTED_BOOKMARK) return
             const stored_comic = await ComicStorage.getByTag(SELECTED_BOOKMARK)
-            console.log(stored_comic)
             SET_COMIC_DATA(stored_comic)
         })()
-    },[SELECTED_BOOKMARK,onRefresh])
+    },[SELECTED_BOOKMARK]))
 
     useFocusEffect(useCallback(() => {
         (async ()=>{
             setIsLoading(true)
             const stored_bookmark = await Storage.get("bookmark") || []
-            console.log(stored_bookmark)
             
             SET_BOOKMARK_DATA(stored_bookmark)
-            console.log("AA",stored_bookmark.length )
             if (stored_bookmark.length) {
                 SET_SELECTED_BOOKMARK(stored_bookmark[0])
             }
@@ -91,11 +118,11 @@ const Index = ({}:any) => {
         setIsLoading(true)
         setShowMenuContext(true)
         setStyles(__styles(themeTypeContext,Dimensions))
-
         return () => {
             controller.abort();
         };
     },[]))
+    // <- End
 
     return (<>{styles && ! isLoading
         ? <>
@@ -167,7 +194,6 @@ const Index = ({}:any) => {
                         contentContainerStyle={{
                             flex:1,
                             flexGrow: 1,
-                            justifyContent: 'center',
                         }}
                         horizontal={true}
                         data={BOOKMARK_DATA}
@@ -188,7 +214,7 @@ const Index = ({}:any) => {
                             setWidgetContext({state:true,component:
                                 <BookmarkWidget
                                     setIsLoading={setIsLoading}
-                                    onRefresh={()=>{setOnRefresh(!onRefresh)}}
+                                    onRefresh={()=>{setOnRefresh(true)}}
 
                                 />
                             })
@@ -197,85 +223,85 @@ const Index = ({}:any) => {
                         <Icon source={require("@/assets/icons/tag-edit-outline.png")} size={((Dimensions.width+Dimensions.height)/2)*0.0325} color={Theme[themeTypeContext].icon_color}/>
                     </TouchableRipple>
                 </View>
-                <FlatList
-                    contentContainerStyle={{
-                        flexGrow: 1,
-                        padding:12,
-                        flexDirection:"row",
-                        gap:Math.max((Dimensions.width+Dimensions.height)/2*0.015,8),
-                        flexWrap:"wrap",
-                    }}
-                    renderItem={renderComicComponent}
-                    ItemSeparatorComponent={undefined}
-                    data={COMIC_DATA.filter((item:any) => item.info.title.toLowerCase().includes(search.text.toLowerCase()))}
-                    ListEmptyComponent={
-                        <View
-                            style={{
-                                width:"100%",
-                                height:"100%",
-                                backgroundColor:"transparent",
-                                display:"flex",
-                                justifyContent:"center",
-                                alignItems:"center",
-                                flexDirection:"row",
-                                gap:12,
-                            }}
-                        >
-                            <>{BOOKMARK_DATA.length 
-                                ? <>
-                                    {search.text && COMIC_DATA.length
-                                        ? <>
-                                            <Icon source={"magnify-scan"} color={Theme[themeTypeContext].icon_color} size={((Dimensions.width+Dimensions.height)/2)*0.03}/>
-                                            <Text selectable={false}
-                                                style={{
-                                                    fontFamily:"roboto-bold",
-                                                    fontSize:((Dimensions.width+Dimensions.height)/2)*0.025,
-                                                    color:Theme[themeTypeContext].text_color,
-                                                }}
-                                            >Search no result</Text>
-                                        </>
-                                        : <>
-                                            <Icon source={require("@/assets/icons/tag-hidden.png")} color={Theme[themeTypeContext].icon_color} size={((Dimensions.width+Dimensions.height)/2)*0.03}/>
-                                            <Text selectable={false}
-                                                style={{
-                                                    fontFamily:"roboto-bold",
-                                                    fontSize:((Dimensions.width+Dimensions.height)/2)*0.025,
-                                                    color:Theme[themeTypeContext].text_color,
-                                                }}
-                                            >This tag is empty.</Text>
-                                        </>
-                                    }
-                                </>
-                                : <View style={{
+                <>{!reRender && (
+                    <FlatList
+                        contentContainerStyle={{
+                            padding:12,
+                            marginHorizontal:"auto",
+                        }}
+                        numColumns={max_column.current}
+                        renderItem={renderComicComponent}
+                        ItemSeparatorComponent={undefined}
+                        data={COMIC_DATA.filter((item:any) => item.info.title.toLowerCase().includes(search.text.toLowerCase()))}
+                        ListEmptyComponent={
+                            <View
+                                style={{
+                                    width:"100%",
+                                    height:"100%",
+                                    backgroundColor:"transparent",
                                     display:"flex",
-                                    flexDirection:"column", 
                                     justifyContent:"center",
                                     alignItems:"center",
-                                    width:"100%",
-                                    height:"auto",
-                                }}>
-                                    
-                                    <Text selectable={false}
-                                        style={{
-                                            fontFamily:"roboto-bold",
-                                            fontSize:((Dimensions.width+Dimensions.height)/2)*0.025,
-                                            color:Theme[themeTypeContext].text_color,
-                                            textAlign:"center",
-                                        }}
-                                    >
-                                        No tag found. {"\n"}Press{"  "}
-                                        <Icon source={require("@/assets/icons/tag-edit-outline.png")} color={Theme[themeTypeContext].icon_color} size={((Dimensions.width+Dimensions.height)/2)*0.03}/>  
-                                        {"  "}to create bookmark tag.
-                                    </Text>
-                                </View>
+                                    flexDirection:"row",
+                                    gap:12,
+                                }}
+                            >
+                                <>{BOOKMARK_DATA.length 
+                                    ? <>
+                                        {search.text && COMIC_DATA.length
+                                            ? <>
+                                                <Icon source={"magnify-scan"} color={Theme[themeTypeContext].icon_color} size={((Dimensions.width+Dimensions.height)/2)*0.03}/>
+                                                <Text selectable={false}
+                                                    style={{
+                                                        fontFamily:"roboto-bold",
+                                                        fontSize:((Dimensions.width+Dimensions.height)/2)*0.025,
+                                                        color:Theme[themeTypeContext].text_color,
+                                                    }}
+                                                >Search no result</Text>
+                                            </>
+                                            : <>
+                                                <Icon source={require("@/assets/icons/tag-hidden.png")} color={Theme[themeTypeContext].icon_color} size={((Dimensions.width+Dimensions.height)/2)*0.03}/>
+                                                <Text selectable={false}
+                                                    style={{
+                                                        fontFamily:"roboto-bold",
+                                                        fontSize:((Dimensions.width+Dimensions.height)/2)*0.025,
+                                                        color:Theme[themeTypeContext].text_color,
+                                                    }}
+                                                >This tag is empty.</Text>
+                                            </>
+                                        }
+                                    </>
+                                    : <View style={{
+                                        display:"flex",
+                                        flexDirection:"column", 
+                                        justifyContent:"center",
+                                        alignItems:"center",
+                                        width:"100%",
+                                        height:"auto",
+                                    }}>
+                                        
+                                        <Text selectable={false}
+                                            style={{
+                                                fontFamily:"roboto-bold",
+                                                fontSize:((Dimensions.width+Dimensions.height)/2)*0.025,
+                                                color:Theme[themeTypeContext].text_color,
+                                                textAlign:"center",
+                                            }}
+                                        >
+                                            No tag found. {"\n"}Press{"  "}
+                                            <Icon source={require("@/assets/icons/tag-edit-outline.png")} color={Theme[themeTypeContext].icon_color} size={((Dimensions.width+Dimensions.height)/2)*0.03}/>  
+                                            {"  "}to create bookmark tag.
+                                        </Text>
+                                    </View>
 
-                            }
-                            </>
+                                }
+                                </>
+                                
+                            </View>
                             
-                        </View>
-                        
-                    }
-                />
+                        }
+                    />
+                )}</>
             </View>
 
         </>

frontend/app/read/[source]/[comic_id]/[chapter_idx].tsx

@@ -58,6 +58,7 @@ const Index = ({}:any) => {
     useFocusEffect(useCallback(() => {
         setShowMenuContext(null)
         return () => {
+            SET_DATA([])
         }
     },[]))
 

frontend/app/view/[source]/[comic_id].tsx

@@ -181,6 +181,7 @@ const Index = ({}:any) => {
         setShowMenuContext(null)
         return () => {
             controller.abort();
+            Toast.hide()
         }
     },[]))
 
@@ -188,8 +189,8 @@ const Index = ({}:any) => {
     const Load_Local = async () => {
         Toast.show({
             type: 'info',
-            text1: '💾 Local mode',
-            text2: `Press refresh button to fetch new updates.`,
+            text1: '⚡ Loaded in Local Mode',
+            text2: `Press refresh button to fetch new updates`,
             
             position: "bottom",
             visibilityTime: 6000,

frontend/app/view/componenets/widgets/bookmark.tsx

@@ -90,13 +90,20 @@ const BookmarkWidget: React.FC<BookmarkWidgetProps> = ({
                                         gap:18,
                                     }}
                                 >
-                                    <Text
+                                    <View
                                         style={{
-                                            color:"white",
-                                            fontFamily:"roboto-medium",
-                                            fontSize:(Dimensions.width+Dimensions.height)/2*0.025
+                                            flex:1,
                                         }}
-                                    >{item.label}</Text>
+                                    >
+                                        <Text
+                                            style={{
+                                                color:"white",
+                                                fontFamily:"roboto-medium",
+                                                fontSize:(Dimensions.width+Dimensions.height)/2*0.025,
+                                                width:"100%"
+                                            }}
+                                        >{item.label}</Text>
+                                    </View>
                                     <View
                                         style={{
                                             width:"auto",

frontend/app/view/modules/content.tsx

@@ -32,7 +32,8 @@ export const get = async (setShowCloudflareTurnstile:any,setIsLoading:any,signal
         timeout: 60000,
         signal:signal,
     }).then((response) => {(async () =>{
-        const DATA = response.data.data
+        const DATA = response.data
+        console.log("EEE",response)
         if (Object.keys(DATA).length) setFeedBack("")
         else{
             setFeedBack("No content found!")
@@ -243,10 +244,15 @@ export const download_chapter = async (
         signal:signal,
     }).then(async (response) => {
         const DATA = response.data
+
+        
+
         if (Platform.OS === "web"){
 
             const zip = new JSZip();
             const zipContent = await zip.loadAsync(DATA);
+            const MAX_PAGE = Object.keys(zipContent.files).length
+
             let page = 0;
             for (const fileName of Object.keys(zipContent.files).sort((a,b) => parseInt(a, 10) - parseInt(b, 10))) {
                 if (zipContent.files[fileName].dir) {
@@ -256,7 +262,8 @@ export const download_chapter = async (
                 const fileData = await zipContent.files[fileName].async('blob');
                 const layout = await getImageLayout(await blobToBase64(fileData, "image/png"));
                 await ChapterDataStorage.store(`${source}-${comic_id}-${request_info.chapter_idx}-${page}`,comic_id,request_info.chapter_idx, fileData, layout)
-                
+                const current_progress = progress_lenth + ((total_length-progress_lenth)*page)/MAX_PAGE
+                download_progress.current = {...download_progress.current, [chapter_id]:{progress:current_progress, total:total_length}}
             }
 
             await ChapterStorage.update(`${source}-${comic_id}`,chapter_id, "completed", page)
@@ -271,7 +278,7 @@ export const download_chapter = async (
             // await ChapterStorage.update(`${source}-${comic_id}`,chapter_id,{type:"file_path", value:chapter_dir + `${request_info.chapter_idx}.zip`}, "completed")
         }
         
-        download_progress.current = {...download_progress.current, [chapter_id]:{progress:total_length, total:total_length}}
+        // download_progress.current = {...download_progress.current, [chapter_id]:{progress:total_length, total:total_length}}