Add script to fetch YouTube playlists and videos

cms/cache_registry.py

@@ -0,0 +1,17 @@
+from django.core.cache import cache
+
+REGISTRY_KEY = "_cache_key_registry"
+
+def register_cache_key(key):
+    keys = cache.get(REGISTRY_KEY) or set()
+    keys.add(key)
+    cache.set(REGISTRY_KEY, keys, None)
+
+def unregister_cache_key(key):
+    keys = cache.get(REGISTRY_KEY) or set()
+    if key in keys:
+        keys.remove(key)
+        cache.set(REGISTRY_KEY, keys, None)
+
+def list_cache_keys():
+    return sorted(cache.get(REGISTRY_KEY) or [])

cms/views.py

@@ -4,6 +4,11 @@
 import random
 import string
 
+from django.contrib.auth.models import User
+
+from cms.models import Profile
+from cms.forms import ChangePasswordForm
+
 # Third Party Stuff
 from django.conf import settings
 from django.contrib import messages, auth
@@ -26,7 +31,7 @@
 from mdldjango.models import MdlUser
 from mdldjango.urls import *
 from django.template.context_processors import csrf
-
+from cms.cache_registry import unregister_cache_key,list_cache_keys
 from donate.models import Payee
 
 from django.core.cache import cache
@@ -270,11 +275,13 @@ def account_logout(request):
 @login_required
 def account_profile(request, username):
     user = request.user
-    try:
-      profile = Profile.objects.get(user_id=user.id)
-    except:
-      profile = create_profile(user)
+
+    profile = Profile.objects.filter(user_id=user.id).order_by('id').first()
+    if not profile:
+        profile = create_profile(user)
+
     old_file_path = settings.MEDIA_ROOT + str(profile.picture)
+
     new_file_path = None
     if request.method == 'POST':
         form = ProfileForm(user, request.POST, request.FILES, instance = profile)
@@ -319,7 +326,8 @@ def account_profile(request, username):
     else:
         context = {}
         context.update(csrf(request))
-        instance = Profile.objects.get(user_id=user.id)
+        # instance = Profile.objects.get(user_id=user.id)
+        instance = Profile.objects.filter(user_id=user.id).first()
         context['form'] = ProfileForm(user, instance = instance)
         return render(request, 'cms/templates/profile.html', context)
 
@@ -329,13 +337,10 @@ def account_view_profile(request, username):
         raise PermissionDenied('You are not allowed to view this page!')
 
     user = User.objects.get(username = username)
-    profile = None
-    try:
-      profile = Profile.objects.get(user = user)
-    except:
-      profile = create_profile(user)
+    profile = Profile.objects.filter(user=user).first()
+    if not profile:
+        profile = create_profile(user)
 
-
     context = {
         'profile' : profile,
         'media_url' : settings.MEDIA_URL,
@@ -425,45 +430,62 @@ def password_reset(request):
     return render(request, 'cms/templates/password_reset.html', context)
 
 
-#@login_required
 def change_password(request):
-    # chacking uselogin
-    pcode = request.GET.get('auto', False)
-    username = request.GET.get('username', False)
-    nextUrl = request.GET.get('next', False)
+    pcode = request.GET.get('auto')
+    username = request.GET.get('username')
+    nextUrl = request.GET.get('next')
 
-    # check pcode in profile page
-    if pcode and username and nextUrl:
-        user = User.objects.get(username=username)
-        profile = Profile.objects.get(user=user)
-        if profile.confirmation_code == pcode:
-            user.backend='django.contrib.auth.backends.ModelBackend'
-            login(request,user)
+    profile = None
+
+    if pcode and username:
+        user = User.objects.get(username=username).first()
+        if user:
+            profile = Profile.objects.filter(user=user,confirmation_code=pcode).first()
+
+            if profile:
+                user.backend = 'django.contrib.auth.backends.ModelBackend'
+                login(request, user)
 
     if request.user.is_anonymous():
-        return HttpResponseRedirect('/accounts/login/?next=/accounts/change-password')
+        return HttpResponseRedirect('/accounts/login/?next=/accounts/change-password/')
+
+    if not profile:
+        profile = (Profile.objects.filter(user=request.user).order_by('id').first())
+
+    if not profile:
+        messages.error(request, "Profile not found.")
+        return HttpResponseRedirect('/accounts/login/')
 
-    context = {}
     form = ChangePasswordForm()
+
     if request.method == "POST":
         form = ChangePasswordForm(request.POST)
         if form.is_valid():
-            profile = Profile.objects.get(user_id = form.cleaned_data['userid'], confirmation_code = form.cleaned_data['code'])
+            profile = Profile.objects.get(user_id=form.cleaned_data['userid'],confirmation_code=form.cleaned_data['code'])
+
             user = profile.user
             user.set_password(form.cleaned_data['new_password'])
             user.save()
-            # change if any mdl user pass too
+            # change if any mdl user pass too            
             from mdldjango.views import changeMdlUserPass
             changeMdlUserPass(user.email, form.cleaned_data['new_password'])
 
             if nextUrl:
                 return HttpResponseRedirect(nextUrl.split("?", 1)[0])
 
-            messages.success(request, "Your account password has been updated successfully!")
+            messages.success(request,"Your account password has been updated successfully!")
             return HttpResponseRedirect("/accounts/view-profile/" + user.username)
-    context['form'] = form
+    if profile is None:
+        profile = request.user.profile_set.first()
+    context = {
+        'form': form,
+        'profile': profile,
+        'nextUrl': nextUrl,
+    }
     context.update(csrf(request))
-    return render(request, 'cms/templates/change_password.html', context)
+
+    return render(request,'cms/templates/change_password.html',context)
+
 
 
 def confirm_student(request, token):
@@ -511,7 +533,7 @@ def verify_email(request):
 @login_required
 def manage_cache(request):
 
-    if not request.user.groups.filter(name='Technical-Team').exists():
+    if not request.user.groups.filter(name__in=['Technical-Team', 'Administrator']).exists():
         raise PermissionDenied('You are not allowed to view this page!')
 
     context = {}
@@ -535,6 +557,39 @@ def manage_cache(request):
                     messages.success(request, f"memcache cleared successfully")
                 except Exception as e:
                     messages.error(request, f"An error occurred while clearing cache: {e}")
-                    print(f"cache error -- {e}")
+
+            elif deletion_type == 'homepage':
+                try:
+                    all_keys = list_cache_keys()
+                    homepage_keys = [
+                        key for key in all_keys
+                        if key.startswith((
+                            'tutorial_search_foss:',
+                            'tutorial_search_lang:',
+                        ))
+                    ]
+
+                    if homepage_keys:
+                        for key in homepage_keys:
+                            cache.delete(key)
+                            unregister_cache_key(key)
+                        messages.success(
+                            request,
+                            "Homepage cache cleared successfully.<br>"
+                            "Deleted keys:<br>"
+                            + "<br>".join(homepage_keys)
+                        )
+
+                    else:
+                        messages.warning(
+                            request,
+                            "Homepage cache keys were not found or already expired."
+                        )
+
+                except Exception as e:
+                    messages.error(
+                        request,
+                        "An error occurred while clearing homepage cache: {}".format(e)
+                    )
 
     return render(request, status_template, context=context) # return to payment page site

creation/management/commands/download_course_data.py

@@ -0,0 +1,121 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from pathlib import Path
+import re
+import json
+import requests
+from creation.models import *
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--ids",
+            type=str,
+            required=True,
+            help="Comma-separated integers, e.g. 1,2,3",
+        )
+        parser.add_argument(
+            "--out",
+            type=str,
+            default="output.json",
+            help="Output JSON file path (default: output.json)",
+        )
+
+    def handle(self, *args, **options):
+        # python manage.py my_command --ids 1,2,3,10
+        raw = options["ids"]
+        out_path = Path(options["out"])
+        try:
+            ids = [int(x.strip()) for x in raw.split(",") if x.strip() != ""]
+            for course_id in ids:
+
+                try:
+                    foss = FossCategory.objects.using('stats').get(id=course_id, show_on_homepage=1)
+                    foss_format = foss.foss.strip().replace(' ','+')
+                    qs = TutorialResource.objects.using('stats').filter(tutorial_detail__foss_id=foss.id, status = 1)
+                    languages = list(qs.distinct().order_by('language__name').values_list('language__name', flat=True))
+                    categories = list(foss.category.all().values_list('name', flat=True))
+                    course_data = {
+                        "course_id": foss.id,
+                        "course": foss.foss,
+                        "course_url": f"https://spoken-tutorial.org/tutorial-search/?search_foss={foss_format}&search_language=",
+                        "categories": categories,
+                        "languages": languages
+                    }
+                    course_data['tutorials'] = []
+                    tutorial_resources = qs.select_related('tutorial_detail', 'language', 'tutorial_detail__foss', 'common_content')
+                    for tr in tutorial_resources:
+
+                        title_formatted = tr.tutorial_detail.tutorial.strip().replace(' ', '+')
+                        title_formatted_srt = tr.tutorial_detail.tutorial.strip().replace(' ', '-')
+                        language = tr.language.name
+                        try:
+                            duration = TutorialDuration.objects.using('stats').get(tresource=tr).duration
+                        except:
+                            duration = None
+                        tutorial_script = self.extract_text_one_paragraph(foss.id, tr.tutorial_detail.id, title_formatted_srt, language)
+                        tutorial_data = {
+                            "tutorial_resource_id": tr.id,
+                            "tutorial_detail_id": tr.tutorial_detail.id,
+                            "title": tr.tutorial_detail.tutorial.strip(),
+                            "url": f"https://spoken-tutorial.org/watch/{foss_format}/{title_formatted}/{language}/",
+                            "keywords": tr.common_content.keyword,
+                            "outline": tr.outline,
+                            "language": language,
+                            "script url": f"https://script.spoken-tutorial.org/index.php/{tr.script}",
+                            "tutorial_script": tutorial_script,
+                            "duration": duration,
+                            "level": tr.tutorial_detail.level.level
+
+                        }
+                        course_data['tutorials'].append(tutorial_data)
+                        self.stdout.write(self.style.SUCCESS(f"Added to tutorials: {tr.tutorial_detail.tutorial} - {language}"))
+                except FossCategory.DoesNotExist:
+                    self.stderr.write(self.style.ERROR(f"No foss found with id: {course_id}. show_on_homepage = 1"))
+
+            # Ensure parent folder exists
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+
+            # Write JSON
+            with out_path.open("w", encoding="utf-8") as f:
+                if course_data:
+                    json.dump(course_data, f, ensure_ascii=False, indent=2)
+                else:
+                    self.stderr.write(self.style.ERROR(f"No data found"))
+            self.stdout.write(self.style.SUCCESS(f"Wrote JSON to: {out_path.resolve()}"))
+        except ValueError:
+            raise CommandError(f"--ids must be comma-separated integers. Got: {raw}")
+
+        if not ids:
+            raise CommandError("--ids is empty")
+
+        self.stdout.write(f"IDs: {ids}")
+
+
+    def extract_text_one_paragraph(self, course_id, tr_id,title,lang):
+        srt_path = "/Users/ankita/workspace/projects/spoken/project/spoken-website/media/videos/48/478/Introduction-to-BASH-Shell-Scripting-English.srt"
+        url = f"https://spoken-tutorial.org/media/videos/{course_id}/{tr_id}/{title}-{lang}.srt"
+        try:
+            res = requests.get(url, timeout=30)
+            res.raise_for_status() # raises error for 4xx/5xx
+            srt_text = res.text
+        except: 
+            self.stderr.write(self.style.ERROR(f"Error for srt file: {url}"))
+            srt_text = ""
+        TAG_RE = re.compile(r"<[^>]+>")
+        TIME_RE = re.compile(r"^\d{2}:\d{2}:\d{2}\s*-->\s*\d{2}:\d{2}:\d{2}")
+        # lines = Path(srt_path).read_text(encoding="utf-8", errors="ignore").splitlines()
+        lines = srt_text.splitlines()
+
+        out = []
+        for line in lines:
+            s = line.strip()
+            if not s or s.isdigit() or TIME_RE.match(s):
+                continue
+            s = TAG_RE.sub("", s)
+            s = re.sub(r"\s+", " ", s).strip()
+            if s:
+                out.append(s)
+        data = " ".join(out)
+        return data