feat: integrated lesson markup

- added PageExtension plugin to inject custom styles scripts in a page
- removed the livecode integration and enabled PageExtension plugins for
  learn page
- also merged the profile_tab.py with plugins.py
- added a utility to find the macros from given text
- updated the before_save of lesson to find exercises using the macros
  and update the exercises as before

Issue #115

community/hooks.py

@@ -176,3 +176,15 @@ profile_rules = [
 website_route_rules = primary_rules + whitelist_rules + profile_rules
 
 update_website_context = 'community.widgets.update_website_context'
+
+## Specify the additional tabs to be included in the user profile page.
+## Each entry must be a subclass of community.community.plugins.ProfileTab
+# profile_tabs = []
+
+## Specify the extension to be used to control what scripts and stylesheets
+## to be included in lesson pages. The specified value must be be a
+## subclass of community.community.plugins.PageExtension
+# community_lesson_page_extension = None
+
+## Markdown Macros for Lessons
+# community_markdown_macro_renderers = {"Exercise": "myapp.mymodule.plugins.render_exercise"}

community/lms/doctype/chapter/chapter.py

@@ -10,6 +10,6 @@ class Chapter(Document):
     def get_lessons(self):
         rows = frappe.db.get_all("Lesson",
             filters={"chapter": self.name},
-            fields='*',
+            fields='name',
             order_by="index_")
-        return [frappe.get_doc(dict(row, doctype='Lesson')) for row in rows]
+        return [frappe.get_doc('Lesson', row['name']) for row in rows]

community/lms/doctype/exercise/exercise.json

@@ -15,7 +15,9 @@
   "hints",
   "tests",
   "image",
-  "lesson"
+  "lesson",
+  "index_",
+  "index_label"
  ],
  "fields": [
   {
@@ -27,6 +29,7 @@
   {
    "fieldname": "course",
    "fieldtype": "Link",
+   "in_list_view": 1,
    "label": "Course",
    "options": "LMS Course"
   },
@@ -73,13 +76,27 @@
   {
    "fieldname": "lesson",
    "fieldtype": "Link",
+   "in_list_view": 1,
    "label": "Lesson",
    "options": "Lesson"
+  },
+  {
+   "fieldname": "index_",
+   "fieldtype": "Int",
+   "label": "Index",
+   "read_only": 1
+  },
+  {
+   "fieldname": "index_label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Index Label",
+   "read_only": 1
   }
  ],
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-05-20 13:23:12.340928",
+ "modified": "2021-06-01 05:22:15.656013",
  "modified_by": "Administrator",
  "module": "LMS",
  "name": "Exercise",
@@ -99,8 +116,8 @@
   }
  ],
  "search_fields": "title",
- "sort_field": "modified",
- "sort_order": "DESC",
+ "sort_field": "index_label",
+ "sort_order": "ASC",
  "title_field": "title",
  "track_changes": 1
 }

community/lms/doctype/exercise/exercise.py

@@ -25,7 +25,6 @@ class Exercise(Document):
             order_by="creation desc",
             page_length=1)
 
-        print("get_user_submission", result)
         if result:
             return result[0]
 

community/lms/doctype/lesson/lesson.json

@@ -10,6 +10,7 @@
   "lesson_type",
   "title",
   "index_",
+  "index_label",
   "body",
   "sections"
  ],
@@ -51,11 +52,18 @@
    "fieldtype": "Table",
    "label": "Sections",
    "options": "LMS Section"
+  },
+  {
+   "fieldname": "index_label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Index Label",
+   "read_only": 1
   }
  ],
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-05-13 20:03:51.510605",
+ "modified": "2021-06-01 05:30:48.127494",
  "modified_by": "Administrator",
  "module": "LMS",
  "name": "Lesson",

community/lms/doctype/lesson/lesson.py

@@ -6,20 +6,50 @@ from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
 from ...section_parser import SectionParser
+from ...md import markdown_to_html, find_macros
 
 class Lesson(Document):
     def before_save(self):
-        sections = SectionParser().parse(self.body or "")
-        self.sections = [self.make_lms_section(i, s) for i, s in enumerate(sections)]
-        for s in self.sections:
-            if s.type == "exercise":
-                e = s.get_exercise()
-                e.lesson = self.name
-                e.save()
+        macros = find_macros(self.body)
+        exercises = [value for name, value in macros if name == "Exercise"]
+
+        index = 1
+        for name in exercises:
+            e = frappe.get_doc("Exercise", name)
+            e.lesson = self.name
+            e.index_ = index
+            e.save()
+            index += 1
+        self.update_orphan_exercises(exercises)
+
+    def update_orphan_exercises(self, active_exercises):
+        """Updates the exercises that were previously part of this lesson,
+        but not any more.
+        """
+        linked_exercises = {row['name'] for row in frappe.get_all('Exercise', {"lesson": self.name})}
+        active_exercises = set(active_exercises)
+        orphan_exercises = linked_exercises - active_exercises
+        for name in orphan_exercises:
+            ex = frappe.get_doc("Exercise", name)
+            ex.lesson = None
+            ex.index_ = 0
+            ex.index_label = ""
+            ex.save()
+
+    def render_html(self):
+        return markdown_to_html(self.body)
 
     def get_sections(self):
         return sorted(self.get('sections'), key=lambda s: s.index)
 
+    def get_exercises(self):
+        if not self.body:
+            return []
+
+        macros = find_macros(self.body)
+        exercises = [value for name, value in macros if name == "Exercise"]
+        return [frappe.get_doc("Exercise", name) for name in exercises]
+
     def make_lms_section(self, index, section):
             s = frappe.new_doc('LMS Section', parent_doc=self, parentfield='sections')
             s.type = section.type

community/lms/doctype/lms_course/lms_course.json

@@ -1,5 +1,18 @@
 {
- "actions": [],
+ "actions": [
+  {
+   "action": "community.lms.doctype.lms_course.lms_course.reindex_lessons",
+   "action_type": "Server Action",
+   "group": "Reindex",
+   "label": "Reindex Lessons"
+  },
+  {
+   "action": "community.lms.doctype.lms_course.lms_course.reindex_exercises",
+   "action_type": "Server Action",
+   "group": "Reindex",
+   "label": "Reindex Exercises"
+  }
+ ],
  "allow_guest_to_view": 1,
  "allow_rename": 1,
  "creation": "2021-03-01 16:49:33.622422",
@@ -86,7 +99,7 @@
    "link_fieldname": "course"
   }
  ],
- "modified": "2021-05-23 18:14:32.602647",
+ "modified": "2021-06-01 04:36:45.696776",
  "modified_by": "Administrator",
  "module": "LMS",
  "name": "LMS Course",

community/lms/doctype/lms_course/lms_course.py

@@ -5,6 +5,7 @@
 from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
+import json
 from ...utils import slugify
 from community.query import find, find_all
 
@@ -157,6 +158,35 @@ class LMSCourse(Document):
         chapter = frappe.get_doc("Chapter", lesson.chapter)
         return f"{chapter.index_}.{lesson.index_}"
 
+    def reindex_lessons(self):
+        for i, c in enumerate(self.get_chapters(), start=1):
+            c.index_ = i
+            c.save()
+            self._reindex_lessons_in_chapter(c)
+
+    def _reindex_lessons_in_chapter(self, c):
+        for i, lesson in enumerate(c.get_lessons(), start=1):
+            lesson.index = i
+            lesson.index_label = f"{c.index_}.{i}"
+            lesson.save()
+
+    def reindex_exercises(self):
+        for i, c in enumerate(self.get_chapters(), start=1):
+            if c.index_ != i:
+                c.index_ = i
+                c.save()
+            self._reindex_exercises_in_chapter(c)
+
+    def _reindex_exercises_in_chapter(self, c):
+        i = 1
+        for lesson in c.get_lessons():
+            for exercise in lesson.get_exercises():
+                exercise.index_ = i
+                exercise.index_label = f"{c.index_}.{i}"
+                exercise.save()
+                i += 1
+
+
     def get_outline(self):
         return CourseOutline(self)
 
@@ -187,7 +217,8 @@ class CourseOutline:
     def get_chapters(self):
         return frappe.db.get_all("Chapter",
             filters={"course": self.course.name},
-            fields=["name", "title", "index_"])
+            fields=["name", "title", "index_"],
+            order_by="index_")
 
     def get_lessons(self):
         chapters = [c['name'] for c in self.chapters]
@@ -199,3 +230,17 @@ class CourseOutline:
         for lesson in lessons:
             lesson['number'] = "{}.{}".format(chapter_numbers[lesson['chapter']], lesson['index_'])
         return lessons
+
+@frappe.whitelist()
+def reindex_lessons(doc):
+    course_data = json.loads(doc)
+    course = frappe.get_doc("LMS Course", course_data['name'])
+    course.reindex_lessons()
+    frappe.msgprint("All lessons in this course have been re-indexed.")
+
+@frappe.whitelist()
+def reindex_exercises(doc):
+    course_data = json.loads(doc)
+    course = frappe.get_doc("LMS Course", course_data['name'])
+    course.reindex_exercises()
+    frappe.msgprint("All exercises in this course have been re-indexed.")

community/lms/md.py

@@ -0,0 +1,107 @@
+"""
+The md module extends markdown to add macros.
+
+Macros can be added to the markdown text in the following format.
+
+    {{ MacroName("macro-argument") }}
+
+These macros will be rendered using a pluggable mechanism.
+
+Apps can provide a hook community_markdown_macro_renderers, a
+dictionary mapping the macro name to the function that to render
+that macro. The function will get the argument passed to the macro
+as argument.
+"""
+
+import frappe
+import re
+from bs4 import BeautifulSoup
+import markdown
+from markdown import Extension
+from markdown.inlinepatterns import InlineProcessor
+import xml.etree.ElementTree as etree
+
+def markdown_to_html(text):
+    """Renders markdown text into html.
+    """
+    return markdown.markdown(text, extensions=['fenced_code', MacroExtension()])
+
+def find_macros(text):
+    """Returns all macros in the given text.
+
+        >>> find_macros(text)
+        [
+            ('YouTubeVideo': 'abcd1234')
+            ('Exercise', 'two-circles'),
+            ('Exercise', 'four-circles')
+        ]
+    """
+    macros = re.findall(MACRO_RE, text)
+    # remove the quotes around the argument
+    return [(name, _remove_quotes(arg)) for name, arg in macros]
+
+def _remove_quotes(value):
+    """Removes quotes around a value.
+
+    Also strips the whitespace.
+
+        >>> _remove_quotes('"hello"')
+        'hello'
+        >>> _remove_quotes("'hello'")
+        'hello'
+        >>> _remove_quotes("hello")
+        'hello'
+    """
+    return value.strip(" '\"")
+
+
+def get_macro_registry():
+    d = frappe.get_hooks("community_markdown_macro_renderers") or {}
+    return {name: frappe.get_attr(klass[0]) for name, klass in d.items()}
+
+def render_macro(macro_name, macro_argument):
+    # stripping the quotes on either side of the argument
+    macro_argument = _remove_quotes(macro_argument)
+
+    registry = get_macro_registry()
+    if macro_name in registry:
+        return registry[macro_name](macro_argument)
+    else:
+        return f"<p>Unknown macro: {macro_name}</p>"
+
+MACRO_RE = r'{{ *(\w+)\(([^{}]*)\) *}}'
+
+class MacroExtension(Extension):
+    """MacroExtension is a markdown extension to support macro syntax.
+    """
+    def extendMarkdown(self, md):
+        self.md = md
+        pattern = MacroInlineProcessor(MACRO_RE)
+        pattern.md = md
+        md.inlinePatterns.register(pattern, 'macro', 75)
+
+class MacroInlineProcessor(InlineProcessor):
+    """MacroInlineProcessor is class that is handles the logic
+    of how to render each macro occurence in the markdown text.
+    """
+    def handleMatch(self, m, data):
+        """Handles each macro match and return rendered contents
+        for that macro as an etree node.
+        """
+        macro = m.group(1)
+        arg = m.group(2)
+        html = render_macro(macro, arg)
+        html = sanitize_html(str(html))
+        e = etree.fromstring(html)
+        return e, m.start(0), m.end(0)
+
+def sanitize_html(html):
+    """Sanotize the html using BeautifulSoup.
+
+    The markdown processor request the correct markup and crashes on
+    any broken tags. This makes sures that all those things are fixed
+    before passing to the etree parser.
+    """
+    soup = BeautifulSoup(html, features="lxml")
+    nodes = soup.body.children
+    return "<div>" + "\n".join(str(node) for node in nodes) + "</div>"

community/lms/widgets/Exercise.html

@@ -1,7 +1,7 @@
 {% from "www/macros/livecode.html" import LiveCodeEditorJS, LiveCodeEditor with context %}
 
 <div class="exercise">
-  <h2>{{ exercise.title }}</h2>
+  <h2>Exercise {{exercise.index_label}}: {{ exercise.title }}</h2>
   <div class="exercise-description">{{frappe.utils.md_to_html(exercise.description)}}</div>
 
   {% if exercise.image %}

community/plugins.py

@@ -0,0 +1,66 @@
+"""
+The plugins module provides various plugins to change the default
+behaviour some parts of the community app.
+
+A site specify what plugins to use using appropriate entries in the frappe
+hooks, written in the `hooks.py`.
+
+This module exposes two plugins: ProfileTab and PageExtension.
+
+The ProfileTab is used to specify any additional tabs to be displayed
+on the profile page of the user.
+
+The PageExtension is used to load additinal stylesheets and scripts to
+be loaded in a webpage.
+"""
+
+class PageExtension:
+    """PageExtension is a plugin to inject custom styles and scripts
+    into a web page.
+
+    The subclasses should overwrite the `render_header()` and
+    `render_footer()` methods to inject whatever styles/scripts into
+    the webpage.
+    """
+
+    def render_header(self):
+        """Returns the HTML snippet to be included in the head section
+        of the web page.
+
+        Typically used to include the stylesheets and javascripts to be
+        included in the <head> of the webpage.
+        """
+        return ""
+
+    def render_footer(self):
+        """Returns the HTML snippet to be included in the body tag at
+        the end of web page.
+
+        Typically used to include javascripts that need to be executed
+        after the page is loaded.
+        """
+        return ""
+
+class ProfileTab:
+    """Base class for profile tabs.
+
+    Every subclass of ProfileTab must implement two methods:
+        - get_title()
+        - render()
+    """
+    def __init__(self, user):
+        self.user = user
+
+    def get_title(self):
+        """Returns the title of the tab.
+
+        Every subclass must implement this.
+        """
+        raise NotImplementedError()
+
+    def render(self):
+        """Renders the contents of the tab as HTML.
+
+        Every subclass must implement this.
+        """
+        raise NotImplementedError()

community/www/batch/learn.html

@@ -9,16 +9,13 @@
 </style>
 
 <link rel="stylesheet" href="/assets/frappe/css/font-awesome.css">
-<link rel="stylesheet" href="{{ livecode_url }}/static/codemirror/lib/codemirror.css">
 <link rel="stylesheet" href="/assets/css/lms.css">
 <link rel="stylesheet" href="/assets/frappe/css/hljs-night-owl.css">
 
-<script src="{{ livecode_url }}/static/codemirror/lib/codemirror.js"></script>
-<script src="{{ livecode_url }}/static/codemirror/mode/python/python.js"></script>
-<script src="{{ livecode_url }}/static/codemirror/keymap/sublime.js"></script>
+{% for ext in page_extensions %}
+  {{ ext.render_header() }}
+{% endfor %}
 
-<script src="{{ livecode_url }}/static/codemirror/addon/edit/matchbrackets.js"></script>
-<script src="{{ livecode_url }}/static/codemirror/addon/comment/comment.js"></script>
 {% endblock %}
 
 
@@ -30,11 +27,7 @@
 
     <h2 class="title {% if course.is_mentor(frappe.session.user) %} is_mentor {% endif %}" data-name="{{ lesson.name }}" data-batch="{{ batch.name }}">{{ lesson.title }}</h2>
 
-    {% for s in lesson.get_sections() %}
-    <div class="section section-{{ s.type }}">
-      {{ render_section(s) }}
-    </div>
-    {% endfor %}
+    {{ lesson.render_html() }}
 
     {{ pagination(prev_chap, prev_url, next_chap, next_url) }}
   </div>
@@ -42,30 +35,6 @@
 {% endblock %}
 
 
-{% macro render_section(s) %}
-{% if s.type == "text" %}
-{{ render_section_text(s) }}
-{% elif s.type == "example" or s.type == "code" %}
-{{ LiveCodeEditor(s.name,
-          code=s.get_latest_code_for_user(),
-          reset_code=s.contents,
-          is_exercise=False)
-    }}
-{% elif s.type == "exercise" %}
-{{ widgets.Exercise(exercise=s.get_exercise())}}
-{% else %}
-<div>Unknown section type: {{s.type}}</div>
-{% endif %}
-{% endmacro %}
-
-{% macro render_section_text(s) %}
-<div class="row">
-  <div class="col-md-9">
-    {{ frappe.utils.md_to_html(s.contents) }}
-  </div>
-</div>
-{% endmacro %}
-
 {% macro pagination(prev_chap, prev_url, next_chap, next_url) %}
 <div class="lesson-pagination">
   {% if prev_url %}
@@ -84,8 +53,6 @@
 
 {%- block script %}
 {{ super() }}
-{{ LiveCodeEditorJS() }}
-
   <script type="text/javascript">
     $(function() {
       var batch_name = "{{ batch.name }}";
@@ -98,4 +65,8 @@
     })
   </script>
 
+{% for ext in page_extensions %}
+  {{ ext.render_footer() }}
+{% endfor %}
+
 {%- endblock %}

community/www/batch/learn.py

@@ -28,7 +28,7 @@ def get_context(context):
     context.next_url = context.batch.get_learn_url(next_)
     context.prev_url = context.batch.get_learn_url(prev_)
 
-
+    context.page_extensions = get_page_extensions()
 
 def get_chapter_title(course_name, lesson_number):
     if not lesson_number:
@@ -42,4 +42,8 @@ def get_lesson_index(course, batch, user):
     lesson = batch.get_current_lesson(user)
     return lesson and course.get_lesson_index(lesson)
 
-
+def get_page_extensions():
+    default_value = ["community.community.plugins.PageExtension"]
+    classnames = frappe.get_hooks("community_lesson_page_extensions") or default_value
+    extensions = [frappe.get_attr(name)() for name in classnames]
+    return extensions

community/www/batch/progress.html

@@ -29,7 +29,7 @@
     <h1>Batch Progress</h1>
     {% for exercise in report.exercises %}
     <div class="exercise-submissions">
-      <h2>{{exercise.title}}</h2>
+      <h2>Exercise {{exercise.index_label}}: {{exercise.title}}</h2>
       {% for s in report.get_submissions_of_exercise(exercise.name) %}
       <div class="submission">
         <h4><a href="/{{s.owner.username}}">{{s.owner.full_name}}</a></h4>

community/www/batch/progress.py

@@ -25,7 +25,7 @@ class BatchReport:
             self.submissions_by_exercise[s.exercise].append(s)
 
     def get_exercises(self, course_name):
-        return frappe.get_all("Exercise", {"course": course_name}, ["name", "title"])
+        return frappe.get_all("Exercise", {"course": course_name, "lesson": ["!=", ""]}, ["name", "title", "index_label"], order_by="index_label")
 
     def get_submissions_of_exercise(self, exercise_name):
         return self.submissions_by_exercise[exercise_name]

community/www/profiles/profile.html

@@ -72,29 +72,26 @@
     </div>
     <div>
       <ul class="nav nav-tabs mt-4" id="myTab" role="tablist">
-        <li class="nav-item">
-          <a class="nav-link active" id="home-tab" data-toggle="tab" href="#home" role="tab" aria-controls="home"
-            aria-selected="true">Sketches</a>
-        </li>
+        {% for tab in profile_tabs %}
+          <li class="nav-item">
+            {% set slug = title.lower().replace(" ", "-") %}
+            {% set selected = loop.index == 1 %}
+            {% set active = 'active' if loop.index == 1 else '' %}
+            <a class="nav-link {{ active }}" id="{{ slug }}-tab" data-toggle="tab" href="#{{ slug }}" role="tab" aria-controls="{{ slug }}"
+              aria-selected="{{ selected }}">Sketches</a>
+          </li>
+        {% endfor %}
       </ul>
     </div>
     <div>
-      <div class="tab-content">
-        <div class="tab-pane fade py-4 show active" role="tabpanel" id="home">
-          <div class="row">
-            {% if sketches %}
-            {% for sketch in sketches %}
-            <div class="col-md-4 col-sm-6">
-              {{ widgets.SketchTeaser(sketch=sketch) }}
-            </div>
-            {% endfor %}
-            {% endif %}
+      {% for tab in profile_tabs %}
+        {% set slug = title.lower().replace(" ", "-") %}
+        <div class="tab-content">
+          <div class="tab-pane fade py-4 show active" role="tabpanel" id="slug">
+            {{ tab.render() }}
           </div>
-          {% if not sketches %}
-          <p class="text-center">{{member.full_name}} has not created any skecth yet.</p>
-          {% endif %}
         </div>
-      </div>
+      {% endfor %}
     </div>
   </div>
 </div>

community/www/profiles/profile.py

@@ -3,9 +3,20 @@ from community.lms.models import Sketch
 
 def get_context(context):
     context.no_cache = 1
+
     try:
         context.member = frappe.get_doc("User", {"username": frappe.form_dict["username"]})
     except:
         context.template = "www/404.html"
-    else:
-        context.sketches = Sketch.get_recent_sketches(owner=context.member.email)
+        return
+
+    context.profile_tabs = get_profile_tabs(context.member)
+
+def get_profile_tabs(user):
+    """Returns the enabled ProfileTab objects.
+
+    Each ProfileTab is rendered as a tab on the profile page and the
+    they are specified as profile_tabs hook.
+    """
+    tabs = frappe.get_hooks("profile_tabs") or []
+    return [frappe.get_attr(tab)(user) for tab in tabs]

requirements.txt

@@ -1,2 +1,5 @@
 frappe
 websocket_client
+markdown
+beautifulsoup4
+lxml