feat: Add support for rendering .ipynb Jupyter/IPython notebooks (#491)

internal/git/commands.go

@@ -202,6 +202,11 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*
 			return nil, err
 		}
 
+		// Don't truncate Jupyter notebooks
+		if strings.HasSuffix(file.Name, ".ipynb") {
+			truncate = false
+		}
+
 		sizeToRead := size
 		if truncate && sizeToRead > truncateLimit {
 			sizeToRead = truncateLimit

internal/web/server/renderer.go

@@ -58,6 +58,9 @@ func (s *Server) setFuncMap() {
 		"isMarkdown": func(i string) bool {
 			return strings.ToLower(filepath.Ext(i)) == ".md"
 		},
+		"isJupyter": func(i string) bool {
+			return strings.ToLower(filepath.Ext(i)) == ".ipynb"
+		},
 		"httpStatusText": http.StatusText,
 		"loadedTime": func(startTime time.Time) string {
 			return fmt.Sprint(time.Since(startTime).Nanoseconds()/1e6) + "ms"

package-lock.json

@@ -5,9 +5,6 @@
   "packages": {
     "": {
       "name": "opengist",
-      "dependencies": {
-        "pdfobject": "^2.3.1"
-      },
       "devDependencies": {
         "@codemirror/commands": "^6.2.2",
         "@codemirror/lang-javascript": "^6.1.4",
@@ -22,8 +19,11 @@
         "cssnano": "^5.1.15",
         "dayjs": "^1.11.9",
         "github-markdown-css": "^5.5.0",
+        "highlight.js": "^11.11.1",
         "jdenticon": "^3.3.0",
+        "katex": "^0.16.22",
         "nodemon": "^2.0.22",
+        "pdfobject": "^2.3.1",
         "postcss": "^8.4.32",
         "postcss-cli": "^11.0.0",
         "postcss-cssnext": "^3.1.1",
@@ -31,6 +31,7 @@
         "postcss-loader": "^7.1.0",
         "postcss-selector-namespace": "^3.0.1",
         "sass": "^1.62.1",
+        "showdown": "^2.1.0",
         "sugarss": "^4.0.1",
         "tailwindcss": "^3.2.7",
         "vite": "^4.5.3"
@@ -2431,6 +2432,16 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/highlight.js": {
+      "version": "11.11.1",
+      "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.11.1.tgz",
+      "integrity": "sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w==",
+      "dev": true,
+      "license": "BSD-3-Clause",
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/ignore": {
       "version": "5.3.1",
       "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",
@@ -2708,6 +2719,33 @@
         "graceful-fs": "^4.1.6"
       }
     },
+    "node_modules/katex": {
+      "version": "0.16.22",
+      "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.22.tgz",
+      "integrity": "sha512-XCHRdUw4lf3SKBaJe4EvgqIuWwkPSo9XoeO8GjQW94Bp7TWv9hNhzZjZ+OH9yf1UmLygb7DIT5GSFQiyt16zYg==",
+      "dev": true,
+      "funding": [
+        "https://opencollective.com/katex",
+        "https://github.com/sponsors/katex"
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "commander": "^8.3.0"
+      },
+      "bin": {
+        "katex": "cli.js"
+      }
+    },
+    "node_modules/katex/node_modules/commander": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz",
+      "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 12"
+      }
+    },
     "node_modules/lilconfig": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz",
@@ -2800,14 +2838,11 @@
       "license": "MIT"
     },
     "node_modules/lru-cache": {
-      "version": "10.2.2",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz",
-      "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==",
+      "version": "10.4.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
+      "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
       "dev": true,
-      "license": "ISC",
-      "engines": {
-        "node": "14 || >=16.14"
-      }
+      "license": "ISC"
     },
     "node_modules/math-expression-evaluator": {
       "version": "1.4.0",
@@ -3161,6 +3196,7 @@
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/pdfobject/-/pdfobject-2.3.1.tgz",
       "integrity": "sha512-vluuGiSDmMGpOvWFGiUY4trNB8aGKLDVxIXuuGHjX0kK3bMxCANUVtLivctE7uejLBScWCnbVarKatFVvdwXaQ==",
+      "dev": true,
       "license": "MIT"
     },
     "node_modules/picocolors": {
@@ -5334,6 +5370,33 @@
         "node": ">=8"
       }
     },
+    "node_modules/showdown": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz",
+      "integrity": "sha512-/6NVYu4U819R2pUIk79n67SYgJHWCce0a5xTP979WbNp0FL9MN1I1QK662IDU1b6JzKTvmhgI7T7JYIxBi3kMQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "commander": "^9.0.0"
+      },
+      "bin": {
+        "showdown": "bin/showdown.js"
+      },
+      "funding": {
+        "type": "individual",
+        "url": "https://www.paypal.me/tiviesantos"
+      }
+    },
+    "node_modules/showdown/node_modules/commander": {
+      "version": "9.5.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz",
+      "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": "^12.20.0 || >=14"
+      }
+    },
     "node_modules/signal-exit": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",

package.json

@@ -23,7 +23,9 @@
     "cssnano": "^5.1.15",
     "dayjs": "^1.11.9",
     "github-markdown-css": "^5.5.0",
+    "highlight.js": "^11.11.1",
     "jdenticon": "^3.3.0",
+    "katex": "^0.16.22",
     "nodemon": "^2.0.22",
     "pdfobject": "^2.3.1",
     "postcss": "^8.4.32",
@@ -33,6 +35,7 @@
     "postcss-loader": "^7.1.0",
     "postcss-selector-namespace": "^3.0.1",
     "sass": "^1.62.1",
+    "showdown": "^2.1.0",
     "sugarss": "^4.0.1",
     "tailwindcss": "^3.2.7",
     "vite": "^4.5.3"

public/fs_embed.go

@@ -4,5 +4,5 @@ package public
 
 import "embed"
 
-//go:embed manifest.json assets/*.js assets/*.css assets/*.svg assets/*.png
+//go:embed manifest.json assets/*.js assets/*.css assets/*.svg assets/*.png assets/*.ttf assets/*.woff assets/*.woff2
 var Files embed.FS

public/gist.ts

@@ -1,3 +1,4 @@
+import './ipynb';
 import PDFObject from 'pdfobject';
 
 document.querySelectorAll<HTMLElement>('.table-code').forEach((el) => {
@@ -79,5 +80,4 @@ if (document.getElementById('gist').dataset.own) {
 
 document.querySelectorAll(".pdf").forEach((el) => {
     PDFObject.embed(el.dataset.src || "", el);
-})
-
+})

public/ipynb.css

@@ -0,0 +1,15 @@
+.jupyter.notebook {
+  padding: 0.5rem;
+}
+
+.jupyter.notebook pre {
+  font-size: 0.8em !important;
+}
+
+.jupyter.notebook .jupyter-cell {
+  padding: 0.5rem;
+}
+
+.jupyter.notebook .jupyter-cell.code-cell {
+  filter: drop-shadow(0 0 0.1rem rgba(0, 0, 0, 0.5));
+}

public/ipynb.ts

@@ -0,0 +1,127 @@
+import hljs from 'highlight.js';
+import latex from './latex';
+import showdown from 'showdown';
+
+class IPynb {
+  private element: HTMLElement;
+  private cells: HTMLElement[] = [];
+  private language: string = 'python';
+  private notebook: any;
+
+  constructor(element: HTMLElement) {
+    this.element = element;
+    let notebookContent = element.innerText;
+
+    try {
+      this.notebook = JSON.parse(notebookContent);
+    } catch (e) {
+      console.error('Failed to parse Jupyter notebook content:', e);
+      return;
+    }
+
+    if (!this.notebook) {
+      console.error('Failed to parse Jupyter notebook content:', notebookContent);
+      return;
+    }
+
+    this.language = this.notebook.metadata.kernelspec?.language || 'python';
+    this.cells = this.createCells();
+  }
+
+  mount() {
+    const parent = this.element.parentElement as HTMLElement;
+    parent.removeChild(this.element);
+    parent.innerHTML = this.cells
+      .filter((cell: HTMLElement) => !!cell?.outerHTML)
+      .map((cell: HTMLElement) => cell.outerHTML)
+      .join('');
+  }
+
+  private getOutputs(cell: any): HTMLElement[] {
+    return (cell.outputs || []).map((output: any) => {
+      const outputElement = document.createElement('div');
+      outputElement.classList.add('jupyter-output');
+
+      if (output.output_type === 'stream') {
+        const textElement = document.createElement('pre');
+        textElement.classList.add('stream-output');
+        textElement.textContent = output.text.join('');
+        outputElement.appendChild(textElement);
+      } else if (output.output_type === 'display_data' || output.output_type === 'execute_result') {
+        if (output.data['text/plain']) {
+          outputElement.innerHTML += `\n<pre>${output.data['text/plain']}</pre>`;
+        }
+        if (output.data['text/html']) {
+          outputElement.innerHTML += '\n' + output.data['text/html'];
+        }
+
+        const images = Object.keys(output.data).filter(key => key.startsWith('image/'));
+        if (images.length > 0) {
+          const imgEl = document.createElement('img');
+          const imgType = images[0]; // Use the first image type found
+          imgEl.src = `data:${imgType};base64,${output.data[imgType]}`;
+          outputElement.innerHTML += imgEl.outerHTML;
+        }
+      } else if (output.output_type === 'execute_result') {
+        if (output.data['text/plain']) {
+          outputElement.innerHTML += `<pre>${output.data['text/plain']}</pre>`;
+        }
+        if (output.data['text/html']) {
+          outputElement.innerHTML += output.data['text/html'];
+        }
+        if (output.data['image/png']) {
+          const imgEl = document.createElement('img');
+          imgEl.src = `data:image/png;base64,${output.data['image/png']}`;
+          outputElement.appendChild(imgEl);
+        }
+      } else if (output.output_type === 'error') {
+        outputElement.classList.add('error');
+        outputElement.textContent = `Error: ${output.ename}: ${output.evalue}`;
+      }
+
+      return outputElement;
+    });
+  }
+
+  private createCellElement(cell: any): HTMLElement {
+    const cellElement = document.createElement('div');
+    const source = cell.source.join('');
+    cellElement.classList.add('jupyter-cell');
+
+    switch (cell.cell_type) {
+      case 'markdown':
+        const converter = new showdown.Converter();
+        cellElement.classList.add('markdown-cell');
+        cellElement.innerHTML = `<div class="markdown-body">${converter.makeHtml(latex.render(source))}</div>`;
+        break;
+      case 'code':
+        cellElement.classList.add('code-cell');
+        cellElement.innerHTML = `<pre class="hljs"><code class="language-${this.language}">${source}</code></pre>`;
+        hljs.highlightElement(cellElement.querySelector('code') as HTMLElement);
+        break;
+      default:
+        break;
+    }
+
+    return cellElement;
+  }
+
+
+  private createCells(): HTMLElement[] {
+    return (this.notebook.cells || []).map((cell: any) => {
+      const container = document.createElement('div');
+      const cellElement = this.createCellElement(cell);
+      const outputs = this.getOutputs(cell);
+
+      container.classList.add('jupyter-cell-container');
+      container.appendChild(cellElement);
+      outputs.forEach((output: HTMLElement) => container.appendChild(output));
+      return container;
+    });
+  }
+}
+
+// Process Jupyter notebooks
+document.querySelectorAll<HTMLElement>('.jupyter.notebook pre').forEach((el) => {
+  new IPynb(el).mount();
+});

public/latex.ts

@@ -0,0 +1,66 @@
+import katex from 'katex';
+
+const delimiters = [
+  { left: '\\\$\\\$', right: '\\\$\\\$', multiline: true },
+  { left: '\\\$', right: '\\\$', multiline: false },
+  { left: '\\\\\[', right: '\\\\\]', multiline: true },
+  { left: '\\\\\(', right: '\\\\\)', multiline: false },
+];
+
+const delimiterMatchers = delimiters.map(
+  (delimiter) => new RegExp(
+    `${delimiter.left}(.*?)${delimiter.right}`,
+    `g${delimiter.multiline ? 'ms' : ''}`
+  )
+);
+
+// Replace LaTeX delimiters in a string with KaTeX rendering
+function render(text: string): string {
+  // Step 1: Replace all LaTeX expressions with placeholders
+  const expressions: Array<{ placeholder: string; latex: string; displayMode: boolean }> = [];
+  let modifiedText = text;
+  let placeholderIndex = 0;
+
+  // Process each delimiter type
+  delimiters.forEach((delimiter, i) => {
+    // Find all matches and replace with placeholders
+    modifiedText = modifiedText.replace(delimiterMatchers[i], (match, latex) => {
+      if (!latex.trim()) {
+        return match; // Return original if content is empty
+      }
+
+      const placeholder = `__KATEX_PLACEHOLDER_${placeholderIndex++}__`;
+      expressions.push({
+        placeholder,
+        latex,
+        displayMode: delimiter.multiline,
+      });
+
+      return placeholder;
+    });
+  });
+
+  // Step 2: Replace placeholders with rendered LaTeX
+  for (const { placeholder, latex, displayMode } of expressions) {
+    try {
+      const rendered = katex.renderToString(latex, {
+        throwOnError: false,
+        displayMode,
+      });
+      modifiedText = modifiedText.replace(placeholder, rendered);
+    } catch (error) {
+      console.error('KaTeX rendering error:', error);
+      // Replace placeholder with original LaTeX if rendering fails
+      modifiedText = modifiedText.replace(
+        placeholder,
+        displayMode ? `$$${latex}$$` : `$${latex}$`
+      );
+    }
+  }
+
+  return modifiedText;
+}
+
+export default {
+  render,
+};

public/style.css

@@ -190,6 +190,16 @@ dl.dl-config dd {
     @apply hidden !important;
 }
 
+/*
+ * A hack to ensure that Jupyter output images are always rendered with a
+ * neutral background color, even if the image itself does not have one, since
+ * Jupyter usually outputs images with transparent or light backgrounds.
+ */
+.dark .jupyter-output img {
+  background-color: #888;
+}
+
+
 .pdfobject-container {
     @apply min-h-[700px] h-[700px] !important;
 }

public/style.scss

@@ -1,9 +1,14 @@
+@import "katex/dist/katex.min.css";
+
 :root {
+    @import "highlight.js/scss/github";
     @import "github-markdown-css/github-markdown-light";
     @import './catppuccin-latte';
+    @import './ipynb';
 }
 
 .dark {
+    @import "highlight.js/scss/github-dark";
     @import "github-markdown-css/github-markdown-dark";
     @import './catppuccin-macchiato';
 }

templates/pages/gist.html

@@ -77,6 +77,8 @@
                     <div class="chroma markdown markdown-body p-8">{{ $file.HTML | safe }}</div>
                 {{ else if $file.MimeType.IsSVG }}
                     <div class="p-8 flex justify-center">{{ $file.HTML | safe }}</div>
+                {{ else if isJupyter $file.Filename }}
+                  <div class="p-8 jupyter notebook"><pre>{{ $file.Content }}</pre></div>
                 {{ else }}
                     <div class="code">
                         {{ $fileslug := slug $file.Filename }}