From 192f61bf0ced21b906bfb98102e556498b9196d4 Mon Sep 17 00:00:00 2001 From: Mathieu PATUREL Date: Sat, 16 Nov 2019 10:07:15 +1100 Subject: [PATCH] improve caching of images First, we used two caches. Turns out that lru_cache wasn't needed, the dict works perfectly fine on it's own. Second, we now also cache local images, so that we don't have to read them off the filesystem and convert them to base64 on every keystroke Maybe there should be a maximum size on that cache dict, but I doubt anyone would actually run into any trouble this cache taking too much ram. --- MarkdownLivePreview.py | 4 ++-- README.md | 3 +-- markdown2html.py | 40 +++++++++++++++++++++++----------------- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/MarkdownLivePreview.py b/MarkdownLivePreview.py index 54617e5..932f759 100644 --- a/MarkdownLivePreview.py +++ b/MarkdownLivePreview.py @@ -26,8 +26,8 @@ resources = {} def plugin_loaded(): - resources["base64_loading_image"] = get_resource("loading.base64") resources["base64_404_image"] = get_resource("404.base64") + resources["base64_loading_image"] = get_resource("loading.base64") resources["stylesheet"] = get_resource("stylesheet.css") @@ -222,7 +222,7 @@ class MarkdownLivePreviewListener(sublime_plugin.EventListener): basepath = os.path.dirname(markdown_view.file_name()) html = markdown2html( - markdown, basepath, partial(self._update_preview, markdown_view), resources + markdown, basepath, partial(self._update_preview, markdown_view), resources, ) self.phantom_sets[markdown_view.id()].update( diff --git a/README.md b/README.md index 2efebad..84b6a30 100644 --- a/README.md +++ b/README.md @@ -23,5 +23,4 @@ in `MarkdownLivePreview.py` and `markdown2html.py` (GitHub only shows the top 3. All your code should be formated by black. 4. Send a PR! - - +FIXME: add a git hook to format using black (can the git hook be added on github?) \ No newline at end of file diff --git a/markdown2html.py b/markdown2html.py index 85a7014..928426a 100644 --- a/markdown2html.py +++ b/markdown2html.py @@ -1,11 +1,17 @@ -import copy +""" Notice how this file is completely independent of sublime text + +I think it should be kept this way, just because it gives a bit more organisation, +and makes it a lot easier to think about, and for anyone who would want to, test since +markdown2html is just a pure function +""" + import os.path import concurrent.futures import urllib.request import base64 import bs4 -from functools import lru_cache, partial +from functools import partial from .lib.markdown2 import Markdown @@ -91,35 +97,35 @@ def markdown2html(markdown, basepath, re_render, resources): def get_base64_image(path, re_render): - def callback(url, future): - # this is "safe" to do because callback is called in the same thread as - # add_done_callback: + """ Gets the base64 for the image (local and remote images). re_render is a + callback which is called when we finish loading an image from the internet + to trigger an update of the preview (the image will then be loaded from the cache) + """ + + def callback(path, future): + # altering image_cache is "safe" to do because callback is called in the same + # thread as add_done_callback: # > Added callables are called in the order that they were added and are always # > called in a thread belonging to the process that added them # > --- Python docs - images_cache[url] = future.result() + images_cache[path] = future.result() # we render, which means this function will be called again, but this time, we # will read from the cache re_render() + if path in images_cache: + return images_cache[path] + if path.startswith("http://") or path.startswith("https://"): - if path in images_cache: - return images_cache[path] executor.submit(load_image, path).add_done_callback(partial(callback, path)) raise LoadingError() - # FIXME: use some kind of cache for this as well, because it decodes on every - # keystroke here... with open(path, "rb") as fp: - return "data:image/png;base64," + base64.b64encode(fp.read()).decode("utf-8") + image = "data:image/png;base64," + base64.b64encode(fp.read()).decode("utf-8") + images_cache[path] = image + return image -# FIXME: wait what the hell? Why do I have two caches? (lru and images_cache) -# FIXME: This is an in memory cache. 20 seems like a fair bit of images... Should it be -# bigger? Should the user be allowed to chose? There definitely should be a limit -# because we don't wanna use to much memory, we're a simple markdown preview plugin -# NOTE: > The LRU feature performs best when maxsize is a power-of-two. --- python docs -@lru_cache(maxsize=2 ** 4) def load_image(url): with urllib.request.urlopen(url, timeout=60) as conn: content_type = conn.info().get_content_type()