Vendor bs4 & soupsieve

setting to increase font scale

.gitattributes

@@ -0,0 +1,3 @@
+docs/ export-ignore
+resources/
+!resources/*.base64

.gitignore

@@ -1,3 +1 @@
-Thumbs.db
-__pycache__/
-cache.txt
+__pycache__

.sublime/Default.sublime-keymap

@@ -1,13 +0,0 @@
-[
-    {
-        "keys": ["alt+m"],
-        "command": "new_markdown_live_preview",
-        "context": [
-            {
-                "key": "selector",
-                "operator": "equal",
-                "operand": "text.html.markdown"
-            }
-        ]
-    }
-]

.sublime/Main.sublime-menu

@@ -1,33 +0,0 @@
-[
-    {
-        "id": "preferences",
-        "children": [
-            {
-                "id": "package-settings",
-                "children": [
-                    {
-                        "caption": "MarkdownLivePreview",
-                        "children": [
-                            {
-                                "caption": "Settings",
-                                "command": "edit_settings",
-                                "args": {
-                                    "base_file": "$packages/MarkdownLivePreview/.sublime/MarkdownLivePreview.sublime-settings",
-                                    "default": "// Your settings for MarkdownLivePreview. See the default file to see the different options. \n{\n\t\n}\n"
-                                }
-                            },
-                            {
-                                "caption": "Style - CSS",
-                                "command": "open_file",
-                                "args": {
-                                    "file": "$packages/User/MarkdownLivePreview.css",
-                                    "contents": "/* See http://www.sublimetext.com/docs/3/minihtml.html#css to know which property you're able to use */\n\n$0"
-                                }
-                            },
-                        ]
-                    },
-                ],
-            },
-        ]
-    }
-]

.sublime/MarkdownLivePreview.sublime-commands

@@ -1,18 +0,0 @@
-[
-    {
-        "caption": "MarkdownLivePreview: Edit Current File",
-        "command": "new_markdown_live_preview"
-    },
-    {
-        "caption": "MarkdownLivePreview: Clear Cache",
-        "command": "markdown_live_preview_clear_cache"
-    },
-    {
-        "caption": "Preferences: MarkdownLivePreview Settings",
-        "command": "edit_settings",
-        "args": {
-            "base_file": "${packages}/MarkdownLivePreview/.sublime/MarkdownLivePreview.sublime-settings",
-            "default": "// Your settings for MarkdownLivePreview. See the default file to see the different options. \n{\n\t$0\n}\n"
-        }
-    }
-]

.sublime/MarkdownLivePreview.sublime-settings

@@ -1,4 +0,0 @@
-{
-    "markdown_live_preview_on_open": false,
-    "load_from_internet_when_starts": ["http://", "https://"]
-}

.sublime/MarkdownLivePreviewSyntax.hidden-tmLanguage

@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-    <key>name</key>
-    <string>MarkdownLivePreviewSyntax</string>
-
-    <key>patterns</key>
-    <array>
-    </array>
-    <key>scopeName</key>
-    <string>text.markdown-live-preview</string>
-</dict>
-</plist>

LICENSE

@@ -0,0 +1,7 @@
+Copyright 2017 Mathieu PATUREL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

MLPApi.py

@@ -1,89 +0,0 @@
-# -*- encoding: utf-8 -*-
-
-import sublime
-import sublime_plugin
-
-import os.path
-from html.parser import HTMLParser
-
-from .lib import markdown2 as md2
-from .escape_amp import *
-from .functions import *
-from .setting_names import *
-from .image_manager import CACHE_FILE
-
-__folder__ = os.path.dirname(__file__)
-
-STYLE_FILE = os.path.join(os.path.dirname(__folder__), 'User',
-                            'MarkdownLivePreview.css')
-
-
-def plugin_loaded():
-    global DEFAULT_STYLE_FILE
-    DEFAULT_STYLE_FILE = sublime.load_resource('Packages/MarkdownLivePreview/'
-                                              'default.css')
-
-def get_preview_name(md_view):
-    file_name = md_view.file_name()
-    name = md_view.name() \
-           or os.path.basename(file_name) if file_name else None \
-           or 'Untitled'
-    return name + ' - Preview'
-
-def create_preview(window, file_name):
-    preview = window.new_file()
-
-    preview.set_name(get_preview_name(file_name))
-    preview.set_scratch(True)
-    preview.set_syntax_file('Packages/MarkdownLivePreview/.sublime/'
-                            'MarkdownLivePreviewSyntax.hidden-tmLanguage')
-
-    return preview
-
-def get_style():
-    content = ''.join([line.strip() + ' ' for line in DEFAULT_STYLE_FILE.splitlines()])
-    return content + "pre code .space {color: var(--light-bg)}"
-
-def show_html(md_view, preview):
-    html = []
-    html.append('<style>\n{}\n</style>'.format(get_style()))
-    html.append(pre_with_br(md2.markdown(get_view_content(md_view),
-                                         extras=['fenced-code-blocks',
-                                                 'no-code-highlighting'])))
-    # the option no-code-highlighting does not exists
-    # in the official version of markdown2 for now
-    # I personaly edited the file (markdown2.py:1743)
-    html = '\n'.join(html)
-
-    html = html.replace('&nbsp;', '&nbspespace;') # save where are the spaces
-
-    html = HTMLParser().unescape(html)
-
-    html = escape_amp(html)
-
-    # exception, again, because <pre> aren't supported by the phantoms
-    html = html.replace('&nbspespace;', '<i class="space">.</i>')
-    html = replace_img_src_base64(html, basepath=os.path.dirname(
-                                                md_view.file_name()))
-    preview.erase_phantoms('markdown_preview')
-    preview.add_phantom('markdown_preview',
-                         sublime.Region(-1),
-                         html,
-                         sublime.LAYOUT_BLOCK,
-                         lambda href: sublime.run_command('open_url',
-                                                          {'url': href}))
-
-    return
-    # set viewport position
-    # 0 < y < 1
-    y = md_view.text_to_layout(md_view.sel()[0].begin())[1] / md_view.layout_extent()[1]
-    vector = [0, y * preview.layout_extent()[1]]
-    # remove half of the viewport_extent.y to center it on the screen (verticaly)
-    vector[1] -= preview.viewport_extent()[1] / 2
-    vector[1] = mini(vector[1], 0)
-    vector[1] += preview.line_height()
-    preview.set_viewport_position(vector, animate=False)
-
-def clear_cache():
-    """Removes the cache file"""
-    os.remove(CACHE_FILE)

MarkdownLivePreview.py

@@ -1,112 +1,271 @@
-# -*- encoding: utf-8 -*-
+"""
+Terminology
+original_view: the view in the regular editor, without it's own window
+markdown_view: the markdown view, in the special window
+preview_view: the preview view, in the special window
+original_window: the regular window
+preview_window: the window with the markdown file and the preview
+"""
 
+import time
+import os.path
+import struct
 import sublime
 import sublime_plugin
 
-from .MLPApi import *
-from .setting_names import *
-from .functions import *
+from functools import partial
 
-class NewMarkdownLivePreviewCommand(sublime_plugin.ApplicationCommand):
+from .markdown2html import markdown2html
 
-    def run(self):
+MARKDOWN_VIEW_INFOS = "markdown_view_infos"
+PREVIEW_VIEW_INFOS = "preview_view_infos"
+SETTING_DELAY_BETWEEN_UPDATES = "delay_between_updates"
+SETTING_FONT_SCALE = "font_scale"
 
-        """Inspired by the edit_settings command"""
+resources = {}
 
-        current_view = sublime.active_window().active_view()
-        file_name = current_view.file_name()
-        current_view.close()
-        if file_name is None:
-            return sublime.error_message('MarkdownLivePreview: Not supporting '
-                                         'unsaved file for now')
 
-        sublime.run_command('new_window')
-        self.window = sublime.active_window()
-        self.window.settings().set(PREVIEW_WINDOW, True)
-        self.window.run_command('set_layout', {
-            'cols': [0.0, 0.5, 1.0],
-            'rows': [0.0, 1.0],
-            'cells': [[0, 0, 1, 1], [1, 0, 2, 1]]
-        })
-        self.window.focus_group(1)
-        preview = create_preview(self.window, current_view)
+def plugin_loaded():
+    global DELAY
+    resources["base64_404_image"] = parse_image_resource(get_resource("404.base64"))
+    resources["base64_loading_image"] = parse_image_resource(
+        get_resource("loading.base64")
+    )
+    resources["stylesheet"] = get_resource("stylesheet.css")
+    # FIXME: how could we make this setting update without restarting sublime text
+    #        and not loading it every update as well
+    DELAY = get_settings().get(SETTING_DELAY_BETWEEN_UPDATES)
 
-        self.window.focus_group(0)
-        md_view = self.window.open_file(file_name)
-        mdsettings = md_view.settings()
 
-        mdsettings.set(PREVIEW_ENABLED, True)
-        mdsettings.set(PREVIEW_ID, preview.id())
+class MdlpInsertCommand(sublime_plugin.TextCommand):
+    def run(self, edit, point, string):
+        self.view.insert(edit, point, string)
+
+
+class OpenMarkdownPreviewCommand(sublime_plugin.TextCommand):
+    def run(self, edit):
+
+        """ If the file is saved exists on disk, we close it, and reopen it in a new
+        window. Otherwise, we copy the content, erase it all (to close the file without
+        a dialog) and re-insert it into a new view into a new window """
+
+        original_view = self.view
+        original_window_id = original_view.window().id()
+        file_name = original_view.file_name()
+
+        syntax_file = original_view.settings().get("syntax")
+
+        if file_name:
+            original_view.close()
+        else:
+            # the file isn't saved, we need to restore the content manually
+            total_region = sublime.Region(0, original_view.size())
+            content = original_view.substr(total_region)
+            original_view.erase(edit, total_region)
+            original_view.close()
+            # FIXME: save the document to a temporary file, so that if we crash,
+            #        the user doesn't lose what he wrote
+
+        sublime.run_command("new_window")
+        preview_window = sublime.active_window()
+
+        preview_window.run_command(
+            "set_layout",
+            {
+                "cols": [0.0, 0.5, 1.0],
+                "rows": [0.0, 1.0],
+                "cells": [[0, 0, 1, 1], [1, 0, 2, 1]],
+            },
+        )
+
+        preview_window.focus_group(1)
+        preview_view = preview_window.new_file()
+        preview_view.set_scratch(True)
+        preview_view.settings().set(PREVIEW_VIEW_INFOS, {})
+        preview_view.set_name("Preview")
+        # FIXME: hide number lines on preview
+
+        preview_window.focus_group(0)
+        if file_name:
+            markdown_view = preview_window.open_file(file_name)
+        else:
+            markdown_view = preview_window.new_file()
+            markdown_view.run_command("mdlp_insert", {"point": 0, "string": content})
+            markdown_view.set_scratch(True)
+
+        markdown_view.set_syntax_file(syntax_file)
+        markdown_view.settings().set(
+            MARKDOWN_VIEW_INFOS, {"original_window_id": original_window_id,},
+        )
 
     def is_enabled(self):
-        return is_markdown_view(sublime.active_window().active_view())
+        # FIXME: is this the best way there is to check if the current syntax is markdown?
+        #        should we only support default markdown?
+        #        what about "md"?
+        # FIXME: what about other languages, where markdown preview roughly works?
+        return "markdown" in self.view.settings().get("syntax").lower()
+
 
 class MarkdownLivePreviewListener(sublime_plugin.EventListener):
 
-    def update(self, view):
-        vsettings = view.settings()
-        if not vsettings.get(PREVIEW_ENABLED):
+    phantom_sets = {
+        # markdown_view.id(): phantom set
+    }
+
+    # we schedule an update for every key stroke, with a delay of DELAY
+    # then, we update only if now() - last_update > DELAY
+    last_update = 0
+
+    # FIXME: maybe we shouldn't restore the file in the original window...
+
+    def on_pre_close(self, markdown_view):
+        """ Close the view in the preview window, and store information for the on_close
+        listener (see doc there)
+        """
+        if not markdown_view.settings().get(MARKDOWN_VIEW_INFOS):
             return
-        id = vsettings.get(PREVIEW_ID)
-        if id is None:
-            raise ValueError('The preview id is None')
-        preview = get_view_from_id(view.window(), id)
-        if preview is None:
-            raise ValueError('The preview is None (id: {})'.format(id))
 
-        show_html(view, preview)
-        return view, preview
+        self.markdown_view = markdown_view
+        self.preview_window = markdown_view.window()
+        self.file_name = markdown_view.file_name()
 
-    def on_modified(self, view):
-        if not is_markdown_view(view): # faster than getting the settings
+        if self.file_name is None:
+            total_region = sublime.Region(0, markdown_view.size())
+            self.content = markdown_view.substr(total_region)
+            markdown_view.erase(edit, total_region)
+        else:
+            self.content = None
+
+    def on_load_async(self, markdown_view):
+        infos = markdown_view.settings().get(MARKDOWN_VIEW_INFOS)
+        if not infos:
             return
-        self.update(view)
 
-    def on_window_command(self, window, command, args):
-        if command == 'close' and window.settings().get(PREVIEW_WINDOW):
-            return 'close_window', {}
+        preview_view = markdown_view.window().active_view_in_group(1)
 
-    def on_activated_async(self, view):
-        vsettings = view.settings()
+        self.phantom_sets[markdown_view.id()] = sublime.PhantomSet(preview_view)
+        self._update_preview(markdown_view)
 
-        if (is_markdown_view(view)
-            and get_settings().get('markdown_live_preview_on_open')
-            and not vsettings.get(PREVIEW_ENABLED)
-            and vsettings.get('syntax') != 'Packages/MarkdownLivePreview/'
-                                           '.sublime/MarkdownLivePreviewSyntax'
-                                           '.hidden-tmLanguage'):
-            sublime.run_command('new_markdown_live_preview')
-
-
-    def on_load_async(self, view):
-        """Check the settings to hide menu, minimap, etc"""
-        try:
-            md_view, preview = self.update(view)
-        except TypeError:
-            # the function update has returned None
+    def on_close(self, markdown_view):
+        """ Use the information saved to restore the markdown_view as an original_view
+        """
+        infos = markdown_view.settings().get(MARKDOWN_VIEW_INFOS)
+        if not infos:
             return
-        window = preview.window()
-        psettings = preview.settings()
 
-        show_tabs = psettings.get('show_tabs')
-        show_minimap = psettings.get('show_minimap')
-        show_status_bar = psettings.get('show_status_bar')
-        show_sidebar = psettings.get('show_sidebar')
-        show_menus = psettings.get('show_menus')
+        assert (
+            markdown_view.id() == self.markdown_view.id()
+        ), "pre_close view.id() != close view.id()"
 
-        if show_tabs is not None:
-            window.set_tabs_visible(show_tabs)
-        if show_minimap is not None:
-            window.set_minimap_visible(show_minimap)
-        if show_status_bar is not None:
-            window.set_status_bar_visible(show_status_bar)
-        if show_sidebar is not None:
-            window.set_sidebar_visible(show_sidebar)
-        if show_menus is not None:
-            window.set_menu_visible(show_menus)
+        del self.phantom_sets[markdown_view.id()]
+
+        self.preview_window.run_command("close_window")
+
+        # find the window with the right id
+        original_window = next(
+            window
+            for window in sublime.windows()
+            if window.id() == infos["original_window_id"]
+        )
+        if self.file_name:
+            original_window.open_file(self.file_name)
+        else:
+            assert markdown_view.is_scratch(), (
+                "markdown view of an unsaved file should " "be a scratch"
+            )
+            # note here that this is called original_view, because it's what semantically
+            # makes sense, but this original_view.id() will be different than the one
+            # that we closed first to reopen in the preview window
+            # shouldn't cause any trouble though
+            original_view = original_window.new_file()
+            original_view.run_command(
+                "mdlp_insert", {"point": 0, "string": self.content}
+            )
+
+            original_view.set_syntax_file(markdown_view.settings().get("syntax"))
+
+    # here, views are NOT treated independently, which is theoretically wrong
+    # but in practice, you can only edit one markdown file at a time, so it doesn't really
+    # matter.
+    # @min_time_between_call(.5)
+    def on_modified_async(self, markdown_view):
+
+        infos = markdown_view.settings().get(MARKDOWN_VIEW_INFOS)
+        if not infos:
+            return
+
+        # we schedule an update, which won't run if an
+        sublime.set_timeout(partial(self._update_preview, markdown_view), DELAY)
+
+    def _update_preview(self, markdown_view):
+        # if the buffer id is 0, that means that the markdown_view has been closed
+        # This check is needed since a this function is used as a callback for when images
+        # are loaded from the internet (ie. it could finish loading *after* the user
+        # closes the markdown_view)
+        # Reload settings each time to catch changes
+        settings = get_settings()
+        delay = settings.get(SETTING_DELAY_BETWEEN_UPDATES, 100) # Provide default
+        font_scale = settings.get(SETTING_FONT_SCALE, 1.0) # Provide default
+
+        if time.time() - self.last_update < delay / 1000:
+            return
+
+        if markdown_view.buffer_id() == 0:
+            return
+
+        self.last_update = time.time()
+
+        total_region = sublime.Region(0, markdown_view.size())
+        markdown = markdown_view.substr(total_region)
+
+        preview_view = markdown_view.window().active_view_in_group(1)
+        # Get viewport_width, default to a large value if view isn't ready
+        viewport_extent = preview_view.viewport_extent()
+        viewport_width = viewport_extent[0] if viewport_extent else 1024
 
 
-class MarkdownLivePreviewClearCacheCommand(sublime_plugin.ApplicationCommand):
+        basepath = os.path.dirname(markdown_view.file_name()) if markdown_view.file_name() else '.' # Handle unsaved files
+        html = markdown2html(
+            markdown,
+            basepath,
+            partial(self._update_preview, markdown_view),
+            resources,
+            viewport_width,
+            font_scale,
+        )
 
-    def run(self):
-        clear_cache()
+        self.phantom_sets[markdown_view.id()].update(
+            [
+                sublime.Phantom(
+                    sublime.Region(0),
+                    html,
+                    sublime.LAYOUT_BLOCK,
+                    lambda href: sublime.run_command("open_url", {"url": href}),
+                )
+            ]
+        )
+
+
+def get_settings():
+    return sublime.load_settings("MarkdownLivePreview.sublime-settings")
+
+
+def get_resource(resource):
+    path = "Packages/MarkdownLivePreview/resources/" + resource
+    abs_path = os.path.join(sublime.packages_path(), "..", path)
+    if os.path.isfile(abs_path):
+        with open(abs_path, "r") as fp:
+            return fp.read()
+    return sublime.load_resource(path)
+
+
+def parse_image_resource(text):
+    width, height, base64_image = text.splitlines()
+    return base64_image, (int(width), int(height))
+
+
+# try to reload the resources if we save this file
+try:
+    plugin_loaded()
+except OSError:
+    pass

MarkdownLivePreview.sublime-commands

@@ -0,0 +1,15 @@
+[
+
+	{
+		"caption": "MarkdownLivePreview: Open Preview",
+		"command": "open_markdown_preview"
+	},
+	{
+		"caption": "MarkdownLivePreview: Open Settings",
+		"command": "edit_settings", "args":
+		{
+			"base_file": "${packages}/MarkdownLivePreview/MarkdownLivePreview.sublime-settings",
+			"default": "{\n\t$0\n}\n"
+		},
+	}
+]

MarkdownLivePreview.sublime-settings

@@ -0,0 +1,7 @@
+{
+	// minimum number of milliseconds to wait before updating the preview again
+	"delay_between_updates": 100,
+
+	// scale factor for the font size relative to markdown settings
+	"font_scale": 1.0
+}

MarkdownLivePreview.tasks

@@ -1,35 +0,0 @@
-Fast:
-  ☐ sync scroll @needsUpdate(because of images)
-  ☐ cache image in object when used, so that it's faster @needsTest
-
-Medium:
-  ☐ auto refresh preview if loading images
-  ☐ use alt attribute for 404 error
-
-Long:
-  ☐ support hanchor (TOC) @big
-
-Unknown:
-  ☐ check how many times is the show_html function called
-
-
-＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿＿
-Archive:
- ✔ fix #4 @high @done Mon 09 Jan 2017 at 18:42 @project(Long)
- ✔ use MarkdownLivePreview syntax, so we can use syntax's settings @done Mon 09 Jan 2017 at 18:41 @project(Medium)
- ✔ add clear cache command @done Mon 09 Jan 2017 at 18:41 @project(Fast)
- ✔ update README for settings in view @done Mon 09 Jan 2017 at 18:41 @project(Fast)
- ✔ add edit settings @done Mon 09 Jan 2017 at 18:41 @project(Fast)
- ✘ listen for settings to change @cancelled Mon 09 Jan 2017 at 18:41 @project(Medium)
- ✘ call settings listener on_new too - might be too heavy @cancelled Sun 08 Jan 2017 at 19:33 @project(Fast)
- ✔ fix relative source @done Sun 08 Jan 2017 at 19:22 @project(Medium)
- ✔ add settings for the preview @done Sun 08 Jan 2017 at 17:36 @project(Fast)
- ✔ regive focus to the right markdown view @done Mon 02 Jan 2017 at 18:34 @project(Fast)
- ✔ try/except for 404 @done Mon 02 Jan 2017 at 18:03 @project(Fast)
- ✔ fix bug when empty `src` @done Mon 02 Jan 2017 at 17:15 @project(Fast)
- ✔ preview.set_scratch(True) @done Mon 02 Jan 2017 at 16:58
- ✔ set the title of the preview @done Mon 02 Jan 2017 at 16:58
- ✔ preview.wordWrap => True @done Mon 02 Jan 2017 at 16:58
- ✔ clean the code (syntax) @done Mon 02 Jan 2017 at 16:58
- ✔ add 404 image @done Mon 02 Jan 2017 at 16:57
- ✔ load images from internet (`https:`) @done Mon 02 Jan 2017 at 16:57

README.md

@@ -1,96 +1,65 @@
 # MarkdownLivePreview
 
-This is a sublime text **3** plugin that allows you to preview your markdown instantly *in* it!
+A simple plugin to preview your markdown as you type right in Sublime Text.
+No dependencies!
 
-### Dependencies
+## How to install
 
-**None**! There is no dependency! It uses [markdown2](https://github.com/trentm/python-markdown2) but it's a one file plugin, so it's included in the package.
+It's available on package control!
 
-## Installation
+## Setting a keybinding
 
-MarkdownLivePreview is available on the default channel of [PackageControl](http://packagecontrol.io), which means you just have to
-
-1. Open the command palette (`ctrl+shift+p`)
-2. Search for: `Package Control: Install Package`
-3. Search for: `MarkdownLivePreview`
-4. hit <kbd>enter</kbd>
-
-to have MarkdownLivePreview working on your computer. Cool right? You can [thank package control](https://packagecontrol.io/say_thanks) for this.
-
-### Usage
-
-You can choose to enable MarkdownLivePreview by pressing <kbd>alt+m</kbd> or selecting in the command palette `MarkdownLivePreview: Edit Current File`. Note that you need to be editing (simply having the focus on) a markdown file. Because [Markdown Extended][markdown-extended] did a good job, it's compatible with this plugin.
-
-So, once you've run it, it will open a new window, with only your markdown file, with the preview. Once you're done, close whichever file and it'll close the entire window.
-
-*Notice that it will close the entire window if you close __whichever__ file. It means that if you open a random file in this window, and then close it, it'll close the entire window still*
-
-### Settings
-
-- `markdown_live_preview_on_open`: if set to `true`, as soon as you open a markdown file, the preview window will popup (thanks to [@ooing](https://github.com/ooing) for its [suggestion](https://github.com/math2001/MarkdownLivePreview/issues/7#issue-199464852)). Default to `false`
-- `load_from_internet_when_starts`: every images that starts with any of the string specified in this list will be loaded from internet. Default to `["http://", "https://"]`
-
-Note: To edit your settings, search up in the command palette `Preferences: MarkdownLivePreview Settings`, or by using the menu: `Preferences → Packages Settings → MarkdownLivePreview → Settings` ;. It's not your global settings, but only the `MarkdownLivePreview`'s one
-
-### Syntax Specific Settings
-
-This in an other "type" of setting. :laughing: If you have a look at the syntax of the preview file (not the markdown one, really the preview), you'll see that the syntax is `MarkdownLivePreviewSyntax`. This mean that you can specify specific settings for this specific syntax (such as `word_wrap: true`, `rulers: []`, etc).
-
-To do so, you can
-
-1. focus the *preview* (<kbd>ctrl+2</kbd> to focus the second group, so, by default, the preview's group)
-2. search up in the command palette `Preferences: Settings Syntax Specific`. It's in the *right* file that you can add the settings you want (not the left one).
-
-Note: MarkdownLivePreview will actualy look in this file for settings that aren't supported by default. Here they are:
-
-- `show_tabs`
-- `show_minimap`
-- `show_status_bar`
-- `show_sidebar`
-- `show_menus`
-
-They talk for themself, don't they? All of them takes a boolean (`true` or `false`). Note that those settings are *window* specific, not just view specific (that's why they aren't supported). It means that they'll affect the entire window, and every view in it.
-
-Here is an example of syntax specific settings for MarkdownLivePreviewSyntax:
+The open the preview, you can search up in the command palette
+(<kbd>ctrl+shift+p</kbd>) `MarkdownLivePreview: Open Preview`. But if you
+prefer to have a shortcut, add this to your keybindings file:
 
 ```json
 {
-    "show_menus": false,
-    "show_tabs": false,
-    "show_minimap": false,
-    "gutter": false,
-    "rulers": [],
-    "word_wrap": true
+    "keys": ["alt+m"],
+    "command": "open_markdown_preview"
 }
 ```
 
-And here's what you'll get:
+## How to contribute
 
-![MarkdownLivePreview Screenshoot](screenshoots/syntax-specific-settings.png)
+If you know what feature you want to implement, or what bug you wanna fix, then
+go ahead and hack! Maybe raise an issue before hand so that we can talk about
+it if it's a big feature.
 
-*Note: to close a file, you can do <kbd>ctrl+w</kbd> (on Mac OS, it's <kbd>cmd+w</kbd>)*
+But if you wanna contribute just to say thanks, and don't really know what you
+could be working on, then there are a bunch of `FIXME`s all over this package.
+Just pick one and fix it :-)
 
-### Clear the cache
+```
+$ git clone https://github.com/math2001/MarkdownLivePreview
+$ cd MarkdownLivePreview
+$ grep -R FIXME
+```
 
-MarkdownLivePreview caches every images it loads from internet (otherwise, you'd never see your images, or you'd need to have a *really* fast internet connection :smile:). So, if for some reason you want to clear the cache (a simple file), you can do so from the command palette by running `
+### Hack it!
 
-### Demo
+1. Fork this repo
+2. Make your own branch (the name of the branch should be the feature you are
+   implementing eg. `improve-tables`, `fix-crash-on-multiple-preview`
+3. All your code should be formated by black.
+4. Send a PR!
 
-![demo](demo.gif)
+### Known limitations
 
-### Custom css
+#### Numbered lists are rendered as unordered lists
 
-It is possible to set your own css. But, be carefull, you have to respect [those rules](http://www.sublimetext.com/docs/3/minihtml.html#css). Just go to `Preferences → Package Settings → MarkdownLivePreview → Style - CSS`. It will open a css file, here: `$packages/User/MarkdownLivePreview.css`. Just save it and it will automatically use it instead of the default one.
+```md
+1. first
+2. second
+3. third
+```
 
-### Somethings wrong!!
+will be previewed the exact same way as
 
-If you find that something's wrong with this package, you can let me know by raising an issue on the [GitHub issue tracker][github-issue-tracker]
+```md
+- first
+- second
+- third
+```
 
-### How to open the [README](http://github.com/math2001/MarkdownLivePreview/README.md)
-
-Some of the package add a command in the menus, others in the command palette, or other nowhere. None of those options are really good, especially the last one on ST3 because the packages are compressed. But, fortunately, there is plugin that exists and **will solve this problem** for us (and he has a really cute name, don't you think?): [ReadmePlease](https://packagecontrol.io/packages/ReadmePlease).
-
-
-
-[markdown-extended]: https://packagecontrol.io/packages/Markdown%20Extended
-[github-issue-tracker]: https://github.com/math2001/MarkdownLivePreview/issues
+The issue comes from [Sublime Text's minihtml](https://www.sublimetext.com/docs/3/minihtml.html) which [doesn't support ordered lists](https://github.com/sublimehq/sublime_text/issues/1767). If you think feel like implementing a workaround, feel free to contribute, but it's not something I'm planning on doing. It isn't a critical feature, and support should come with time...

bs4/__init__.py

@@ -0,0 +1,791 @@
+"""Beautiful Soup Elixir and Tonic - "The Screen-Scraper's Friend".
+
+http://www.crummy.com/software/BeautifulSoup/
+
+Beautiful Soup uses a pluggable XML or HTML parser to parse a
+(possibly invalid) document into a tree representation. Beautiful Soup
+provides methods and Pythonic idioms that make it easy to navigate,
+search, and modify the parse tree.
+
+Beautiful Soup works with Python 2.7 and up. It works better if lxml
+and/or html5lib is installed.
+
+For more than you ever wanted to know about Beautiful Soup, see the
+documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/
+"""
+
+__author__ = "Leonard Richardson (leonardr@segfault.org)"
+__version__ = "4.9.3"
+__copyright__ = "Copyright (c) 2004-2020 Leonard Richardson"
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
+__all__ = ['BeautifulSoup']
+
+from collections import Counter
+import os
+import re
+import sys
+import traceback
+import warnings
+
+from .builder import builder_registry, ParserRejectedMarkup
+from .dammit import UnicodeDammit
+from .element import (
+    CData,
+    Comment,
+    DEFAULT_OUTPUT_ENCODING,
+    Declaration,
+    Doctype,
+    NavigableString,
+    PageElement,
+    ProcessingInstruction,
+    PYTHON_SPECIFIC_ENCODINGS,
+    ResultSet,
+    Script,
+    Stylesheet,
+    SoupStrainer,
+    Tag,
+    TemplateString,
+    )
+
+# The very first thing we do is give a useful error if someone is
+# running this code under Python 3 without converting it.
+'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'!='You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
+
+# Define some custom warnings.
+class GuessedAtParserWarning(UserWarning):
+    """The warning issued when BeautifulSoup has to guess what parser to
+    use -- probably because no parser was specified in the constructor.
+    """
+
+class MarkupResemblesLocatorWarning(UserWarning):
+    """The warning issued when BeautifulSoup is given 'markup' that
+    actually looks like a resource locator -- a URL or a path to a file
+    on disk.
+    """
+
+
+class BeautifulSoup(Tag):
+    """A data structure representing a parsed HTML or XML document.
+
+    Most of the methods you'll call on a BeautifulSoup object are inherited from
+    PageElement or Tag.
+
+    Internally, this class defines the basic interface called by the
+    tree builders when converting an HTML/XML document into a data
+    structure. The interface abstracts away the differences between
+    parsers. To write a new tree builder, you'll need to understand
+    these methods as a whole.
+
+    These methods will be called by the BeautifulSoup constructor:
+      * reset()
+      * feed(markup)
+
+    The tree builder may call these methods from its feed() implementation:
+      * handle_starttag(name, attrs) # See note about return value
+      * handle_endtag(name)
+      * handle_data(data) # Appends to the current data node
+      * endData(containerClass) # Ends the current data node
+
+    No matter how complicated the underlying parser is, you should be
+    able to build a tree using 'start tag' events, 'end tag' events,
+    'data' events, and "done with data" events.
+
+    If you encounter an empty-element tag (aka a self-closing tag,
+    like HTML's <br> tag), call handle_starttag and then
+    handle_endtag.
+    """
+
+    # Since BeautifulSoup subclasses Tag, it's possible to treat it as
+    # a Tag with a .name. This name makes it clear the BeautifulSoup
+    # object isn't a real markup tag.
+    ROOT_TAG_NAME = '[document]'
+
+    # If the end-user gives no indication which tree builder they
+    # want, look for one with these features.
+    DEFAULT_BUILDER_FEATURES = ['html', 'fast']
+
+    # A string containing all ASCII whitespace characters, used in
+    # endData() to detect data chunks that seem 'empty'.
+    ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
+
+    NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features=\"%(parser)s\"' to the BeautifulSoup constructor.\n"
+    
+    def __init__(self, markup="", features=None, builder=None,
+                 parse_only=None, from_encoding=None, exclude_encodings=None,
+                 element_classes=None, **kwargs):
+        """Constructor.
+
+        :param markup: A string or a file-like object representing
+         markup to be parsed.
+
+        :param features: Desirable features of the parser to be
+         used. This may be the name of a specific parser ("lxml",
+         "lxml-xml", "html.parser", or "html5lib") or it may be the
+         type of markup to be used ("html", "html5", "xml"). It's
+         recommended that you name a specific parser, so that
+         Beautiful Soup gives you the same results across platforms
+         and virtual environments.
+
+        :param builder: A TreeBuilder subclass to instantiate (or
+         instance to use) instead of looking one up based on
+         `features`. You only need to use this if you've implemented a
+         custom TreeBuilder.
+
+        :param parse_only: A SoupStrainer. Only parts of the document
+         matching the SoupStrainer will be considered. This is useful
+         when parsing part of a document that would otherwise be too
+         large to fit into memory.
+
+        :param from_encoding: A string indicating the encoding of the
+         document to be parsed. Pass this in if Beautiful Soup is
+         guessing wrongly about the document's encoding.
+
+        :param exclude_encodings: A list of strings indicating
+         encodings known to be wrong. Pass this in if you don't know
+         the document's encoding but you know Beautiful Soup's guess is
+         wrong.
+
+        :param element_classes: A dictionary mapping BeautifulSoup
+         classes like Tag and NavigableString, to other classes you'd
+         like to be instantiated instead as the parse tree is
+         built. This is useful for subclassing Tag or NavigableString
+         to modify default behavior.
+
+        :param kwargs: For backwards compatibility purposes, the
+         constructor accepts certain keyword arguments used in
+         Beautiful Soup 3. None of these arguments do anything in
+         Beautiful Soup 4; they will result in a warning and then be
+         ignored.
+         
+         Apart from this, any keyword arguments passed into the
+         BeautifulSoup constructor are propagated to the TreeBuilder
+         constructor. This makes it possible to configure a
+         TreeBuilder by passing in arguments, not just by saying which
+         one to use.
+        """
+        if 'convertEntities' in kwargs:
+            del kwargs['convertEntities']
+            warnings.warn(
+                "BS4 does not respect the convertEntities argument to the "
+                "BeautifulSoup constructor. Entities are always converted "
+                "to Unicode characters.")
+
+        if 'markupMassage' in kwargs:
+            del kwargs['markupMassage']
+            warnings.warn(
+                "BS4 does not respect the markupMassage argument to the "
+                "BeautifulSoup constructor. The tree builder is responsible "
+                "for any necessary markup massage.")
+
+        if 'smartQuotesTo' in kwargs:
+            del kwargs['smartQuotesTo']
+            warnings.warn(
+                "BS4 does not respect the smartQuotesTo argument to the "
+                "BeautifulSoup constructor. Smart quotes are always converted "
+                "to Unicode characters.")
+
+        if 'selfClosingTags' in kwargs:
+            del kwargs['selfClosingTags']
+            warnings.warn(
+                "BS4 does not respect the selfClosingTags argument to the "
+                "BeautifulSoup constructor. The tree builder is responsible "
+                "for understanding self-closing tags.")
+
+        if 'isHTML' in kwargs:
+            del kwargs['isHTML']
+            warnings.warn(
+                "BS4 does not respect the isHTML argument to the "
+                "BeautifulSoup constructor. Suggest you use "
+                "features='lxml' for HTML and features='lxml-xml' for "
+                "XML.")
+
+        def deprecated_argument(old_name, new_name):
+            if old_name in kwargs:
+                warnings.warn(
+                    'The "%s" argument to the BeautifulSoup constructor '
+                    'has been renamed to "%s."' % (old_name, new_name))
+                value = kwargs[old_name]
+                del kwargs[old_name]
+                return value
+            return None
+
+        parse_only = parse_only or deprecated_argument(
+            "parseOnlyThese", "parse_only")
+
+        from_encoding = from_encoding or deprecated_argument(
+            "fromEncoding", "from_encoding")
+
+        if from_encoding and isinstance(markup, str):
+            warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.")
+            from_encoding = None
+
+        self.element_classes = element_classes or dict()
+
+        # We need this information to track whether or not the builder
+        # was specified well enough that we can omit the 'you need to
+        # specify a parser' warning.
+        original_builder = builder
+        original_features = features
+            
+        if isinstance(builder, type):
+            # A builder class was passed in; it needs to be instantiated.
+            builder_class = builder
+            builder = None
+        elif builder is None:
+            if isinstance(features, str):
+                features = [features]
+            if features is None or len(features) == 0:
+                features = self.DEFAULT_BUILDER_FEATURES
+            builder_class = builder_registry.lookup(*features)
+            if builder_class is None:
+                raise FeatureNotFound(
+                    "Couldn't find a tree builder with the features you "
+                    "requested: %s. Do you need to install a parser library?"
+                    % ",".join(features))
+
+        # At this point either we have a TreeBuilder instance in
+        # builder, or we have a builder_class that we can instantiate
+        # with the remaining **kwargs.
+        if builder is None:
+            builder = builder_class(**kwargs)
+            if not original_builder and not (
+                    original_features == builder.NAME or
+                    original_features in builder.ALTERNATE_NAMES
+            ) and markup:
+                # The user did not tell us which TreeBuilder to use,
+                # and we had to guess. Issue a warning.
+                if builder.is_xml:
+                    markup_type = "XML"
+                else:
+                    markup_type = "HTML"
+
+                # This code adapted from warnings.py so that we get the same line
+                # of code as our warnings.warn() call gets, even if the answer is wrong
+                # (as it may be in a multithreading situation).
+                caller = None
+                try:
+                    caller = sys._getframe(1)
+                except ValueError:
+                    pass
+                if caller:
+                    globals = caller.f_globals
+                    line_number = caller.f_lineno
+                else:
+                    globals = sys.__dict__
+                    line_number= 1                    
+                filename = globals.get('__file__')
+                if filename:
+                    fnl = filename.lower()
+                    if fnl.endswith((".pyc", ".pyo")):
+                        filename = filename[:-1]
+                if filename:
+                    # If there is no filename at all, the user is most likely in a REPL,
+                    # and the warning is not necessary.
+                    values = dict(
+                        filename=filename,
+                        line_number=line_number,
+                        parser=builder.NAME,
+                        markup_type=markup_type
+                    )
+                    warnings.warn(
+                        self.NO_PARSER_SPECIFIED_WARNING % values,
+                        GuessedAtParserWarning, stacklevel=2
+                    )
+        else:
+            if kwargs:
+                warnings.warn("Keyword arguments to the BeautifulSoup constructor will be ignored. These would normally be passed into the TreeBuilder constructor, but a TreeBuilder instance was passed in as `builder`.")
+                    
+        self.builder = builder
+        self.is_xml = builder.is_xml
+        self.known_xml = self.is_xml
+        self._namespaces = dict()
+        self.parse_only = parse_only
+
+        self.builder.initialize_soup(self)
+
+        if hasattr(markup, 'read'):        # It's a file-type object.
+            markup = markup.read()
+        elif len(markup) <= 256 and (
+                (isinstance(markup, bytes) and not b'<' in markup)
+                or (isinstance(markup, str) and not '<' in markup)
+        ):
+            # Print out warnings for a couple beginner problems
+            # involving passing non-markup to Beautiful Soup.
+            # Beautiful Soup will still parse the input as markup,
+            # just in case that's what the user really wants.
+            if (isinstance(markup, str)
+                and not os.path.supports_unicode_filenames):
+                possible_filename = markup.encode("utf8")
+            else:
+                possible_filename = markup
+            is_file = False
+            try:
+                is_file = os.path.exists(possible_filename)
+            except Exception as e:
+                # This is almost certainly a problem involving
+                # characters not valid in filenames on this
+                # system. Just let it go.
+                pass
+            if is_file:
+                warnings.warn(
+                    '"%s" looks like a filename, not markup. You should'
+                    ' probably open this file and pass the filehandle into'
+                    ' Beautiful Soup.' % self._decode_markup(markup),
+                    MarkupResemblesLocatorWarning
+                )
+            self._check_markup_is_url(markup)
+
+        rejections = []
+        success = False
+        for (self.markup, self.original_encoding, self.declared_html_encoding,
+         self.contains_replacement_characters) in (
+             self.builder.prepare_markup(
+                 markup, from_encoding, exclude_encodings=exclude_encodings)):
+            self.reset()
+            try:
+                self._feed()
+                success = True
+                break
+            except ParserRejectedMarkup as e:
+                rejections.append(e)
+                pass
+
+        if not success:
+            other_exceptions = [str(e) for e in rejections]
+            raise ParserRejectedMarkup(
+                "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.\n\nOriginal exception(s) from parser:\n " + "\n ".join(other_exceptions)
+            )
+
+        # Clear out the markup and remove the builder's circular
+        # reference to this object.
+        self.markup = None
+        self.builder.soup = None
+
+    def __copy__(self):
+        """Copy a BeautifulSoup object by converting the document to a string and parsing it again."""
+        copy = type(self)(
+            self.encode('utf-8'), builder=self.builder, from_encoding='utf-8'
+        )
+
+        # Although we encoded the tree to UTF-8, that may not have
+        # been the encoding of the original markup. Set the copy's
+        # .original_encoding to reflect the original object's
+        # .original_encoding.
+        copy.original_encoding = self.original_encoding
+        return copy
+
+    def __getstate__(self):
+        # Frequently a tree builder can't be pickled.
+        d = dict(self.__dict__)
+        if 'builder' in d and not self.builder.picklable:
+            d['builder'] = None
+        return d
+
+    @classmethod
+    def _decode_markup(cls, markup):
+        """Ensure `markup` is bytes so it's safe to send into warnings.warn.
+
+        TODO: warnings.warn had this problem back in 2010 but it might not
+        anymore.
+        """
+        if isinstance(markup, bytes):
+            decoded = markup.decode('utf-8', 'replace')
+        else:
+            decoded = markup
+        return decoded
+
+    @classmethod
+    def _check_markup_is_url(cls, markup):
+        """Error-handling method to raise a warning if incoming markup looks
+        like a URL.
+
+        :param markup: A string.
+        """
+        if isinstance(markup, bytes):
+            space = b' '
+            cant_start_with = (b"http:", b"https:")
+        elif isinstance(markup, str):
+            space = ' '
+            cant_start_with = ("http:", "https:")
+        else:
+            return
+
+        if any(markup.startswith(prefix) for prefix in cant_start_with):
+            if not space in markup:
+                warnings.warn(
+                    '"%s" looks like a URL. Beautiful Soup is not an'
+                    ' HTTP client. You should probably use an HTTP client like'
+                    ' requests to get the document behind the URL, and feed'
+                    ' that document to Beautiful Soup.' % cls._decode_markup(
+                        markup
+                    ),
+                    MarkupResemblesLocatorWarning
+                )
+
+    def _feed(self):
+        """Internal method that parses previously set markup, creating a large
+        number of Tag and NavigableString objects.
+        """
+        # Convert the document to Unicode.
+        self.builder.reset()
+
+        self.builder.feed(self.markup)
+        # Close out any unfinished strings and close all the open tags.
+        self.endData()
+        while self.currentTag.name != self.ROOT_TAG_NAME:
+            self.popTag()
+
+    def reset(self):
+        """Reset this object to a state as though it had never parsed any
+        markup.
+        """
+        Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
+        self.hidden = 1
+        self.builder.reset()
+        self.current_data = []
+        self.currentTag = None
+        self.tagStack = []
+        self.open_tag_counter = Counter()
+        self.preserve_whitespace_tag_stack = []
+        self.string_container_stack = []
+        self.pushTag(self)
+
+    def new_tag(self, name, namespace=None, nsprefix=None, attrs={},
+                sourceline=None, sourcepos=None, **kwattrs):
+        """Create a new Tag associated with this BeautifulSoup object.
+
+        :param name: The name of the new Tag.
+        :param namespace: The URI of the new Tag's XML namespace, if any.
+        :param prefix: The prefix for the new Tag's XML namespace, if any.
+        :param attrs: A dictionary of this Tag's attribute values; can
+            be used instead of `kwattrs` for attributes like 'class'
+            that are reserved words in Python.
+        :param sourceline: The line number where this tag was
+            (purportedly) found in its source document.
+        :param sourcepos: The character position within `sourceline` where this
+            tag was (purportedly) found.
+        :param kwattrs: Keyword arguments for the new Tag's attribute values.
+
+        """
+        kwattrs.update(attrs)
+        return self.element_classes.get(Tag, Tag)(
+            None, self.builder, name, namespace, nsprefix, kwattrs,
+            sourceline=sourceline, sourcepos=sourcepos
+        )
+
+    def string_container(self, base_class=None):
+        container = base_class or NavigableString
+        
+        # There may be a general override of NavigableString.
+        container = self.element_classes.get(
+            container, container
+        )
+
+        # On top of that, we may be inside a tag that needs a special
+        # container class.
+        if self.string_container_stack:
+            container = self.builder.string_containers.get(
+                self.string_container_stack[-1].name, container
+            )
+        return container
+        
+    def new_string(self, s, subclass=None):
+        """Create a new NavigableString associated with this BeautifulSoup
+        object.
+        """
+        container = self.string_container(subclass)
+        return container(s)
+
+    def insert_before(self, *args):
+        """This method is part of the PageElement API, but `BeautifulSoup` doesn't implement
+        it because there is nothing before or after it in the parse tree.
+        """
+        raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
+
+    def insert_after(self, *args):
+        """This method is part of the PageElement API, but `BeautifulSoup` doesn't implement
+        it because there is nothing before or after it in the parse tree.
+        """
+        raise NotImplementedError("BeautifulSoup objects don't support insert_after().")
+
+    def popTag(self):
+        """Internal method called by _popToTag when a tag is closed."""
+        tag = self.tagStack.pop()
+        if tag.name in self.open_tag_counter:
+            self.open_tag_counter[tag.name] -= 1
+        if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]:
+            self.preserve_whitespace_tag_stack.pop()
+        if self.string_container_stack and tag == self.string_container_stack[-1]:
+            self.string_container_stack.pop()
+        #print("Pop", tag.name)
+        if self.tagStack:
+            self.currentTag = self.tagStack[-1]
+        return self.currentTag
+
+    def pushTag(self, tag):
+        """Internal method called by handle_starttag when a tag is opened."""
+        #print("Push", tag.name)
+        if self.currentTag is not None:
+            self.currentTag.contents.append(tag)
+        self.tagStack.append(tag)
+        self.currentTag = self.tagStack[-1]
+        if tag.name != self.ROOT_TAG_NAME:
+            self.open_tag_counter[tag.name] += 1
+        if tag.name in self.builder.preserve_whitespace_tags:
+            self.preserve_whitespace_tag_stack.append(tag)
+        if tag.name in self.builder.string_containers:
+            self.string_container_stack.append(tag)
+
+    def endData(self, containerClass=None):
+        """Method called by the TreeBuilder when the end of a data segment
+        occurs.
+        """
+        containerClass = self.string_container(containerClass)
+        
+        if self.current_data:
+            current_data = ''.join(self.current_data)
+            # If whitespace is not preserved, and this string contains
+            # nothing but ASCII spaces, replace it with a single space
+            # or newline.
+            if not self.preserve_whitespace_tag_stack:
+                strippable = True
+                for i in current_data:
+                    if i not in self.ASCII_SPACES:
+                        strippable = False
+                        break
+                if strippable:
+                    if '\n' in current_data:
+                        current_data = '\n'
+                    else:
+                        current_data = ' '
+
+            # Reset the data collector.
+            self.current_data = []
+
+            # Should we add this string to the tree at all?
+            if self.parse_only and len(self.tagStack) <= 1 and \
+                   (not self.parse_only.text or \
+                    not self.parse_only.search(current_data)):
+                return
+
+            o = containerClass(current_data)
+            self.object_was_parsed(o)
+
+    def object_was_parsed(self, o, parent=None, most_recent_element=None):
+        """Method called by the TreeBuilder to integrate an object into the parse tree."""
+        if parent is None:
+            parent = self.currentTag
+        if most_recent_element is not None:
+            previous_element = most_recent_element
+        else:
+            previous_element = self._most_recent_element
+
+        next_element = previous_sibling = next_sibling = None
+        if isinstance(o, Tag):
+            next_element = o.next_element
+            next_sibling = o.next_sibling
+            previous_sibling = o.previous_sibling
+            if previous_element is None:
+                previous_element = o.previous_element
+
+        fix = parent.next_element is not None
+
+        o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)
+
+        self._most_recent_element = o
+        parent.contents.append(o)
+
+        # Check if we are inserting into an already parsed node.
+        if fix:
+            self._linkage_fixer(parent)
+
+    def _linkage_fixer(self, el):
+        """Make sure linkage of this fragment is sound."""
+
+        first = el.contents[0]
+        child = el.contents[-1]
+        descendant = child
+
+        if child is first and el.parent is not None:
+            # Parent should be linked to first child
+            el.next_element = child
+            # We are no longer linked to whatever this element is
+            prev_el = child.previous_element
+            if prev_el is not None and prev_el is not el:
+                prev_el.next_element = None
+            # First child should be linked to the parent, and no previous siblings.
+            child.previous_element = el
+            child.previous_sibling = None
+
+        # We have no sibling as we've been appended as the last.
+        child.next_sibling = None
+
+        # This index is a tag, dig deeper for a "last descendant"
+        if isinstance(child, Tag) and child.contents:
+            descendant = child._last_descendant(False)
+
+        # As the final step, link last descendant. It should be linked
+        # to the parent's next sibling (if found), else walk up the chain
+        # and find a parent with a sibling. It should have no next sibling.
+        descendant.next_element = None
+        descendant.next_sibling = None
+        target = el
+        while True:
+            if target is None:
+                break
+            elif target.next_sibling is not None:
+                descendant.next_element = target.next_sibling
+                target.next_sibling.previous_element = child
+                break
+            target = target.parent
+
+    def _popToTag(self, name, nsprefix=None, inclusivePop=True):
+        """Pops the tag stack up to and including the most recent
+        instance of the given tag.
+
+        If there are no open tags with the given name, nothing will be
+        popped.
+
+        :param name: Pop up to the most recent tag with this name.
+        :param nsprefix: The namespace prefix that goes with `name`.
+        :param inclusivePop: It this is false, pops the tag stack up
+          to but *not* including the most recent instqance of the
+          given tag.
+
+        """
+        #print("Popping to %s" % name)
+        if name == self.ROOT_TAG_NAME:
+            # The BeautifulSoup object itself can never be popped.
+            return
+
+        most_recently_popped = None
+
+        stack_size = len(self.tagStack)
+        for i in range(stack_size - 1, 0, -1):
+            if not self.open_tag_counter.get(name):
+                break
+            t = self.tagStack[i]
+            if (name == t.name and nsprefix == t.prefix):
+                if inclusivePop:
+                    most_recently_popped = self.popTag()
+                break
+            most_recently_popped = self.popTag()
+
+        return most_recently_popped
+
+    def handle_starttag(self, name, namespace, nsprefix, attrs, sourceline=None,
+                        sourcepos=None):
+        """Called by the tree builder when a new tag is encountered.
+
+        :param name: Name of the tag.
+        :param nsprefix: Namespace prefix for the tag.
+        :param attrs: A dictionary of attribute values.
+        :param sourceline: The line number where this tag was found in its
+            source document.
+        :param sourcepos: The character position within `sourceline` where this
+            tag was found.
+
+        If this method returns None, the tag was rejected by an active
+        SoupStrainer. You should proceed as if the tag had not occurred
+        in the document. For instance, if this was a self-closing tag,
+        don't call handle_endtag.
+        """
+        # print("Start tag %s: %s" % (name, attrs))
+        self.endData()
+
+        if (self.parse_only and len(self.tagStack) <= 1
+            and (self.parse_only.text
+                 or not self.parse_only.search_tag(name, attrs))):
+            return None
+
+        tag = self.element_classes.get(Tag, Tag)(
+            self, self.builder, name, namespace, nsprefix, attrs,
+            self.currentTag, self._most_recent_element,
+            sourceline=sourceline, sourcepos=sourcepos
+        )
+        if tag is None:
+            return tag
+        if self._most_recent_element is not None:
+            self._most_recent_element.next_element = tag
+        self._most_recent_element = tag
+        self.pushTag(tag)
+        return tag
+
+    def handle_endtag(self, name, nsprefix=None):
+        """Called by the tree builder when an ending tag is encountered.
+
+        :param name: Name of the tag.
+        :param nsprefix: Namespace prefix for the tag.
+        """
+        #print("End tag: " + name)
+        self.endData()
+        self._popToTag(name, nsprefix)
+
+    def handle_data(self, data):
+        """Called by the tree builder when a chunk of textual data is encountered."""
+        self.current_data.append(data)
+       
+    def decode(self, pretty_print=False,
+               eventual_encoding=DEFAULT_OUTPUT_ENCODING,
+               formatter="minimal"):
+        """Returns a string or Unicode representation of the parse tree
+            as an HTML or XML document.
+
+        :param pretty_print: If this is True, indentation will be used to
+            make the document more readable.
+        :param eventual_encoding: The encoding of the final document.
+            If this is None, the document will be a Unicode string.
+        """
+        if self.is_xml:
+            # Print the XML declaration
+            encoding_part = ''
+            if eventual_encoding in PYTHON_SPECIFIC_ENCODINGS:
+                # This is a special Python encoding; it can't actually
+                # go into an XML document because it means nothing
+                # outside of Python.
+                eventual_encoding = None
+            if eventual_encoding != None:
+                encoding_part = ' encoding="%s"' % eventual_encoding
+            prefix = '<?xml version="1.0"%s?>\n' % encoding_part
+        else:
+            prefix = ''
+        if not pretty_print:
+            indent_level = None
+        else:
+            indent_level = 0
+        return prefix + super(BeautifulSoup, self).decode(
+            indent_level, eventual_encoding, formatter)
+
+# Aliases to make it easier to get started quickly, e.g. 'from bs4 import _soup'
+_s = BeautifulSoup
+_soup = BeautifulSoup
+
+class BeautifulStoneSoup(BeautifulSoup):
+    """Deprecated interface to an XML parser."""
+
+    def __init__(self, *args, **kwargs):
+        kwargs['features'] = 'xml'
+        warnings.warn(
+            'The BeautifulStoneSoup class is deprecated. Instead of using '
+            'it, pass features="xml" into the BeautifulSoup constructor.')
+        super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
+
+
+class StopParsing(Exception):
+    """Exception raised by a TreeBuilder if it's unable to continue parsing."""
+    pass
+
+class FeatureNotFound(ValueError):
+    """Exception raised by the BeautifulSoup constructor if no parser with the
+    requested features is found.
+    """
+    pass
+
+
+#If this file is run as a script, act as an HTML pretty-printer.
+if __name__ == '__main__':
+    import sys
+    soup = BeautifulSoup(sys.stdin)
+    print((soup.prettify()))

bs4/builder/__init__.py

@@ -0,0 +1,519 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
+from collections import defaultdict
+import itertools
+import sys
+from bs4.element import (
+    CharsetMetaAttributeValue,
+    ContentMetaAttributeValue,
+    Stylesheet,
+    Script,
+    TemplateString,
+    nonwhitespace_re
+)
+
+__all__ = [
+    'HTMLTreeBuilder',
+    'SAXTreeBuilder',
+    'TreeBuilder',
+    'TreeBuilderRegistry',
+    ]
+
+# Some useful features for a TreeBuilder to have.
+FAST = 'fast'
+PERMISSIVE = 'permissive'
+STRICT = 'strict'
+XML = 'xml'
+HTML = 'html'
+HTML_5 = 'html5'
+
+
+class TreeBuilderRegistry(object):
+    """A way of looking up TreeBuilder subclasses by their name or by desired
+    features.
+    """
+    
+    def __init__(self):
+        self.builders_for_feature = defaultdict(list)
+        self.builders = []
+
+    def register(self, treebuilder_class):
+        """Register a treebuilder based on its advertised features.
+
+        :param treebuilder_class: A subclass of Treebuilder. its .features
+           attribute should list its features.
+        """
+        for feature in treebuilder_class.features:
+            self.builders_for_feature[feature].insert(0, treebuilder_class)
+        self.builders.insert(0, treebuilder_class)
+
+    def lookup(self, *features):
+        """Look up a TreeBuilder subclass with the desired features.
+
+        :param features: A list of features to look for. If none are
+            provided, the most recently registered TreeBuilder subclass
+            will be used.
+        :return: A TreeBuilder subclass, or None if there's no
+            registered subclass with all the requested features.
+        """
+        if len(self.builders) == 0:
+            # There are no builders at all.
+            return None
+
+        if len(features) == 0:
+            # They didn't ask for any features. Give them the most
+            # recently registered builder.
+            return self.builders[0]
+
+        # Go down the list of features in order, and eliminate any builders
+        # that don't match every feature.
+        features = list(features)
+        features.reverse()
+        candidates = None
+        candidate_set = None
+        while len(features) > 0:
+            feature = features.pop()
+            we_have_the_feature = self.builders_for_feature.get(feature, [])
+            if len(we_have_the_feature) > 0:
+                if candidates is None:
+                    candidates = we_have_the_feature
+                    candidate_set = set(candidates)
+                else:
+                    # Eliminate any candidates that don't have this feature.
+                    candidate_set = candidate_set.intersection(
+                        set(we_have_the_feature))
+
+        # The only valid candidates are the ones in candidate_set.
+        # Go through the original list of candidates and pick the first one
+        # that's in candidate_set.
+        if candidate_set is None:
+            return None
+        for candidate in candidates:
+            if candidate in candidate_set:
+                return candidate
+        return None
+
+# The BeautifulSoup class will take feature lists from developers and use them
+# to look up builders in this registry.
+builder_registry = TreeBuilderRegistry()
+
+class TreeBuilder(object):
+    """Turn a textual document into a Beautiful Soup object tree."""
+
+    NAME = "[Unknown tree builder]"
+    ALTERNATE_NAMES = []
+    features = []
+
+    is_xml = False
+    picklable = False
+    empty_element_tags = None # A tag will be considered an empty-element
+                              # tag when and only when it has no contents.
+    
+    # A value for these tag/attribute combinations is a space- or
+    # comma-separated list of CDATA, rather than a single CDATA.
+    DEFAULT_CDATA_LIST_ATTRIBUTES = {}
+
+    # Whitespace should be preserved inside these tags.
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set()
+
+    # The textual contents of tags with these names should be
+    # instantiated with some class other than NavigableString.
+    DEFAULT_STRING_CONTAINERS = {}
+    
+    USE_DEFAULT = object()
+
+    # Most parsers don't keep track of line numbers.
+    TRACKS_LINE_NUMBERS = False
+    
+    def __init__(self, multi_valued_attributes=USE_DEFAULT,
+                 preserve_whitespace_tags=USE_DEFAULT,
+                 store_line_numbers=USE_DEFAULT,
+                 string_containers=USE_DEFAULT,
+    ):
+        """Constructor.
+
+        :param multi_valued_attributes: If this is set to None, the
+         TreeBuilder will not turn any values for attributes like
+         'class' into lists. Setting this to a dictionary will
+         customize this behavior; look at DEFAULT_CDATA_LIST_ATTRIBUTES
+         for an example.
+
+         Internally, these are called "CDATA list attributes", but that
+         probably doesn't make sense to an end-user, so the argument name
+         is `multi_valued_attributes`.
+
+        :param preserve_whitespace_tags: A list of tags to treat
+         the way <pre> tags are treated in HTML. Tags in this list
+         are immune from pretty-printing; their contents will always be
+         output as-is.
+
+        :param string_containers: A dictionary mapping tag names to
+        the classes that should be instantiated to contain the textual
+        contents of those tags. The default is to use NavigableString
+        for every tag, no matter what the name. You can override the
+        default by changing DEFAULT_STRING_CONTAINERS.
+
+        :param store_line_numbers: If the parser keeps track of the
+         line numbers and positions of the original markup, that
+         information will, by default, be stored in each corresponding
+         `Tag` object. You can turn this off by passing
+         store_line_numbers=False. If the parser you're using doesn't 
+         keep track of this information, then setting store_line_numbers=True
+         will do nothing.
+        """
+        self.soup = None
+        if multi_valued_attributes is self.USE_DEFAULT:
+            multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES
+        self.cdata_list_attributes = multi_valued_attributes
+        if preserve_whitespace_tags is self.USE_DEFAULT:
+            preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS
+        self.preserve_whitespace_tags = preserve_whitespace_tags
+        if store_line_numbers == self.USE_DEFAULT:
+            store_line_numbers = self.TRACKS_LINE_NUMBERS
+        self.store_line_numbers = store_line_numbers 
+        if string_containers == self.USE_DEFAULT:
+            string_containers = self.DEFAULT_STRING_CONTAINERS
+        self.string_containers = string_containers
+        
+    def initialize_soup(self, soup):
+        """The BeautifulSoup object has been initialized and is now
+        being associated with the TreeBuilder.
+
+        :param soup: A BeautifulSoup object.
+        """
+        self.soup = soup
+        
+    def reset(self):
+        """Do any work necessary to reset the underlying parser
+        for a new document.
+
+        By default, this does nothing.
+        """
+        pass
+
+    def can_be_empty_element(self, tag_name):
+        """Might a tag with this name be an empty-element tag?
+
+        The final markup may or may not actually present this tag as
+        self-closing.
+
+        For instance: an HTMLBuilder does not consider a <p> tag to be
+        an empty-element tag (it's not in
+        HTMLBuilder.empty_element_tags). This means an empty <p> tag
+        will be presented as "<p></p>", not "<p/>" or "<p>".
+
+        The default implementation has no opinion about which tags are
+        empty-element tags, so a tag will be presented as an
+        empty-element tag if and only if it has no children.
+        "<foo></foo>" will become "<foo/>", and "<foo>bar</foo>" will
+        be left alone.
+
+        :param tag_name: The name of a markup tag.
+        """
+        if self.empty_element_tags is None:
+            return True
+        return tag_name in self.empty_element_tags
+    
+    def feed(self, markup):
+        """Run some incoming markup through some parsing process,
+        populating the `BeautifulSoup` object in self.soup.
+
+        This method is not implemented in TreeBuilder; it must be
+        implemented in subclasses.
+
+        :return: None.
+        """
+        raise NotImplementedError()
+
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None, exclude_encodings=None):
+        """Run any preliminary steps necessary to make incoming markup
+        acceptable to the parser.
+
+        :param markup: Some markup -- probably a bytestring.
+        :param user_specified_encoding: The user asked to try this encoding.
+        :param document_declared_encoding: The markup itself claims to be
+            in this encoding.
+        :param exclude_encodings: The user asked _not_ to try any of
+            these encodings.
+
+        :yield: A series of 4-tuples:
+         (markup, encoding, declared encoding,
+          has undergone character replacement)
+
+         Each 4-tuple represents a strategy for converting the
+         document to Unicode and parsing it. Each strategy will be tried 
+         in turn.
+
+         By default, the only strategy is to parse the markup
+         as-is. See `LXMLTreeBuilderForXML` and
+         `HTMLParserTreeBuilder` for implementations that take into
+         account the quirks of particular parsers.
+        """
+        yield markup, None, None, False
+
+    def test_fragment_to_document(self, fragment):
+        """Wrap an HTML fragment to make it look like a document.
+
+        Different parsers do this differently. For instance, lxml
+        introduces an empty <head> tag, and html5lib
+        doesn't. Abstracting this away lets us write simple tests
+        which run HTML fragments through the parser and compare the
+        results against other HTML fragments.
+
+        This method should not be used outside of tests.
+
+        :param fragment: A string -- fragment of HTML.
+        :return: A string -- a full HTML document.
+        """
+        return fragment
+
+    def set_up_substitutions(self, tag):
+        """Set up any substitutions that will need to be performed on 
+        a `Tag` when it's output as a string.
+
+        By default, this does nothing. See `HTMLTreeBuilder` for a
+        case where this is used.
+
+        :param tag: A `Tag`
+        :return: Whether or not a substitution was performed.
+        """
+        return False
+
+    def _replace_cdata_list_attribute_values(self, tag_name, attrs):
+        """When an attribute value is associated with a tag that can
+        have multiple values for that attribute, convert the string
+        value to a list of strings.
+
+        Basically, replaces class="foo bar" with class=["foo", "bar"]
+
+        NOTE: This method modifies its input in place.
+
+        :param tag_name: The name of a tag.
+        :param attrs: A dictionary containing the tag's attributes.
+           Any appropriate attribute values will be modified in place.
+        """
+        if not attrs:
+            return attrs
+        if self.cdata_list_attributes:
+            universal = self.cdata_list_attributes.get('*', [])
+            tag_specific = self.cdata_list_attributes.get(
+                tag_name.lower(), None)
+            for attr in list(attrs.keys()):
+                if attr in universal or (tag_specific and attr in tag_specific):
+                    # We have a "class"-type attribute whose string
+                    # value is a whitespace-separated list of
+                    # values. Split it into a list.
+                    value = attrs[attr]
+                    if isinstance(value, str):
+                        values = nonwhitespace_re.findall(value)
+                    else:
+                        # html5lib sometimes calls setAttributes twice
+                        # for the same tag when rearranging the parse
+                        # tree. On the second call the attribute value
+                        # here is already a list.  If this happens,
+                        # leave the value alone rather than trying to
+                        # split it again.
+                        values = value
+                    attrs[attr] = values
+        return attrs
+
+class SAXTreeBuilder(TreeBuilder):
+    """A Beautiful Soup treebuilder that listens for SAX events.
+
+    This is not currently used for anything, but it demonstrates
+    how a simple TreeBuilder would work.
+    """
+
+    def feed(self, markup):
+        raise NotImplementedError()
+
+    def close(self):
+        pass
+
+    def startElement(self, name, attrs):
+        attrs = dict((key[1], value) for key, value in list(attrs.items()))
+        #print("Start %s, %r" % (name, attrs))
+        self.soup.handle_starttag(name, attrs)
+
+    def endElement(self, name):
+        #print("End %s" % name)
+        self.soup.handle_endtag(name)
+
+    def startElementNS(self, nsTuple, nodeName, attrs):
+        # Throw away (ns, nodeName) for now.
+        self.startElement(nodeName, attrs)
+
+    def endElementNS(self, nsTuple, nodeName):
+        # Throw away (ns, nodeName) for now.
+        self.endElement(nodeName)
+        #handler.endElementNS((ns, node.nodeName), node.nodeName)
+
+    def startPrefixMapping(self, prefix, nodeValue):
+        # Ignore the prefix for now.
+        pass
+
+    def endPrefixMapping(self, prefix):
+        # Ignore the prefix for now.
+        # handler.endPrefixMapping(prefix)
+        pass
+
+    def characters(self, content):
+        self.soup.handle_data(content)
+
+    def startDocument(self):
+        pass
+
+    def endDocument(self):
+        pass
+
+
+class HTMLTreeBuilder(TreeBuilder):
+    """This TreeBuilder knows facts about HTML.
+
+    Such as which tags are empty-element tags.
+    """
+
+    empty_element_tags = set([
+        # These are from HTML5.
+        'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
+        
+        # These are from earlier versions of HTML and are removed in HTML5.
+        'basefont', 'bgsound', 'command', 'frame', 'image', 'isindex', 'nextid', 'spacer'
+    ])
+
+    # The HTML standard defines these as block-level elements. Beautiful
+    # Soup does not treat these elements differently from other elements,
+    # but it may do so eventually, and this information is available if
+    # you need to use it.
+    block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"])
+
+    # The HTML standard defines an unusual content model for these tags.
+    # We represent this by using a string class other than NavigableString
+    # inside these tags.
+    #
+    # I made this list by going through the HTML spec
+    # (https://html.spec.whatwg.org/#metadata-content) and looking for
+    # "metadata content" elements that can contain strings.
+    #
+    # TODO: Arguably <noscript> could go here but it seems
+    # qualitatively different from the other tags.
+    DEFAULT_STRING_CONTAINERS = {
+        'style': Stylesheet,
+        'script': Script,
+        'template': TemplateString,
+    }    
+    
+    # The HTML standard defines these attributes as containing a
+    # space-separated list of values, not a single value. That is,
+    # class="foo bar" means that the 'class' attribute has two values,
+    # 'foo' and 'bar', not the single value 'foo bar'.  When we
+    # encounter one of these attributes, we will parse its value into
+    # a list of values if possible. Upon output, the list will be
+    # converted back into a string.
+    DEFAULT_CDATA_LIST_ATTRIBUTES = {
+        "*" : ['class', 'accesskey', 'dropzone'],
+        "a" : ['rel', 'rev'],
+        "link" :  ['rel', 'rev'],
+        "td" : ["headers"],
+        "th" : ["headers"],
+        "td" : ["headers"],
+        "form" : ["accept-charset"],
+        "object" : ["archive"],
+
+        # These are HTML5 specific, as are *.accesskey and *.dropzone above.
+        "area" : ["rel"],
+        "icon" : ["sizes"],
+        "iframe" : ["sandbox"],
+        "output" : ["for"],
+        }
+
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
+    
+    def set_up_substitutions(self, tag):
+        """Replace the declared encoding in a <meta> tag with a placeholder,
+        to be substituted when the tag is output to a string.
+
+        An HTML document may come in to Beautiful Soup as one
+        encoding, but exit in a different encoding, and the <meta> tag
+        needs to be changed to reflect this.
+
+        :param tag: A `Tag`
+        :return: Whether or not a substitution was performed.
+        """
+        # We are only interested in <meta> tags
+        if tag.name != 'meta':
+            return False
+
+        http_equiv = tag.get('http-equiv')
+        content = tag.get('content')
+        charset = tag.get('charset')
+
+        # We are interested in <meta> tags that say what encoding the
+        # document was originally in. This means HTML 5-style <meta>
+        # tags that provide the "charset" attribute. It also means
+        # HTML 4-style <meta> tags that provide the "content"
+        # attribute and have "http-equiv" set to "content-type".
+        #
+        # In both cases we will replace the value of the appropriate
+        # attribute with a standin object that can take on any
+        # encoding.
+        meta_encoding = None
+        if charset is not None:
+            # HTML 5 style:
+            # <meta charset="utf8">
+            meta_encoding = charset
+            tag['charset'] = CharsetMetaAttributeValue(charset)
+
+        elif (content is not None and http_equiv is not None
+              and http_equiv.lower() == 'content-type'):
+            # HTML 4 style:
+            # <meta http-equiv="content-type" content="text/html; charset=utf8">
+            tag['content'] = ContentMetaAttributeValue(content)
+
+        return (meta_encoding is not None)
+
+def register_treebuilders_from(module):
+    """Copy TreeBuilders from the given module into this module."""
+    this_module = sys.modules[__name__]
+    for name in module.__all__:
+        obj = getattr(module, name)
+
+        if issubclass(obj, TreeBuilder):
+            setattr(this_module, name, obj)
+            this_module.__all__.append(name)
+            # Register the builder while we're at it.
+            this_module.builder_registry.register(obj)
+
+class ParserRejectedMarkup(Exception):
+    """An Exception to be raised when the underlying parser simply
+    refuses to parse the given markup.
+    """
+    def __init__(self, message_or_exception):
+        """Explain why the parser rejected the given markup, either
+        with a textual explanation or another exception.
+        """
+        if isinstance(message_or_exception, Exception):
+            e = message_or_exception
+            message_or_exception = "%s: %s" % (e.__class__.__name__, str(e))
+        super(ParserRejectedMarkup, self).__init__(message_or_exception)
+            
+# Builders are registered in reverse order of priority, so that custom
+# builder registrations will take precedence. In general, we want lxml
+# to take precedence over html5lib, because it's faster. And we only
+# want to use HTMLParser as a last resort.
+from . import _htmlparser
+register_treebuilders_from(_htmlparser)
+try:
+    from . import _html5lib
+    register_treebuilders_from(_html5lib)
+except ImportError:
+    # They don't have html5lib installed.
+    pass
+try:
+    from . import _lxml
+    register_treebuilders_from(_lxml)
+except ImportError:
+    # They don't have lxml installed.
+    pass

bs4/builder/_html5lib.py

@@ -0,0 +1,467 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
+__all__ = [
+    'HTML5TreeBuilder',
+    ]
+
+import warnings
+import re
+from bs4.builder import (
+    PERMISSIVE,
+    HTML,
+    HTML_5,
+    HTMLTreeBuilder,
+    )
+from bs4.element import (
+    NamespacedAttribute,
+    nonwhitespace_re,
+)
+import html5lib
+from html5lib.constants import (
+    namespaces,
+    prefixes,
+    )
+from bs4.element import (
+    Comment,
+    Doctype,
+    NavigableString,
+    Tag,
+    )
+
+try:
+    # Pre-0.99999999
+    from html5lib.treebuilders import _base as treebuilder_base
+    new_html5lib = False
+except ImportError as e:
+    # 0.99999999 and up
+    from html5lib.treebuilders import base as treebuilder_base
+    new_html5lib = True
+
+class HTML5TreeBuilder(HTMLTreeBuilder):
+    """Use html5lib to build a tree.
+
+    Note that this TreeBuilder does not support some features common
+    to HTML TreeBuilders. Some of these features could theoretically
+    be implemented, but at the very least it's quite difficult,
+    because html5lib moves the parse tree around as it's being built.
+
+    * This TreeBuilder doesn't use different subclasses of NavigableString
+      based on the name of the tag in which the string was found.
+
+    * You can't use a SoupStrainer to parse only part of a document.
+    """
+
+    NAME = "html5lib"
+
+    features = [NAME, PERMISSIVE, HTML_5, HTML]
+
+    # html5lib can tell us which line number and position in the
+    # original file is the source of an element.
+    TRACKS_LINE_NUMBERS = True
+    
+    def prepare_markup(self, markup, user_specified_encoding,
+                       document_declared_encoding=None, exclude_encodings=None):
+        # Store the user-specified encoding for use later on.
+        self.user_specified_encoding = user_specified_encoding
+
+        # document_declared_encoding and exclude_encodings aren't used
+        # ATM because the html5lib TreeBuilder doesn't use
+        # UnicodeDammit.
+        if exclude_encodings:
+            warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
+        yield (markup, None, None, False)
+
+    # These methods are defined by Beautiful Soup.
+    def feed(self, markup):
+        if self.soup.parse_only is not None:
+            warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
+        parser = html5lib.HTMLParser(tree=self.create_treebuilder)
+        self.underlying_builder.parser = parser
+        extra_kwargs = dict()
+        if not isinstance(markup, str):
+            if new_html5lib:
+                extra_kwargs['override_encoding'] = self.user_specified_encoding
+            else:
+                extra_kwargs['encoding'] = self.user_specified_encoding
+        doc = parser.parse(markup, **extra_kwargs)
+        
+        # Set the character encoding detected by the tokenizer.
+        if isinstance(markup, str):
+            # We need to special-case this because html5lib sets
+            # charEncoding to UTF-8 if it gets Unicode input.
+            doc.original_encoding = None
+        else:
+            original_encoding = parser.tokenizer.stream.charEncoding[0]
+            if not isinstance(original_encoding, str):
+                # In 0.99999999 and up, the encoding is an html5lib
+                # Encoding object. We want to use a string for compatibility
+                # with other tree builders.
+                original_encoding = original_encoding.name
+            doc.original_encoding = original_encoding
+        self.underlying_builder.parser = None
+            
+    def create_treebuilder(self, namespaceHTMLElements):
+        self.underlying_builder = TreeBuilderForHtml5lib(
+            namespaceHTMLElements, self.soup,
+            store_line_numbers=self.store_line_numbers
+        )
+        return self.underlying_builder
+
+    def test_fragment_to_document(self, fragment):
+        """See `TreeBuilder`."""
+        return '<html><head></head><body>%s</body></html>' % fragment
+
+
+class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
+    
+    def __init__(self, namespaceHTMLElements, soup=None,
+                 store_line_numbers=True, **kwargs):
+        if soup:
+            self.soup = soup
+        else:
+            from bs4 import BeautifulSoup
+            # TODO: Why is the parser 'html.parser' here? To avoid an
+            # infinite loop?
+            self.soup = BeautifulSoup(
+                "", "html.parser", store_line_numbers=store_line_numbers,
+                **kwargs
+            )
+        # TODO: What are **kwargs exactly? Should they be passed in
+        # here in addition to/instead of being passed to the BeautifulSoup
+        # constructor?
+        super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
+
+        # This will be set later to an html5lib.html5parser.HTMLParser
+        # object, which we can use to track the current line number.
+        self.parser = None
+        self.store_line_numbers = store_line_numbers
+        
+    def documentClass(self):
+        self.soup.reset()
+        return Element(self.soup, self.soup, None)
+
+    def insertDoctype(self, token):
+        name = token["name"]
+        publicId = token["publicId"]
+        systemId = token["systemId"]
+
+        doctype = Doctype.for_name_and_ids(name, publicId, systemId)
+        self.soup.object_was_parsed(doctype)
+
+    def elementClass(self, name, namespace):
+        kwargs = {}
+        if self.parser and self.store_line_numbers:
+            # This represents the point immediately after the end of the
+            # tag. We don't know when the tag started, but we do know
+            # where it ended -- the character just before this one.
+            sourceline, sourcepos = self.parser.tokenizer.stream.position()
+            kwargs['sourceline'] = sourceline
+            kwargs['sourcepos'] = sourcepos-1
+        tag = self.soup.new_tag(name, namespace, **kwargs)
+
+        return Element(tag, self.soup, namespace)
+
+    def commentClass(self, data):
+        return TextNode(Comment(data), self.soup)
+
+    def fragmentClass(self):
+        from bs4 import BeautifulSoup
+        # TODO: Why is the parser 'html.parser' here? To avoid an
+        # infinite loop?
+        self.soup = BeautifulSoup("", "html.parser")
+        self.soup.name = "[document_fragment]"
+        return Element(self.soup, self.soup, None)
+
+    def appendChild(self, node):
+        # XXX This code is not covered by the BS4 tests.
+        self.soup.append(node.element)
+
+    def getDocument(self):
+        return self.soup
+
+    def getFragment(self):
+        return treebuilder_base.TreeBuilder.getFragment(self).element
+
+    def testSerializer(self, element):
+        from bs4 import BeautifulSoup
+        rv = []
+        doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$')
+
+        def serializeElement(element, indent=0):
+            if isinstance(element, BeautifulSoup):
+                pass
+            if isinstance(element, Doctype):
+                m = doctype_re.match(element)
+                if m:
+                    name = m.group(1)
+                    if m.lastindex > 1:
+                        publicId = m.group(2) or ""
+                        systemId = m.group(3) or m.group(4) or ""
+                        rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (' ' * indent, name, publicId, systemId))
+                    else:
+                        rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name))
+                else:
+                    rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
+            elif isinstance(element, Comment):
+                rv.append("|%s<!-- %s -->" % (' ' * indent, element))
+            elif isinstance(element, NavigableString):
+                rv.append("|%s\"%s\"" % (' ' * indent, element))
+            else:
+                if element.namespace:
+                    name = "%s %s" % (prefixes[element.namespace],
+                                      element.name)
+                else:
+                    name = element.name
+                rv.append("|%s<%s>" % (' ' * indent, name))
+                if element.attrs:
+                    attributes = []
+                    for name, value in list(element.attrs.items()):
+                        if isinstance(name, NamespacedAttribute):
+                            name = "%s %s" % (prefixes[name.namespace], name.name)
+                        if isinstance(value, list):
+                            value = " ".join(value)
+                        attributes.append((name, value))
+
+                    for name, value in sorted(attributes):
+                        rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
+                indent += 2
+                for child in element.children:
+                    serializeElement(child, indent)
+        serializeElement(element, 0)
+
+        return "\n".join(rv)
+
+class AttrList(object):
+    def __init__(self, element):
+        self.element = element
+        self.attrs = dict(self.element.attrs)
+    def __iter__(self):
+        return list(self.attrs.items()).__iter__()
+    def __setitem__(self, name, value):
+        # If this attribute is a multi-valued attribute for this element,
+        # turn its value into a list.
+        list_attr = self.element.cdata_list_attributes
+        if (name in list_attr['*']
+            or (self.element.name in list_attr
+                and name in list_attr[self.element.name])):
+            # A node that is being cloned may have already undergone
+            # this procedure.
+            if not isinstance(value, list):
+                value = nonwhitespace_re.findall(value)
+        self.element[name] = value
+    def items(self):
+        return list(self.attrs.items())
+    def keys(self):
+        return list(self.attrs.keys())
+    def __len__(self):
+        return len(self.attrs)
+    def __getitem__(self, name):
+        return self.attrs[name]
+    def __contains__(self, name):
+        return name in list(self.attrs.keys())
+
+
+class Element(treebuilder_base.Node):
+    def __init__(self, element, soup, namespace):
+        treebuilder_base.Node.__init__(self, element.name)
+        self.element = element
+        self.soup = soup
+        self.namespace = namespace
+
+    def appendChild(self, node):
+        string_child = child = None
+        if isinstance(node, str):
+            # Some other piece of code decided to pass in a string
+            # instead of creating a TextElement object to contain the
+            # string.
+            string_child = child = node
+        elif isinstance(node, Tag):
+            # Some other piece of code decided to pass in a Tag
+            # instead of creating an Element object to contain the
+            # Tag.
+            child = node
+        elif node.element.__class__ == NavigableString:
+            string_child = child = node.element
+            node.parent = self
+        else:
+            child = node.element
+            node.parent = self
+
+        if not isinstance(child, str) and child.parent is not None:
+            node.element.extract()
+
+        if (string_child is not None and self.element.contents
+            and self.element.contents[-1].__class__ == NavigableString):
+            # We are appending a string onto another string.
+            # TODO This has O(n^2) performance, for input like
+            # "a</a>a</a>a</a>..."
+            old_element = self.element.contents[-1]
+            new_element = self.soup.new_string(old_element + string_child)
+            old_element.replace_with(new_element)
+            self.soup._most_recent_element = new_element
+        else:
+            if isinstance(node, str):
+                # Create a brand new NavigableString from this string.
+                child = self.soup.new_string(node)
+
+            # Tell Beautiful Soup to act as if it parsed this element
+            # immediately after the parent's last descendant. (Or
+            # immediately after the parent, if it has no children.)
+            if self.element.contents:
+                most_recent_element = self.element._last_descendant(False)
+            elif self.element.next_element is not None:
+                # Something from further ahead in the parse tree is
+                # being inserted into this earlier element. This is
+                # very annoying because it means an expensive search
+                # for the last element in the tree.
+                most_recent_element = self.soup._last_descendant()
+            else:
+                most_recent_element = self.element
+
+            self.soup.object_was_parsed(
+                child, parent=self.element,
+                most_recent_element=most_recent_element)
+
+    def getAttributes(self):
+        if isinstance(self.element, Comment):
+            return {}
+        return AttrList(self.element)
+
+    def setAttributes(self, attributes):
+        if attributes is not None and len(attributes) > 0:
+            converted_attributes = []
+            for name, value in list(attributes.items()):
+                if isinstance(name, tuple):
+                    new_name = NamespacedAttribute(*name)
+                    del attributes[name]
+                    attributes[new_name] = value
+
+            self.soup.builder._replace_cdata_list_attribute_values(
+                self.name, attributes)
+            for name, value in list(attributes.items()):
+                self.element[name] = value
+
+            # The attributes may contain variables that need substitution.
+            # Call set_up_substitutions manually.
+            #
+            # The Tag constructor called this method when the Tag was created,
+            # but we just set/changed the attributes, so call it again.
+            self.soup.builder.set_up_substitutions(self.element)
+    attributes = property(getAttributes, setAttributes)
+
+    def insertText(self, data, insertBefore=None):
+        text = TextNode(self.soup.new_string(data), self.soup)
+        if insertBefore:
+            self.insertBefore(text, insertBefore)
+        else:
+            self.appendChild(text)
+
+    def insertBefore(self, node, refNode):
+        index = self.element.index(refNode.element)
+        if (node.element.__class__ == NavigableString and self.element.contents
+            and self.element.contents[index-1].__class__ == NavigableString):
+            # (See comments in appendChild)
+            old_node = self.element.contents[index-1]
+            new_str = self.soup.new_string(old_node + node.element)
+            old_node.replace_with(new_str)
+        else:
+            self.element.insert(index, node.element)
+            node.parent = self
+
+    def removeChild(self, node):
+        node.element.extract()
+
+    def reparentChildren(self, new_parent):
+        """Move all of this tag's children into another tag."""
+        # print("MOVE", self.element.contents)
+        # print("FROM", self.element)
+        # print("TO", new_parent.element)
+
+        element = self.element
+        new_parent_element = new_parent.element
+        # Determine what this tag's next_element will be once all the children
+        # are removed.
+        final_next_element = element.next_sibling
+
+        new_parents_last_descendant = new_parent_element._last_descendant(False, False)
+        if len(new_parent_element.contents) > 0:
+            # The new parent already contains children. We will be
+            # appending this tag's children to the end.
+            new_parents_last_child = new_parent_element.contents[-1]
+            new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
+        else:
+            # The new parent contains no children.
+            new_parents_last_child = None
+            new_parents_last_descendant_next_element = new_parent_element.next_element
+
+        to_append = element.contents
+        if len(to_append) > 0:
+            # Set the first child's previous_element and previous_sibling
+            # to elements within the new parent
+            first_child = to_append[0]
+            if new_parents_last_descendant is not None:
+                first_child.previous_element = new_parents_last_descendant
+            else:
+                first_child.previous_element = new_parent_element
+            first_child.previous_sibling = new_parents_last_child
+            if new_parents_last_descendant is not None:
+                new_parents_last_descendant.next_element = first_child
+            else:
+                new_parent_element.next_element = first_child
+            if new_parents_last_child is not None:
+                new_parents_last_child.next_sibling = first_child
+
+            # Find the very last element being moved. It is now the
+            # parent's last descendant. It has no .next_sibling and
+            # its .next_element is whatever the previous last
+            # descendant had.
+            last_childs_last_descendant = to_append[-1]._last_descendant(False, True)
+
+            last_childs_last_descendant.next_element = new_parents_last_descendant_next_element
+            if new_parents_last_descendant_next_element is not None:
+                # TODO: This code has no test coverage and I'm not sure
+                # how to get html5lib to go through this path, but it's
+                # just the other side of the previous line.
+                new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant
+            last_childs_last_descendant.next_sibling = None
+
+        for child in to_append:
+            child.parent = new_parent_element
+            new_parent_element.contents.append(child)
+
+        # Now that this element has no children, change its .next_element.
+        element.contents = []
+        element.next_element = final_next_element
+
+        # print("DONE WITH MOVE")
+        # print("FROM", self.element)
+        # print("TO", new_parent_element)
+
+    def cloneNode(self):
+        tag = self.soup.new_tag(self.element.name, self.namespace)
+        node = Element(tag, self.soup, self.namespace)
+        for key,value in self.attributes:
+            node.attributes[key] = value
+        return node
+
+    def hasContent(self):
+        return self.element.contents
+
+    def getNameTuple(self):
+        if self.namespace == None:
+            return namespaces["html"], self.name
+        else:
+            return self.namespace, self.name
+
+    nameTuple = property(getNameTuple)
+
+class TextNode(Element):
+    def __init__(self, element, soup):
+        treebuilder_base.Node.__init__(self, None)
+        self.element = element
+        self.soup = soup
+
+    def cloneNode(self):
+        raise NotImplementedError

bs4/builder/_htmlparser.py

@@ -0,0 +1,477 @@
+# encoding: utf-8
+"""Use the HTMLParser library to parse HTML files that aren't too bad."""
+
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
+__all__ = [
+    'HTMLParserTreeBuilder',
+    ]
+
+from html.parser import HTMLParser
+
+try:
+    from html.parser import HTMLParseError
+except ImportError as e:
+    # HTMLParseError is removed in Python 3.5. Since it can never be
+    # thrown in 3.5, we can just define our own class as a placeholder.
+    class HTMLParseError(Exception):
+        pass
+
+import sys
+import warnings
+
+# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
+# argument, which we'd like to set to False. Unfortunately,
+# http://bugs.python.org/issue13273 makes strict=True a better bet
+# before Python 3.2.3.
+#
+# At the end of this file, we monkeypatch HTMLParser so that
+# strict=True works well on Python 3.2.2.
+major, minor, release = sys.version_info[:3]
+CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3
+CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3
+CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4
+
+
+from bs4.element import (
+    CData,
+    Comment,
+    Declaration,
+    Doctype,
+    ProcessingInstruction,
+    )
+from bs4.dammit import EntitySubstitution, UnicodeDammit
+
+from bs4.builder import (
+    HTML,
+    HTMLTreeBuilder,
+    STRICT,
+    )
+
+
+HTMLPARSER = 'html.parser'
+
+class BeautifulSoupHTMLParser(HTMLParser):
+    """A subclass of the Python standard library's HTMLParser class, which
+    listens for HTMLParser events and translates them into calls
+    to Beautiful Soup's tree construction API.
+    """
+
+    # Strategies for handling duplicate attributes
+    IGNORE = 'ignore'
+    REPLACE = 'replace'
+    
+    def __init__(self, *args, **kwargs):
+        """Constructor.
+
+        :param on_duplicate_attribute: A strategy for what to do if a
+            tag includes the same attribute more than once. Accepted
+            values are: REPLACE (replace earlier values with later
+            ones, the default), IGNORE (keep the earliest value
+            encountered), or a callable. A callable must take three
+            arguments: the dictionary of attributes already processed,
+            the name of the duplicate attribute, and the most recent value
+            encountered.           
+        """
+        self.on_duplicate_attribute = kwargs.pop(
+            'on_duplicate_attribute', self.REPLACE
+        )
+        HTMLParser.__init__(self, *args, **kwargs)
+
+        # Keep a list of empty-element tags that were encountered
+        # without an explicit closing tag. If we encounter a closing tag
+        # of this type, we'll associate it with one of those entries.
+        #
+        # This isn't a stack because we don't care about the
+        # order. It's a list of closing tags we've already handled and
+        # will ignore, assuming they ever show up.
+        self.already_closed_empty_element = []
+
+    def error(self, msg):
+        """In Python 3, HTMLParser subclasses must implement error(), although
+        this requirement doesn't appear to be documented.
+
+        In Python 2, HTMLParser implements error() by raising an exception,
+        which we don't want to do.
+
+        In any event, this method is called only on very strange
+        markup and our best strategy is to pretend it didn't happen
+        and keep going.
+        """
+        warnings.warn(msg)
+        
+    def handle_startendtag(self, name, attrs):
+        """Handle an incoming empty-element tag.
+
+        This is only called when the markup looks like <tag/>.
+
+        :param name: Name of the tag.
+        :param attrs: Dictionary of the tag's attributes.
+        """
+        # is_startend() tells handle_starttag not to close the tag
+        # just because its name matches a known empty-element tag. We
+        # know that this is an empty-element tag and we want to call
+        # handle_endtag ourselves.
+        tag = self.handle_starttag(name, attrs, handle_empty_element=False)
+        self.handle_endtag(name)
+        
+    def handle_starttag(self, name, attrs, handle_empty_element=True):
+        """Handle an opening tag, e.g. '<tag>'
+
+        :param name: Name of the tag.
+        :param attrs: Dictionary of the tag's attributes.
+        :param handle_empty_element: True if this tag is known to be
+            an empty-element tag (i.e. there is not expected to be any
+            closing tag).
+        """
+        # XXX namespace
+        attr_dict = {}
+        for key, value in attrs:
+            # Change None attribute values to the empty string
+            # for consistency with the other tree builders.
+            if value is None:
+                value = ''
+            if key in attr_dict:
+                # A single attribute shows up multiple times in this
+                # tag. How to handle it depends on the
+                # on_duplicate_attribute setting.
+                on_dupe = self.on_duplicate_attribute
+                if on_dupe == self.IGNORE:
+                    pass
+                elif on_dupe in (None, self.REPLACE):
+                    attr_dict[key] = value
+                else:
+                    on_dupe(attr_dict, key, value)
+            else:
+                attr_dict[key] = value
+            attrvalue = '""'
+        #print("START", name)
+        sourceline, sourcepos = self.getpos()
+        tag = self.soup.handle_starttag(
+            name, None, None, attr_dict, sourceline=sourceline,
+            sourcepos=sourcepos
+        )
+        if tag and tag.is_empty_element and handle_empty_element:
+            # Unlike other parsers, html.parser doesn't send separate end tag
+            # events for empty-element tags. (It's handled in
+            # handle_startendtag, but only if the original markup looked like
+            # <tag/>.)
+            #
+            # So we need to call handle_endtag() ourselves. Since we
+            # know the start event is identical to the end event, we
+            # don't want handle_endtag() to cross off any previous end
+            # events for tags of this name.
+            self.handle_endtag(name, check_already_closed=False)
+
+            # But we might encounter an explicit closing tag for this tag
+            # later on. If so, we want to ignore it.
+            self.already_closed_empty_element.append(name)
+            
+    def handle_endtag(self, name, check_already_closed=True):
+        """Handle a closing tag, e.g. '</tag>'
+        
+        :param name: A tag name.
+        :param check_already_closed: True if this tag is expected to
+           be the closing portion of an empty-element tag,
+           e.g. '<tag></tag>'.
+        """
+        #print("END", name)
+        if check_already_closed and name in self.already_closed_empty_element:
+            # This is a redundant end tag for an empty-element tag.
+            # We've already called handle_endtag() for it, so just
+            # check it off the list.
+            #print("ALREADY CLOSED", name)
+            self.already_closed_empty_element.remove(name)
+        else:
+            self.soup.handle_endtag(name)
+
+    def handle_data(self, data):
+        """Handle some textual data that shows up between tags."""
+        self.soup.handle_data(data)
+
+    def handle_charref(self, name):
+        """Handle a numeric character reference by converting it to the
+        corresponding Unicode character and treating it as textual
+        data.
+
+        :param name: Character number, possibly in hexadecimal.
+        """
+        # XXX workaround for a bug in HTMLParser. Remove this once
+        # it's fixed in all supported versions.
+        # http://bugs.python.org/issue13633
+        if name.startswith('x'):
+            real_name = int(name.lstrip('x'), 16)
+        elif name.startswith('X'):
+            real_name = int(name.lstrip('X'), 16)
+        else:
+            real_name = int(name)
+
+        data = None
+        if real_name < 256:
+            # HTML numeric entities are supposed to reference Unicode
+            # code points, but sometimes they reference code points in
+            # some other encoding (ahem, Windows-1252). E.g. &#147;
+            # instead of &#201; for LEFT DOUBLE QUOTATION MARK. This
+            # code tries to detect this situation and compensate.
+            for encoding in (self.soup.original_encoding, 'windows-1252'):
+                if not encoding:
+                    continue
+                try:
+                    data = bytearray([real_name]).decode(encoding)
+                except UnicodeDecodeError as e:
+                    pass
+        if not data:
+            try:
+                data = chr(real_name)
+            except (ValueError, OverflowError) as e:
+                pass
+        data = data or "\N{REPLACEMENT CHARACTER}"
+        self.handle_data(data)
+
+    def handle_entityref(self, name):
+        """Handle a named entity reference by converting it to the
+        corresponding Unicode character and treating it as textual
+        data.
+
+        :param name: Name of the entity reference.
+        """
+        character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
+        if character is not None:
+            data = character
+        else:
+            # If this were XML, it would be ambiguous whether "&foo"
+            # was an character entity reference with a missing
+            # semicolon or the literal string "&foo". Since this is
+            # HTML, we have a complete list of all character entity references,
+            # and this one wasn't found, so assume it's the literal string "&foo".
+            data = "&%s" % name
+        self.handle_data(data)
+
+    def handle_comment(self, data):
+        """Handle an HTML comment.
+
+        :param data: The text of the comment.
+        """
+        self.soup.endData()
+        self.soup.handle_data(data)
+        self.soup.endData(Comment)
+
+    def handle_decl(self, data):
+        """Handle a DOCTYPE declaration.
+
+        :param data: The text of the declaration.
+        """
+        self.soup.endData()
+        data = data[len("DOCTYPE "):]
+        self.soup.handle_data(data)
+        self.soup.endData(Doctype)
+
+    def unknown_decl(self, data):
+        """Handle a declaration of unknown type -- probably a CDATA block.
+
+        :param data: The text of the declaration.
+        """
+        if data.upper().startswith('CDATA['):
+            cls = CData
+            data = data[len('CDATA['):]
+        else:
+            cls = Declaration
+        self.soup.endData()
+        self.soup.handle_data(data)
+        self.soup.endData(cls)
+
+    def handle_pi(self, data):
+        """Handle a processing instruction.
+
+        :param data: The text of the instruction.
+        """
+        self.soup.endData()
+        self.soup.handle_data(data)
+        self.soup.endData(ProcessingInstruction)
+
+
+class HTMLParserTreeBuilder(HTMLTreeBuilder):
+    """A Beautiful soup `TreeBuilder` that uses the `HTMLParser` parser,
+    found in the Python standard library.
+    """
+    is_xml = False
+    picklable = True
+    NAME = HTMLPARSER
+    features = [NAME, HTML, STRICT]
+
+    # The html.parser knows which line number and position in the
+    # original file is the source of an element.
+    TRACKS_LINE_NUMBERS = True
+
+    def __init__(self, parser_args=None, parser_kwargs=None, **kwargs):
+        """Constructor.
+
+        :param parser_args: Positional arguments to pass into 
+            the BeautifulSoupHTMLParser constructor, once it's
+            invoked.
+        :param parser_kwargs: Keyword arguments to pass into 
+            the BeautifulSoupHTMLParser constructor, once it's
+            invoked.
+        :param kwargs: Keyword arguments for the superclass constructor.
+        """
+        # Some keyword arguments will be pulled out of kwargs and placed
+        # into parser_kwargs.
+        extra_parser_kwargs = dict()
+        for arg in ('on_duplicate_attribute',):
+            if arg in kwargs:
+                value = kwargs.pop(arg)
+                extra_parser_kwargs[arg] = value
+        super(HTMLParserTreeBuilder, self).__init__(**kwargs)
+        parser_args = parser_args or []
+        parser_kwargs = parser_kwargs or {}
+        parser_kwargs.update(extra_parser_kwargs)
+        if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED:
+            parser_kwargs['strict'] = False
+        if CONSTRUCTOR_TAKES_CONVERT_CHARREFS:
+            parser_kwargs['convert_charrefs'] = False
+        self.parser_args = (parser_args, parser_kwargs)
+        
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None, exclude_encodings=None):
+
+        """Run any preliminary steps necessary to make incoming markup
+        acceptable to the parser.
+
+        :param markup: Some markup -- probably a bytestring.
+        :param user_specified_encoding: The user asked to try this encoding.
+        :param document_declared_encoding: The markup itself claims to be
+            in this encoding.
+        :param exclude_encodings: The user asked _not_ to try any of
+            these encodings.
+
+        :yield: A series of 4-tuples:
+         (markup, encoding, declared encoding,
+          has undergone character replacement)
+
+         Each 4-tuple represents a strategy for converting the
+         document to Unicode and parsing it. Each strategy will be tried 
+         in turn.
+        """
+        if isinstance(markup, str):
+            # Parse Unicode as-is.
+            yield (markup, None, None, False)
+            return
+
+        # Ask UnicodeDammit to sniff the most likely encoding.
+        try_encodings = [user_specified_encoding, document_declared_encoding]
+        dammit = UnicodeDammit(markup, try_encodings, is_html=True,
+                               exclude_encodings=exclude_encodings)
+        yield (dammit.markup, dammit.original_encoding,
+               dammit.declared_html_encoding,
+               dammit.contains_replacement_characters)
+
+    def feed(self, markup):
+        """Run some incoming markup through some parsing process,
+        populating the `BeautifulSoup` object in self.soup.
+        """
+        args, kwargs = self.parser_args
+        parser = BeautifulSoupHTMLParser(*args, **kwargs)
+        parser.soup = self.soup
+        try:
+            parser.feed(markup)
+            parser.close()
+        except HTMLParseError as e:
+            warnings.warn(RuntimeWarning(
+                "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
+            raise e
+        parser.already_closed_empty_element = []
+
+# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
+# 3.2.3 code. This ensures they don't treat markup like <p></p> as a
+# string.
+#
+# XXX This code can be removed once most Python 3 users are on 3.2.3.
+if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
+    import re
+    attrfind_tolerant = re.compile(
+        r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
+        r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
+    HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant
+
+    locatestarttagend = re.compile(r"""
+  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
+  (?:\s+                             # whitespace before attribute name
+    (?:[a-zA-Z_][-.:a-zA-Z0-9_]*     # attribute name
+      (?:\s*=\s*                     # value indicator
+        (?:'[^']*'                   # LITA-enclosed value
+          |\"[^\"]*\"                # LIT-enclosed value
+          |[^'\">\s]+                # bare value
+         )
+       )?
+     )
+   )*
+  \s*                                # trailing whitespace
+""", re.VERBOSE)
+    BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
+
+    from html.parser import tagfind, attrfind
+
+    def parse_starttag(self, i):
+        self.__starttag_text = None
+        endpos = self.check_for_whole_start_tag(i)
+        if endpos < 0:
+            return endpos
+        rawdata = self.rawdata
+        self.__starttag_text = rawdata[i:endpos]
+
+        # Now parse the data between i+1 and j into a tag and attrs
+        attrs = []
+        match = tagfind.match(rawdata, i+1)
+        assert match, 'unexpected call to parse_starttag()'
+        k = match.end()
+        self.lasttag = tag = rawdata[i+1:k].lower()
+        while k < endpos:
+            if self.strict:
+                m = attrfind.match(rawdata, k)
+            else:
+                m = attrfind_tolerant.match(rawdata, k)
+            if not m:
+                break
+            attrname, rest, attrvalue = m.group(1, 2, 3)
+            if not rest:
+                attrvalue = None
+            elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
+                 attrvalue[:1] == '"' == attrvalue[-1:]:
+                attrvalue = attrvalue[1:-1]
+            if attrvalue:
+                attrvalue = self.unescape(attrvalue)
+            attrs.append((attrname.lower(), attrvalue))
+            k = m.end()
+
+        end = rawdata[k:endpos].strip()
+        if end not in (">", "/>"):
+            lineno, offset = self.getpos()
+            if "\n" in self.__starttag_text:
+                lineno = lineno + self.__starttag_text.count("\n")
+                offset = len(self.__starttag_text) \
+                         - self.__starttag_text.rfind("\n")
+            else:
+                offset = offset + len(self.__starttag_text)
+            if self.strict:
+                self.error("junk characters in start tag: %r"
+                           % (rawdata[k:endpos][:20],))
+            self.handle_data(rawdata[i:endpos])
+            return endpos
+        if end.endswith('/>'):
+            # XHTML-style empty tag: <span attr="value" />
+            self.handle_startendtag(tag, attrs)
+        else:
+            self.handle_starttag(tag, attrs)
+            if tag in self.CDATA_CONTENT_ELEMENTS:
+                self.set_cdata_mode(tag)
+        return endpos
+
+    def set_cdata_mode(self, elem):
+        self.cdata_elem = elem.lower()
+        self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
+
+    BeautifulSoupHTMLParser.parse_starttag = parse_starttag
+    BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
+
+    CONSTRUCTOR_TAKES_STRICT = True

bs4/builder/_lxml.py

@@ -0,0 +1,332 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
+__all__ = [
+    'LXMLTreeBuilderForXML',
+    'LXMLTreeBuilder',
+    ]
+
+try:
+    from collections.abc import Callable # Python 3.6
+except ImportError as e:
+    from collections import Callable
+
+from io import BytesIO
+from io import StringIO
+from lxml import etree
+from bs4.element import (
+    Comment,
+    Doctype,
+    NamespacedAttribute,
+    ProcessingInstruction,
+    XMLProcessingInstruction,
+)
+from bs4.builder import (
+    FAST,
+    HTML,
+    HTMLTreeBuilder,
+    PERMISSIVE,
+    ParserRejectedMarkup,
+    TreeBuilder,
+    XML)
+from bs4.dammit import EncodingDetector
+
+LXML = 'lxml'
+
+def _invert(d):
+    "Invert a dictionary."
+    return dict((v,k) for k, v in list(d.items()))
+
+class LXMLTreeBuilderForXML(TreeBuilder):
+    DEFAULT_PARSER_CLASS = etree.XMLParser
+
+    is_xml = True
+    processing_instruction_class = XMLProcessingInstruction
+
+    NAME = "lxml-xml"
+    ALTERNATE_NAMES = ["xml"]
+
+    # Well, it's permissive by XML parser standards.
+    features = [NAME, LXML, XML, FAST, PERMISSIVE]
+
+    CHUNK_SIZE = 512
+
+    # This namespace mapping is specified in the XML Namespace
+    # standard.
+    DEFAULT_NSMAPS = dict(xml='http://www.w3.org/XML/1998/namespace')
+
+    DEFAULT_NSMAPS_INVERTED = _invert(DEFAULT_NSMAPS)
+
+    # NOTE: If we parsed Element objects and looked at .sourceline,
+    # we'd be able to see the line numbers from the original document.
+    # But instead we build an XMLParser or HTMLParser object to serve
+    # as the target of parse messages, and those messages don't include
+    # line numbers.
+    # See: https://bugs.launchpad.net/lxml/+bug/1846906
+    
+    def initialize_soup(self, soup):
+        """Let the BeautifulSoup object know about the standard namespace
+        mapping.
+
+        :param soup: A `BeautifulSoup`.
+        """
+        super(LXMLTreeBuilderForXML, self).initialize_soup(soup)
+        self._register_namespaces(self.DEFAULT_NSMAPS)
+
+    def _register_namespaces(self, mapping):
+        """Let the BeautifulSoup object know about namespaces encountered
+        while parsing the document.
+
+        This might be useful later on when creating CSS selectors.
+
+        :param mapping: A dictionary mapping namespace prefixes to URIs.
+        """
+        for key, value in list(mapping.items()):
+            if key and key not in self.soup._namespaces:
+                # Let the BeautifulSoup object know about a new namespace.
+                # If there are multiple namespaces defined with the same
+                # prefix, the first one in the document takes precedence.
+                self.soup._namespaces[key] = value
+
+    def default_parser(self, encoding):
+        """Find the default parser for the given encoding.
+
+        :param encoding: A string.
+        :return: Either a parser object or a class, which
+          will be instantiated with default arguments.
+        """
+        if self._default_parser is not None:
+            return self._default_parser
+        return etree.XMLParser(
+            target=self, strip_cdata=False, recover=True, encoding=encoding)
+
+    def parser_for(self, encoding):
+        """Instantiate an appropriate parser for the given encoding.
+
+        :param encoding: A string.
+        :return: A parser object such as an `etree.XMLParser`.
+        """
+        # Use the default parser.
+        parser = self.default_parser(encoding)
+
+        if isinstance(parser, Callable):
+            # Instantiate the parser with default arguments
+            parser = parser(
+                target=self, strip_cdata=False, recover=True, encoding=encoding
+            )
+        return parser
+
+    def __init__(self, parser=None, empty_element_tags=None, **kwargs):
+        # TODO: Issue a warning if parser is present but not a
+        # callable, since that means there's no way to create new
+        # parsers for different encodings.
+        self._default_parser = parser
+        if empty_element_tags is not None:
+            self.empty_element_tags = set(empty_element_tags)
+        self.soup = None
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
+        super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
+        
+    def _getNsTag(self, tag):
+        # Split the namespace URL out of a fully-qualified lxml tag
+        # name. Copied from lxml's src/lxml/sax.py.
+        if tag[0] == '{':
+            return tuple(tag[1:].split('}', 1))
+        else:
+            return (None, tag)
+
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       exclude_encodings=None,
+                       document_declared_encoding=None):
+        """Run any preliminary steps necessary to make incoming markup
+        acceptable to the parser.
+
+        lxml really wants to get a bytestring and convert it to
+        Unicode itself. So instead of using UnicodeDammit to convert
+        the bytestring to Unicode using different encodings, this
+        implementation uses EncodingDetector to iterate over the
+        encodings, and tell lxml to try to parse the document as each
+        one in turn.
+
+        :param markup: Some markup -- hopefully a bytestring.
+        :param user_specified_encoding: The user asked to try this encoding.
+        :param document_declared_encoding: The markup itself claims to be
+            in this encoding.
+        :param exclude_encodings: The user asked _not_ to try any of
+            these encodings.
+
+        :yield: A series of 4-tuples:
+         (markup, encoding, declared encoding,
+          has undergone character replacement)
+
+         Each 4-tuple represents a strategy for converting the
+         document to Unicode and parsing it. Each strategy will be tried 
+         in turn.
+        """
+        is_html = not self.is_xml
+        if is_html:
+            self.processing_instruction_class = ProcessingInstruction
+        else:
+            self.processing_instruction_class = XMLProcessingInstruction
+
+        if isinstance(markup, str):
+            # We were given Unicode. Maybe lxml can parse Unicode on
+            # this system?
+            yield markup, None, document_declared_encoding, False
+
+        if isinstance(markup, str):
+            # No, apparently not. Convert the Unicode to UTF-8 and
+            # tell lxml to parse it as UTF-8.
+            yield (markup.encode("utf8"), "utf8",
+                   document_declared_encoding, False)
+
+        try_encodings = [user_specified_encoding, document_declared_encoding]
+        detector = EncodingDetector(
+            markup, try_encodings, is_html, exclude_encodings)
+        for encoding in detector.encodings:
+            yield (detector.markup, encoding, document_declared_encoding, False)
+
+    def feed(self, markup):
+        if isinstance(markup, bytes):
+            markup = BytesIO(markup)
+        elif isinstance(markup, str):
+            markup = StringIO(markup)
+
+        # Call feed() at least once, even if the markup is empty,
+        # or the parser won't be initialized.
+        data = markup.read(self.CHUNK_SIZE)
+        try:
+            self.parser = self.parser_for(self.soup.original_encoding)
+            self.parser.feed(data)
+            while len(data) != 0:
+                # Now call feed() on the rest of the data, chunk by chunk.
+                data = markup.read(self.CHUNK_SIZE)
+                if len(data) != 0:
+                    self.parser.feed(data)
+            self.parser.close()
+        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
+            raise ParserRejectedMarkup(e)
+
+    def close(self):
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
+
+    def start(self, name, attrs, nsmap={}):
+        # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
+        attrs = dict(attrs)
+        nsprefix = None
+        # Invert each namespace map as it comes in.
+        if len(nsmap) == 0 and len(self.nsmaps) > 1:
+                # There are no new namespaces for this tag, but
+                # non-default namespaces are in play, so we need a
+                # separate tag stack to know when they end.
+                self.nsmaps.append(None)
+        elif len(nsmap) > 0:
+            # A new namespace mapping has come into play.
+
+            # First, Let the BeautifulSoup object know about it.
+            self._register_namespaces(nsmap)
+
+            # Then, add it to our running list of inverted namespace
+            # mappings.
+            self.nsmaps.append(_invert(nsmap))
+
+            # Also treat the namespace mapping as a set of attributes on the
+            # tag, so we can recreate it later.
+            attrs = attrs.copy()
+            for prefix, namespace in list(nsmap.items()):
+                attribute = NamespacedAttribute(
+                    "xmlns", prefix, "http://www.w3.org/2000/xmlns/")
+                attrs[attribute] = namespace
+
+        # Namespaces are in play. Find any attributes that came in
+        # from lxml with namespaces attached to their names, and
+        # turn then into NamespacedAttribute objects.
+        new_attrs = {}
+        for attr, value in list(attrs.items()):
+            namespace, attr = self._getNsTag(attr)
+            if namespace is None:
+                new_attrs[attr] = value
+            else:
+                nsprefix = self._prefix_for_namespace(namespace)
+                attr = NamespacedAttribute(nsprefix, attr, namespace)
+                new_attrs[attr] = value
+        attrs = new_attrs
+
+        namespace, name = self._getNsTag(name)
+        nsprefix = self._prefix_for_namespace(namespace)
+        self.soup.handle_starttag(name, namespace, nsprefix, attrs)
+
+    def _prefix_for_namespace(self, namespace):
+        """Find the currently active prefix for the given namespace."""
+        if namespace is None:
+            return None
+        for inverted_nsmap in reversed(self.nsmaps):
+            if inverted_nsmap is not None and namespace in inverted_nsmap:
+                return inverted_nsmap[namespace]
+        return None
+
+    def end(self, name):
+        self.soup.endData()
+        completed_tag = self.soup.tagStack[-1]
+        namespace, name = self._getNsTag(name)
+        nsprefix = None
+        if namespace is not None:
+            for inverted_nsmap in reversed(self.nsmaps):
+                if inverted_nsmap is not None and namespace in inverted_nsmap:
+                    nsprefix = inverted_nsmap[namespace]
+                    break
+        self.soup.handle_endtag(name, nsprefix)
+        if len(self.nsmaps) > 1:
+            # This tag, or one of its parents, introduced a namespace
+            # mapping, so pop it off the stack.
+            self.nsmaps.pop()
+
+    def pi(self, target, data):
+        self.soup.endData()
+        self.soup.handle_data(target + ' ' + data)
+        self.soup.endData(self.processing_instruction_class)
+
+    def data(self, content):
+        self.soup.handle_data(content)
+
+    def doctype(self, name, pubid, system):
+        self.soup.endData()
+        doctype = Doctype.for_name_and_ids(name, pubid, system)
+        self.soup.object_was_parsed(doctype)
+
+    def comment(self, content):
+        "Handle comments as Comment objects."
+        self.soup.endData()
+        self.soup.handle_data(content)
+        self.soup.endData(Comment)
+
+    def test_fragment_to_document(self, fragment):
+        """See `TreeBuilder`."""
+        return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
+
+
+class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
+
+    NAME = LXML
+    ALTERNATE_NAMES = ["lxml-html"]
+
+    features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE]
+    is_xml = False
+    processing_instruction_class = ProcessingInstruction
+
+    def default_parser(self, encoding):
+        return etree.HTMLParser
+
+    def feed(self, markup):
+        encoding = self.soup.original_encoding
+        try:
+            self.parser = self.parser_for(encoding)
+            self.parser.feed(markup)
+            self.parser.close()
+        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
+            raise ParserRejectedMarkup(e)
+
+
+    def test_fragment_to_document(self, fragment):
+        """See `TreeBuilder`."""
+        return '<html><body>%s</body></html>' % fragment

bs4/dammit.py

@@ -0,0 +1,939 @@
+# -*- coding: utf-8 -*-
+"""Beautiful Soup bonus library: Unicode, Dammit
+
+This library converts a bytestream to Unicode through any means
+necessary. It is heavily based on code from Mark Pilgrim's Universal
+Feed Parser. It works best on XML and HTML, but it does not rewrite the
+XML or HTML to reflect a new encoding; that's the tree builder's job.
+"""
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
+import codecs
+from html.entities import codepoint2name
+import re
+import logging
+import string
+
+# Import a library to autodetect character encodings.
+chardet_type = None
+try:
+    # First try the fast C implementation.
+    #  PyPI package: cchardet
+    import cchardet
+    def chardet_dammit(s):
+        if isinstance(s, str):
+            return None
+        return cchardet.detect(s)['encoding']
+except ImportError:
+    try:
+        # Fall back to the pure Python implementation
+        #  Debian package: python-chardet
+        #  PyPI package: chardet
+        import chardet
+        def chardet_dammit(s):
+            if isinstance(s, str):
+                return None
+            return chardet.detect(s)['encoding']
+        #import chardet.constants
+        #chardet.constants._debug = 1
+    except ImportError:
+        # No chardet available.
+        def chardet_dammit(s):
+            return None
+
+# Available from http://cjkpython.i18n.org/.
+#
+# TODO: This doesn't work anymore and the closest thing, iconv_codecs,
+# is GPL-licensed. Check whether this is still necessary.
+try:
+    import iconv_codec
+except ImportError:
+    pass
+
+# Build bytestring and Unicode versions of regular expressions for finding
+# a declared encoding inside an XML or HTML document.
+xml_encoding = '^\\s*<\\?.*encoding=[\'"](.*?)[\'"].*\\?>'
+html_meta = '<\\s*meta[^>]+charset\\s*=\\s*["\']?([^>]*?)[ /;\'">]'
+encoding_res = dict()
+encoding_res[bytes] = {
+    'html' : re.compile(html_meta.encode("ascii"), re.I),
+    'xml' : re.compile(xml_encoding.encode("ascii"), re.I),
+}
+encoding_res[str] = {
+    'html' : re.compile(html_meta, re.I),
+    'xml' : re.compile(xml_encoding, re.I)
+}
+
+class EntitySubstitution(object):
+    """The ability to substitute XML or HTML entities for certain characters."""
+
+    def _populate_class_variables():
+        lookup = {}
+        reverse_lookup = {}
+        characters_for_re = []
+
+        # &apos is an XHTML entity and an HTML 5, but not an HTML 4
+        # entity. We don't want to use it, but we want to recognize it on the way in.
+        #
+        # TODO: Ideally we would be able to recognize all HTML 5 named
+        # entities, but that's a little tricky.
+        extra = [(39, 'apos')]
+        for codepoint, name in list(codepoint2name.items()) + extra:
+            character = chr(codepoint)
+            if codepoint not in (34, 39):
+                # There's no point in turning the quotation mark into
+                # &quot; or the single quote into &apos;, unless it
+                # happens within an attribute value, which is handled
+                # elsewhere.
+                characters_for_re.append(character)
+                lookup[character] = name
+            # But we do want to recognize those entities on the way in and
+            # convert them to Unicode characters.
+            reverse_lookup[name] = character
+        re_definition = "[%s]" % "".join(characters_for_re)
+        return lookup, reverse_lookup, re.compile(re_definition)
+    (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
+     CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
+
+    CHARACTER_TO_XML_ENTITY = {
+        "'": "apos",
+        '"': "quot",
+        "&": "amp",
+        "<": "lt",
+        ">": "gt",
+        }
+
+    BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
+                                           "&(?!#\\d+;|#x[0-9a-fA-F]+;|\\w+;)"
+                                           ")")
+
+    AMPERSAND_OR_BRACKET = re.compile("([<>&])")
+
+    @classmethod
+    def _substitute_html_entity(cls, matchobj):
+        """Used with a regular expression to substitute the
+        appropriate HTML entity for a special character."""
+        entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
+        return "&%s;" % entity
+
+    @classmethod
+    def _substitute_xml_entity(cls, matchobj):
+        """Used with a regular expression to substitute the
+        appropriate XML entity for a special character."""
+        entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
+        return "&%s;" % entity
+
+    @classmethod
+    def quoted_attribute_value(self, value):
+        """Make a value into a quoted XML attribute, possibly escaping it.
+
+         Most strings will be quoted using double quotes.
+
+          Bob's Bar -> "Bob's Bar"
+
+         If a string contains double quotes, it will be quoted using
+         single quotes.
+
+          Welcome to "my bar" -> 'Welcome to "my bar"'
+
+         If a string contains both single and double quotes, the
+         double quotes will be escaped, and the string will be quoted
+         using double quotes.
+
+          Welcome to "Bob's Bar" -> "Welcome to &quot;Bob's bar&quot;
+        """
+        quote_with = '"'
+        if '"' in value:
+            if "'" in value:
+                # The string contains both single and double
+                # quotes.  Turn the double quotes into
+                # entities. We quote the double quotes rather than
+                # the single quotes because the entity name is
+                # "&quot;" whether this is HTML or XML.  If we
+                # quoted the single quotes, we'd have to decide
+                # between &apos; and &squot;.
+                replace_with = "&quot;"
+                value = value.replace('"', replace_with)
+            else:
+                # There are double quotes but no single quotes.
+                # We can use single quotes to quote the attribute.
+                quote_with = "'"
+        return quote_with + value + quote_with
+
+    @classmethod
+    def substitute_xml(cls, value, make_quoted_attribute=False):
+        """Substitute XML entities for special XML characters.
+
+        :param value: A string to be substituted. The less-than sign
+          will become &lt;, the greater-than sign will become &gt;,
+          and any ampersands will become &amp;. If you want ampersands
+          that appear to be part of an entity definition to be left
+          alone, use substitute_xml_containing_entities() instead.
+
+        :param make_quoted_attribute: If True, then the string will be
+         quoted, as befits an attribute value.
+        """
+        # Escape angle brackets and ampersands.
+        value = cls.AMPERSAND_OR_BRACKET.sub(
+            cls._substitute_xml_entity, value)
+
+        if make_quoted_attribute:
+            value = cls.quoted_attribute_value(value)
+        return value
+
+    @classmethod
+    def substitute_xml_containing_entities(
+        cls, value, make_quoted_attribute=False):
+        """Substitute XML entities for special XML characters.
+
+        :param value: A string to be substituted. The less-than sign will
+          become &lt;, the greater-than sign will become &gt;, and any
+          ampersands that are not part of an entity defition will
+          become &amp;.
+
+        :param make_quoted_attribute: If True, then the string will be
+         quoted, as befits an attribute value.
+        """
+        # Escape angle brackets, and ampersands that aren't part of
+        # entities.
+        value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
+            cls._substitute_xml_entity, value)
+
+        if make_quoted_attribute:
+            value = cls.quoted_attribute_value(value)
+        return value
+
+    @classmethod
+    def substitute_html(cls, s):
+        """Replace certain Unicode characters with named HTML entities.
+
+        This differs from data.encode(encoding, 'xmlcharrefreplace')
+        in that the goal is to make the result more readable (to those
+        with ASCII displays) rather than to recover from
+        errors. There's absolutely nothing wrong with a UTF-8 string
+        containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
+        character with "&eacute;" will make it more readable to some
+        people.
+
+        :param s: A Unicode string.
+        """
+        return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
+            cls._substitute_html_entity, s)
+
+
+class EncodingDetector:
+    """Suggests a number of possible encodings for a bytestring.
+
+    Order of precedence:
+
+    1. Encodings you specifically tell EncodingDetector to try first
+    (the override_encodings argument to the constructor).
+
+    2. An encoding declared within the bytestring itself, either in an
+    XML declaration (if the bytestring is to be interpreted as an XML
+    document), or in a <meta> tag (if the bytestring is to be
+    interpreted as an HTML document.)
+
+    3. An encoding detected through textual analysis by chardet,
+    cchardet, or a similar external library.
+
+    4. UTF-8.
+
+    5. Windows-1252.
+    """
+    def __init__(self, markup, override_encodings=None, is_html=False,
+                 exclude_encodings=None):
+        """Constructor.
+
+        :param markup: Some markup in an unknown encoding.
+        :param override_encodings: These encodings will be tried first.
+        :param is_html: If True, this markup is considered to be HTML. Otherwise
+            it's assumed to be XML.
+        :param exclude_encodings: These encodings will not be tried, even
+            if they otherwise would be.
+        """
+        self.override_encodings = override_encodings or []
+        exclude_encodings = exclude_encodings or []
+        self.exclude_encodings = set([x.lower() for x in exclude_encodings])
+        self.chardet_encoding = None
+        self.is_html = is_html
+        self.declared_encoding = None
+
+        # First order of business: strip a byte-order mark.
+        self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup)
+
+    def _usable(self, encoding, tried):
+        """Should we even bother to try this encoding?
+
+        :param encoding: Name of an encoding.
+        :param tried: Encodings that have already been tried. This will be modified
+            as a side effect.
+        """
+        if encoding is not None:
+            encoding = encoding.lower()
+            if encoding in self.exclude_encodings:
+                return False
+            if encoding not in tried:
+                tried.add(encoding)
+                return True
+        return False
+
+    @property
+    def encodings(self):
+        """Yield a number of encodings that might work for this markup.
+
+        :yield: A sequence of strings.
+        """
+        tried = set()
+        for e in self.override_encodings:
+            if self._usable(e, tried):
+                yield e
+
+        # Did the document originally start with a byte-order mark
+        # that indicated its encoding?
+        if self._usable(self.sniffed_encoding, tried):
+            yield self.sniffed_encoding
+
+        # Look within the document for an XML or HTML encoding
+        # declaration.
+        if self.declared_encoding is None:
+            self.declared_encoding = self.find_declared_encoding(
+                self.markup, self.is_html)
+        if self._usable(self.declared_encoding, tried):
+            yield self.declared_encoding
+
+        # Use third-party character set detection to guess at the
+        # encoding.
+        if self.chardet_encoding is None:
+            self.chardet_encoding = chardet_dammit(self.markup)
+        if self._usable(self.chardet_encoding, tried):
+            yield self.chardet_encoding
+
+        # As a last-ditch effort, try utf-8 and windows-1252.
+        for e in ('utf-8', 'windows-1252'):
+            if self._usable(e, tried):
+                yield e
+
+    @classmethod
+    def strip_byte_order_mark(cls, data):
+        """If a byte-order mark is present, strip it and return the encoding it implies.
+
+        :param data: Some markup.
+        :return: A 2-tuple (modified data, implied encoding)
+        """
+        encoding = None
+        if isinstance(data, str):
+            # Unicode data cannot have a byte-order mark.
+            return data, encoding
+        if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
+               and (data[2:4] != '\x00\x00'):
+            encoding = 'utf-16be'
+            data = data[2:]
+        elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \
+                 and (data[2:4] != '\x00\x00'):
+            encoding = 'utf-16le'
+            data = data[2:]
+        elif data[:3] == b'\xef\xbb\xbf':
+            encoding = 'utf-8'
+            data = data[3:]
+        elif data[:4] == b'\x00\x00\xfe\xff':
+            encoding = 'utf-32be'
+            data = data[4:]
+        elif data[:4] == b'\xff\xfe\x00\x00':
+            encoding = 'utf-32le'
+            data = data[4:]
+        return data, encoding
+
+    @classmethod
+    def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False):
+        """Given a document, tries to find its declared encoding.
+
+        An XML encoding is declared at the beginning of the document.
+
+        An HTML encoding is declared in a <meta> tag, hopefully near the
+        beginning of the document.
+
+        :param markup: Some markup.
+        :param is_html: If True, this markup is considered to be HTML. Otherwise
+            it's assumed to be XML.
+        :param search_entire_document: Since an encoding is supposed to declared near the beginning
+            of the document, most of the time it's only necessary to search a few kilobytes of data.
+            Set this to True to force this method to search the entire document.
+        """
+        if search_entire_document:
+            xml_endpos = html_endpos = len(markup)
+        else:
+            xml_endpos = 1024
+            html_endpos = max(2048, int(len(markup) * 0.05))
+
+        if isinstance(markup, bytes):
+            res = encoding_res[bytes]
+        else:
+            res = encoding_res[str]
+
+        xml_re = res['xml']
+        html_re = res['html']
+        declared_encoding = None
+        declared_encoding_match = xml_re.search(markup, endpos=xml_endpos)
+        if not declared_encoding_match and is_html:
+            declared_encoding_match = html_re.search(markup, endpos=html_endpos)
+        if declared_encoding_match is not None:
+            declared_encoding = declared_encoding_match.groups()[0]
+        if declared_encoding:
+            if isinstance(declared_encoding, bytes):
+                declared_encoding = declared_encoding.decode('ascii', 'replace')
+            return declared_encoding.lower()
+        return None
+
+class UnicodeDammit:
+    """A class for detecting the encoding of a *ML document and
+    converting it to a Unicode string. If the source encoding is
+    windows-1252, can replace MS smart quotes with their HTML or XML
+    equivalents."""
+
+    # This dictionary maps commonly seen values for "charset" in HTML
+    # meta tags to the corresponding Python codec names. It only covers
+    # values that aren't in Python's aliases and can't be determined
+    # by the heuristics in find_codec.
+    CHARSET_ALIASES = {"macintosh": "mac-roman",
+                       "x-sjis": "shift-jis"}
+
+    ENCODINGS_WITH_SMART_QUOTES = [
+        "windows-1252",
+        "iso-8859-1",
+        "iso-8859-2",
+        ]
+
+    def __init__(self, markup, override_encodings=[],
+                 smart_quotes_to=None, is_html=False, exclude_encodings=[]):
+        """Constructor.
+
+        :param markup: A bytestring representing markup in an unknown encoding.
+        :param override_encodings: These encodings will be tried first,
+           before any sniffing code is run.
+
+        :param smart_quotes_to: By default, Microsoft smart quotes will, like all other characters, be converted
+           to Unicode characters. Setting this to 'ascii' will convert them to ASCII quotes instead.
+           Setting it to 'xml' will convert them to XML entity references, and setting it to 'html'
+           will convert them to HTML entity references.
+        :param is_html: If True, this markup is considered to be HTML. Otherwise
+            it's assumed to be XML.
+        :param exclude_encodings: These encodings will not be considered, even
+            if the sniffing code thinks they might make sense.
+        """
+        self.smart_quotes_to = smart_quotes_to
+        self.tried_encodings = []
+        self.contains_replacement_characters = False
+        self.is_html = is_html
+        self.log = logging.getLogger(__name__)
+        self.detector = EncodingDetector(
+            markup, override_encodings, is_html, exclude_encodings)
+
+        # Short-circuit if the data is in Unicode to begin with.
+        if isinstance(markup, str) or markup == '':
+            self.markup = markup
+            self.unicode_markup = str(markup)
+            self.original_encoding = None
+            return
+
+        # The encoding detector may have stripped a byte-order mark.
+        # Use the stripped markup from this point on.
+        self.markup = self.detector.markup
+
+        u = None
+        for encoding in self.detector.encodings:
+            markup = self.detector.markup
+            u = self._convert_from(encoding)
+            if u is not None:
+                break
+
+        if not u:
+            # None of the encodings worked. As an absolute last resort,
+            # try them again with character replacement.
+
+            for encoding in self.detector.encodings:
+                if encoding != "ascii":
+                    u = self._convert_from(encoding, "replace")
+                if u is not None:
+                    self.log.warning(
+                            "Some characters could not be decoded, and were "
+                            "replaced with REPLACEMENT CHARACTER."
+                    )
+                    self.contains_replacement_characters = True
+                    break
+
+        # If none of that worked, we could at this point force it to
+        # ASCII, but that would destroy so much data that I think
+        # giving up is better.
+        self.unicode_markup = u
+        if not u:
+            self.original_encoding = None
+
+    def _sub_ms_char(self, match):
+        """Changes a MS smart quote character to an XML or HTML
+        entity, or an ASCII character."""
+        orig = match.group(1)
+        if self.smart_quotes_to == 'ascii':
+            sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
+        else:
+            sub = self.MS_CHARS.get(orig)
+            if type(sub) == tuple:
+                if self.smart_quotes_to == 'xml':
+                    sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
+                else:
+                    sub = '&'.encode() + sub[0].encode() + ';'.encode()
+            else:
+                sub = sub.encode()
+        return sub
+
+    def _convert_from(self, proposed, errors="strict"):
+        """Attempt to convert the markup to the proposed encoding.
+
+        :param proposed: The name of a character encoding.
+        """
+        proposed = self.find_codec(proposed)
+        if not proposed or (proposed, errors) in self.tried_encodings:
+            return None
+        self.tried_encodings.append((proposed, errors))
+        markup = self.markup
+        # Convert smart quotes to HTML if coming from an encoding
+        # that might have them.
+        if (self.smart_quotes_to is not None
+            and proposed in self.ENCODINGS_WITH_SMART_QUOTES):
+            smart_quotes_re = b"([\x80-\x9f])"
+            smart_quotes_compiled = re.compile(smart_quotes_re)
+            markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
+
+        try:
+            #print("Trying to convert document to %s (errors=%s)" % (
+            #    proposed, errors))
+            u = self._to_unicode(markup, proposed, errors)
+            self.markup = u
+            self.original_encoding = proposed
+        except Exception as e:
+            #print("That didn't work!")
+            #print(e)
+            return None
+        #print("Correct encoding: %s" % proposed)
+        return self.markup
+
+    def _to_unicode(self, data, encoding, errors="strict"):
+        """Given a string and its encoding, decodes the string into Unicode.
+
+        :param encoding: The name of an encoding.
+        """
+        return str(data, encoding, errors)
+
+    @property
+    def declared_html_encoding(self):
+        """If the markup is an HTML document, returns the encoding declared _within_
+        the document.
+        """
+        if not self.is_html:
+            return None
+        return self.detector.declared_encoding
+
+    def find_codec(self, charset):
+        """Convert the name of a character set to a codec name.
+
+        :param charset: The name of a character set.
+        :return: The name of a codec.
+        """
+        value = (self._codec(self.CHARSET_ALIASES.get(charset, charset))
+               or (charset and self._codec(charset.replace("-", "")))
+               or (charset and self._codec(charset.replace("-", "_")))
+               or (charset and charset.lower())
+               or charset
+                )
+        if value:
+            return value.lower()
+        return None
+
+    def _codec(self, charset):
+        if not charset:
+            return charset
+        codec = None
+        try:
+            codecs.lookup(charset)
+            codec = charset
+        except (LookupError, ValueError):
+            pass
+        return codec
+
+
+    # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
+    MS_CHARS = {b'\x80': ('euro', '20AC'),
+                b'\x81': ' ',
+                b'\x82': ('sbquo', '201A'),
+                b'\x83': ('fnof', '192'),
+                b'\x84': ('bdquo', '201E'),
+                b'\x85': ('hellip', '2026'),
+                b'\x86': ('dagger', '2020'),
+                b'\x87': ('Dagger', '2021'),
+                b'\x88': ('circ', '2C6'),
+                b'\x89': ('permil', '2030'),
+                b'\x8A': ('Scaron', '160'),
+                b'\x8B': ('lsaquo', '2039'),
+                b'\x8C': ('OElig', '152'),
+                b'\x8D': '?',
+                b'\x8E': ('#x17D', '17D'),
+                b'\x8F': '?',
+                b'\x90': '?',
+                b'\x91': ('lsquo', '2018'),
+                b'\x92': ('rsquo', '2019'),
+                b'\x93': ('ldquo', '201C'),
+                b'\x94': ('rdquo', '201D'),
+                b'\x95': ('bull', '2022'),
+                b'\x96': ('ndash', '2013'),
+                b'\x97': ('mdash', '2014'),
+                b'\x98': ('tilde', '2DC'),
+                b'\x99': ('trade', '2122'),
+                b'\x9a': ('scaron', '161'),
+                b'\x9b': ('rsaquo', '203A'),
+                b'\x9c': ('oelig', '153'),
+                b'\x9d': '?',
+                b'\x9e': ('#x17E', '17E'),
+                b'\x9f': ('Yuml', ''),}
+
+    # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
+    # horrors like stripping diacritical marks to turn á into a, but also
+    # contains non-horrors like turning “ into ".
+    MS_CHARS_TO_ASCII = {
+        b'\x80' : 'EUR',
+        b'\x81' : ' ',
+        b'\x82' : ',',
+        b'\x83' : 'f',
+        b'\x84' : ',,',
+        b'\x85' : '...',
+        b'\x86' : '+',
+        b'\x87' : '++',
+        b'\x88' : '^',
+        b'\x89' : '%',
+        b'\x8a' : 'S',
+        b'\x8b' : '<',
+        b'\x8c' : 'OE',
+        b'\x8d' : '?',
+        b'\x8e' : 'Z',
+        b'\x8f' : '?',
+        b'\x90' : '?',
+        b'\x91' : "'",
+        b'\x92' : "'",
+        b'\x93' : '"',
+        b'\x94' : '"',
+        b'\x95' : '*',
+        b'\x96' : '-',
+        b'\x97' : '--',
+        b'\x98' : '~',
+        b'\x99' : '(TM)',
+        b'\x9a' : 's',
+        b'\x9b' : '>',
+        b'\x9c' : 'oe',
+        b'\x9d' : '?',
+        b'\x9e' : 'z',
+        b'\x9f' : 'Y',
+        b'\xa0' : ' ',
+        b'\xa1' : '!',
+        b'\xa2' : 'c',
+        b'\xa3' : 'GBP',
+        b'\xa4' : '$', #This approximation is especially parochial--this is the
+                       #generic currency symbol.
+        b'\xa5' : 'YEN',
+        b'\xa6' : '|',
+        b'\xa7' : 'S',
+        b'\xa8' : '..',
+        b'\xa9' : '',
+        b'\xaa' : '(th)',
+        b'\xab' : '<<',
+        b'\xac' : '!',
+        b'\xad' : ' ',
+        b'\xae' : '(R)',
+        b'\xaf' : '-',
+        b'\xb0' : 'o',
+        b'\xb1' : '+-',
+        b'\xb2' : '2',
+        b'\xb3' : '3',
+        b'\xb4' : ("'", 'acute'),
+        b'\xb5' : 'u',
+        b'\xb6' : 'P',
+        b'\xb7' : '*',
+        b'\xb8' : ',',
+        b'\xb9' : '1',
+        b'\xba' : '(th)',
+        b'\xbb' : '>>',
+        b'\xbc' : '1/4',
+        b'\xbd' : '1/2',
+        b'\xbe' : '3/4',
+        b'\xbf' : '?',
+        b'\xc0' : 'A',
+        b'\xc1' : 'A',
+        b'\xc2' : 'A',
+        b'\xc3' : 'A',
+        b'\xc4' : 'A',
+        b'\xc5' : 'A',
+        b'\xc6' : 'AE',
+        b'\xc7' : 'C',
+        b'\xc8' : 'E',
+        b'\xc9' : 'E',
+        b'\xca' : 'E',
+        b'\xcb' : 'E',
+        b'\xcc' : 'I',
+        b'\xcd' : 'I',
+        b'\xce' : 'I',
+        b'\xcf' : 'I',
+        b'\xd0' : 'D',
+        b'\xd1' : 'N',
+        b'\xd2' : 'O',
+        b'\xd3' : 'O',
+        b'\xd4' : 'O',
+        b'\xd5' : 'O',
+        b'\xd6' : 'O',
+        b'\xd7' : '*',
+        b'\xd8' : 'O',
+        b'\xd9' : 'U',
+        b'\xda' : 'U',
+        b'\xdb' : 'U',
+        b'\xdc' : 'U',
+        b'\xdd' : 'Y',
+        b'\xde' : 'b',
+        b'\xdf' : 'B',
+        b'\xe0' : 'a',
+        b'\xe1' : 'a',
+        b'\xe2' : 'a',
+        b'\xe3' : 'a',
+        b'\xe4' : 'a',
+        b'\xe5' : 'a',
+        b'\xe6' : 'ae',
+        b'\xe7' : 'c',
+        b'\xe8' : 'e',
+        b'\xe9' : 'e',
+        b'\xea' : 'e',
+        b'\xeb' : 'e',
+        b'\xec' : 'i',
+        b'\xed' : 'i',
+        b'\xee' : 'i',
+        b'\xef' : 'i',
+        b'\xf0' : 'o',
+        b'\xf1' : 'n',
+        b'\xf2' : 'o',
+        b'\xf3' : 'o',
+        b'\xf4' : 'o',
+        b'\xf5' : 'o',
+        b'\xf6' : 'o',
+        b'\xf7' : '/',
+        b'\xf8' : 'o',
+        b'\xf9' : 'u',
+        b'\xfa' : 'u',
+        b'\xfb' : 'u',
+        b'\xfc' : 'u',
+        b'\xfd' : 'y',
+        b'\xfe' : 'b',
+        b'\xff' : 'y',
+        }
+
+    # A map used when removing rogue Windows-1252/ISO-8859-1
+    # characters in otherwise UTF-8 documents.
+    #
+    # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
+    # Windows-1252.
+    WINDOWS_1252_TO_UTF8 = {
+        0x80 : b'\xe2\x82\xac', # €
+        0x82 : b'\xe2\x80\x9a', # ‚
+        0x83 : b'\xc6\x92',     # ƒ
+        0x84 : b'\xe2\x80\x9e', # „
+        0x85 : b'\xe2\x80\xa6', # …
+        0x86 : b'\xe2\x80\xa0', # †
+        0x87 : b'\xe2\x80\xa1', # ‡
+        0x88 : b'\xcb\x86',     # ˆ
+        0x89 : b'\xe2\x80\xb0', # ‰
+        0x8a : b'\xc5\xa0',     # Š
+        0x8b : b'\xe2\x80\xb9', # ‹
+        0x8c : b'\xc5\x92',     # Œ
+        0x8e : b'\xc5\xbd',     # Ž
+        0x91 : b'\xe2\x80\x98', # ‘
+        0x92 : b'\xe2\x80\x99', # ’
+        0x93 : b'\xe2\x80\x9c', # “
+        0x94 : b'\xe2\x80\x9d', # ”
+        0x95 : b'\xe2\x80\xa2', # •
+        0x96 : b'\xe2\x80\x93', # –
+        0x97 : b'\xe2\x80\x94', # —
+        0x98 : b'\xcb\x9c',     # ˜
+        0x99 : b'\xe2\x84\xa2', # ™
+        0x9a : b'\xc5\xa1',     # š
+        0x9b : b'\xe2\x80\xba', # ›
+        0x9c : b'\xc5\x93',     # œ
+        0x9e : b'\xc5\xbe',     # ž
+        0x9f : b'\xc5\xb8',     # Ÿ
+        0xa0 : b'\xc2\xa0',     #  
+        0xa1 : b'\xc2\xa1',     # ¡
+        0xa2 : b'\xc2\xa2',     # ¢
+        0xa3 : b'\xc2\xa3',     # £
+        0xa4 : b'\xc2\xa4',     # ¤
+        0xa5 : b'\xc2\xa5',     # ¥
+        0xa6 : b'\xc2\xa6',     # ¦
+        0xa7 : b'\xc2\xa7',     # §
+        0xa8 : b'\xc2\xa8',     # ¨
+        0xa9 : b'\xc2\xa9',     # ©
+        0xaa : b'\xc2\xaa',     # ª
+        0xab : b'\xc2\xab',     # «
+        0xac : b'\xc2\xac',     # ¬
+        0xad : b'\xc2\xad',     # ­
+        0xae : b'\xc2\xae',     # ®
+        0xaf : b'\xc2\xaf',     # ¯
+        0xb0 : b'\xc2\xb0',     # °
+        0xb1 : b'\xc2\xb1',     # ±
+        0xb2 : b'\xc2\xb2',     # ²
+        0xb3 : b'\xc2\xb3',     # ³
+        0xb4 : b'\xc2\xb4',     # ´
+        0xb5 : b'\xc2\xb5',     # µ
+        0xb6 : b'\xc2\xb6',     # ¶
+        0xb7 : b'\xc2\xb7',     # ·
+        0xb8 : b'\xc2\xb8',     # ¸
+        0xb9 : b'\xc2\xb9',     # ¹
+        0xba : b'\xc2\xba',     # º
+        0xbb : b'\xc2\xbb',     # »
+        0xbc : b'\xc2\xbc',     # ¼
+        0xbd : b'\xc2\xbd',     # ½
+        0xbe : b'\xc2\xbe',     # ¾
+        0xbf : b'\xc2\xbf',     # ¿
+        0xc0 : b'\xc3\x80',     # À
+        0xc1 : b'\xc3\x81',     # Á
+        0xc2 : b'\xc3\x82',     # Â
+        0xc3 : b'\xc3\x83',     # Ã
+        0xc4 : b'\xc3\x84',     # Ä
+        0xc5 : b'\xc3\x85',     # Å
+        0xc6 : b'\xc3\x86',     # Æ
+        0xc7 : b'\xc3\x87',     # Ç
+        0xc8 : b'\xc3\x88',     # È
+        0xc9 : b'\xc3\x89',     # É
+        0xca : b'\xc3\x8a',     # Ê
+        0xcb : b'\xc3\x8b',     # Ë
+        0xcc : b'\xc3\x8c',     # Ì
+        0xcd : b'\xc3\x8d',     # Í
+        0xce : b'\xc3\x8e',     # Î
+        0xcf : b'\xc3\x8f',     # Ï
+        0xd0 : b'\xc3\x90',     # Ð
+        0xd1 : b'\xc3\x91',     # Ñ
+        0xd2 : b'\xc3\x92',     # Ò
+        0xd3 : b'\xc3\x93',     # Ó
+        0xd4 : b'\xc3\x94',     # Ô
+        0xd5 : b'\xc3\x95',     # Õ
+        0xd6 : b'\xc3\x96',     # Ö
+        0xd7 : b'\xc3\x97',     # ×
+        0xd8 : b'\xc3\x98',     # Ø
+        0xd9 : b'\xc3\x99',     # Ù
+        0xda : b'\xc3\x9a',     # Ú
+        0xdb : b'\xc3\x9b',     # Û
+        0xdc : b'\xc3\x9c',     # Ü
+        0xdd : b'\xc3\x9d',     # Ý
+        0xde : b'\xc3\x9e',     # Þ
+        0xdf : b'\xc3\x9f',     # ß
+        0xe0 : b'\xc3\xa0',     # à
+        0xe1 : b'\xa1',         # á
+        0xe2 : b'\xc3\xa2',     # â
+        0xe3 : b'\xc3\xa3',     # ã
+        0xe4 : b'\xc3\xa4',     # ä
+        0xe5 : b'\xc3\xa5',     # å
+        0xe6 : b'\xc3\xa6',     # æ
+        0xe7 : b'\xc3\xa7',     # ç
+        0xe8 : b'\xc3\xa8',     # è
+        0xe9 : b'\xc3\xa9',     # é
+        0xea : b'\xc3\xaa',     # ê
+        0xeb : b'\xc3\xab',     # ë
+        0xec : b'\xc3\xac',     # ì
+        0xed : b'\xc3\xad',     # í
+        0xee : b'\xc3\xae',     # î
+        0xef : b'\xc3\xaf',     # ï
+        0xf0 : b'\xc3\xb0',     # ð
+        0xf1 : b'\xc3\xb1',     # ñ
+        0xf2 : b'\xc3\xb2',     # ò
+        0xf3 : b'\xc3\xb3',     # ó
+        0xf4 : b'\xc3\xb4',     # ô
+        0xf5 : b'\xc3\xb5',     # õ
+        0xf6 : b'\xc3\xb6',     # ö
+        0xf7 : b'\xc3\xb7',     # ÷
+        0xf8 : b'\xc3\xb8',     # ø
+        0xf9 : b'\xc3\xb9',     # ù
+        0xfa : b'\xc3\xba',     # ú
+        0xfb : b'\xc3\xbb',     # û
+        0xfc : b'\xc3\xbc',     # ü
+        0xfd : b'\xc3\xbd',     # ý
+        0xfe : b'\xc3\xbe',     # þ
+        }
+
+    MULTIBYTE_MARKERS_AND_SIZES = [
+        (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
+        (0xe0, 0xef, 3), # 3-byte characters start with E0-EF
+        (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
+        ]
+
+    FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
+    LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
+
+    @classmethod
+    def detwingle(cls, in_bytes, main_encoding="utf8",
+                  embedded_encoding="windows-1252"):
+        """Fix characters from one encoding embedded in some other encoding.
+
+        Currently the only situation supported is Windows-1252 (or its
+        subset ISO-8859-1), embedded in UTF-8.
+
+        :param in_bytes: A bytestring that you suspect contains
+            characters from multiple encodings. Note that this _must_
+            be a bytestring. If you've already converted the document
+            to Unicode, you're too late.
+        :param main_encoding: The primary encoding of `in_bytes`.
+        :param embedded_encoding: The encoding that was used to embed characters
+            in the main document.
+        :return: A bytestring in which `embedded_encoding`
+          characters have been converted to their `main_encoding`
+          equivalents.
+        """
+        if embedded_encoding.replace('_', '-').lower() not in (
+            'windows-1252', 'windows_1252'):
+            raise NotImplementedError(
+                "Windows-1252 and ISO-8859-1 are the only currently supported "
+                "embedded encodings.")
+
+        if main_encoding.lower() not in ('utf8', 'utf-8'):
+            raise NotImplementedError(
+                "UTF-8 is the only currently supported main encoding.")
+
+        byte_chunks = []
+
+        chunk_start = 0
+        pos = 0
+        while pos < len(in_bytes):
+            byte = in_bytes[pos]
+            if not isinstance(byte, int):
+                # Python 2.x
+                byte = ord(byte)
+            if (byte >= cls.FIRST_MULTIBYTE_MARKER
+                and byte <= cls.LAST_MULTIBYTE_MARKER):
+                # This is the start of a UTF-8 multibyte character. Skip
+                # to the end.
+                for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
+                    if byte >= start and byte <= end:
+                        pos += size
+                        break
+            elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
+                # We found a Windows-1252 character!
+                # Save the string up to this point as a chunk.
+                byte_chunks.append(in_bytes[chunk_start:pos])
+
+                # Now translate the Windows-1252 character into UTF-8
+                # and add it as another, one-byte chunk.
+                byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
+                pos += 1
+                chunk_start = pos
+            else:
+                # Go on to the next character.
+                pos += 1
+        if chunk_start == 0:
+            # The string is unchanged.
+            return in_bytes
+        else:
+            # Store the final chunk.
+            byte_chunks.append(in_bytes[chunk_start:])
+        return b''.join(byte_chunks)
+

bs4/diagnose.py

@@ -0,0 +1,242 @@
+"""Diagnostic functions, mainly for use when doing tech support."""
+
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
+import cProfile
+from io import StringIO
+from html.parser import HTMLParser
+import bs4
+from bs4 import BeautifulSoup, __version__
+from bs4.builder import builder_registry
+
+import os
+import pstats
+import random
+import tempfile
+import time
+import traceback
+import sys
+import cProfile
+
+def diagnose(data):
+    """Diagnostic suite for isolating common problems.
+
+    :param data: A string containing markup that needs to be explained.
+    :return: None; diagnostics are printed to standard output.
+    """
+    print(("Diagnostic running on Beautiful Soup %s" % __version__))
+    print(("Python version %s" % sys.version))
+
+    basic_parsers = ["html.parser", "html5lib", "lxml"]
+    for name in basic_parsers:
+        for builder in builder_registry.builders:
+            if name in builder.features:
+                break
+        else:
+            basic_parsers.remove(name)
+            print((
+                "I noticed that %s is not installed. Installing it may help." %
+                name))
+
+    if 'lxml' in basic_parsers:
+        basic_parsers.append("lxml-xml")
+        try:
+            from lxml import etree
+            print(("Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))))
+        except ImportError as e:
+            print(
+                "lxml is not installed or couldn't be imported.")
+
+
+    if 'html5lib' in basic_parsers:
+        try:
+            import html5lib
+            print(("Found html5lib version %s" % html5lib.__version__))
+        except ImportError as e:
+            print(
+                "html5lib is not installed or couldn't be imported.")
+
+    if hasattr(data, 'read'):
+        data = data.read()
+    elif data.startswith("http:") or data.startswith("https:"):
+        print(('"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data))
+        print("You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup.")
+        return
+    else:
+        try:
+            if os.path.exists(data):
+                print(('"%s" looks like a filename. Reading data from the file.' % data))
+                with open(data) as fp:
+                    data = fp.read()
+        except ValueError:
+            # This can happen on some platforms when the 'filename' is
+            # too long. Assume it's data and not a filename.
+            pass
+        print("")
+
+    for parser in basic_parsers:
+        print(("Trying to parse your markup with %s" % parser))
+        success = False
+        try:
+            soup = BeautifulSoup(data, features=parser)
+            success = True
+        except Exception as e:
+            print(("%s could not parse the markup." % parser))
+            traceback.print_exc()
+        if success:
+            print(("Here's what %s did with the markup:" % parser))
+            print((soup.prettify()))
+
+        print(("-" * 80))
+
+def lxml_trace(data, html=True, **kwargs):
+    """Print out the lxml events that occur during parsing.
+
+    This lets you see how lxml parses a document when no Beautiful
+    Soup code is running. You can use this to determine whether
+    an lxml-specific problem is in Beautiful Soup's lxml tree builders
+    or in lxml itself.
+
+    :param data: Some markup.
+    :param html: If True, markup will be parsed with lxml's HTML parser.
+       if False, lxml's XML parser will be used.
+    """
+    from lxml import etree
+    for event, element in etree.iterparse(StringIO(data), html=html, **kwargs):
+        print(("%s, %4s, %s" % (event, element.tag, element.text)))
+
+class AnnouncingParser(HTMLParser):
+    """Subclass of HTMLParser that announces parse events, without doing
+    anything else.
+
+    You can use this to get a picture of how html.parser sees a given
+    document. The easiest way to do this is to call `htmlparser_trace`.
+    """
+
+    def _p(self, s):
+        print(s)
+
+    def handle_starttag(self, name, attrs):
+        self._p("%s START" % name)
+
+    def handle_endtag(self, name):
+        self._p("%s END" % name)
+
+    def handle_data(self, data):
+        self._p("%s DATA" % data)
+
+    def handle_charref(self, name):
+        self._p("%s CHARREF" % name)
+
+    def handle_entityref(self, name):
+        self._p("%s ENTITYREF" % name)
+
+    def handle_comment(self, data):
+        self._p("%s COMMENT" % data)
+
+    def handle_decl(self, data):
+        self._p("%s DECL" % data)
+
+    def unknown_decl(self, data):
+        self._p("%s UNKNOWN-DECL" % data)
+
+    def handle_pi(self, data):
+        self._p("%s PI" % data)
+
+def htmlparser_trace(data):
+    """Print out the HTMLParser events that occur during parsing.
+
+    This lets you see how HTMLParser parses a document when no
+    Beautiful Soup code is running.
+
+    :param data: Some markup.
+    """
+    parser = AnnouncingParser()
+    parser.feed(data)
+
+_vowels = "aeiou"
+_consonants = "bcdfghjklmnpqrstvwxyz"
+
+def rword(length=5):
+    "Generate a random word-like string."
+    s = ''
+    for i in range(length):
+        if i % 2 == 0:
+            t = _consonants
+        else:
+            t = _vowels
+        s += random.choice(t)
+    return s
+
+def rsentence(length=4):
+    "Generate a random sentence-like string."
+    return " ".join(rword(random.randint(4,9)) for i in range(length))
+        
+def rdoc(num_elements=1000):
+    """Randomly generate an invalid HTML document."""
+    tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table']
+    elements = []
+    for i in range(num_elements):
+        choice = random.randint(0,3)
+        if choice == 0:
+            # New tag.
+            tag_name = random.choice(tag_names)
+            elements.append("<%s>" % tag_name)
+        elif choice == 1:
+            elements.append(rsentence(random.randint(1,4)))
+        elif choice == 2:
+            # Close a tag.
+            tag_name = random.choice(tag_names)
+            elements.append("</%s>" % tag_name)
+    return "<html>" + "\n".join(elements) + "</html>"
+
+def benchmark_parsers(num_elements=100000):
+    """Very basic head-to-head performance benchmark."""
+    print(("Comparative parser benchmark on Beautiful Soup %s" % __version__))
+    data = rdoc(num_elements)
+    print(("Generated a large invalid HTML document (%d bytes)." % len(data)))
+    
+    for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
+        success = False
+        try:
+            a = time.time()
+            soup = BeautifulSoup(data, parser)
+            b = time.time()
+            success = True
+        except Exception as e:
+            print(("%s could not parse the markup." % parser))
+            traceback.print_exc()
+        if success:
+            print(("BS4+%s parsed the markup in %.2fs." % (parser, b-a)))
+
+    from lxml import etree
+    a = time.time()
+    etree.HTML(data)
+    b = time.time()
+    print(("Raw lxml parsed the markup in %.2fs." % (b-a)))
+
+    import html5lib
+    parser = html5lib.HTMLParser()
+    a = time.time()
+    parser.parse(data)
+    b = time.time()
+    print(("Raw html5lib parsed the markup in %.2fs." % (b-a)))
+
+def profile(num_elements=100000, parser="lxml"):
+    """Use Python's profiler on a randomly generated document."""
+    filehandle = tempfile.NamedTemporaryFile()
+    filename = filehandle.name
+
+    data = rdoc(num_elements)
+    vars = dict(bs4=bs4, data=data, parser=parser)
+    cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename)
+
+    stats = pstats.Stats(filename)
+    # stats.strip_dirs()
+    stats.sort_stats("cumulative")
+    stats.print_stats('_html5lib|bs4', 50)
+
+# If this file is run as a script, standard input is diagnosed.
+if __name__ == '__main__':
+    diagnose(sys.stdin.read())

bs4/formatter.py

@@ -0,0 +1,152 @@
+from bs4.dammit import EntitySubstitution
+
+class Formatter(EntitySubstitution):
+    """Describes a strategy to use when outputting a parse tree to a string.
+
+    Some parts of this strategy come from the distinction between
+    HTML4, HTML5, and XML. Others are configurable by the user.
+
+    Formatters are passed in as the `formatter` argument to methods
+    like `PageElement.encode`. Most people won't need to think about
+    formatters, and most people who need to think about them can pass
+    in one of these predefined strings as `formatter` rather than
+    making a new Formatter object:
+
+    For HTML documents:
+     * 'html' - HTML entity substitution for generic HTML documents. (default)
+     * 'html5' - HTML entity substitution for HTML5 documents.
+     * 'minimal' - Only make the substitutions necessary to guarantee
+                   valid HTML.
+     * None - Do not perform any substitution. This will be faster
+              but may result in invalid markup.
+
+    For XML documents:
+     * 'html' - Entity substitution for XHTML documents.
+     * 'minimal' - Only make the substitutions necessary to guarantee
+                   valid XML. (default)
+     * None - Do not perform any substitution. This will be faster
+              but may result in invalid markup.
+    """
+    # Registries of XML and HTML formatters.
+    XML_FORMATTERS = {}
+    HTML_FORMATTERS = {}
+
+    HTML = 'html'
+    XML = 'xml'
+
+    HTML_DEFAULTS = dict(
+        cdata_containing_tags=set(["script", "style"]),
+    )
+
+    def _default(self, language, value, kwarg):
+        if value is not None:
+            return value
+        if language == self.XML:
+            return set()
+        return self.HTML_DEFAULTS[kwarg]
+
+    def __init__(
+            self, language=None, entity_substitution=None,
+            void_element_close_prefix='/', cdata_containing_tags=None,
+    ):
+        """Constructor.
+
+        :param language: This should be Formatter.XML if you are formatting
+           XML markup and Formatter.HTML if you are formatting HTML markup.
+
+        :param entity_substitution: A function to call to replace special
+           characters with XML/HTML entities. For examples, see 
+           bs4.dammit.EntitySubstitution.substitute_html and substitute_xml.
+        :param void_element_close_prefix: By default, void elements
+           are represented as <tag/> (XML rules) rather than <tag>
+           (HTML rules). To get <tag>, pass in the empty string.
+        :param cdata_containing_tags: The list of tags that are defined
+           as containing CDATA in this dialect. For example, in HTML,
+           <script> and <style> tags are defined as containing CDATA,
+           and their contents should not be formatted.
+        """
+        self.language = language
+        self.entity_substitution = entity_substitution
+        self.void_element_close_prefix = void_element_close_prefix
+        self.cdata_containing_tags = self._default(
+            language, cdata_containing_tags, 'cdata_containing_tags'
+        )
+            
+    def substitute(self, ns):
+        """Process a string that needs to undergo entity substitution.
+        This may be a string encountered in an attribute value or as
+        text.
+
+        :param ns: A string.
+        :return: A string with certain characters replaced by named
+           or numeric entities.
+        """
+        if not self.entity_substitution:
+            return ns
+        from .element import NavigableString
+        if (isinstance(ns, NavigableString)
+            and ns.parent is not None
+            and ns.parent.name in self.cdata_containing_tags):
+            # Do nothing.
+            return ns
+        # Substitute.
+        return self.entity_substitution(ns)
+
+    def attribute_value(self, value):
+        """Process the value of an attribute.
+
+        :param ns: A string.
+        :return: A string with certain characters replaced by named
+           or numeric entities.
+        """
+        return self.substitute(value)
+    
+    def attributes(self, tag):
+        """Reorder a tag's attributes however you want.
+        
+        By default, attributes are sorted alphabetically. This makes
+        behavior consistent between Python 2 and Python 3, and preserves
+        backwards compatibility with older versions of Beautiful Soup.
+        """
+        if tag.attrs is None:
+            return []
+        return sorted(tag.attrs.items())
+
+   
+class HTMLFormatter(Formatter):
+    """A generic Formatter for HTML."""
+    REGISTRY = {}
+    def __init__(self, *args, **kwargs):
+        return super(HTMLFormatter, self).__init__(self.HTML, *args, **kwargs)
+
+    
+class XMLFormatter(Formatter):
+    """A generic Formatter for XML."""
+    REGISTRY = {}
+    def __init__(self, *args, **kwargs):
+        return super(XMLFormatter, self).__init__(self.XML, *args, **kwargs)
+
+
+# Set up aliases for the default formatters.
+HTMLFormatter.REGISTRY['html'] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html
+)
+HTMLFormatter.REGISTRY["html5"] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html,
+    void_element_close_prefix = None
+)
+HTMLFormatter.REGISTRY["minimal"] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_xml
+)
+HTMLFormatter.REGISTRY[None] = HTMLFormatter(
+    entity_substitution=None
+)
+XMLFormatter.REGISTRY["html"] =  XMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html
+)
+XMLFormatter.REGISTRY["minimal"] = XMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_xml
+)
+XMLFormatter.REGISTRY[None] = Formatter(
+    Formatter(Formatter.XML, entity_substitution=None)
+)

bs4/tests/__init__.py

@@ -0,0 +1 @@
+"The beautifulsoup tests."

bs4/tests/test_builder_registry.py

@@ -0,0 +1,147 @@
+"""Tests of the builder registry."""
+
+import unittest
+import warnings
+
+from bs4 import BeautifulSoup
+from bs4.builder import (
+    builder_registry as registry,
+    HTMLParserTreeBuilder,
+    TreeBuilderRegistry,
+)
+
+try:
+    from bs4.builder import HTML5TreeBuilder
+    HTML5LIB_PRESENT = True
+except ImportError:
+    HTML5LIB_PRESENT = False
+
+try:
+    from bs4.builder import (
+        LXMLTreeBuilderForXML,
+        LXMLTreeBuilder,
+        )
+    LXML_PRESENT = True
+except ImportError:
+    LXML_PRESENT = False
+
+
+class BuiltInRegistryTest(unittest.TestCase):
+    """Test the built-in registry with the default builders registered."""
+
+    def test_combination(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('fast', 'html'),
+                             LXMLTreeBuilder)
+
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('permissive', 'xml'),
+                             LXMLTreeBuilderForXML)
+        self.assertEqual(registry.lookup('strict', 'html'),
+                          HTMLParserTreeBuilder)
+        if HTML5LIB_PRESENT:
+            self.assertEqual(registry.lookup('html5lib', 'html'),
+                              HTML5TreeBuilder)
+
+    def test_lookup_by_markup_type(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('html'), LXMLTreeBuilder)
+            self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML)
+        else:
+            self.assertEqual(registry.lookup('xml'), None)
+            if HTML5LIB_PRESENT:
+                self.assertEqual(registry.lookup('html'), HTML5TreeBuilder)
+            else:
+                self.assertEqual(registry.lookup('html'), HTMLParserTreeBuilder)
+
+    def test_named_library(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('lxml', 'xml'),
+                             LXMLTreeBuilderForXML)
+            self.assertEqual(registry.lookup('lxml', 'html'),
+                             LXMLTreeBuilder)
+        if HTML5LIB_PRESENT:
+            self.assertEqual(registry.lookup('html5lib'),
+                              HTML5TreeBuilder)
+
+        self.assertEqual(registry.lookup('html.parser'),
+                          HTMLParserTreeBuilder)
+
+    def test_beautifulsoup_constructor_does_lookup(self):
+
+        with warnings.catch_warnings(record=True) as w:
+            # This will create a warning about not explicitly
+            # specifying a parser, but we'll ignore it.
+
+            # You can pass in a string.
+            BeautifulSoup("", features="html")
+            # Or a list of strings.
+            BeautifulSoup("", features=["html", "fast"])
+
+        # You'll get an exception if BS can't find an appropriate
+        # builder.
+        self.assertRaises(ValueError, BeautifulSoup,
+                          "", features="no-such-feature")
+
+class RegistryTest(unittest.TestCase):
+    """Test the TreeBuilderRegistry class in general."""
+
+    def setUp(self):
+        self.registry = TreeBuilderRegistry()
+
+    def builder_for_features(self, *feature_list):
+        cls = type('Builder_' + '_'.join(feature_list),
+                   (object,), {'features' : feature_list})
+
+        self.registry.register(cls)
+        return cls
+
+    def test_register_with_no_features(self):
+        builder = self.builder_for_features()
+
+        # Since the builder advertises no features, you can't find it
+        # by looking up features.
+        self.assertEqual(self.registry.lookup('foo'), None)
+
+        # But you can find it by doing a lookup with no features, if
+        # this happens to be the only registered builder.
+        self.assertEqual(self.registry.lookup(), builder)
+
+    def test_register_with_features_makes_lookup_succeed(self):
+        builder = self.builder_for_features('foo', 'bar')
+        self.assertEqual(self.registry.lookup('foo'), builder)
+        self.assertEqual(self.registry.lookup('bar'), builder)
+
+    def test_lookup_fails_when_no_builder_implements_feature(self):
+        builder = self.builder_for_features('foo', 'bar')
+        self.assertEqual(self.registry.lookup('baz'), None)
+
+    def test_lookup_gets_most_recent_registration_when_no_feature_specified(self):
+        builder1 = self.builder_for_features('foo')
+        builder2 = self.builder_for_features('bar')
+        self.assertEqual(self.registry.lookup(), builder2)
+
+    def test_lookup_fails_when_no_tree_builders_registered(self):
+        self.assertEqual(self.registry.lookup(), None)
+
+    def test_lookup_gets_most_recent_builder_supporting_all_features(self):
+        has_one = self.builder_for_features('foo')
+        has_the_other = self.builder_for_features('bar')
+        has_both_early = self.builder_for_features('foo', 'bar', 'baz')
+        has_both_late = self.builder_for_features('foo', 'bar', 'quux')
+        lacks_one = self.builder_for_features('bar')
+        has_the_other = self.builder_for_features('foo')
+
+        # There are two builders featuring 'foo' and 'bar', but
+        # the one that also features 'quux' was registered later.
+        self.assertEqual(self.registry.lookup('foo', 'bar'),
+                          has_both_late)
+
+        # There is only one builder featuring 'foo', 'bar', and 'baz'.
+        self.assertEqual(self.registry.lookup('foo', 'bar', 'baz'),
+                          has_both_early)
+
+    def test_lookup_fails_when_cannot_reconcile_requested_features(self):
+        builder1 = self.builder_for_features('foo', 'bar')
+        builder2 = self.builder_for_features('foo', 'baz')
+        self.assertEqual(self.registry.lookup('bar', 'baz'), None)

bs4/tests/test_docs.py

@@ -0,0 +1,36 @@
+"Test harness for doctests."
+
+# pylint: disable-msg=E0611,W0142
+
+__metaclass__ = type
+__all__ = [
+    'additional_tests',
+    ]
+
+import atexit
+import doctest
+import os
+#from pkg_resources import (
+#    resource_filename, resource_exists, resource_listdir, cleanup_resources)
+import unittest
+
+DOCTEST_FLAGS = (
+    doctest.ELLIPSIS |
+    doctest.NORMALIZE_WHITESPACE |
+    doctest.REPORT_NDIFF)
+
+
+# def additional_tests():
+#     "Run the doc tests (README.txt and docs/*, if any exist)"
+#     doctest_files = [
+#         os.path.abspath(resource_filename('bs4', 'README.txt'))]
+#     if resource_exists('bs4', 'docs'):
+#         for name in resource_listdir('bs4', 'docs'):
+#             if name.endswith('.txt'):
+#                 doctest_files.append(
+#                     os.path.abspath(
+#                         resource_filename('bs4', 'docs/%s' % name)))
+#     kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
+#     atexit.register(cleanup_resources)
+#     return unittest.TestSuite((
+#         doctest.DocFileSuite(*doctest_files, **kwargs)))

bs4/tests/test_html5lib.py

@@ -0,0 +1,190 @@
+"""Tests to ensure that the html5lib tree builder generates good trees."""
+
+import warnings
+
+try:
+    from bs4.builder import HTML5TreeBuilder
+    HTML5LIB_PRESENT = True
+except ImportError as e:
+    HTML5LIB_PRESENT = False
+from bs4.element import SoupStrainer
+from bs4.testing import (
+    HTML5TreeBuilderSmokeTest,
+    SoupTest,
+    skipIf,
+)
+
+@skipIf(
+    not HTML5LIB_PRESENT,
+    "html5lib seems not to be present, not testing its tree builder.")
+class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest):
+    """See ``HTML5TreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return HTML5TreeBuilder
+
+    def test_soupstrainer(self):
+        # The html5lib tree builder does not support SoupStrainers.
+        strainer = SoupStrainer("b")
+        markup = "<p>A <b>bold</b> statement.</p>"
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup(markup, parse_only=strainer)
+        self.assertEqual(
+            soup.decode(), self.document_for(markup))
+
+        self.assertTrue(
+            "the html5lib tree builder doesn't support parse_only" in
+            str(w[0].message))
+
+    def test_correctly_nested_tables(self):
+        """html5lib inserts <tbody> tags where other parsers don't."""
+        markup = ('<table id="1">'
+                  '<tr>'
+                  "<td>Here's another table:"
+                  '<table id="2">'
+                  '<tr><td>foo</td></tr>'
+                  '</table></td>')
+
+        self.assertSoupEquals(
+            markup,
+            '<table id="1"><tbody><tr><td>Here\'s another table:'
+            '<table id="2"><tbody><tr><td>foo</td></tr></tbody></table>'
+            '</td></tr></tbody></table>')
+
+        self.assertSoupEquals(
+            "<table><thead><tr><td>Foo</td></tr></thead>"
+            "<tbody><tr><td>Bar</td></tr></tbody>"
+            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
+
+    def test_xml_declaration_followed_by_doctype(self):
+        markup = '''<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html>
+<html>
+  <head>
+  </head>
+  <body>
+   <p>foo</p>
+  </body>
+</html>'''
+        soup = self.soup(markup)
+        # Verify that we can reach the <p> tag; this means the tree is connected.
+        self.assertEqual(b"<p>foo</p>", soup.p.encode())
+
+    def test_reparented_markup(self):
+        markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>'
+        soup = self.soup(markup)
+        self.assertEqual("<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p></body>", soup.body.decode())
+        self.assertEqual(2, len(soup.find_all('p')))
+
+
+    def test_reparented_markup_ends_with_whitespace(self):
+        markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>\n'
+        soup = self.soup(markup)
+        self.assertEqual("<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p>\n</body>", soup.body.decode())
+        self.assertEqual(2, len(soup.find_all('p')))
+
+    def test_reparented_markup_containing_identical_whitespace_nodes(self):
+        """Verify that we keep the two whitespace nodes in this
+        document distinct when reparenting the adjacent <tbody> tags.
+        """
+        markup = '<table> <tbody><tbody><ims></tbody> </table>'
+        soup = self.soup(markup)
+        space1, space2 = soup.find_all(string=' ')
+        tbody1, tbody2 = soup.find_all('tbody')
+        assert space1.next_element is tbody1
+        assert tbody2.next_element is space2
+
+    def test_reparented_markup_containing_children(self):
+        markup = '<div><a>aftermath<p><noscript>target</noscript>aftermath</a></p></div>'
+        soup = self.soup(markup)
+        noscript = soup.noscript
+        self.assertEqual("target", noscript.next_element)
+        target = soup.find(string='target')
+
+        # The 'aftermath' string was duplicated; we want the second one.
+        final_aftermath = soup.find_all(string='aftermath')[-1]
+
+        # The <noscript> tag was moved beneath a copy of the <a> tag,
+        # but the 'target' string within is still connected to the
+        # (second) 'aftermath' string.
+        self.assertEqual(final_aftermath, target.next_element)
+        self.assertEqual(target, final_aftermath.previous_element)
+        
+    def test_processing_instruction(self):
+        """Processing instructions become comments."""
+        markup = b"""<?PITarget PIContent?>"""
+        soup = self.soup(markup)
+        assert str(soup).startswith("<!--?PITarget PIContent?-->")
+
+    def test_cloned_multivalue_node(self):
+        markup = b"""<a class="my_class"><p></a>"""
+        soup = self.soup(markup)
+        a1, a2 = soup.find_all('a')
+        self.assertEqual(a1, a2)
+        assert a1 is not a2
+
+    def test_foster_parenting(self):
+        markup = b"""<table><td></tbody>A"""
+        soup = self.soup(markup)
+        self.assertEqual("<body>A<table><tbody><tr><td></td></tr></tbody></table></body>", soup.body.decode())
+
+    def test_extraction(self):
+        """
+        Test that extraction does not destroy the tree.
+
+        https://bugs.launchpad.net/beautifulsoup/+bug/1782928
+        """
+
+        markup = """
+<html><head></head>
+<style>
+</style><script></script><body><p>hello</p></body></html>
+"""
+        soup = self.soup(markup)
+        [s.extract() for s in soup('script')]
+        [s.extract() for s in soup('style')]
+
+        self.assertEqual(len(soup.find_all("p")), 1)
+
+    def test_empty_comment(self):
+        """
+        Test that empty comment does not break structure.
+
+        https://bugs.launchpad.net/beautifulsoup/+bug/1806598
+        """
+
+        markup = """
+<html>
+<body>
+<form>
+<!----><input type="text">
+</form>
+</body>
+</html>
+"""
+        soup = self.soup(markup)
+        inputs = []
+        for form in soup.find_all('form'):
+            inputs.extend(form.find_all('input'))
+        self.assertEqual(len(inputs), 1)
+
+    def test_tracking_line_numbers(self):
+        # The html.parser TreeBuilder keeps track of line number and
+        # position of each element.
+        markup = "\n   <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>"
+        soup = self.soup(markup)
+        self.assertEqual(2, soup.p.sourceline)
+        self.assertEqual(5, soup.p.sourcepos)
+        self.assertEqual("sourceline", soup.p.find('sourceline').name)
+
+        # You can deactivate this behavior.
+        soup = self.soup(markup, store_line_numbers=False)
+        self.assertEqual("sourceline", soup.p.sourceline.name)
+        self.assertEqual("sourcepos", soup.p.sourcepos.name)
+
+    def test_special_string_containers(self):
+        # The html5lib tree builder doesn't support this standard feature,
+        # because there's no way of knowing, when a string is created,
+        # where in the tree it will eventually end up.
+        pass

bs4/tests/test_htmlparser.py

@@ -0,0 +1,97 @@
+"""Tests to ensure that the html.parser tree builder generates good
+trees."""
+
+from pdb import set_trace
+import pickle
+from bs4.testing import SoupTest, HTMLTreeBuilderSmokeTest
+from bs4.builder import HTMLParserTreeBuilder
+from bs4.builder._htmlparser import BeautifulSoupHTMLParser
+
+class HTMLParserTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
+
+    default_builder = HTMLParserTreeBuilder
+
+    def test_namespaced_system_doctype(self):
+        # html.parser can't handle namespaced doctypes, so skip this one.
+        pass
+
+    def test_namespaced_public_doctype(self):
+        # html.parser can't handle namespaced doctypes, so skip this one.
+        pass
+
+    def test_builder_is_pickled(self):
+        """Unlike most tree builders, HTMLParserTreeBuilder and will
+        be restored after pickling.
+        """
+        tree = self.soup("<a><b>foo</a>")
+        dumped = pickle.dumps(tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertTrue(isinstance(loaded.builder, type(tree.builder)))
+
+    def test_redundant_empty_element_closing_tags(self):
+        self.assertSoupEquals('<br></br><br></br><br></br>', "<br/><br/><br/>")
+        self.assertSoupEquals('</br></br></br>', "")
+
+    def test_empty_element(self):
+        # This verifies that any buffered data present when the parser
+        # finishes working is handled.
+        self.assertSoupEquals("foo &# bar", "foo &amp;# bar")
+
+    def test_tracking_line_numbers(self):
+        # The html.parser TreeBuilder keeps track of line number and
+        # position of each element.
+        markup = "\n   <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>"
+        soup = self.soup(markup)
+        self.assertEqual(2, soup.p.sourceline)
+        self.assertEqual(3, soup.p.sourcepos)
+        self.assertEqual("sourceline", soup.p.find('sourceline').name)
+
+        # You can deactivate this behavior.
+        soup = self.soup(markup, store_line_numbers=False)
+        self.assertEqual("sourceline", soup.p.sourceline.name)
+        self.assertEqual("sourcepos", soup.p.sourcepos.name)
+
+    def test_on_duplicate_attribute(self):
+        # The html.parser tree builder has a variety of ways of
+        # handling a tag that contains the same attribute multiple times.
+
+        markup = '<a class="cls" href="url1" href="url2" href="url3" id="id">'
+
+        # If you don't provide any particular value for
+        # on_duplicate_attribute, later values replace earlier values.
+        soup = self.soup(markup)
+        self.assertEqual("url3", soup.a['href'])
+        self.assertEqual(["cls"], soup.a['class'])
+        self.assertEqual("id", soup.a['id'])
+        
+        # You can also get this behavior explicitly.
+        def assert_attribute(on_duplicate_attribute, expected):
+            soup = self.soup(
+                markup, on_duplicate_attribute=on_duplicate_attribute
+            )
+            self.assertEqual(expected, soup.a['href'])
+
+            # Verify that non-duplicate attributes are treated normally.
+            self.assertEqual(["cls"], soup.a['class'])
+            self.assertEqual("id", soup.a['id'])
+        assert_attribute(None, "url3")
+        assert_attribute(BeautifulSoupHTMLParser.REPLACE, "url3")
+
+        # You can ignore subsequent values in favor of the first.
+        assert_attribute(BeautifulSoupHTMLParser.IGNORE, "url1")
+
+        # And you can pass in a callable that does whatever you want.
+        def accumulate(attrs, key, value):
+            if not isinstance(attrs[key], list):
+                attrs[key] = [attrs[key]]
+            attrs[key].append(value)
+        assert_attribute(accumulate, ["url1", "url2", "url3"])            
+
+
+class TestHTMLParserSubclass(SoupTest):
+    def test_error(self):
+        """Verify that our HTMLParser subclass implements error() in a way
+        that doesn't cause a crash.
+        """
+        parser = BeautifulSoupHTMLParser()
+        parser.error("don't crash")

bs4/tests/test_lxml.py

@@ -0,0 +1,115 @@
+"""Tests to ensure that the lxml tree builder generates good trees."""
+
+import re
+import warnings
+
+try:
+    import lxml.etree
+    LXML_PRESENT = True
+    LXML_VERSION = lxml.etree.LXML_VERSION
+except ImportError as e:
+    LXML_PRESENT = False
+    LXML_VERSION = (0,)
+
+if LXML_PRESENT:
+    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
+
+from bs4 import (
+    BeautifulSoup,
+    BeautifulStoneSoup,
+    )
+from bs4.element import Comment, Doctype, SoupStrainer
+from bs4.testing import skipIf
+from bs4.tests import test_htmlparser
+from bs4.testing import (
+    HTMLTreeBuilderSmokeTest,
+    XMLTreeBuilderSmokeTest,
+    SoupTest,
+    skipIf,
+)
+
+@skipIf(
+    not LXML_PRESENT,
+    "lxml seems not to be present, not testing its tree builder.")
+class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
+    """See ``HTMLTreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return LXMLTreeBuilder
+
+    def test_out_of_range_entity(self):
+        self.assertSoupEquals(
+            "<p>foo&#10000000000000;bar</p>", "<p>foobar</p>")
+        self.assertSoupEquals(
+            "<p>foo&#x10000000000000;bar</p>", "<p>foobar</p>")
+        self.assertSoupEquals(
+            "<p>foo&#1000000000;bar</p>", "<p>foobar</p>")
+
+    def test_entities_in_foreign_document_encoding(self):
+        # We can't implement this case correctly because by the time we
+        # hear about markup like "&#147;", it's been (incorrectly) converted into
+        # a string like u'\x93'
+        pass
+        
+    # In lxml < 2.3.5, an empty doctype causes a segfault. Skip this
+    # test if an old version of lxml is installed.
+
+    @skipIf(
+        not LXML_PRESENT or LXML_VERSION < (2,3,5,0),
+        "Skipping doctype test for old version of lxml to avoid segfault.")
+    def test_empty_doctype(self):
+        soup = self.soup("<!DOCTYPE>")
+        doctype = soup.contents[0]
+        self.assertEqual("", doctype.strip())
+
+    def test_beautifulstonesoup_is_xml_parser(self):
+        # Make sure that the deprecated BSS class uses an xml builder
+        # if one is installed.
+        with warnings.catch_warnings(record=True) as w:
+            soup = BeautifulStoneSoup("<b />")
+        self.assertEqual("<b/>", str(soup.b))
+        self.assertTrue("BeautifulStoneSoup class is deprecated" in str(w[0].message))
+
+    def test_tracking_line_numbers(self):
+        # The lxml TreeBuilder cannot keep track of line numbers from
+        # the original markup. Even if you ask for line numbers, we
+        # don't have 'em.
+        #
+        # This means that if you have a tag like <sourceline> or
+        # <sourcepos>, attribute access will find it rather than
+        # giving you a numeric answer.
+        soup = self.soup(
+            "\n   <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>",
+            store_line_numbers=True
+        )
+        self.assertEqual("sourceline", soup.p.sourceline.name)
+        self.assertEqual("sourcepos", soup.p.sourcepos.name)
+        
+@skipIf(
+    not LXML_PRESENT,
+    "lxml seems not to be present, not testing its XML tree builder.")
+class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest):
+    """See ``HTMLTreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return LXMLTreeBuilderForXML
+
+    def test_namespace_indexing(self):
+        # We should not track un-prefixed namespaces as we can only hold one
+        # and it will be recognized as the default namespace by soupsieve,
+        # which may be confusing in some situations. When no namespace is provided
+        # for a selector, the default namespace (if defined) is assumed.
+
+        soup = self.soup(
+            '<?xml version="1.1"?>\n'
+            '<root>'
+            '<tag xmlns="http://unprefixed-namespace.com">content</tag>'
+            '<prefix:tag xmlns:prefix="http://prefixed-namespace.com">content</tag>'
+            '</root>'
+        )
+        self.assertEqual(
+            soup._namespaces,
+            {'xml': 'http://www.w3.org/XML/1998/namespace', 'prefix': 'http://prefixed-namespace.com'}
+        )

bs4/tests/test_soup.py

@@ -0,0 +1,728 @@
+# -*- coding: utf-8 -*-
+"""Tests of Beautiful Soup as a whole."""
+
+from pdb import set_trace
+import logging
+import unittest
+import sys
+import tempfile
+
+from bs4 import (
+    BeautifulSoup,
+    BeautifulStoneSoup,
+    GuessedAtParserWarning,
+    MarkupResemblesLocatorWarning,
+)
+from bs4.builder import (
+    TreeBuilder,
+    ParserRejectedMarkup,
+)
+from bs4.element import (
+    CharsetMetaAttributeValue,
+    Comment,
+    ContentMetaAttributeValue,
+    SoupStrainer,
+    NamespacedAttribute,
+    Tag,
+    NavigableString,
+    )
+
+import bs4.dammit
+from bs4.dammit import (
+    EntitySubstitution,
+    UnicodeDammit,
+    EncodingDetector,
+)
+from bs4.testing import (
+    default_builder,
+    SoupTest,
+    skipIf,
+)
+import warnings
+
+try:
+    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
+    LXML_PRESENT = True
+except ImportError as e:
+    LXML_PRESENT = False
+
+PYTHON_3_PRE_3_2 = (sys.version_info[0] == 3 and sys.version_info < (3,2))
+
+class TestConstructor(SoupTest):
+
+    def test_short_unicode_input(self):
+        data = "<h1>éé</h1>"
+        soup = self.soup(data)
+        self.assertEqual("éé", soup.h1.string)
+
+    def test_embedded_null(self):
+        data = "<h1>foo\0bar</h1>"
+        soup = self.soup(data)
+        self.assertEqual("foo\0bar", soup.h1.string)
+
+    def test_exclude_encodings(self):
+        utf8_data = "Räksmörgås".encode("utf-8")
+        soup = self.soup(utf8_data, exclude_encodings=["utf-8"])
+        self.assertEqual("windows-1252", soup.original_encoding)
+
+    def test_custom_builder_class(self):
+        # Verify that you can pass in a custom Builder class and
+        # it'll be instantiated with the appropriate keyword arguments.
+        class Mock(object):
+            def __init__(self, **kwargs):
+                self.called_with = kwargs
+                self.is_xml = True
+                self.store_line_numbers = False
+                self.cdata_list_attributes = []
+                self.preserve_whitespace_tags = []
+                self.string_containers = {}
+            def initialize_soup(self, soup):
+                pass
+            def feed(self, markup):
+                self.fed = markup
+            def reset(self):
+                pass
+            def ignore(self, ignore):
+                pass
+            set_up_substitutions = can_be_empty_element = ignore
+            def prepare_markup(self, *args, **kwargs):
+                yield "prepared markup", "original encoding", "declared encoding", "contains replacement characters"
+                
+        kwargs = dict(
+            var="value",
+            # This is a deprecated BS3-era keyword argument, which
+            # will be stripped out.
+            convertEntities=True,
+        )
+        with warnings.catch_warnings(record=True):
+            soup = BeautifulSoup('', builder=Mock, **kwargs)
+        assert isinstance(soup.builder, Mock)
+        self.assertEqual(dict(var="value"), soup.builder.called_with)
+        self.assertEqual("prepared markup", soup.builder.fed)
+        
+        # You can also instantiate the TreeBuilder yourself. In this
+        # case, that specific object is used and any keyword arguments
+        # to the BeautifulSoup constructor are ignored.
+        builder = Mock(**kwargs)
+        with warnings.catch_warnings(record=True) as w:
+            soup = BeautifulSoup(
+                '', builder=builder, ignored_value=True,
+            )
+        msg = str(w[0].message)
+        assert msg.startswith("Keyword arguments to the BeautifulSoup constructor will be ignored.")
+        self.assertEqual(builder, soup.builder)
+        self.assertEqual(kwargs, builder.called_with)
+
+    def test_parser_markup_rejection(self):
+        # If markup is completely rejected by the parser, an
+        # explanatory ParserRejectedMarkup exception is raised.
+        class Mock(TreeBuilder):
+            def feed(self, *args, **kwargs):
+                raise ParserRejectedMarkup("Nope.")
+
+        def prepare_markup(self, *args, **kwargs):
+            # We're going to try two different ways of preparing this markup,
+            # but feed() will reject both of them.
+            yield markup, None, None, False
+            yield markup, None, None, False
+            
+        import re
+        self.assertRaisesRegex(
+            ParserRejectedMarkup,
+            "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.",
+            BeautifulSoup, '', builder=Mock,
+        )
+        
+    def test_cdata_list_attributes(self):
+        # Most attribute values are represented as scalars, but the
+        # HTML standard says that some attributes, like 'class' have
+        # space-separated lists as values.
+        markup = '<a id=" an id " class=" a class "></a>'
+        soup = self.soup(markup)
+
+        # Note that the spaces are stripped for 'class' but not for 'id'.
+        a = soup.a
+        self.assertEqual(" an id ", a['id'])
+        self.assertEqual(["a", "class"], a['class'])
+
+        # TreeBuilder takes an argument called 'mutli_valued_attributes'  which lets
+        # you customize or disable this. As always, you can customize the TreeBuilder
+        # by passing in a keyword argument to the BeautifulSoup constructor.
+        soup = self.soup(markup, builder=default_builder, multi_valued_attributes=None)
+        self.assertEqual(" a class ", soup.a['class'])
+
+        # Here are two ways of saying that `id` is a multi-valued
+        # attribute in this context, but 'class' is not.
+        for switcheroo in ({'*': 'id'}, {'a': 'id'}):
+            with warnings.catch_warnings(record=True) as w:
+                # This will create a warning about not explicitly
+                # specifying a parser, but we'll ignore it.
+                soup = self.soup(markup, builder=None, multi_valued_attributes=switcheroo)
+            a = soup.a
+            self.assertEqual(["an", "id"], a['id'])
+            self.assertEqual(" a class ", a['class'])
+
+    def test_replacement_classes(self):
+        # Test the ability to pass in replacements for element classes
+        # which will be used when building the tree.
+        class TagPlus(Tag):
+            pass
+
+        class StringPlus(NavigableString):
+            pass
+
+        class CommentPlus(Comment):
+            pass
+        
+        soup = self.soup(
+            "<a><b>foo</b>bar</a><!--whee-->",
+            element_classes = {
+                Tag: TagPlus,
+                NavigableString: StringPlus,
+                Comment: CommentPlus,
+            }
+        )
+
+        # The tree was built with TagPlus, StringPlus, and CommentPlus objects,
+        # rather than Tag, String, and Comment objects.
+        assert all(
+            isinstance(x, (TagPlus, StringPlus, CommentPlus))
+            for x in soup.recursiveChildGenerator()
+        )
+
+    def test_alternate_string_containers(self):
+        # Test the ability to customize the string containers for
+        # different types of tags.
+        class PString(NavigableString):
+            pass
+
+        class BString(NavigableString):
+            pass
+
+        soup = self.soup(
+            "<div>Hello.<p>Here is <b>some <i>bolded</i></b> text",
+            string_containers = {
+                'b': BString,
+                'p': PString,
+            }
+        )
+
+        # The string before the <p> tag is a regular NavigableString.
+        assert isinstance(soup.div.contents[0], NavigableString)
+        
+        # The string inside the <p> tag, but not inside the <i> tag,
+        # is a PString.
+        assert isinstance(soup.p.contents[0], PString)
+
+        # Every string inside the <b> tag is a BString, even the one that
+        # was also inside an <i> tag.
+        for s in soup.b.strings:
+            assert isinstance(s, BString)
+
+        # Now that parsing was complete, the string_container_stack
+        # (where this information was kept) has been cleared out.
+        self.assertEqual([], soup.string_container_stack)
+
+
+class TestWarnings(SoupTest):
+
+    def _assert_warning(self, warnings, cls):
+        for w in warnings:
+            if isinstance(w.message, cls):
+                return w
+        raise Exception("%s warning not found in %r" % cls, warnings)
+    
+    def _assert_no_parser_specified(self, w):
+        warning = self._assert_warning(w, GuessedAtParserWarning)
+        message = str(warning.message)
+        self.assertTrue(
+            message.startswith(BeautifulSoup.NO_PARSER_SPECIFIED_WARNING[:60])
+        )
+
+    def test_warning_if_no_parser_specified(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = BeautifulSoup("<a><b></b></a>")
+        self._assert_no_parser_specified(w)
+
+    def test_warning_if_parser_specified_too_vague(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = BeautifulSoup("<a><b></b></a>", "html")
+        self._assert_no_parser_specified(w)
+
+    def test_no_warning_if_explicit_parser_specified(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = BeautifulSoup("<a><b></b></a>", "html.parser")
+        self.assertEqual([], w)
+
+    def test_parseOnlyThese_renamed_to_parse_only(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup("<a><b></b></a>", parseOnlyThese=SoupStrainer("b"))
+        msg = str(w[0].message)
+        self.assertTrue("parseOnlyThese" in msg)
+        self.assertTrue("parse_only" in msg)
+        self.assertEqual(b"<b></b>", soup.encode())
+
+    def test_fromEncoding_renamed_to_from_encoding(self):
+        with warnings.catch_warnings(record=True) as w:
+            utf8 = b"\xc3\xa9"
+            soup = self.soup(utf8, fromEncoding="utf8")
+        msg = str(w[0].message)
+        self.assertTrue("fromEncoding" in msg)
+        self.assertTrue("from_encoding" in msg)
+        self.assertEqual("utf8", soup.original_encoding)
+
+    def test_unrecognized_keyword_argument(self):
+        self.assertRaises(
+            TypeError, self.soup, "<a>", no_such_argument=True)
+
+    def test_disk_file_warning(self):
+        filehandle = tempfile.NamedTemporaryFile()
+        filename = filehandle.name
+        try:
+            with warnings.catch_warnings(record=True) as w:
+                soup = self.soup(filename)
+            warning = self._assert_warning(w, MarkupResemblesLocatorWarning)
+            self.assertTrue("looks like a filename" in str(warning.message))
+        finally:
+            filehandle.close()
+
+        # The file no longer exists, so Beautiful Soup will no longer issue the warning.
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup(filename)
+        self.assertEqual([], w)
+
+    def test_url_warning_with_bytes_url(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            soup = self.soup(b"http://www.crummybytes.com/")
+        warning = self._assert_warning(
+            warning_list, MarkupResemblesLocatorWarning
+        )
+        self.assertTrue("looks like a URL" in str(warning.message))
+
+    def test_url_warning_with_unicode_url(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            # note - this url must differ from the bytes one otherwise
+            # python's warnings system swallows the second warning
+            soup = self.soup("http://www.crummyunicode.com/")
+        warning = self._assert_warning(
+            warning_list, MarkupResemblesLocatorWarning
+        )
+        self.assertTrue("looks like a URL" in str(warning.message))
+
+    def test_url_warning_with_bytes_and_space(self):
+        # Here the markup contains something besides a URL, so no warning
+        # is issued.
+        with warnings.catch_warnings(record=True) as warning_list:
+            soup = self.soup(b"http://www.crummybytes.com/ is great")
+        self.assertFalse(any("looks like a URL" in str(w.message) 
+            for w in warning_list))
+
+    def test_url_warning_with_unicode_and_space(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            soup = self.soup("http://www.crummyuncode.com/ is great")
+        self.assertFalse(any("looks like a URL" in str(w.message) 
+            for w in warning_list))
+
+
+class TestSelectiveParsing(SoupTest):
+
+    def test_parse_with_soupstrainer(self):
+        markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>"
+        strainer = SoupStrainer("b")
+        soup = self.soup(markup, parse_only=strainer)
+        self.assertEqual(soup.encode(), b"<b>Yes</b><b>Yes <c>Yes</c></b>")
+
+
+class TestEntitySubstitution(unittest.TestCase):
+    """Standalone tests of the EntitySubstitution class."""
+    def setUp(self):
+        self.sub = EntitySubstitution
+
+    def test_simple_html_substitution(self):
+        # Unicode characters corresponding to named HTML entites
+        # are substituted, and no others.
+        s = "foo\u2200\N{SNOWMAN}\u00f5bar"
+        self.assertEqual(self.sub.substitute_html(s),
+                          "foo&forall;\N{SNOWMAN}&otilde;bar")
+
+    def test_smart_quote_substitution(self):
+        # MS smart quotes are a common source of frustration, so we
+        # give them a special test.
+        quotes = b"\x91\x92foo\x93\x94"
+        dammit = UnicodeDammit(quotes)
+        self.assertEqual(self.sub.substitute_html(dammit.markup),
+                          "&lsquo;&rsquo;foo&ldquo;&rdquo;")
+
+    def test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false(self):
+        s = 'Welcome to "my bar"'
+        self.assertEqual(self.sub.substitute_xml(s, False), s)
+
+    def test_xml_attribute_quoting_normally_uses_double_quotes(self):
+        self.assertEqual(self.sub.substitute_xml("Welcome", True),
+                          '"Welcome"')
+        self.assertEqual(self.sub.substitute_xml("Bob's Bar", True),
+                          '"Bob\'s Bar"')
+
+    def test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes(self):
+        s = 'Welcome to "my bar"'
+        self.assertEqual(self.sub.substitute_xml(s, True),
+                          "'Welcome to \"my bar\"'")
+
+    def test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes(self):
+        s = 'Welcome to "Bob\'s Bar"'
+        self.assertEqual(
+            self.sub.substitute_xml(s, True),
+            '"Welcome to &quot;Bob\'s Bar&quot;"')
+
+    def test_xml_quotes_arent_escaped_when_value_is_not_being_quoted(self):
+        quoted = 'Welcome to "Bob\'s Bar"'
+        self.assertEqual(self.sub.substitute_xml(quoted), quoted)
+
+    def test_xml_quoting_handles_angle_brackets(self):
+        self.assertEqual(
+            self.sub.substitute_xml("foo<bar>"),
+            "foo&lt;bar&gt;")
+
+    def test_xml_quoting_handles_ampersands(self):
+        self.assertEqual(self.sub.substitute_xml("AT&T"), "AT&amp;T")
+
+    def test_xml_quoting_including_ampersands_when_they_are_part_of_an_entity(self):
+        self.assertEqual(
+            self.sub.substitute_xml("&Aacute;T&T"),
+            "&amp;Aacute;T&amp;T")
+
+    def test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity(self):
+        self.assertEqual(
+            self.sub.substitute_xml_containing_entities("&Aacute;T&T"),
+            "&Aacute;T&amp;T")
+       
+    def test_quotes_not_html_substituted(self):
+        """There's no need to do this except inside attribute values."""
+        text = 'Bob\'s "bar"'
+        self.assertEqual(self.sub.substitute_html(text), text)
+
+
+class TestEncodingConversion(SoupTest):
+    # Test Beautiful Soup's ability to decode and encode from various
+    # encodings.
+
+    def setUp(self):
+        super(TestEncodingConversion, self).setUp()
+        self.unicode_data = '<html><head><meta charset="utf-8"/></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>'
+        self.utf8_data = self.unicode_data.encode("utf-8")
+        # Just so you know what it looks like.
+        self.assertEqual(
+            self.utf8_data,
+            b'<html><head><meta charset="utf-8"/></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>')
+
+    def test_ascii_in_unicode_out(self):
+        # ASCII input is converted to Unicode. The original_encoding
+        # attribute is set to 'utf-8', a superset of ASCII.
+        chardet = bs4.dammit.chardet_dammit
+        logging.disable(logging.WARNING)
+        try:
+            def noop(str):
+                return None
+            # Disable chardet, which will realize that the ASCII is ASCII.
+            bs4.dammit.chardet_dammit = noop
+            ascii = b"<foo>a</foo>"
+            soup_from_ascii = self.soup(ascii)
+            unicode_output = soup_from_ascii.decode()
+            self.assertTrue(isinstance(unicode_output, str))
+            self.assertEqual(unicode_output, self.document_for(ascii.decode()))
+            self.assertEqual(soup_from_ascii.original_encoding.lower(), "utf-8")
+        finally:
+            logging.disable(logging.NOTSET)
+            bs4.dammit.chardet_dammit = chardet
+
+    def test_unicode_in_unicode_out(self):
+        # Unicode input is left alone. The original_encoding attribute
+        # is not set.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEqual(soup_from_unicode.decode(), self.unicode_data)
+        self.assertEqual(soup_from_unicode.foo.string, 'Sacr\xe9 bleu!')
+        self.assertEqual(soup_from_unicode.original_encoding, None)
+
+    def test_utf8_in_unicode_out(self):
+        # UTF-8 input is converted to Unicode. The original_encoding
+        # attribute is set.
+        soup_from_utf8 = self.soup(self.utf8_data)
+        self.assertEqual(soup_from_utf8.decode(), self.unicode_data)
+        self.assertEqual(soup_from_utf8.foo.string, 'Sacr\xe9 bleu!')
+
+    def test_utf8_out(self):
+        # The internal data structures can be encoded as UTF-8.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data)
+
+    @skipIf(
+        PYTHON_3_PRE_3_2,
+        "Bad HTMLParser detected; skipping test of non-ASCII characters in attribute name.")
+    def test_attribute_name_containing_unicode_characters(self):
+        markup = '<div><a \N{SNOWMAN}="snowman"></a></div>'
+        self.assertEqual(self.soup(markup).div.encode("utf8"), markup.encode("utf8"))
+
+class TestUnicodeDammit(unittest.TestCase):
+    """Standalone tests of UnicodeDammit."""
+
+    def test_unicode_input(self):
+        markup = "I'm already Unicode! \N{SNOWMAN}"
+        dammit = UnicodeDammit(markup)
+        self.assertEqual(dammit.unicode_markup, markup)
+
+    def test_smart_quotes_to_unicode(self):
+        markup = b"<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup)
+        self.assertEqual(
+            dammit.unicode_markup, "<foo>\u2018\u2019\u201c\u201d</foo>")
+
+    def test_smart_quotes_to_xml_entities(self):
+        markup = b"<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup, smart_quotes_to="xml")
+        self.assertEqual(
+            dammit.unicode_markup, "<foo>&#x2018;&#x2019;&#x201C;&#x201D;</foo>")
+
+    def test_smart_quotes_to_html_entities(self):
+        markup = b"<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup, smart_quotes_to="html")
+        self.assertEqual(
+            dammit.unicode_markup, "<foo>&lsquo;&rsquo;&ldquo;&rdquo;</foo>")
+
+    def test_smart_quotes_to_ascii(self):
+        markup = b"<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup, smart_quotes_to="ascii")
+        self.assertEqual(
+            dammit.unicode_markup, """<foo>''""</foo>""")
+
+    def test_detect_utf8(self):
+        utf8 = b"Sacr\xc3\xa9 bleu! \xe2\x98\x83"
+        dammit = UnicodeDammit(utf8)
+        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+        self.assertEqual(dammit.unicode_markup, 'Sacr\xe9 bleu! \N{SNOWMAN}')
+
+
+    def test_convert_hebrew(self):
+        hebrew = b"\xed\xe5\xec\xf9"
+        dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
+        self.assertEqual(dammit.original_encoding.lower(), 'iso-8859-8')
+        self.assertEqual(dammit.unicode_markup, '\u05dd\u05d5\u05dc\u05e9')
+
+    def test_dont_see_smart_quotes_where_there_are_none(self):
+        utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
+        dammit = UnicodeDammit(utf_8)
+        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+        self.assertEqual(dammit.unicode_markup.encode("utf-8"), utf_8)
+
+    def test_ignore_inappropriate_codecs(self):
+        utf8_data = "Räksmörgås".encode("utf-8")
+        dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
+        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+
+    def test_ignore_invalid_codecs(self):
+        utf8_data = "Räksmörgås".encode("utf-8")
+        for bad_encoding in ['.utf8', '...', 'utF---16.!']:
+            dammit = UnicodeDammit(utf8_data, [bad_encoding])
+            self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+
+    def test_exclude_encodings(self):
+        # This is UTF-8.
+        utf8_data = "Räksmörgås".encode("utf-8")
+
+        # But if we exclude UTF-8 from consideration, the guess is
+        # Windows-1252.
+        dammit = UnicodeDammit(utf8_data, exclude_encodings=["utf-8"])
+        self.assertEqual(dammit.original_encoding.lower(), 'windows-1252')
+
+        # And if we exclude that, there is no valid guess at all.
+        dammit = UnicodeDammit(
+            utf8_data, exclude_encodings=["utf-8", "windows-1252"])
+        self.assertEqual(dammit.original_encoding, None)
+
+    def test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_character(self):
+        detected = EncodingDetector(
+            b'<?xml version="1.0" encoding="UTF-\xdb" ?>')
+        encodings = list(detected.encodings)
+        assert 'utf-\N{REPLACEMENT CHARACTER}' in encodings
+
+    def test_detect_html5_style_meta_tag(self):
+
+        for data in (
+            b'<html><meta charset="euc-jp" /></html>',
+            b"<html><meta charset='euc-jp' /></html>",
+            b"<html><meta charset=euc-jp /></html>",
+            b"<html><meta charset=euc-jp/></html>"):
+            dammit = UnicodeDammit(data, is_html=True)
+            self.assertEqual(
+                "euc-jp", dammit.original_encoding)
+
+    def test_last_ditch_entity_replacement(self):
+        # This is a UTF-8 document that contains bytestrings
+        # completely incompatible with UTF-8 (ie. encoded with some other
+        # encoding).
+        #
+        # Since there is no consistent encoding for the document,
+        # Unicode, Dammit will eventually encode the document as UTF-8
+        # and encode the incompatible characters as REPLACEMENT
+        # CHARACTER.
+        #
+        # If chardet is installed, it will detect that the document
+        # can be converted into ISO-8859-1 without errors. This happens
+        # to be the wrong encoding, but it is a consistent encoding, so the
+        # code we're testing here won't run.
+        #
+        # So we temporarily disable chardet if it's present.
+        doc = b"""\357\273\277<?xml version="1.0" encoding="UTF-8"?>
+<html><b>\330\250\330\252\330\261</b>
+<i>\310\322\321\220\312\321\355\344</i></html>"""
+        chardet = bs4.dammit.chardet_dammit
+        logging.disable(logging.WARNING)
+        try:
+            def noop(str):
+                return None
+            bs4.dammit.chardet_dammit = noop
+            dammit = UnicodeDammit(doc)
+            self.assertEqual(True, dammit.contains_replacement_characters)
+            self.assertTrue("\ufffd" in dammit.unicode_markup)
+
+            soup = BeautifulSoup(doc, "html.parser")
+            self.assertTrue(soup.contains_replacement_characters)
+        finally:
+            logging.disable(logging.NOTSET)
+            bs4.dammit.chardet_dammit = chardet
+
+    def test_byte_order_mark_removed(self):
+        # A document written in UTF-16LE will have its byte order marker stripped.
+        data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
+        dammit = UnicodeDammit(data)
+        self.assertEqual("<a>áé</a>", dammit.unicode_markup)
+        self.assertEqual("utf-16le", dammit.original_encoding)
+
+    def test_detwingle(self):
+        # Here's a UTF8 document.
+        utf8 = ("\N{SNOWMAN}" * 3).encode("utf8")
+
+        # Here's a Windows-1252 document.
+        windows_1252 = (
+            "\N{LEFT DOUBLE QUOTATION MARK}Hi, I like Windows!"
+            "\N{RIGHT DOUBLE QUOTATION MARK}").encode("windows_1252")
+
+        # Through some unholy alchemy, they've been stuck together.
+        doc = utf8 + windows_1252 + utf8
+
+        # The document can't be turned into UTF-8:
+        self.assertRaises(UnicodeDecodeError, doc.decode, "utf8")
+
+        # Unicode, Dammit thinks the whole document is Windows-1252,
+        # and decodes it into "☃☃☃“Hi, I like Windows!”☃☃☃"
+
+        # But if we run it through fix_embedded_windows_1252, it's fixed:
+
+        fixed = UnicodeDammit.detwingle(doc)
+        self.assertEqual(
+            "☃☃☃“Hi, I like Windows!”☃☃☃", fixed.decode("utf8"))
+
+    def test_detwingle_ignores_multibyte_characters(self):
+        # Each of these characters has a UTF-8 representation ending
+        # in \x93. \x93 is a smart quote if interpreted as
+        # Windows-1252. But our code knows to skip over multibyte
+        # UTF-8 characters, so they'll survive the process unscathed.
+        for tricky_unicode_char in (
+            "\N{LATIN SMALL LIGATURE OE}", # 2-byte char '\xc5\x93'
+            "\N{LATIN SUBSCRIPT SMALL LETTER X}", # 3-byte char '\xe2\x82\x93'
+            "\xf0\x90\x90\x93", # This is a CJK character, not sure which one.
+            ):
+            input = tricky_unicode_char.encode("utf8")
+            self.assertTrue(input.endswith(b'\x93'))
+            output = UnicodeDammit.detwingle(input)
+            self.assertEqual(output, input)
+
+    def test_find_declared_encoding(self):
+        # Test our ability to find a declared encoding inside an
+        # XML or HTML document.
+        #
+        # Even if the document comes in as Unicode, it may be
+        # interesting to know what encoding was claimed
+        # originally.
+
+        html_unicode = '<html><head><meta charset="utf-8"></head></html>'
+        html_bytes = html_unicode.encode("ascii")
+
+        xml_unicode= '<?xml version="1.0" encoding="ISO-8859-1" ?>'
+        xml_bytes = xml_unicode.encode("ascii")
+
+        m = EncodingDetector.find_declared_encoding
+        self.assertEqual(None, m(html_unicode, is_html=False))
+        self.assertEqual("utf-8", m(html_unicode, is_html=True))
+        self.assertEqual("utf-8", m(html_bytes, is_html=True))
+
+        self.assertEqual("iso-8859-1", m(xml_unicode))
+        self.assertEqual("iso-8859-1", m(xml_bytes))
+
+        # Normally, only the first few kilobytes of a document are checked for
+        # an encoding.
+        spacer = b' ' * 5000
+        self.assertEqual(None, m(spacer + html_bytes))
+        self.assertEqual(None, m(spacer + xml_bytes))
+
+        # But you can tell find_declared_encoding to search an entire
+        # HTML document.
+        self.assertEqual(
+            "utf-8",
+            m(spacer + html_bytes, is_html=True, search_entire_document=True)
+        )
+
+        # The XML encoding declaration has to be the very first thing
+        # in the document. We'll allow whitespace before the document
+        # starts, but nothing else.
+        self.assertEqual(
+            "iso-8859-1",
+            m(xml_bytes, search_entire_document=True)
+        )
+        self.assertEqual(
+            None, m(b'a' + xml_bytes, search_entire_document=True)
+        )
+            
+class TestNamedspacedAttribute(SoupTest):
+
+    def test_name_may_be_none_or_missing(self):
+        a = NamespacedAttribute("xmlns", None)
+        self.assertEqual(a, "xmlns")
+
+        a = NamespacedAttribute("xmlns")
+        self.assertEqual(a, "xmlns")
+        
+    def test_attribute_is_equivalent_to_colon_separated_string(self):
+        a = NamespacedAttribute("a", "b")
+        self.assertEqual("a:b", a)
+
+    def test_attributes_are_equivalent_if_prefix_and_name_identical(self):
+        a = NamespacedAttribute("a", "b", "c")
+        b = NamespacedAttribute("a", "b", "c")
+        self.assertEqual(a, b)
+
+        # The actual namespace is not considered.
+        c = NamespacedAttribute("a", "b", None)
+        self.assertEqual(a, c)
+
+        # But name and prefix are important.
+        d = NamespacedAttribute("a", "z", "c")
+        self.assertNotEqual(a, d)
+
+        e = NamespacedAttribute("z", "b", "c")
+        self.assertNotEqual(a, e)
+
+
+class TestAttributeValueWithCharsetSubstitution(unittest.TestCase):
+
+    def test_content_meta_attribute_value(self):
+        value = CharsetMetaAttributeValue("euc-jp")
+        self.assertEqual("euc-jp", value)
+        self.assertEqual("euc-jp", value.original_value)
+        self.assertEqual("utf8", value.encode("utf8"))
+
+
+    def test_content_meta_attribute_value(self):
+        value = ContentMetaAttributeValue("text/html; charset=euc-jp")
+        self.assertEqual("text/html; charset=euc-jp", value)
+        self.assertEqual("text/html; charset=euc-jp", value.original_value)
+        self.assertEqual("text/html; charset=utf8", value.encode("utf8"))

default.css

@@ -1,39 +0,0 @@
-html {
-    --light-bg: color(var(--background) blend(#999 85%));
-}
-body {
-    padding:10px;
-    padding-top: 0px;
-    font-family: "Open Sans", sans-serif;
-    background-color: var(--background);
-    font-size: 15px;
-}
-
-blockquote {
-    font-style: italic;
-    display: block;
-    margin-left: 30px;
-    border: 1px solid red;
-}
-
-code {
-    padding-left: 0.2rem;
-    padding-right: 0.2rem;
-    background-color: var(--light-bg);
-    margin: 0;
-    border-radius: 3px;
-    margin: 5px;
-}
-
-pre {
-    display: block;
-    margin-top: 20px;
-    line-height: 1.7;
-    background-color: var(--light-bg);
-    padding-left: 10px;
-    width: 100%;
-    border-radius: 3px;
-}
-pre code {
-    padding-left: 0;
-}

dependencies.json

@@ -0,0 +1,7 @@
+{
+    "*": {
+        "*": [
+            "bs4"
+        ]
+    }
+}

devListener.py

@@ -1,20 +0,0 @@
-# -*- encoding: utf-8 -*-
-
-import sublime
-import sublime_plugin
-import os.path
-
-class MLPDevListener(sublime_plugin.EventListener):
-
-    def on_post_save(self, view):
-        return
-        if not (os.path.dirname(__file__) in view.file_name() and
-            view.file_name().endswith('.py')):
-            return
-        sublime.run_command('reload_plugin', {
-            'main': os.path.join(sublime.packages_path(),
-                                 'MarkdownLivePreview', 'md_in_popup.py'),
-            'scripts': ['image_manager', 'functions', 'MLPApi',
-                        'setting_names'],
-            'quiet': True
-        })

escape_amp.py

@@ -1,39 +0,0 @@
-# -*- encoding: utf-8 -*-
-
-import re
-
-__all__ = ['escape_amp']
-
-RE_REPLACE_AMPERSAND = re.compile(r'&(\w*)(;)?')
-
-def replace(matchobj):
-    if matchobj.group(2):
-        return matchobj.group(0)
-    else:
-        return matchobj.group(0).replace('&', '&')
-
-def escape_amp(text):
-    return RE_REPLACE_AMPERSAND.sub(replace, text)
-
-def run_tests():
-    tests = [
-        ['&', '&'],
-        ['&amp', '&amp'],
-        ['&', '&'],
-        ['& &hello &bonjour;', '& &hello &bonjour;']
-    ]
-    fails = 0
-    for i, (subject, result) in enumerate(tests):
-        if RE_REPLACE_AMPERSAND.sub(replace, subject) != result:
-            # CSW: ignore
-            print('TEST FAIL ({i}): {subject!r} escaped did not match {result!r}'.format(**locals()))
-            fails += 1
-    if fails == 0:
-        # CSW: ignore
-        print("SUCCESS: every tests ({}) passed successfully!".format(len(tests)))
-    else:
-        # CSW: ignore
-        print("{} test{} failed".format(fails, 's' if fails > 1 else ''))
-
-if __name__ == '__main__':
-    run_tests()

functions.py

@@ -1,102 +0,0 @@
-# -*- encoding: utf-8 -*-
-import base64
-import os.path
-import sublime
-import re
-from .image_manager import ImageManager
-
-def plugin_loaded():
-    global error404, loading
-    loading = sublime.load_resource('Packages/MarkdownLivePreview/loading.txt')
-    error404 = sublime.load_resource('Packages/MarkdownLivePreview/404.txt')
-
-
-def replace_img_src_base64(html, basepath):
-    """Really messy, but it works (should be updated)"""
-    index = -1
-    tag_start = '<img src="'
-    shtml, html = html, list(html)
-    while True:
-        index = shtml.find(tag_start, index + 1)
-        if index == -1:
-            break
-        path, end = get_content_till(html, '"', start=index + len(tag_start))
-        if ''.join(path).startswith('data:image/'):
-            continue
-        if ''.join(path).startswith(tuple(get_settings().get('load_from_internet'
-                                                    '_when_starts'))):
-            image = ImageManager.get(''.join(path))
-            image = image or loading
-
-        else:
-            # local image
-            path = ''.join(path)
-            path = os.path.join(basepath, path)
-            image = to_base64(path)
-        html[index+len(tag_start):end] = image
-        shtml = ''.join(html)
-    return ''.join(html)
-
-def is_markdown_view(view):
-        return 'markdown' in view.scope_name(0)
-
-def to_base64(path=None, content=None):
-    if path is None and content is None:
-        return error404
-    elif content is None and path is not None:
-        try:
-            with open(path, 'rb') as fp:
-                content = fp.read()
-        except (FileNotFoundError, OSError):
-            return error404
-
-    return 'data:image/png;base64,' + ''.join([chr(el) for el in list(base64.standard_b64encode(content))])
-
-def md(*t, **kwargs):
-    sublime.message_dialog(kwargs.get('sep', '\n').join([str(el) for el in t]))
-
-def sm(*t, **kwargs):
-    sublime.status_message(kwargs.get('sep', ' ').join([str(el) for el in t]))
-
-def em(*t, **kwargs):
-    sublime.error_message(kwargs.get('sep', ' ').join([str(el) for el in t]))
-
-def mini(val, min):
-    if val < min:
-        return min
-    return val
-
-def get_content_till(string, char_to_look_for, start=0):
-    i = start
-    while i < len(string):
-        if string[i] == char_to_look_for:
-            return string[start:i], i
-        i += 1
-
-def get_view_content(view):
-    return view.substr(sublime.Region(0, view.size()))
-
-def get_view_from_id(window, id):
-    if not isinstance(id, int):
-        return
-    for view in window.views():
-        if view.id() == id:
-            return view
-
-def get_settings():
-    return sublime.load_settings('MarkdownLivePreview.sublime-settings')
-
-def pre_with_br(html):
-    """Because the phantoms of sublime text does not support <pre> blocks
-    this function replaces every \n with a <br> in a <pre>"""
-
-    while True:
-        obj = re.search(r'<pre>(.*?)</pre>', html, re.DOTALL)
-        if not obj:
-            break
-        html = list(html)
-        html[obj.start(0):obj.end(0)] = '<pre >' + ''.join(html[obj.start(1):obj.end(1)]) \
-                                            .replace('\n', '<br>') \
-                                            .replace(' ', '&nbsp;') + '</pre>'
-        html = ''.join(html)
-    return html

image_manager.py

@@ -1,98 +0,0 @@
-# -*- encoding: utf-8 -*-
-
-import os.path
-import tempfile
-import sublime
-from threading import Thread
-import urllib.request, urllib.error
-from .functions import *
-
-CACHE_FILE = os.path.join(tempfile.gettempdir(),
-                          'MarkdownLivePreviewCache.txt')
-TIMEOUT = 20 # seconds
-
-SEPARATOR = '---%cache%--'
-
-def get_base64_saver(loading, url):
-    def callback(content):
-        if isinstance(content, urllib.error.HTTPError):
-            if content.getcode() == 404:
-                loading[url] = 404
-                return
-        elif isinstance(content, urllib.error.URLError):
-            if (content.reason.errno == 11001 and
-                content.reason.strerror == 'getaddrinfo failed'):
-                loading[url] = 404
-                return
-            return sublime.error_message('An unexpected error has occured: ' +
-                                         str(content))
-        loading[url] = to_base64(content=content)
-
-    return callback
-
-def get_cache_for(imageurl):
-    if not os.path.exists(CACHE_FILE):
-        return
-    with open(CACHE_FILE) as fp:
-        for line in fp.read().splitlines():
-            url, base64 = line.split(SEPARATOR, 1)
-            if url == imageurl:
-                return base64
-
-def cache(imageurl, base64):
-    with open(CACHE_FILE, 'a') as fp:
-        fp.write(imageurl + SEPARATOR + base64 + '\n')
-
-class ImageLoader(Thread):
-
-    def __init__(self, url, callback):
-        Thread.__init__(self)
-        self.url = url
-        self.callback = callback
-
-    def run(self):
-        try:
-            page = urllib.request.urlopen(self.url, None, TIMEOUT)
-        except Exception as e:
-            self.callback(e)
-        else:
-            self.callback(page.read())
-
-
-class ImageManager(object):
-
-    """
-        Usage:
-
-        >>> image = ImageManager.get('http://domain.com/image.png')
-        >>> image = ImageManager.get('http://domain.com/image.png')
-        # still loading (this is a comment, no an outputed text), it doesn't
-        # run an other request
-        >>> image = ImageManager.get('http://domain.com/image.png')
-        'data:image/png;base64,....'
-    """
-    loading = {}
-
-    @staticmethod
-    def get(imageurl, user_callback=None):
-
-        cached = get_cache_for(imageurl)
-        if cached:
-            return cached
-        elif imageurl in ImageManager.loading.keys():
-            # return None (the file is still loading, already made a request)
-            # return string the base64 of the url (which is going to be cached)
-            temp_cached = ImageManager.loading[imageurl]
-            if temp_cached == 404:
-                return to_base64('404.png')
-            if temp_cached:
-                cache(imageurl, temp_cached)
-                del ImageManager.loading[imageurl]
-            return temp_cached
-        else:
-            # load from internet
-            ImageManager.loading[imageurl] = None
-            callback = get_base64_saver(ImageManager.loading, imageurl)
-            loader = ImageLoader(imageurl, callback)
-            loader.start()
-            sublime.set_timeout_async(lambda: loader.join(), TIMEOUT * 1000)

imgs/README.md

@@ -1 +0,0 @@
-There is images here, allthough they aren't of any use for the plugin. They're just the image that I used to generate the base64 (404.txt and loading.txt)

live-testing/images.md

@@ -0,0 +1,18 @@
+I'm not sure that it **actually** going to work, but it seems nicer than the [previous version][prev]
+
+this is a test, hello world
+
+This is the first image from the local file system (absolute path, sorry, it's not going
+to work on your system unless your username is math2001):
+
+![The sublime text logo!](file:///home/math2001/.config/sublime-text-3/Packages/MarkdownLivePreview/live-testing/sublime_text.png)
+
+This is the first image from the local file system, *relative* path!
+
+![The sublime text logo!](sublime_merge.png)
+
+This is the first image from the internet!
+
+![math2001's logo](https://avatars1.githubusercontent.com/u/15224242?s=400&u=53324cf4e303d15032ba53aa41673a2046b3284b&v=4)
+
+[prev]: https://github.com/math2001/MarkdownLivePreview/tree/d4c477749ce7e77b8e9fc85464a2488f003c45bc

live-testing/test.md

@@ -0,0 +1,52 @@
+# Why?
+Why do you want to use fancy symbols in your standard monospace font? Obviously to have a fancy prompt like mine :-)
+
+<!-- ![prompt](https://github.com/gabrielelana/awesome-terminal-fonts/raw/master/why.png) -->
+
+And because when you live in a terminal a symbol can convey more informations in less space creating a dense and beautiful (for those who have a certain aesthetic taste) informative workspace
+
+Heavily inspired by <https://github.com/Lokaltog/vim-powerline> and the relative patch script from **Kim Silkebækken** (kim.silkebaekken+vim@gmail.com)
+
+## Patching vs Fallback
+There are two strategies that could be used to have symbols in a terminal
+* you can take a bunch of symbol fonts, your favourite monospace font and merge them together (patching strategy)
+* you can use a feature of `freetype2` font engine, basically you can say that whenever the current font doesn't have a glyph for a certain codepoint then fallback and go look into other fonts (fallback strategy)
+
+Initially I used the first strategy, later I switched to the second. The patching strategy it's more reliable and portable, the problem is that you need to patch every monospace font you want to use and patching a single font it's a lot of manual fine tuning. If you want you can find all previous patched fonts in [patching-strategy branch](https://github.com/gabrielelana/awesome-terminal-fonts/tree/patching-strategy)
+
+## Font Maps
+Referring to glyphs by codepints (eg. `\uf00c`) in your scripts or shell configuration it's not recommended because icon fonts like [Font Awesome](http://fontawesome.io/) use [code points ranges](https://en.wikipedia.org/wiki/Private_Use_Areas) those ranges are not disciplined by the unicode consortium, every font can associate every glyphs to those codepoints. This means that [Font Awesome](http://fontawesome.io/) can choose to move glyphs around freely, today `\uf00c` is associated to the `check` symbol, tomorrow it can be associated to something else. Moreover, more than one icon font can use the same codepoint for different glyphs and if we want to use them both we need to move one of them. So, if you use a codepoint to refer to a glyph after an update that codepoint can point to another glyph. To avoid this situation you can use the font maps in the `./build` directory, font maps are scripts which define shell variables that give names to glyphs, by sourcing those files in your shell you can refer to glyphs by name (eg. `$CODEPOINT_OF_AWESOME_CHECK`).
+
+TLDR: don't refer to glyphs by codepoints (eg. `\uf00c`) but by name (eg. `$CODEPOINT_OF_AWESOME_CHECK`) to make your scripts and shell configurations resilient to future updates. To do that don't forget to copy font maps (`*.sh` files) in the `./build` directory in your home directory and to source them in your shell startup
+
+## Included Fonts
+In this repository you can find a bunch of fonts that I use as symbol fonts with the relative font maps
+* **Font Awesome 4.7.0**: `./fonts/fontawesome-regular.ttf`, for further informations and license see http://fortawesome.github.io/Font-Awesome
+* **Devicons 1.8.0**: `./fonts/devicons-regular.ttf`, for further informations and license see https://github.com/vorillaz/devicons
+* **Octicons 1.0.0**: `./fonts/octicons-regular.ttf`, for further informations and license see https://github.com/blog/1135-the-making-of-octicons
+* **Pomicons 1.0.0**: `./fonts/pomicons-regular.ttf`, for further informations and license see https://github.com/gabrielelana/pomicons
+
+## How to install (Linux)
+* copy all the fonts from `./build` directory to `~/.fonts` directory
+* copy all the font maps (all `*.sh` files) from `./build` directory to `~/.fonts` directory
+* run `fc-cache -fv ~/.fonts` to let freetype2 know of those fonts
+* customize the configuration file `./config/10-symbols.conf` replacing `PragmataPro` with the name of the font you want to use in the terminal (I will add more fonts in the future so that this step could be skippable)
+* copy the above configuration file to `~/.config/fontconfig/conf.d` directory
+* source the font maps (`source ~/.fonts/*.sh`) in your shell startup script (eg. `~/.bashrc` or `~/.zshrc`)
+
+### Arch Linux
+We have been included in the [official repositories](https://www.archlinux.org/packages/community/any/awesome-terminal-fonts/), so if you are running an Arch Linux
+* run `pacman -Syu awesome-terminal-fonts`
+
+## How to install (OSX)
+* follow [this detailed instructions](https://github.com/gabrielelana/awesome-terminal-fonts/wiki/OS-X) contributed by [@inkrement](https://github.com/inkrement)
+* copy all the fonts maps (all `*.sh` files) from `./build` directory to `~/.fonts` directory
+* source the font maps (`source ~/.fonts/*.sh`) in your shell startup script (eg. `~/.bashrc` or `~/.zshrc`)
+* If it still doesn't work, consider to use the [patching strategy](#patching-vs-fallback)
+
+## How to install (Windows)
+* make sure you have permissions to execute Powershell scripts in your machine. To do so, open Windows Powershell as Administrator and paste & run the following command `Set-ExecutionPolicy RemoteSigned`
+* then run the install script `./install.ps1`
+
+## License
+[MIT](https://github.com/gabrielelana/awesome-terminal-fonts/blob/master/LICENSE)

loading.txt

@@ -1 +0,0 @@
-data:image/png;base64,R0lGODlhZABkAPU+AAAAAAwNDRweHyYpKzg8Pzo+QUBFSERJTEdMT05UV1NYXFVbX1hfY1lfZGFobWJpbmhvdGxzeHF5fnJ6gHV9g3Z+hHqDiXuEin+IjoCIjoKLkYKMkoSNk4eQl4iSmIqTmouUm42XnY+ZoJKco5OdpJOepZSeppahqJeiqZmjqpumrZ6psJ+qsqOutqSvt6SwuKezu6i0vKm1vay4wK66wq+7w6+8xLK+xrK/x7TAybXCy7bDy7jEzbjFzrzJ0gAAACH5BAUAAD4AIf8LTkVUU0NBUEUyLjADAQAAACwAAAAAZABkAAAG/kCfcEgsGo/IpHLJbDqf0Kh0Sq1ar9isdsvter/gsHhMLpvP6LR6zW673/C4fE6v2+/4vH7P7/v/gIGCg4SFhoeIiW8IAAAUilUpjQABkEsmMUchkwBIOTOQBQICGUabk0ctFhYdiSajAgOZRKeNRjkYqxaghyuwAgxFtZ1FJBe6NokHvya0nEUzuhYgijG/B86oRCDSOZAPv6VCw0SquiiWNwOwAzfjz0I8uasYPIMvDQ0kRhm/Ee/afKiQ1sIIDBAgkuUxQKDhA29ERMHK9GJSJR85pLUiwkOELgx6Goo0sG/IK1gVhCig9MjHimOreAmBMU+XngciRTrAMSQB/qxmR6KtEjGko7Shey7kbGgA6A0GBz4oOUjCno8YNXWp6NOCwVICD6UYPQqiBiANDHNOkILiqIYVg2Y0yPlAikddICASQtuwJJQY9OAimqFCZpRPei0pPnKjg4fHkB936JBYyg4VmDNrVlH5zYMFoEOLZgDBSoejqDfQEc1atBXUsOl8bi26bpUNsKWpnlPjg+PIj32brZJjs/HOi5PjiMFzCo4ZyAWpqCBhwgspMFa9NZRjg4TvEjZCEQFzWvQ9KiiA/+73SVtpGAT7mcFh/XcPVqH0uCsNhDs+J9gnAQXX+cADCSDMggRVVtGE2lZ6fCAgfkPcdYFhRAhlAVHxxfCnC4d42EdghtII1hYGLgjxki6GOSiNHtR990F+QpymizcZ0SNEjquI1+FHetDHQYFEuCANhBpaMMRAuqRYxEEJDSLPR1YlWVRN9Vjy3ioFCWHlEC6Uh44iOcB0gQck2kSEB90o4sEFx1yY5irQ9JdIDdIANcSXRBiDzGAfVcbnELiwmEgHx3Q5p5JGmOPjIdAF9eIRnyRnhA1AWvqEn4pq6umnoIYq6qiklmrqqaimquqqrLbq6quwxirrrLTWauuttwYBADs=

markdown2html.py

@@ -0,0 +1,226 @@
+""" Notice how this file is completely independent of sublime text
+
+I think it should be kept this way, just because it gives a bit more organisation,
+and makes it a lot easier to think about, and for anyone who would want to, test since
+markdown2html is just a pure function
+"""
+
+import io
+import struct
+import os.path
+import concurrent.futures
+import urllib.request
+import base64
+import bs4
+
+from functools import partial
+
+from .lib.markdown2 import Markdown
+
+__all__ = ("markdown2html",)
+
+markdowner = Markdown(extras=["fenced-code-blocks", "cuddled-lists"])
+
+# FIXME: how do I choose how many workers I want? Does thread pool reuse threads or
+#        does it stupidly throw them out? (we could implement something of our own)
+executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)
+
+def markdown2html(markdown, basepath, re_render, resources, viewport_width, font_scale=1.0):
+    """ converts the markdown to html, loads the images and puts in base64 for sublime
+    to understand them correctly. That means that we are responsible for loading the
+    images from the internet. Hence, we take in re_render, which is just a function we 
+    call when an image has finished loading to retrigger a render (see #90)
+    """
+    html = markdowner.convert(markdown)
+
+    soup = bs4.BeautifulSoup(html, "html.parser")
+    for img_element in soup.find_all("img"):
+        src = img_element["src"]
+
+        # already in base64, or something of the like
+        # FIXME: what other types are possible? Are they handled by ST? If not, could we
+        #        convert it into base64? is it worth the effort?
+        if src.startswith("data:image/"):
+            continue
+
+        if src.startswith("http://") or src.startswith("https://"):
+            path = src
+        elif src.startswith("file://"):
+            path = src[len("file://") :]
+        else:
+            # expanduser: ~ -> /home/math2001
+            # realpath: simplify that paths so that we don't have duplicated caches
+            path = os.path.realpath(os.path.expanduser(os.path.join(basepath, src)))
+
+        base64, (width, height) = get_base64_image(path, re_render, resources)
+
+        img_element["src"] = base64
+        if width > viewport_width:
+            img_element["width"] = viewport_width
+            img_element["height"] = viewport_width * (height / width)
+
+    # remove comments, because they pollute the console with error messages
+    for comment_element in soup.find_all(
+        text=lambda text: isinstance(text, bs4.Comment)
+    ):
+        comment_element.extract()
+
+    # FIXME: how do tables look? should we use ascii tables?
+
+    # pre aren't handled by ST3. The require manual adjustment
+    for pre_element in soup.find_all("pre"):
+        # select the first child, <code>
+        code_element = next(pre_element.children)
+
+        # FIXME: this method sucks, but can we do better?
+        fixed_pre = (
+            str(code_element)
+            .replace(" ", '<i class="space">.</i>')
+            .replace("\n", "<br />")
+        )
+
+        code_element.replace_with(bs4.BeautifulSoup(fixed_pre, "html.parser"))
+
+    # FIXME: highlight the code using Sublime's syntax
+
+    # FIXME: report that ST doesn't support <br/> but does work with <br />... WTF?
+    # Add font scaling CSS rule
+    font_scale_css = "body {{ font-size: {}em; }}\n".format(font_scale)
+    stylesheet = font_scale_css + resources["stylesheet"]
+
+    return "<style>\n{}\n</style>\n\n{}".format(stylesheet, soup).replace(
+        "<br/>", "<br />"
+    )
+
+images_cache = {}
+images_loading = []
+
+def get_base64_image(path, re_render, resources):
+    """ Gets the base64 for the image (local and remote images). re_render is a
+    callback which is called when we finish loading an image from the internet
+    to trigger an update of the preview (the image will then be loaded from the cache)
+
+    return base64_data, (width, height)
+    """
+
+    def callback(path, resources, future):
+        # altering images_cache is "safe" to do because callback is called in the same
+        # thread as add_done_callback:
+        # > Added callables are called in the order that they were added and are always
+        # > called in a thread belonging to the process that added them
+        # > --- Python docs
+        try:
+            images_cache[path] = future.result()
+        except urllib.error.HTTPError as e:
+            images_cache[path] = resources['base64_404_image']
+            print("Error loading {!r}: {!r}".format(path, e))
+
+        images_loading.remove(path)
+
+        # we render, which means this function will be called again, but this time, we
+        # will read from the cache
+        re_render()
+
+    if path in images_cache:
+        return images_cache[path]
+
+    if path.startswith("http://") or path.startswith("https://"):
+        # FIXME: submiting a load of loaders, we should only have one
+        if path not in images_loading:
+            executor.submit(load_image, path).add_done_callback(partial(callback, path, resources))
+            images_loading.append(path)
+        return resources['base64_loading_image']
+
+    with open(path, "rb") as fhandle:
+        image_content = fhandle.read()
+        width, height = get_image_size(io.BytesIO(image_content), path)
+
+        image = "data:image/png;base64," + base64.b64encode(image_content).decode(
+            "utf-8"
+        )
+        images_cache[path] = image, (width, height)
+        return images_cache[path]
+
+
+def load_image(url):
+    with urllib.request.urlopen(url, timeout=60) as conn:
+
+        image_content = conn.read()
+        width, height = get_image_size(io.BytesIO(image_content), url)
+
+        content_type = conn.info().get_content_type()
+        if "image" not in content_type:
+            raise ValueError(
+                "{!r} doesn't point to an image, but to a {!r}".format(
+                    url, content_type
+                )
+            )
+        return (
+            "data:image/png;base64," + base64.b64encode(image_content).decode("utf-8"),
+            (width, height),
+        )
+
+
+def get_image_size(fhandle, pathlike):
+    """ Thanks to https://stackoverflow.com/a/20380514/6164984 for providing the basis
+        of a working solution.
+
+    fhandle should be a seekable stream. It's not the best for non-seekable streams,
+    but in our case, we have to load the whole stream into memory anyway because base64
+    library only accepts bytes-like objects, and not streams.
+
+    pathlike is the filename/path/url of the image so that we can guess the file format
+    """
+
+    format_ = os.path.splitext(os.path.basename(pathlike))[1][1:]
+
+    head = fhandle.read(24)
+    if len(head) != 24:
+        return "invalid head"
+    if format_ == "png":
+        check = struct.unpack(">i", head[4:8])[0]
+        if check != 0x0D0A1A0A:
+            return
+        width, height = struct.unpack(">ii", head[16:24])
+    elif format_ == "gif":
+        width, height = struct.unpack("<HH", head[6:10])
+    elif format_ == "jpeg":
+        try:
+            fhandle.seek(0)  # Read 0xff next
+
+            size = 2
+            ftype = 0
+            while not 0xC0 <= ftype <= 0xCF:
+                fhandle.seek(size, 1)
+                byte = fhandle.read(1)
+                if byte == b"":
+                    fhandle = end
+                    byte = fhandle.read(1)
+
+                while ord(byte) == 0xFF:
+                    byte = fhandle.read(1)
+                ftype = ord(byte)
+                size = struct.unpack(">H", fhandle.read(2))[0] - 2
+            # We are at a SOFn block
+            fhandle.seek(1, 1)  # Skip `precision' byte.
+            height, width = struct.unpack(">HH", fhandle.read(4))
+        except Exception as e:  # IGNORE:W0703
+            raise e
+    else:
+        return "unknown format {!r}".format(format_)
+    return width, height
+
+
+def independent_markdown2html(markdown):
+    return markdown2html(
+        markdown,
+        ".",
+        lambda: None,
+        {
+            "base64_404_image": ("", (0, 0)),
+            "base64_loading_image": ("", (0, 0)),
+            "stylesheet": "",
+        },
+        960,
+        1.0, # Add default font_scale for independent call
+    )

messages.json

@@ -2,5 +2,6 @@
     "install": "messages/install.txt",
     "1.1.2": "messages/1.1.2.txt",
     "2.0.1": "messages/2.0.1.txt",
-    "2.1.1": "messages/2.0.1.txt"
+    "2.2.1": "messages/2.2.0.txt",
+    "2.4.1": "messages/2.4.txt"
 }

messages/2.1.0.txt

@@ -3,3 +3,6 @@ Sorry to interrupt you... :(
 A settings is now available: `markdown_live_preview_on_open`. If set to true,
 it opens the window preview as soon as you open a markdown file. See the
 README for more infos.
+
+Tip of the day: `ctrl+w`: closes the current file
+                `ctrl+shift+w`: closes the current window

messages/2.2.0.txt

@@ -0,0 +1,8 @@
+Sorry to interrupt you... :(
+
+Something changed on MarkdownLivePreview: the preview is now scrolled to where
+you are editing your markdown file! It doesn't scroll back up all the time any
+more!
+
+Tip of the day: You can center the screen on your cursor by pressing:
+               `ctrl+k, ctrl+v` (on OSX `cmd+k, cmd+v`)

messages/2.4.txt

@@ -0,0 +1,10 @@
+Sorry to interrupt you... :(
+
+Some stuff changed on MarkdownLivePreview. It now supports YAML/TOML front matters. You can hide it,
+or show it in a pre block (edit your settings for this).
+
+Hope you'll enjoy it!
+
+Tip of the day: If you want a VIM-like search feature, then just press 'ctrl/cmd+i'
+                (Find → Incremental find). You can still go the next match by pressing 'f3', and to
+                the previous one by pressing 'shift+f3'

messages/install.txt

@@ -10,10 +10,10 @@ Thanks for installing MarkdownLivePreview! I hope you'll enjoy using it!
 Quick Start:
 ~~~~~~~~~~~~
 
-To enable MarkdownLivePreview, you need to me on a markdown view (works with
+To enable MarkdownLivePreview, you need to be on a markdown view (works with
 Markdown Extended). Then just press `alt+m`, or search up in the command
 palette: 'MarkdownLivePreview: Toggle'. Hit enter and you're done. As soon as
-you'll type anything in, it'll show up the preview in a new group.
+you type anything in, it'll show up with the preview in a new group.
 
 Say thanks:
 ~~~~~~~~~~~

repository.json

@@ -0,0 +1,15 @@
+{
+  "schema_version": "3",
+  "packages": [
+    {
+      "name": "MarkdownLivePreview-Fork",
+      "description": "My enhanced live-preview fork of MarkdownLivePreview",
+      "releases": [
+        {
+          "version": "1.0.0",
+          "url": "https://git.0x42.cloud/christian.morpurgo/MarkdownLivePreview/archive/v0.0.0.zip"
+        }
+      ]
+    }
+  ]
+}

resources/convertresources.py

@@ -0,0 +1,34 @@
+""" A small script to convert the images into base64 data """
+
+import struct
+from base64 import b64encode
+
+
+def get_image_size(fhandle):
+    """https://stackoverflow.com/a/20380514/6164984"""
+    head = fhandle.read(24)
+    if len(head) != 24:
+        return
+
+    # always going to be png
+    check = struct.unpack(">i", head[4:8])[0]
+    if check != 0x0D0A1A0A:
+        raise ValueError("invalid check (?)")
+
+    width, height = struct.unpack(">ii", head[16:24])
+    return width, height
+
+
+def make_cache(image_name):
+    with open("{}.png".format(image_name), "rb") as png, open(
+        "{}.base64".format(image_name), "wb"
+    ) as base64:
+        width, height = get_image_size(png)
+        png.seek(0)
+        base64.write(bytes("{}\n{}\n".format(width, height), encoding="utf-8"))
+        base64.write(b'data:image/png;base64,')
+        base64.write(b64encode(png.read()))
+
+
+make_cache("404")
+make_cache("loading")

resources/stylesheet.css

@@ -0,0 +1,45 @@
+html {
+    --light-bg: color(var(--background) blend(#fff 90%));
+    --very-light-bg: color(var(--background) blend(#fff 85%));
+}
+
+body {
+    font-family: "Ubuntu", "DejaVu Sans", "Open Sans", sans-serif;
+    font-size: 15px;
+}
+
+blockquote {
+    font-style: italic;
+    display: block;
+    margin-left: 30px;
+}
+
+pre {
+    display: block;
+    background-color: var(--very-light-bg);
+    margin: 10px 0;
+}
+
+code {
+    padding-left: 0.2rem;
+    padding-right: 0.2rem;
+    margin: 0;
+    border-radius: 3px;
+    background-color: red;
+}
+
+code {
+    padding-left: 0.2rem;
+    padding-right: 0.2rem;
+    background-color: var(--very-light-bg);
+    margin: 0;
+    border-radius: 3px;
+}
+
+pre code {
+    padding: 0;
+}
+
+i.space {
+    color: var(--very-light-bg);
+}

sample.md

@@ -1,15 +0,0 @@
-# DuckDuckGo - The Search engine you'll fall in love
-
-Hope you'll enjoy using MarkdownLivePreview!
-
-![Sublime Text Forum](https://forum.sublimetext.com/uploads/st-forum-wide.png)
-
-```python
-print('Hello world')
-if DEBUG:
-    print('DEBUG_MODE on')
-```
-
-> Only a fool knows everything. A wise man knows how little he knows
-
-The only think I know right now is that Boxy Theme's just awesome.

setting_names.py

@@ -1,8 +0,0 @@
-# -*- encoding: utf-8 -*-
-
-PREVIEW_ENABLED = 'markdown_live_preview_enabled'
-PREVIEW_ID = 'markdown_live_preview_id'
-IS_PREVIEW = 'is_markdown_live_preview'
-IS_HIDDEN = 'is_hidden_markdown_live_preview'
-MD_VIEW_ID = 'markdown_live_preview_md_id'
-PREVIEW_WINDOW = 'markdown_live_preview_window'

soupsieve/__init__.py

@@ -0,0 +1,111 @@
+"""
+Soup Sieve.
+
+A CSS selector filter for BeautifulSoup4.
+
+MIT License
+
+Copyright (c) 2018 Isaac Muse
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+from .__meta__ import __version__, __version_info__  # noqa: F401
+from . import css_parser as cp
+from . import css_match as cm
+from . import css_types as ct
+from .util import DEBUG, SelectorSyntaxError  # noqa: F401
+
+__all__ = (
+    'DEBUG', 'SelectorSyntaxError', 'SoupSieve',
+    'closest', 'compile', 'filter', 'iselect',
+    'match', 'select', 'select_one'
+)
+
+SoupSieve = cm.SoupSieve
+
+
+def compile(pattern, namespaces=None, flags=0, **kwargs):  # noqa: A001
+    """Compile CSS pattern."""
+
+    if namespaces is not None:
+        namespaces = ct.Namespaces(namespaces)
+
+    custom = kwargs.get('custom')
+    if custom is not None:
+        custom = ct.CustomSelectors(custom)
+
+    if isinstance(pattern, SoupSieve):
+        if flags:
+            raise ValueError("Cannot process 'flags' argument on a compiled selector list")
+        elif namespaces is not None:
+            raise ValueError("Cannot process 'namespaces' argument on a compiled selector list")
+        elif custom is not None:
+            raise ValueError("Cannot process 'custom' argument on a compiled selector list")
+        return pattern
+
+    return cp._cached_css_compile(pattern, namespaces, custom, flags)
+
+
+def purge():
+    """Purge cached patterns."""
+
+    cp._purge_cache()
+
+
+def closest(select, tag, namespaces=None, flags=0, **kwargs):
+    """Match closest ancestor."""
+
+    return compile(select, namespaces, flags, **kwargs).closest(tag)
+
+
+def match(select, tag, namespaces=None, flags=0, **kwargs):
+    """Match node."""
+
+    return compile(select, namespaces, flags, **kwargs).match(tag)
+
+
+def filter(select, iterable, namespaces=None, flags=0, **kwargs):  # noqa: A001
+    """Filter list of nodes."""
+
+    return compile(select, namespaces, flags, **kwargs).filter(iterable)
+
+
+def select_one(select, tag, namespaces=None, flags=0, **kwargs):
+    """Select a single tag."""
+
+    return compile(select, namespaces, flags, **kwargs).select_one(tag)
+
+
+def select(select, tag, namespaces=None, limit=0, flags=0, **kwargs):
+    """Select the specified tags."""
+
+    return compile(select, namespaces, flags, **kwargs).select(tag, limit)
+
+
+def iselect(select, tag, namespaces=None, limit=0, flags=0, **kwargs):
+    """Iterate the specified tags."""
+
+    for el in compile(select, namespaces, flags, **kwargs).iselect(tag, limit):
+        yield el
+
+
+def escape(ident):
+    """Escape identifier."""
+
+    return cp.escape(ident)

soupsieve/__meta__.py

@@ -0,0 +1,192 @@
+"""Meta related things."""
+from collections import namedtuple
+import re
+
+RE_VER = re.compile(
+    r'''(?x)
+    (?P<major>\d+)(?:\.(?P<minor>\d+))?(?:\.(?P<micro>\d+))?
+    (?:(?P<type>a|b|rc)(?P<pre>\d+))?
+    (?:\.post(?P<post>\d+))?
+    (?:\.dev(?P<dev>\d+))?
+    '''
+)
+
+REL_MAP = {
+    ".dev": "",
+    ".dev-alpha": "a",
+    ".dev-beta": "b",
+    ".dev-candidate": "rc",
+    "alpha": "a",
+    "beta": "b",
+    "candidate": "rc",
+    "final": ""
+}
+
+DEV_STATUS = {
+    ".dev": "2 - Pre-Alpha",
+    ".dev-alpha": "2 - Pre-Alpha",
+    ".dev-beta": "2 - Pre-Alpha",
+    ".dev-candidate": "2 - Pre-Alpha",
+    "alpha": "3 - Alpha",
+    "beta": "4 - Beta",
+    "candidate": "4 - Beta",
+    "final": "5 - Production/Stable"
+}
+
+PRE_REL_MAP = {"a": 'alpha', "b": 'beta', "rc": 'candidate'}
+
+
+class Version(namedtuple("Version", ["major", "minor", "micro", "release", "pre", "post", "dev"])):
+    """
+    Get the version (PEP 440).
+
+    A biased approach to the PEP 440 semantic version.
+
+    Provides a tuple structure which is sorted for comparisons `v1 > v2` etc.
+      (major, minor, micro, release type, pre-release build, post-release build, development release build)
+    Release types are named in is such a way they are comparable with ease.
+    Accessors to check if a development, pre-release, or post-release build. Also provides accessor to get
+    development status for setup files.
+
+    How it works (currently):
+
+    - You must specify a release type as either `final`, `alpha`, `beta`, or `candidate`.
+    - To define a development release, you can use either `.dev`, `.dev-alpha`, `.dev-beta`, or `.dev-candidate`.
+      The dot is used to ensure all development specifiers are sorted before `alpha`.
+      You can specify a `dev` number for development builds, but do not have to as implicit development releases
+      are allowed.
+    - You must specify a `pre` value greater than zero if using a prerelease as this project (not PEP 440) does not
+      allow implicit prereleases.
+    - You can optionally set `post` to a value greater than zero to make the build a post release. While post releases
+      are technically allowed in prereleases, it is strongly discouraged, so we are rejecting them. It should be
+      noted that we do not allow `post0` even though PEP 440 does not restrict this. This project specifically
+      does not allow implicit post releases.
+    - It should be noted that we do not support epochs `1!` or local versions `+some-custom.version-1`.
+
+    Acceptable version releases:
+
+    ```
+    Version(1, 0, 0, "final")                    1.0
+    Version(1, 2, 0, "final")                    1.2
+    Version(1, 2, 3, "final")                    1.2.3
+    Version(1, 2, 0, ".dev-alpha", pre=4)        1.2a4
+    Version(1, 2, 0, ".dev-beta", pre=4)         1.2b4
+    Version(1, 2, 0, ".dev-candidate", pre=4)    1.2rc4
+    Version(1, 2, 0, "final", post=1)            1.2.post1
+    Version(1, 2, 3, ".dev")                     1.2.3.dev0
+    Version(1, 2, 3, ".dev", dev=1)              1.2.3.dev1
+    ```
+
+    """
+
+    def __new__(cls, major, minor, micro, release="final", pre=0, post=0, dev=0):
+        """Validate version info."""
+
+        # Ensure all parts are positive integers.
+        for value in (major, minor, micro, pre, post):
+            if not (isinstance(value, int) and value >= 0):
+                raise ValueError("All version parts except 'release' should be integers.")
+
+        if release not in REL_MAP:
+            raise ValueError("'{}' is not a valid release type.".format(release))
+
+        # Ensure valid pre-release (we do not allow implicit pre-releases).
+        if ".dev-candidate" < release < "final":
+            if pre == 0:
+                raise ValueError("Implicit pre-releases not allowed.")
+            elif dev:
+                raise ValueError("Version is not a development release.")
+            elif post:
+                raise ValueError("Post-releases are not allowed with pre-releases.")
+
+        # Ensure valid development or development/pre release
+        elif release < "alpha":
+            if release > ".dev" and pre == 0:
+                raise ValueError("Implicit pre-release not allowed.")
+            elif post:
+                raise ValueError("Post-releases are not allowed with pre-releases.")
+
+        # Ensure a valid normal release
+        else:
+            if pre:
+                raise ValueError("Version is not a pre-release.")
+            elif dev:
+                raise ValueError("Version is not a development release.")
+
+        return super(Version, cls).__new__(cls, major, minor, micro, release, pre, post, dev)
+
+    def _is_pre(self):
+        """Is prerelease."""
+
+        return self.pre > 0
+
+    def _is_dev(self):
+        """Is development."""
+
+        return bool(self.release < "alpha")
+
+    def _is_post(self):
+        """Is post."""
+
+        return self.post > 0
+
+    def _get_dev_status(self):  # pragma: no cover
+        """Get development status string."""
+
+        return DEV_STATUS[self.release]
+
+    def _get_canonical(self):
+        """Get the canonical output string."""
+
+        # Assemble major, minor, micro version and append `pre`, `post`, or `dev` if needed..
+        if self.micro == 0:
+            ver = "{}.{}".format(self.major, self.minor)
+        else:
+            ver = "{}.{}.{}".format(self.major, self.minor, self.micro)
+        if self._is_pre():
+            ver += '{}{}'.format(REL_MAP[self.release], self.pre)
+        if self._is_post():
+            ver += ".post{}".format(self.post)
+        if self._is_dev():
+            ver += ".dev{}".format(self.dev)
+
+        return ver
+
+
+def parse_version(ver):
+    """Parse version into a comparable Version tuple."""
+
+    m = RE_VER.match(ver)
+
+    if m is None:
+        raise ValueError("'{}' is not a valid version".format(ver))
+
+    # Handle major, minor, micro
+    major = int(m.group('major'))
+    minor = int(m.group('minor')) if m.group('minor') else 0
+    micro = int(m.group('micro')) if m.group('micro') else 0
+
+    # Handle pre releases
+    if m.group('type'):
+        release = PRE_REL_MAP[m.group('type')]
+        pre = int(m.group('pre'))
+    else:
+        release = "final"
+        pre = 0
+
+    # Handle development releases
+    dev = m.group('dev') if m.group('dev') else 0
+    if m.group('dev'):
+        dev = int(m.group('dev'))
+        release = '.dev-' + release if pre else '.dev'
+    else:
+        dev = 0
+
+    # Handle post
+    post = int(m.group('post')) if m.group('post') else 0
+
+    return Version(major, minor, micro, release, pre, post, dev)
+
+
+__version_info__ = Version(2, 2, 1, "final")
+__version__ = __version_info__._get_canonical()

soupsieve/css_types.py

@@ -0,0 +1,344 @@
+"""CSS selector structure items."""
+import copyreg
+from collections.abc import Hashable, Mapping
+
+__all__ = (
+    'Selector',
+    'SelectorNull',
+    'SelectorTag',
+    'SelectorAttribute',
+    'SelectorContains',
+    'SelectorNth',
+    'SelectorLang',
+    'SelectorList',
+    'Namespaces',
+    'CustomSelectors'
+)
+
+
+SEL_EMPTY = 0x1
+SEL_ROOT = 0x2
+SEL_DEFAULT = 0x4
+SEL_INDETERMINATE = 0x8
+SEL_SCOPE = 0x10
+SEL_DIR_LTR = 0x20
+SEL_DIR_RTL = 0x40
+SEL_IN_RANGE = 0x80
+SEL_OUT_OF_RANGE = 0x100
+SEL_DEFINED = 0x200
+SEL_PLACEHOLDER_SHOWN = 0x400
+
+
+class Immutable(object):
+    """Immutable."""
+
+    __slots__ = ('_hash',)
+
+    def __init__(self, **kwargs):
+        """Initialize."""
+
+        temp = []
+        for k, v in kwargs.items():
+            temp.append(type(v))
+            temp.append(v)
+            super(Immutable, self).__setattr__(k, v)
+        super(Immutable, self).__setattr__('_hash', hash(tuple(temp)))
+
+    @classmethod
+    def __base__(cls):
+        """Get base class."""
+
+        return cls
+
+    def __eq__(self, other):
+        """Equal."""
+
+        return (
+            isinstance(other, self.__base__()) and
+            all([getattr(other, key) == getattr(self, key) for key in self.__slots__ if key != '_hash'])
+        )
+
+    def __ne__(self, other):
+        """Equal."""
+
+        return (
+            not isinstance(other, self.__base__()) or
+            any([getattr(other, key) != getattr(self, key) for key in self.__slots__ if key != '_hash'])
+        )
+
+    def __hash__(self):
+        """Hash."""
+
+        return self._hash
+
+    def __setattr__(self, name, value):
+        """Prevent mutability."""
+
+        raise AttributeError("'{}' is immutable".format(self.__class__.__name__))
+
+    def __repr__(self):  # pragma: no cover
+        """Representation."""
+
+        return "{}({})".format(
+            self.__base__(), ', '.join(["{}={!r}".format(k, getattr(self, k)) for k in self.__slots__[:-1]])
+        )
+
+    __str__ = __repr__
+
+
+class ImmutableDict(Mapping):
+    """Hashable, immutable dictionary."""
+
+    def __init__(self, arg):
+        """Initialize."""
+
+        arg
+        is_dict = isinstance(arg, dict)
+        if (
+            is_dict and not all([isinstance(v, Hashable) for v in arg.values()]) or
+            not is_dict and not all([isinstance(k, Hashable) and isinstance(v, Hashable) for k, v in arg])
+        ):
+            raise TypeError('All values must be hashable')
+
+        self._d = dict(arg)
+        self._hash = hash(tuple([(type(x), x, type(y), y) for x, y in sorted(self._d.items())]))
+
+    def __iter__(self):
+        """Iterator."""
+
+        return iter(self._d)
+
+    def __len__(self):
+        """Length."""
+
+        return len(self._d)
+
+    def __getitem__(self, key):
+        """Get item: `namespace['key']`."""
+        return self._d[key]
+
+    def __hash__(self):
+        """Hash."""
+
+        return self._hash
+
+    def __repr__(self):  # pragma: no cover
+        """Representation."""
+
+        return "{!r}".format(self._d)
+
+    __str__ = __repr__
+
+
+class Namespaces(ImmutableDict):
+    """Namespaces."""
+
+    def __init__(self, arg):
+        """Initialize."""
+
+        # If there are arguments, check the first index.
+        # `super` should fail if the user gave multiple arguments,
+        # so don't bother checking that.
+        is_dict = isinstance(arg, dict)
+        if is_dict and not all([isinstance(k, str) and isinstance(v, str) for k, v in arg.items()]):
+            raise TypeError('Namespace keys and values must be Unicode strings')
+        elif not is_dict and not all([isinstance(k, str) and isinstance(v, str) for k, v in arg]):
+            raise TypeError('Namespace keys and values must be Unicode strings')
+
+        super(Namespaces, self).__init__(arg)
+
+
+class CustomSelectors(ImmutableDict):
+    """Custom selectors."""
+
+    def __init__(self, arg):
+        """Initialize."""
+
+        # If there are arguments, check the first index.
+        # `super` should fail if the user gave multiple arguments,
+        # so don't bother checking that.
+        is_dict = isinstance(arg, dict)
+        if is_dict and not all([isinstance(k, str) and isinstance(v, str) for k, v in arg.items()]):
+            raise TypeError('CustomSelectors keys and values must be Unicode strings')
+        elif not is_dict and not all([isinstance(k, str) and isinstance(v, str) for k, v in arg]):
+            raise TypeError('CustomSelectors keys and values must be Unicode strings')
+
+        super(CustomSelectors, self).__init__(arg)
+
+
+class Selector(Immutable):
+    """Selector."""
+
+    __slots__ = (
+        'tag', 'ids', 'classes', 'attributes', 'nth', 'selectors',
+        'relation', 'rel_type', 'contains', 'lang', 'flags', '_hash'
+    )
+
+    def __init__(
+        self, tag, ids, classes, attributes, nth, selectors,
+        relation, rel_type, contains, lang, flags
+    ):
+        """Initialize."""
+
+        super(Selector, self).__init__(
+            tag=tag,
+            ids=ids,
+            classes=classes,
+            attributes=attributes,
+            nth=nth,
+            selectors=selectors,
+            relation=relation,
+            rel_type=rel_type,
+            contains=contains,
+            lang=lang,
+            flags=flags
+        )
+
+
+class SelectorNull(Immutable):
+    """Null Selector."""
+
+    def __init__(self):
+        """Initialize."""
+
+        super(SelectorNull, self).__init__()
+
+
+class SelectorTag(Immutable):
+    """Selector tag."""
+
+    __slots__ = ("name", "prefix", "_hash")
+
+    def __init__(self, name, prefix):
+        """Initialize."""
+
+        super(SelectorTag, self).__init__(
+            name=name,
+            prefix=prefix
+        )
+
+
+class SelectorAttribute(Immutable):
+    """Selector attribute rule."""
+
+    __slots__ = ("attribute", "prefix", "pattern", "xml_type_pattern", "_hash")
+
+    def __init__(self, attribute, prefix, pattern, xml_type_pattern):
+        """Initialize."""
+
+        super(SelectorAttribute, self).__init__(
+            attribute=attribute,
+            prefix=prefix,
+            pattern=pattern,
+            xml_type_pattern=xml_type_pattern
+        )
+
+
+class SelectorContains(Immutable):
+    """Selector contains rule."""
+
+    __slots__ = ("text", "own", "_hash")
+
+    def __init__(self, text, own):
+        """Initialize."""
+
+        super(SelectorContains, self).__init__(
+            text=text,
+            own=own
+        )
+
+
+class SelectorNth(Immutable):
+    """Selector nth type."""
+
+    __slots__ = ("a", "n", "b", "of_type", "last", "selectors", "_hash")
+
+    def __init__(self, a, n, b, of_type, last, selectors):
+        """Initialize."""
+
+        super(SelectorNth, self).__init__(
+            a=a,
+            n=n,
+            b=b,
+            of_type=of_type,
+            last=last,
+            selectors=selectors
+        )
+
+
+class SelectorLang(Immutable):
+    """Selector language rules."""
+
+    __slots__ = ("languages", "_hash",)
+
+    def __init__(self, languages):
+        """Initialize."""
+
+        super(SelectorLang, self).__init__(
+            languages=tuple(languages)
+        )
+
+    def __iter__(self):
+        """Iterator."""
+
+        return iter(self.languages)
+
+    def __len__(self):  # pragma: no cover
+        """Length."""
+
+        return len(self.languages)
+
+    def __getitem__(self, index):  # pragma: no cover
+        """Get item."""
+
+        return self.languages[index]
+
+
+class SelectorList(Immutable):
+    """Selector list."""
+
+    __slots__ = ("selectors", "is_not", "is_html", "_hash")
+
+    def __init__(self, selectors=tuple(), is_not=False, is_html=False):
+        """Initialize."""
+
+        super(SelectorList, self).__init__(
+            selectors=tuple(selectors),
+            is_not=is_not,
+            is_html=is_html
+        )
+
+    def __iter__(self):
+        """Iterator."""
+
+        return iter(self.selectors)
+
+    def __len__(self):
+        """Length."""
+
+        return len(self.selectors)
+
+    def __getitem__(self, index):
+        """Get item."""
+
+        return self.selectors[index]
+
+
+def _pickle(p):
+    return p.__base__(), tuple([getattr(p, s) for s in p.__slots__[:-1]])
+
+
+def pickle_register(obj):
+    """Allow object to be pickled."""
+
+    copyreg.pickle(obj, _pickle)
+
+
+pickle_register(Selector)
+pickle_register(SelectorNull)
+pickle_register(SelectorTag)
+pickle_register(SelectorAttribute)
+pickle_register(SelectorContains)
+pickle_register(SelectorNth)
+pickle_register(SelectorLang)
+pickle_register(SelectorList)

soupsieve/util.py

@@ -0,0 +1,110 @@
+"""Utility."""
+from functools import wraps, lru_cache
+import warnings
+import re
+
+DEBUG = 0x00001
+
+RE_PATTERN_LINE_SPLIT = re.compile(r'(?:\r\n|(?!\r\n)[\n\r])|$')
+
+UC_A = ord('A')
+UC_Z = ord('Z')
+
+
+@lru_cache(maxsize=512)
+def lower(string):
+    """Lower."""
+
+    new_string = []
+    for c in string:
+        o = ord(c)
+        new_string.append(chr(o + 32) if UC_A <= o <= UC_Z else c)
+    return ''.join(new_string)
+
+
+class SelectorSyntaxError(Exception):
+    """Syntax error in a CSS selector."""
+
+    def __init__(self, msg, pattern=None, index=None):
+        """Initialize."""
+
+        self.line = None
+        self.col = None
+        self.context = None
+
+        if pattern is not None and index is not None:
+            # Format pattern to show line and column position
+            self.context, self.line, self.col = get_pattern_context(pattern, index)
+            msg = '{}\n  line {}:\n{}'.format(msg, self.line, self.context)
+
+        super(SelectorSyntaxError, self).__init__(msg)
+
+
+def deprecated(message, stacklevel=2):  # pragma: no cover
+    """
+    Raise a `DeprecationWarning` when wrapped function/method is called.
+
+    Borrowed from https://stackoverflow.com/a/48632082/866026
+    """
+
+    def _decorator(func):
+        @wraps(func)
+        def _func(*args, **kwargs):
+            warnings.warn(
+                "'{}' is deprecated. {}".format(func.__name__, message),
+                category=DeprecationWarning,
+                stacklevel=stacklevel
+            )
+            return func(*args, **kwargs)
+        return _func
+    return _decorator
+
+
+def warn_deprecated(message, stacklevel=2):  # pragma: no cover
+    """Warn deprecated."""
+
+    warnings.warn(
+        message,
+        category=DeprecationWarning,
+        stacklevel=stacklevel
+    )
+
+
+def get_pattern_context(pattern, index):
+    """Get the pattern context."""
+
+    last = 0
+    current_line = 1
+    col = 1
+    text = []
+    line = 1
+
+    # Split pattern by newline and handle the text before the newline
+    for m in RE_PATTERN_LINE_SPLIT.finditer(pattern):
+        linetext = pattern[last:m.start(0)]
+        if not len(m.group(0)) and not len(text):
+            indent = ''
+            offset = -1
+            col = index - last + 1
+        elif last <= index < m.end(0):
+            indent = '--> '
+            offset = (-1 if index > m.start(0) else 0) + 3
+            col = index - last + 1
+        else:
+            indent = '    '
+            offset = None
+        if len(text):
+            # Regardless of whether we are presented with `\r\n`, `\r`, or `\n`,
+            # we will render the output with just `\n`. We will still log the column
+            # correctly though.
+            text.append('\n')
+        text.append('{}{}'.format(indent, linetext))
+        if offset is not None:
+            text.append('\n')
+            text.append(' ' * (col + offset) + '^')
+            line = current_line
+
+        current_line += 1
+        last = m.end(0)
+
+    return ''.join(text), line, col