htmlcontentconverter

Convert html content to the Giella xml format.

HTMLBeautifier

Convert html documents to the Giella xml format.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

class HTMLBeautifier:
    """Convert html documents to the Giella xml format."""

    def __init__(self, html_elt):
        for elt in html_elt.iter("script"):
            elt.getparent().remove(elt)

        c_clean = self.superclean(etree.tostring(html_elt, encoding="unicode"))
        self.soup = html.document_fromstring(c_clean)

    def superclean(self, content):
        """Remove unwanted elements from an html document.

        Args:
            content (str): a string containing an html document.

        Returns:
            (str): a string containing the cleaned up html document.
        """
        cleaner = clean.Cleaner(
            page_structure=False,
            scripts=True,
            javascript=True,
            comments=True,
            style=True,
            processing_instructions=True,
            remove_unknown_tags=True,
            embedded=True,
            kill_tags=[
                "img",
                "area",
                "address",
                "hr",
                "cite",
                "footer",
                "figcaption",
                "aside",
                "time",
                "figure",
                "nav",
                "noscript",
                "map",
                "ins",
                "s",
                "colgroup",
            ],
        )

        return cleaner.clean_html(self.remove_cruft(content))

    @staticmethod
    def remove_cruft(content):
        """Remove cruft from svenskakyrkan.se documents.

        Args:
            content (str): the content of a document.

        Returns:
            (str): The content of the document without the cruft.
        """
        replacements = [("//<script", "<script"), ("&nbsp;", " "), (" ", " ")]
        return util.replace_all(replacements, content)

    def simplify_tags(self):
        """Turn tags to divs.

        We don't care about the difference between <fieldsets>, <legend>
        etc. – treat them all as <div>'s for xhtml2corpus
        """
        superfluously_named_tags = self.soup.xpath(
            "//fieldset | //legend | //article | //hgroup "
            "| //section | //dl | //dd | //dt"
            "| //menu"
        )
        for elt in superfluously_named_tags:
            elt.tag = "div"

    def fix_spans_as_divs(self):
        """Turn div like elements into div.

        XHTML doesn't allow (and xhtml2corpus doesn't handle) span-like
        elements with div-like elements inside them; fix this and
        similar issues by turning them into divs.
        """
        spans_as_divs = self.soup.xpath(
            "//*[( descendant::div or descendant::p"
            "      or descendant::h1 or descendant::h2"
            "      or descendant::h3 or descendant::h4"
            "      or descendant::h5 or descendant::h6 ) "
            "and ( self::span or self::b or self::i"
            "      or self::em or self::strong "
            "      or self::a )"
            "    ]"
        )
        for elt in spans_as_divs:
            elt.tag = "div"

        ps_as_divs = self.soup.xpath("//p[descendant::div]")
        for elt in ps_as_divs:
            elt.tag = "div"

        lists_as_divs = self.soup.xpath(
            "//*[( child::ul or child::ol ) " "and ( self::ul or self::ol )" "    ]"
        )
        for elt in lists_as_divs:
            elt.tag = "div"

    def remove_empty_p(self):
        """Remove empty p elements."""
        paragraphs = self.soup.xpath("//p")

        for elt in paragraphs:
            if elt.text is None and elt.tail is None and not len(elt):
                elt.getparent().remove(elt)

    def remove_empty_class(self):
        """Delete empty class attributes."""
        for element in self.soup.xpath('.//*[@class=""]'):
            del element.attrib["class"]

    def remove_elements(self):
        """Remove unwanted tags from a html document.

        The point with this exercise is to remove all but the main content of
        the document.
        """
        unwanted_classes_ids = {
            "div": {
                "class": [
                    "skiplinks",  # yle.fi
                    "AddThis",  # lansstyrelsen.se
                    "InnholdForfatter",  # unginordland
                    "NavigationLeft",  # lansstyrelsen.se
                    "QuickNav",
                    "ad",
                    "andrenyheter",  # tysfjord.kommune.no
                    "art-layout-cell art-sidebar2",  # gaaltije.se
                    "art-postheadericons art-metadata-icons",  # gaaltije.se
                    "article-ad",
                    "article-bottom-element",
                    "article-column",
                    (
                        "article-dateline article-dateline-footer "
                        "meta-widget-content"
                    ),  # nrk.no
                    (
                        "article-dateline article-footer " "container-widget-content cf"
                    ),  # nrk.no
                    "article-heading-wrapper",  # 1177.se
                    "article-info",  # regjeringen.no
                    "article-related",
                    "article-toolbar__tool",  # umo.se
                    "article-universe-teaser container-widget-content",
                    "articleImageRig",
                    "articlegooglemap",  # tysfjord.kommune.no
                    "articleTags",  # nord-salten.no
                    "attribute-related_object",  # samediggi.no
                    "authors",
                    "authors ui-helper-clearfix",  # nord-salten.no
                    "back_button",
                    "banner-element",
                    "bl_linktext",
                    "bottom-center",
                    "breadcrumbs ",
                    "breadcrumbs",
                    "breadcrums span-12",
                    "btm_menu",
                    "byline",  # arran.no
                    "c1",  # jll.se
                    "art-bar art-nav",  # gaaltije.se
                    "art-layout-cell art-sidebar1",  # gaaltije.se
                    "clearfix breadcrumbsAndSocial noindex",  # udir.no
                    "complexDocumentBottom",  # regjeringen.no
                    "container-widget-content",  # nrk.no
                    "container_full",
                    "content-body attribute-vnd.openxmlformats-"
                    "officedocument.spreadsheetml.sheet",  # samediggi.no
                    "content-language-links",  # metsa.fi
                    "content-wrapper",  # siida.fi
                    "control-group field-wrapper tiedotteet-period",  # metsa.fi
                    "control-group form-inline",  # metsa.fi
                    "date",  # samediggi.no, 2019 ->
                    "documentInfoEm",
                    "documentPaging",
                    "documentPaging PagingBtm",  # regjeringen.no
                    "documentTop",  # regjeringen.no
                    "dotList",  # nord-salten.no
                    "dropmenudiv",  # calliidlagadus.org
                    "embedFile",  # samediggi.no -> 2019
                    "embedded-breadcrumbs",
                    "egavpi",  # calliidlagadus.org
                    "egavpi_fiskes",  # calliidlagadus.org
                    "esite_footer",
                    "esite_header",
                    "expandable",
                    "feedbackContainer noindex",  # udir.no
                    "file",  # samediggi.no
                    "fixed-header",
                    "g100 col fc s18 sg6 sg9 sg12 menu-reference",  # nrk.no
                    "g100 col fc s18 sg6 sg9 sg12 flow-reference",  # nrk.no
                    "g11 col fl s2 sl6 sl9 sl12 sl18",  # nrk.no
                    "g22 col fl s4 sl6 sl9 sl12 sl18 "
                    "article-header-sidebar",  # nrk.no
                    "g94 col fl s17 sl18 sg6 sg9 sg12 meta-widget",  # nrk.no
                    "globmenu",  # visitstetind.no
                    "grid cf",  # nrk.no
                    "help closed hidden-xs",
                    "historic-info",  # regjeringen.no
                    "historic-label",  # regjeringen.no
                    "imagecontainer",
                    "innholdsfortegenlse-child",
                    "inside",  # samas.no
                    "latestnews_uutisarkisto",
                    "ld-navbar",
                    "listArticleLink",  # samediggi.no -> 2019
                    "logo-links",  # metsa.fi
                    "meta",
                    "meta ui-helper-clearfix",  # nord-salten.no
                    "authors ui-helper-clearfix",  # nord-salten.no
                    "menu",  # visitstetind.no
                    "metaWrapper",
                    "mini-frontpage",  # yle.fi
                    "moduletable_oikopolut",
                    "moduletable_etulinkki",  # www.samediggi.fi
                    "navigation",  # latex2html docs
                    "nav-menu nav-menu-style-dots",  # metsa.fi
                    "naviHlp",  # visitstetind.no
                    "noindex",  # ntfk
                    "nrk-globalfooter",  # nrk.no
                    "nrk-globalfooter-dk lp_globalfooter",  # nrk.no
                    "nrk-globalnavigation",  # nrk.no
                    "nrkno-share bulletin-share",  # nrk.no
                    "outer-column",
                    "page-inner",  # samas.no
                    "person_info",  # samediggi.no
                    "plug-teaser",  # nrk.no
                    "post-footer",
                    "printbutton-wrapper",  # 1177.se
                    "printContact",
                    "right",  # ntfk
                    "rightverticalgradient",  # udir.no
                    "sharebutton-wrapper",  # 1177.se
                    "sharing",
                    "sidebar",
                    "spalte300",  # osko.no
                    "span12 tiedotteet-show",
                    "subpage-bottom",
                    "subfooter",  # visitstetind.no
                    "subnavigation",  # oikeusministeriö
                    "tabbedmenu",
                    "tipformcontainer",  # tysfjord.kommune.no
                    "tipsarad mt6 selfClear",
                    "titlepage",
                    "toc-placeholder",  # 1177.se
                    "toc",
                    "tools",  # arran.no
                    "trail",  # siida.fi
                    "translations",  # siida.fi
                    "upperheader",
                ],
                "id": [
                    "oikea_palsta",  # yle.fi
                    "ylefifooter",  # yle.fi
                    "print-logo-wrapper",  # 1177.se
                    "AreaLeft",
                    "AreaLeftNav",
                    "AreaRight",
                    "AreaTopRight",
                    "AreaTopSiteNav",
                    "NAVbreadcrumbContainer",
                    "NAVfooterContainer",
                    "NAVheaderContainer",
                    "NAVrelevantContentContainer",
                    "NAVsubmenuContainer",
                    "PageFooter",
                    "PageLanguageInfo",  # regjeringen.no
                    "PrintDocHead",
                    "SamiDisclaimer",
                    "ShareArticle",
                    "WIPSELEMENT_CALENDAR",  # learoevierhtieh.no
                    "WIPSELEMENT_HEADING",  # learoevierhtieh.no
                    "WIPSELEMENT_MENU",  # learoevierhtieh.no
                    "WIPSELEMENT_MENURIGHT",  # learoevierhtieh.no
                    "WIPSELEMENT_NEWS",  # learoevierhtieh.no
                    "WebPartZone1",  # lansstyrelsen.se
                    "aa",
                    "andrenyheter",  # tysfjord.kommune.no
                    "article_footer",
                    "attached",  # tysfjord.kommune.no
                    "blog-pager",
                    "bottom",  # samas.no
                    "breadcrumbs-bottom",
                    "bunninformasjon",  # unginordland
                    "chatBox",
                    "chromemenu",  # calliidlagadus.org
                    "crumbs",  # visitstetind.no
                    "ctl00_AccesskeyShortcuts",  # lansstyrelsen.se
                    "ctl00_ctl00_ArticleFormContentRegion_"
                    "ArticleBodyContentRegion_ctl00_"
                    "PageToolWrapper",  # 1177.se
                    "ctl00_ctl00_ArticleFormContentRegion_"
                    "ArticleBodyContentRegion_ctl03_"
                    "PageToolWrapper",  # 1177.se
                    "ctl00_Cookies",  # lansstyrelsen.se
                    "ctl00_FullRegion_CenterAndRightRegion_HitsControl_"
                    "ctl00_FullRegion_CenterAndRightRegion_Sorting_sortByDiv",
                    "ctl00_LSTPlaceHolderFeedback_"
                    "editmodepanel31",  # lansstyrelsen.se
                    "ctl00_LSTPlaceHolderSearch_"
                    "SearchBoxControl",  # lansstyrelsen.se
                    "ctl00_MidtSone_ucArtikkel_ctl00_ctl00_ctl01_divRessurser",
                    "ctl00_MidtSone_ucArtikkel_ctl00_divNavigasjon",
                    "ctl00_PlaceHolderMain_EditModePanel1",  # lansstyrelsen.se
                    "ctl00_PlaceHolderTitleBreadcrumb_"
                    "DefaultBreadcrumb",  # lansstyrelsen.se
                    "ctl00_TopLinks",  # lansstyrelsen.se
                    "deleModal",
                    "document-header",
                    "errorMessageContainer",  # nord-salten.no
                    "final-footer-wrapper",  # 1177.se
                    "flu-vaccination",  # 1177.se
                    "footer",  # forrest, too, tysfjord.kommune.no
                    "footer-wrapper",
                    "frontgallery",  # visitstetind.no
                    "header",
                    "headerBar",
                    "headWrapper",  # osko.no
                    "hoyre",  # unginordland
                    "innholdsfortegnelse",  # regjeringen.no
                    "leftMenu",
                    "leftPanel",
                    "leftbar",  # forrest (divvun and giellatekno sites)
                    "leftcol",  # new samediggi.no
                    "leftmenu",
                    "main_navi_main",  # www.samediggi.fi
                    "mainContentBookmark",  # udir.no
                    "mainsidebar",  # arran.no
                    "menu",
                    "mobile-header",
                    "mobile-subnavigation",
                    "murupolku",  # www.samediggi.fi
                    "nav-content",
                    "navbar",  # tysfjord.kommune.no
                    "ncFooter",  # visitstetind.no
                    "ntfkFooter",  # ntfk
                    "ntfkHeader",  # ntfk
                    "ntfkNavBreadcrumb",  # ntfk
                    "ntfkNavMain",  # ntfk
                    "pageFooter",
                    "path",  # new samediggi.no, tysfjord.kommune.no
                    "phone-bar",  # 1177.se
                    "publishinfo",  # 1177.se
                    "readspeaker_button1",
                    "right-wrapper",  # ndla
                    "rightAds",
                    "rightCol",
                    "rightside",
                    "s4-leftpanel",  # ntfk
                    "searchBox",
                    "searchHitSummary",
                    "sendReminder",
                    "share-article",
                    "sidebar",  # finlex.fi, too
                    "sidebar-wrapper",
                    "sitemap",
                    "skipLinks",  # udir.no
                    "skiplink",  # tysfjord.kommune.no
                    "spraakvelger",  # osko.no
                    "subfoote",  # visitstetind.no
                    "submenu",  # nord-salten.no
                    "svid10_49531bad1412ceb82564aea",  # ostersund.se
                    "svid10_6ba9fa711d2575a2a7800024318",  # jll.se
                    "svid10_6c1eb18a13ec7d9b5b82ee7",  # ostersund.se
                    "svid10_b0dabad141b6aeaf101229",  # ostersund.se
                    "svid10_49531bad1412ceb82564af3",  # ostersund.se
                    "svid10_6ba9fa711d2575a2a7800032145",  # jll.se
                    "svid10_6ba9fa711d2575a2a7800032151",  # jll.se
                    "svid10_6ba9fa711d2575a2a7800024344",  # jll.se
                    "svid10_6ba9fa711d2575a2a7800032135",  # jll.se
                    "svid10_6c1eb18a13ec7d9b5b82ee3",  # ostersund.se
                    "svid10_6c1eb18a13ec7d9b5b82edf",  # ostersund.se
                    "svid10_6c1eb18a13ec7d9b5b82edd",  # ostersund.se
                    "svid10_6c1eb18a13ec7d9b5b82eda",  # ostersund.se
                    "svid10_6c1eb18a13ec7d9b5b82ed5",  # ostersund.se
                    "svid12_6ba9fa711d2575a2a7800032140",  # jll.se
                    "theme-area-label-wrapper",  # 1177.se
                    "tipafriend",
                    "tools",  # arran.no
                    "topHeader",  # nord-salten.no
                    "topMenu",
                    "topUserMenu",
                    "top",  # arran.no
                    "topnav",  # tysfjord.kommune.no
                    "toppsone",  # unginordland
                    "vedleggogregistre",  # regjeringen.no
                    "venstre",  # unginordland
                    "static-menu-inner",  # arran.no
                ],
            },
            "p": {
                "class": [
                    "WebPartReadMoreParagraph",
                    "breadcrumbs",
                    "langs",  # oahpa.no
                    "art-page-footer",  # gaaltije.se
                ],
                "id": ["skip-link"],  # samas.no
            },
            "ul": {
                "id": [
                    "AreaTopLanguageNav",
                    "AreaTopPrintMeny",
                    "skiplinks",  # umo.se
                    "mainmenu",  # admin/tysfjord
                ],
                "class": [
                    "QuickNav",
                    "article-tools",
                    "article-universe-list",  # nrk.no
                    "byline",
                    "chapter-index",  # lovdata.no
                    "footer-nav",  # lovdata.no
                    "hidden",  # unginordland
                    "mainmenu menu menulevel0",  # admin/tysfjord
                ],
            },
            "span": {
                "id": ["skiplinks"],
                "class": [
                    "K-NOTE-FOTNOTE",
                    "graytext",  # svenskakyrkan.se
                    "breadcrumbs pathway",  # gaaltije.se
                    "meta",  # yle.fi
                ],
            },
            "a": {
                "id": ["ctl00_IdWelcome_ExplicitLogin", "leftPanelTab"],  # ntfk
                "class": [
                    "addthis_button_print",  # ntfk
                    "mainlevel",
                    "share-paragraf",  # lovdata.no
                    "mainlevel_alavalikko",  # www.samediggi.fi
                    "sublevel_alavalikko",  # www.samediggi.fi
                    "skip-link",  # 1177.se
                    "toggle-link expanded",  # 1177.se
                ],
                "name": ["footnote-ref"],  # footnotes in running text
            },
            "td": {
                "id": [
                    "hakulomake",  # www.samediggi.fi
                    "paavalikko_linkit",  # www.samediggi.fi
                    "sg_oikea",  # www.samediggi.fi
                    "sg_vasen",  # www.samediggi.fi
                ],
                "class": ["modifydate"],
            },
            "tr": {"id": ["sg_ylaosa1", "sg_ylaosa2"]},
            "header": {
                "id": ["header"],  # umo.se
                "class": [
                    "nrk-masthead-content cf",  # nrk.no
                    "pageHeader ",  # regjeringen.no
                    "singleton widget rich nrk-masthead lp_masthead",  # nrk.no
                ],
            },
            "section": {
                "class": [
                    "recents-on-this-topic",  # yle.fi
                    "section-theme-sub-nav",  # 1177.se
                    "span3",  # samernas.se
                    "tree-menu current",  # umo.se
                    "tree-menu",  # umo.se
                ]
            },
            "table": {"id": ["Table_01"]},
        }

        namespace = {"html": "http://www.w3.org/1999/xhtml"}
        for tag, attribs in unwanted_classes_ids.items():
            for key, values in attribs.items():
                for value in values:
                    search = f'.//{tag}[@{key}="{value}"]'
                    for unwanted in self.soup.xpath(search, namespaces=namespace):
                        unwanted.getparent().remove(unwanted)

    def add_p_around_text(self):
        """Add p around text after an hX element."""
        stop_tags = ["p", "h3", "h2", "div", "table"]
        for tag in self.soup.xpath(".//body/*"):
            if tag.tail is not None and tag.tail.strip() != "":
                paragraph = etree.Element("p")
                paragraph.text = tag.tail
                tag.tail = None
                for next_element in iter(tag.getnext, None):
                    if next_element.tag in stop_tags:
                        break
                    paragraph.append(next_element)

                tag_parent = tag.getparent()
                tag_parent.insert(tag_parent.index(tag) + 1, paragraph)

        # br's are not allowed right under body in XHTML:
        for elt in self.soup.xpath(".//body/br"):
            elt.tag = "p"
            elt.text = " "

    def center2div(self):
        """Convert center to div in tidy style."""
        for center in self.soup.xpath(".//center"):
            center.tag = "div"
            center.set("class", "c1")

    def body_i(self):
        """Wrap bare elements inside a p element."""
        for tag in ["a", "i", "em", "u", "strong", "span"]:
            for body_tag in self.soup.xpath(f".//body/{tag}"):
                paragraph = etree.Element("p")
                bi_parent = body_tag.getparent()
                bi_parent.insert(bi_parent.index(body_tag), paragraph)
                paragraph.append(body_tag)

    @staticmethod
    def handle_font_text(font_elt):
        """Incorporate font.text into correct element.

        Args:
            font_elt (etree.Element): a font element.
        """
        font_parent = font_elt.getparent()
        font_index = font_parent.index(font_elt)

        if font_elt.text is not None:
            if font_index > 0:
                previous_element = font_parent[font_index - 1]
                if previous_element.tail is not None:
                    previous_element.tail += font_elt.text
                else:
                    previous_element.tail = font_elt.text
            else:
                if font_elt.text is not None:
                    if font_parent.text is not None:
                        font_parent.text += font_elt.text
                    else:
                        font_parent.text = font_elt.text

    @staticmethod
    def handle_font_children(font_elt):
        """Incorporate font children into correct element.

        Args:
            font_elt (etree.Element): a font element.
        """
        font_parent = font_elt.getparent()
        font_index = font_parent.index(font_elt)

        for position, font_child in enumerate(font_elt, start=font_index):
            if font_elt.tail is not None:
                if font_elt[-1].tail is not None:
                    font_elt[-1].tail += font_elt.tail
                else:
                    font_elt[-1].tail = font_elt.tail
            font_parent.insert(position, font_child)

    @staticmethod
    def handle_font_tail(font_elt):
        """Incorporate font.tail into correct element.

        Args:
            font_elt (etree.Element): a font element.
        """
        font_parent = font_elt.getparent()
        font_index = font_parent.index(font_elt)
        previous_element = font_parent[font_index - 1]

        if font_elt.tail is not None:
            if font_index > 0:
                if previous_element.tail is not None:
                    previous_element.tail += font_elt.tail
                else:
                    previous_element.tail = font_elt.tail
            else:
                if font_parent.text is not None:
                    font_parent.text += font_elt.tail
                else:
                    font_parent.text = font_elt.tail

    def remove_font(self):
        """Remove font elements, incorporate content into it's parent."""
        for font_elt in reversed(list(self.soup.iter("{*}font"))):
            self.handle_font_text(font_elt)

            if len(font_elt) > 0:
                self.handle_font_children(font_elt)
            else:
                self.handle_font_tail(font_elt)

            font_elt.getparent().remove(font_elt)

    def body_text(self):
        """Wrap bare text inside a p element."""
        body = self.soup.find(".//body")

        if body.text is not None:
            paragraph = etree.Element("p")
            paragraph.text = body.text
            body.text = None
            body.insert(0, paragraph)

    def beautify(self):
        """Clean up the html document.

        Destructively modifies self.soup, trying
        to create strict xhtml for xhtml2corpus.xsl
        """
        self.remove_empty_class()
        self.remove_empty_p()
        self.remove_elements()
        self.remove_font()
        self.add_p_around_text()
        self.center2div()
        self.body_i()
        self.body_text()
        self.simplify_tags()
        self.fix_spans_as_divs()

        return self.soup

add_p_around_text()

Add p around text after an hX element.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def add_p_around_text(self):
    """Add p around text after an hX element."""
    stop_tags = ["p", "h3", "h2", "div", "table"]
    for tag in self.soup.xpath(".//body/*"):
        if tag.tail is not None and tag.tail.strip() != "":
            paragraph = etree.Element("p")
            paragraph.text = tag.tail
            tag.tail = None
            for next_element in iter(tag.getnext, None):
                if next_element.tag in stop_tags:
                    break
                paragraph.append(next_element)

            tag_parent = tag.getparent()
            tag_parent.insert(tag_parent.index(tag) + 1, paragraph)

    # br's are not allowed right under body in XHTML:
    for elt in self.soup.xpath(".//body/br"):
        elt.tag = "p"
        elt.text = " "

beautify()

Clean up the html document.

Destructively modifies self.soup, trying to create strict xhtml for xhtml2corpus.xsl

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def beautify(self):
    """Clean up the html document.

    Destructively modifies self.soup, trying
    to create strict xhtml for xhtml2corpus.xsl
    """
    self.remove_empty_class()
    self.remove_empty_p()
    self.remove_elements()
    self.remove_font()
    self.add_p_around_text()
    self.center2div()
    self.body_i()
    self.body_text()
    self.simplify_tags()
    self.fix_spans_as_divs()

    return self.soup

body_i()

Wrap bare elements inside a p element.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def body_i(self):
    """Wrap bare elements inside a p element."""
    for tag in ["a", "i", "em", "u", "strong", "span"]:
        for body_tag in self.soup.xpath(f".//body/{tag}"):
            paragraph = etree.Element("p")
            bi_parent = body_tag.getparent()
            bi_parent.insert(bi_parent.index(body_tag), paragraph)
            paragraph.append(body_tag)

body_text()

Wrap bare text inside a p element.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def body_text(self):
    """Wrap bare text inside a p element."""
    body = self.soup.find(".//body")

    if body.text is not None:
        paragraph = etree.Element("p")
        paragraph.text = body.text
        body.text = None
        body.insert(0, paragraph)

center2div()

Convert center to div in tidy style.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def center2div(self):
    """Convert center to div in tidy style."""
    for center in self.soup.xpath(".//center"):
        center.tag = "div"
        center.set("class", "c1")

fix_spans_as_divs()

Turn div like elements into div.

XHTML doesn't allow (and xhtml2corpus doesn't handle) span-like elements with div-like elements inside them; fix this and similar issues by turning them into divs.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def fix_spans_as_divs(self):
    """Turn div like elements into div.

    XHTML doesn't allow (and xhtml2corpus doesn't handle) span-like
    elements with div-like elements inside them; fix this and
    similar issues by turning them into divs.
    """
    spans_as_divs = self.soup.xpath(
        "//*[( descendant::div or descendant::p"
        "      or descendant::h1 or descendant::h2"
        "      or descendant::h3 or descendant::h4"
        "      or descendant::h5 or descendant::h6 ) "
        "and ( self::span or self::b or self::i"
        "      or self::em or self::strong "
        "      or self::a )"
        "    ]"
    )
    for elt in spans_as_divs:
        elt.tag = "div"

    ps_as_divs = self.soup.xpath("//p[descendant::div]")
    for elt in ps_as_divs:
        elt.tag = "div"

    lists_as_divs = self.soup.xpath(
        "//*[( child::ul or child::ol ) " "and ( self::ul or self::ol )" "    ]"
    )
    for elt in lists_as_divs:
        elt.tag = "div"

handle_font_children(font_elt) staticmethod

Incorporate font children into correct element.

Parameters:

Name	Type	Description	Default
font_elt	etree.Element	a font element.	required

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

@staticmethod
def handle_font_children(font_elt):
    """Incorporate font children into correct element.

    Args:
        font_elt (etree.Element): a font element.
    """
    font_parent = font_elt.getparent()
    font_index = font_parent.index(font_elt)

    for position, font_child in enumerate(font_elt, start=font_index):
        if font_elt.tail is not None:
            if font_elt[-1].tail is not None:
                font_elt[-1].tail += font_elt.tail
            else:
                font_elt[-1].tail = font_elt.tail
        font_parent.insert(position, font_child)

handle_font_tail(font_elt) staticmethod

Incorporate font.tail into correct element.

Parameters:

Name	Type	Description	Default
font_elt	etree.Element	a font element.	required

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

@staticmethod
def handle_font_tail(font_elt):
    """Incorporate font.tail into correct element.

    Args:
        font_elt (etree.Element): a font element.
    """
    font_parent = font_elt.getparent()
    font_index = font_parent.index(font_elt)
    previous_element = font_parent[font_index - 1]

    if font_elt.tail is not None:
        if font_index > 0:
            if previous_element.tail is not None:
                previous_element.tail += font_elt.tail
            else:
                previous_element.tail = font_elt.tail
        else:
            if font_parent.text is not None:
                font_parent.text += font_elt.tail
            else:
                font_parent.text = font_elt.tail

handle_font_text(font_elt) staticmethod

Incorporate font.text into correct element.

Parameters:

Name	Type	Description	Default
font_elt	etree.Element	a font element.	required

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

@staticmethod
def handle_font_text(font_elt):
    """Incorporate font.text into correct element.

    Args:
        font_elt (etree.Element): a font element.
    """
    font_parent = font_elt.getparent()
    font_index = font_parent.index(font_elt)

    if font_elt.text is not None:
        if font_index > 0:
            previous_element = font_parent[font_index - 1]
            if previous_element.tail is not None:
                previous_element.tail += font_elt.text
            else:
                previous_element.tail = font_elt.text
        else:
            if font_elt.text is not None:
                if font_parent.text is not None:
                    font_parent.text += font_elt.text
                else:
                    font_parent.text = font_elt.text

remove_cruft(content) staticmethod

Remove cruft from svenskakyrkan.se documents.

Parameters:

Name	Type	Description	Default
content	str	the content of a document.	required

Returns:

Type	Description
str	The content of the document without the cruft.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

@staticmethod
def remove_cruft(content):
    """Remove cruft from svenskakyrkan.se documents.

    Args:
        content (str): the content of a document.

    Returns:
        (str): The content of the document without the cruft.
    """
    replacements = [("//<script", "<script"), ("&nbsp;", " "), (" ", " ")]
    return util.replace_all(replacements, content)

remove_elements()

Remove unwanted tags from a html document.

The point with this exercise is to remove all but the main content of the document.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def remove_elements(self):
    """Remove unwanted tags from a html document.

    The point with this exercise is to remove all but the main content of
    the document.
    """
    unwanted_classes_ids = {
        "div": {
            "class": [
                "skiplinks",  # yle.fi
                "AddThis",  # lansstyrelsen.se
                "InnholdForfatter",  # unginordland
                "NavigationLeft",  # lansstyrelsen.se
                "QuickNav",
                "ad",
                "andrenyheter",  # tysfjord.kommune.no
                "art-layout-cell art-sidebar2",  # gaaltije.se
                "art-postheadericons art-metadata-icons",  # gaaltije.se
                "article-ad",
                "article-bottom-element",
                "article-column",
                (
                    "article-dateline article-dateline-footer "
                    "meta-widget-content"
                ),  # nrk.no
                (
                    "article-dateline article-footer " "container-widget-content cf"
                ),  # nrk.no
                "article-heading-wrapper",  # 1177.se
                "article-info",  # regjeringen.no
                "article-related",
                "article-toolbar__tool",  # umo.se
                "article-universe-teaser container-widget-content",
                "articleImageRig",
                "articlegooglemap",  # tysfjord.kommune.no
                "articleTags",  # nord-salten.no
                "attribute-related_object",  # samediggi.no
                "authors",
                "authors ui-helper-clearfix",  # nord-salten.no
                "back_button",
                "banner-element",
                "bl_linktext",
                "bottom-center",
                "breadcrumbs ",
                "breadcrumbs",
                "breadcrums span-12",
                "btm_menu",
                "byline",  # arran.no
                "c1",  # jll.se
                "art-bar art-nav",  # gaaltije.se
                "art-layout-cell art-sidebar1",  # gaaltije.se
                "clearfix breadcrumbsAndSocial noindex",  # udir.no
                "complexDocumentBottom",  # regjeringen.no
                "container-widget-content",  # nrk.no
                "container_full",
                "content-body attribute-vnd.openxmlformats-"
                "officedocument.spreadsheetml.sheet",  # samediggi.no
                "content-language-links",  # metsa.fi
                "content-wrapper",  # siida.fi
                "control-group field-wrapper tiedotteet-period",  # metsa.fi
                "control-group form-inline",  # metsa.fi
                "date",  # samediggi.no, 2019 ->
                "documentInfoEm",
                "documentPaging",
                "documentPaging PagingBtm",  # regjeringen.no
                "documentTop",  # regjeringen.no
                "dotList",  # nord-salten.no
                "dropmenudiv",  # calliidlagadus.org
                "embedFile",  # samediggi.no -> 2019
                "embedded-breadcrumbs",
                "egavpi",  # calliidlagadus.org
                "egavpi_fiskes",  # calliidlagadus.org
                "esite_footer",
                "esite_header",
                "expandable",
                "feedbackContainer noindex",  # udir.no
                "file",  # samediggi.no
                "fixed-header",
                "g100 col fc s18 sg6 sg9 sg12 menu-reference",  # nrk.no
                "g100 col fc s18 sg6 sg9 sg12 flow-reference",  # nrk.no
                "g11 col fl s2 sl6 sl9 sl12 sl18",  # nrk.no
                "g22 col fl s4 sl6 sl9 sl12 sl18 "
                "article-header-sidebar",  # nrk.no
                "g94 col fl s17 sl18 sg6 sg9 sg12 meta-widget",  # nrk.no
                "globmenu",  # visitstetind.no
                "grid cf",  # nrk.no
                "help closed hidden-xs",
                "historic-info",  # regjeringen.no
                "historic-label",  # regjeringen.no
                "imagecontainer",
                "innholdsfortegenlse-child",
                "inside",  # samas.no
                "latestnews_uutisarkisto",
                "ld-navbar",
                "listArticleLink",  # samediggi.no -> 2019
                "logo-links",  # metsa.fi
                "meta",
                "meta ui-helper-clearfix",  # nord-salten.no
                "authors ui-helper-clearfix",  # nord-salten.no
                "menu",  # visitstetind.no
                "metaWrapper",
                "mini-frontpage",  # yle.fi
                "moduletable_oikopolut",
                "moduletable_etulinkki",  # www.samediggi.fi
                "navigation",  # latex2html docs
                "nav-menu nav-menu-style-dots",  # metsa.fi
                "naviHlp",  # visitstetind.no
                "noindex",  # ntfk
                "nrk-globalfooter",  # nrk.no
                "nrk-globalfooter-dk lp_globalfooter",  # nrk.no
                "nrk-globalnavigation",  # nrk.no
                "nrkno-share bulletin-share",  # nrk.no
                "outer-column",
                "page-inner",  # samas.no
                "person_info",  # samediggi.no
                "plug-teaser",  # nrk.no
                "post-footer",
                "printbutton-wrapper",  # 1177.se
                "printContact",
                "right",  # ntfk
                "rightverticalgradient",  # udir.no
                "sharebutton-wrapper",  # 1177.se
                "sharing",
                "sidebar",
                "spalte300",  # osko.no
                "span12 tiedotteet-show",
                "subpage-bottom",
                "subfooter",  # visitstetind.no
                "subnavigation",  # oikeusministeriö
                "tabbedmenu",
                "tipformcontainer",  # tysfjord.kommune.no
                "tipsarad mt6 selfClear",
                "titlepage",
                "toc-placeholder",  # 1177.se
                "toc",
                "tools",  # arran.no
                "trail",  # siida.fi
                "translations",  # siida.fi
                "upperheader",
            ],
            "id": [
                "oikea_palsta",  # yle.fi
                "ylefifooter",  # yle.fi
                "print-logo-wrapper",  # 1177.se
                "AreaLeft",
                "AreaLeftNav",
                "AreaRight",
                "AreaTopRight",
                "AreaTopSiteNav",
                "NAVbreadcrumbContainer",
                "NAVfooterContainer",
                "NAVheaderContainer",
                "NAVrelevantContentContainer",
                "NAVsubmenuContainer",
                "PageFooter",
                "PageLanguageInfo",  # regjeringen.no
                "PrintDocHead",
                "SamiDisclaimer",
                "ShareArticle",
                "WIPSELEMENT_CALENDAR",  # learoevierhtieh.no
                "WIPSELEMENT_HEADING",  # learoevierhtieh.no
                "WIPSELEMENT_MENU",  # learoevierhtieh.no
                "WIPSELEMENT_MENURIGHT",  # learoevierhtieh.no
                "WIPSELEMENT_NEWS",  # learoevierhtieh.no
                "WebPartZone1",  # lansstyrelsen.se
                "aa",
                "andrenyheter",  # tysfjord.kommune.no
                "article_footer",
                "attached",  # tysfjord.kommune.no
                "blog-pager",
                "bottom",  # samas.no
                "breadcrumbs-bottom",
                "bunninformasjon",  # unginordland
                "chatBox",
                "chromemenu",  # calliidlagadus.org
                "crumbs",  # visitstetind.no
                "ctl00_AccesskeyShortcuts",  # lansstyrelsen.se
                "ctl00_ctl00_ArticleFormContentRegion_"
                "ArticleBodyContentRegion_ctl00_"
                "PageToolWrapper",  # 1177.se
                "ctl00_ctl00_ArticleFormContentRegion_"
                "ArticleBodyContentRegion_ctl03_"
                "PageToolWrapper",  # 1177.se
                "ctl00_Cookies",  # lansstyrelsen.se
                "ctl00_FullRegion_CenterAndRightRegion_HitsControl_"
                "ctl00_FullRegion_CenterAndRightRegion_Sorting_sortByDiv",
                "ctl00_LSTPlaceHolderFeedback_"
                "editmodepanel31",  # lansstyrelsen.se
                "ctl00_LSTPlaceHolderSearch_"
                "SearchBoxControl",  # lansstyrelsen.se
                "ctl00_MidtSone_ucArtikkel_ctl00_ctl00_ctl01_divRessurser",
                "ctl00_MidtSone_ucArtikkel_ctl00_divNavigasjon",
                "ctl00_PlaceHolderMain_EditModePanel1",  # lansstyrelsen.se
                "ctl00_PlaceHolderTitleBreadcrumb_"
                "DefaultBreadcrumb",  # lansstyrelsen.se
                "ctl00_TopLinks",  # lansstyrelsen.se
                "deleModal",
                "document-header",
                "errorMessageContainer",  # nord-salten.no
                "final-footer-wrapper",  # 1177.se
                "flu-vaccination",  # 1177.se
                "footer",  # forrest, too, tysfjord.kommune.no
                "footer-wrapper",
                "frontgallery",  # visitstetind.no
                "header",
                "headerBar",
                "headWrapper",  # osko.no
                "hoyre",  # unginordland
                "innholdsfortegnelse",  # regjeringen.no
                "leftMenu",
                "leftPanel",
                "leftbar",  # forrest (divvun and giellatekno sites)
                "leftcol",  # new samediggi.no
                "leftmenu",
                "main_navi_main",  # www.samediggi.fi
                "mainContentBookmark",  # udir.no
                "mainsidebar",  # arran.no
                "menu",
                "mobile-header",
                "mobile-subnavigation",
                "murupolku",  # www.samediggi.fi
                "nav-content",
                "navbar",  # tysfjord.kommune.no
                "ncFooter",  # visitstetind.no
                "ntfkFooter",  # ntfk
                "ntfkHeader",  # ntfk
                "ntfkNavBreadcrumb",  # ntfk
                "ntfkNavMain",  # ntfk
                "pageFooter",
                "path",  # new samediggi.no, tysfjord.kommune.no
                "phone-bar",  # 1177.se
                "publishinfo",  # 1177.se
                "readspeaker_button1",
                "right-wrapper",  # ndla
                "rightAds",
                "rightCol",
                "rightside",
                "s4-leftpanel",  # ntfk
                "searchBox",
                "searchHitSummary",
                "sendReminder",
                "share-article",
                "sidebar",  # finlex.fi, too
                "sidebar-wrapper",
                "sitemap",
                "skipLinks",  # udir.no
                "skiplink",  # tysfjord.kommune.no
                "spraakvelger",  # osko.no
                "subfoote",  # visitstetind.no
                "submenu",  # nord-salten.no
                "svid10_49531bad1412ceb82564aea",  # ostersund.se
                "svid10_6ba9fa711d2575a2a7800024318",  # jll.se
                "svid10_6c1eb18a13ec7d9b5b82ee7",  # ostersund.se
                "svid10_b0dabad141b6aeaf101229",  # ostersund.se
                "svid10_49531bad1412ceb82564af3",  # ostersund.se
                "svid10_6ba9fa711d2575a2a7800032145",  # jll.se
                "svid10_6ba9fa711d2575a2a7800032151",  # jll.se
                "svid10_6ba9fa711d2575a2a7800024344",  # jll.se
                "svid10_6ba9fa711d2575a2a7800032135",  # jll.se
                "svid10_6c1eb18a13ec7d9b5b82ee3",  # ostersund.se
                "svid10_6c1eb18a13ec7d9b5b82edf",  # ostersund.se
                "svid10_6c1eb18a13ec7d9b5b82edd",  # ostersund.se
                "svid10_6c1eb18a13ec7d9b5b82eda",  # ostersund.se
                "svid10_6c1eb18a13ec7d9b5b82ed5",  # ostersund.se
                "svid12_6ba9fa711d2575a2a7800032140",  # jll.se
                "theme-area-label-wrapper",  # 1177.se
                "tipafriend",
                "tools",  # arran.no
                "topHeader",  # nord-salten.no
                "topMenu",
                "topUserMenu",
                "top",  # arran.no
                "topnav",  # tysfjord.kommune.no
                "toppsone",  # unginordland
                "vedleggogregistre",  # regjeringen.no
                "venstre",  # unginordland
                "static-menu-inner",  # arran.no
            ],
        },
        "p": {
            "class": [
                "WebPartReadMoreParagraph",
                "breadcrumbs",
                "langs",  # oahpa.no
                "art-page-footer",  # gaaltije.se
            ],
            "id": ["skip-link"],  # samas.no
        },
        "ul": {
            "id": [
                "AreaTopLanguageNav",
                "AreaTopPrintMeny",
                "skiplinks",  # umo.se
                "mainmenu",  # admin/tysfjord
            ],
            "class": [
                "QuickNav",
                "article-tools",
                "article-universe-list",  # nrk.no
                "byline",
                "chapter-index",  # lovdata.no
                "footer-nav",  # lovdata.no
                "hidden",  # unginordland
                "mainmenu menu menulevel0",  # admin/tysfjord
            ],
        },
        "span": {
            "id": ["skiplinks"],
            "class": [
                "K-NOTE-FOTNOTE",
                "graytext",  # svenskakyrkan.se
                "breadcrumbs pathway",  # gaaltije.se
                "meta",  # yle.fi
            ],
        },
        "a": {
            "id": ["ctl00_IdWelcome_ExplicitLogin", "leftPanelTab"],  # ntfk
            "class": [
                "addthis_button_print",  # ntfk
                "mainlevel",
                "share-paragraf",  # lovdata.no
                "mainlevel_alavalikko",  # www.samediggi.fi
                "sublevel_alavalikko",  # www.samediggi.fi
                "skip-link",  # 1177.se
                "toggle-link expanded",  # 1177.se
            ],
            "name": ["footnote-ref"],  # footnotes in running text
        },
        "td": {
            "id": [
                "hakulomake",  # www.samediggi.fi
                "paavalikko_linkit",  # www.samediggi.fi
                "sg_oikea",  # www.samediggi.fi
                "sg_vasen",  # www.samediggi.fi
            ],
            "class": ["modifydate"],
        },
        "tr": {"id": ["sg_ylaosa1", "sg_ylaosa2"]},
        "header": {
            "id": ["header"],  # umo.se
            "class": [
                "nrk-masthead-content cf",  # nrk.no
                "pageHeader ",  # regjeringen.no
                "singleton widget rich nrk-masthead lp_masthead",  # nrk.no
            ],
        },
        "section": {
            "class": [
                "recents-on-this-topic",  # yle.fi
                "section-theme-sub-nav",  # 1177.se
                "span3",  # samernas.se
                "tree-menu current",  # umo.se
                "tree-menu",  # umo.se
            ]
        },
        "table": {"id": ["Table_01"]},
    }

    namespace = {"html": "http://www.w3.org/1999/xhtml"}
    for tag, attribs in unwanted_classes_ids.items():
        for key, values in attribs.items():
            for value in values:
                search = f'.//{tag}[@{key}="{value}"]'
                for unwanted in self.soup.xpath(search, namespaces=namespace):
                    unwanted.getparent().remove(unwanted)

remove_empty_class()

Delete empty class attributes.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def remove_empty_class(self):
    """Delete empty class attributes."""
    for element in self.soup.xpath('.//*[@class=""]'):
        del element.attrib["class"]

remove_empty_p()

Remove empty p elements.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def remove_empty_p(self):
    """Remove empty p elements."""
    paragraphs = self.soup.xpath("//p")

    for elt in paragraphs:
        if elt.text is None and elt.tail is None and not len(elt):
            elt.getparent().remove(elt)

remove_font()

Remove font elements, incorporate content into it's parent.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def remove_font(self):
    """Remove font elements, incorporate content into it's parent."""
    for font_elt in reversed(list(self.soup.iter("{*}font"))):
        self.handle_font_text(font_elt)

        if len(font_elt) > 0:
            self.handle_font_children(font_elt)
        else:
            self.handle_font_tail(font_elt)

        font_elt.getparent().remove(font_elt)

simplify_tags()

Turn tags to divs.

We don't care about the difference between ,

etc. – treat them all as

's for xhtml2corpus

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def simplify_tags(self):
    """Turn tags to divs.

    We don't care about the difference between <fieldsets>, <legend>
    etc. – treat them all as <div>'s for xhtml2corpus
    """
    superfluously_named_tags = self.soup.xpath(
        "//fieldset | //legend | //article | //hgroup "
        "| //section | //dl | //dd | //dt"
        "| //menu"
    )
    for elt in superfluously_named_tags:
        elt.tag = "div"

superclean(content)

Remove unwanted elements from an html document.

Parameters:

Name	Type	Description	Default
content	str	a string containing an html document.	required

Returns:

Type	Description
str	a string containing the cleaned up html document.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def superclean(self, content):
    """Remove unwanted elements from an html document.

    Args:
        content (str): a string containing an html document.

    Returns:
        (str): a string containing the cleaned up html document.
    """
    cleaner = clean.Cleaner(
        page_structure=False,
        scripts=True,
        javascript=True,
        comments=True,
        style=True,
        processing_instructions=True,
        remove_unknown_tags=True,
        embedded=True,
        kill_tags=[
            "img",
            "area",
            "address",
            "hr",
            "cite",
            "footer",
            "figcaption",
            "aside",
            "time",
            "figure",
            "nav",
            "noscript",
            "map",
            "ins",
            "s",
            "colgroup",
        ],
    )

    return cleaner.clean_html(self.remove_cruft(content))

add_p_instead_of_tail(intermediate)

Convert tail in list and p to a p element.

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def add_p_instead_of_tail(intermediate):
    """Convert tail in list and p to a p element."""
    for element in ["list", "p"]:
        for found_element in intermediate.findall(".//" + element):
            if found_element.tail is not None and found_element.tail.strip() != "":
                new_p = etree.Element("p")
                new_p.text = found_element.tail
                found_element.tail = None
                found_element.addnext(new_p)

convert2intermediate(filename)

Convert a webpage to Giella xml.

Parameters:

Name	Type	Description	Default
filename	str	name of the file	required

Returns:

Type	Description
lxml.etree.Element	the root element of the Giella xml document

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def convert2intermediate(filename):
    """Convert a webpage to Giella xml.

    Args:
        filename (str): name of the file

    Returns:
        (lxml.etree.Element): the root element of the Giella xml document
    """
    return xhtml2intermediate(to_html_elt(filename))

replace_bare_text(body)

Replace bare text in body with a p element.

Parameters:

Name	Type	Description	Default
body	etree.Element	the body element of the html document	required

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def replace_bare_text(body):
    """Replace bare text in body with a p element.

    Args:
        body (etree.Element): the body element of the html document
    """
    if body.text is not None and body.text.strip() != "":
        new_p = etree.Element("p")
        new_p.text = body.text
        body.text = None
        body.insert(0, new_p)

xhtml2intermediate(content_xml)

Convert xhtml to Giella xml.

Parameters:

Name	Type	Description	Default
content_xml	etree.Element	the result of convert2xhtml	required

Returns:

Type	Description
lxml.etree.Element	the root element of the Giella xml document

Source code in /home/anders/projects/CorpusTools/corpustools/htmlcontentconverter.py

def xhtml2intermediate(content_xml):
    """Convert xhtml to Giella xml.

    Args:
        content_xml (etree.Element): the result of convert2xhtml

    Returns:
        (lxml.etree.Element): the root element of the Giella xml document
    """
    converter_xsl = os.path.join(HERE, "xslt/xhtml2corpus.xsl")

    html_xslt_root = etree.parse(converter_xsl)
    transform = etree.XSLT(html_xslt_root)

    intermediate = transform(HTMLBeautifier(content_xml).beautify())
    beautify_intermediate(intermediate)

    return intermediate.getroot()