Skip to content

ocrcontentconverter

Convert ocr text files to the Giella xml format.

These files have the .ocr suffix. They are plain text files where * lines starting with # are headings * empty lines separate paragraphs * words are often split over two lines, and must be joined again

OcrContentConverter

Bases: BasicConverter

Convert ocr text files to the Giella xml format.

Source code in corpustools/ocrcontentconverter.py 
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
class OcrContentConverter(basicconverter.BasicConverter):
    """Convert ocr text files to the Giella xml format."""

    def to_unicode(self) -> str:
        """Read a file into a unicode string.

        If the content of the file is not utf-8, pretend the encoding is
        latin1. The real encoding will be detected later.

        Returns:
            The decoded string.
        """
        try:
            content = codecs.open(self.orig.as_posix(), encoding="utf8").read()
        except ValueError:
            content = codecs.open(self.orig.as_posix(), encoding="latin1").read()

        return content.replace("\r\n", "\n")

    @staticmethod
    def make_element(text: str, is_heading: bool = False) -> etree._Element:
        """Make a p element.

        Args:
            text: the text the element should contain.
            is_heading: whether the text is a heading.

        Returns:
            An etree element.
        """
        element = etree.Element("p")
        if is_heading:
            element.set("type", "title")
        element.text = text

        return element

    def lines2xml(self, content: io.StringIO) -> Iterable[etree._Element]:
        """Turn headings and paragraphs into etree elements.

        Args:
            content: the content of the ocr_corrected document.

        Yields:
            An etree element.
        """
        valid_lines = (
            line.strip()
            for line_no, line in enumerate(content, start=1)
            if line_no not in self.metadata.skip_lines
        )

        buffer: list[str] = []
        for line in valid_lines:
            heading = HEADING_RE.fullmatch(line)

            if (not line or heading) and buffer:
                yield self.make_element(join_lines(buffer))
                buffer.clear()

            if heading:
                title = heading.group("title").strip()
                if title:
                    yield self.make_element(title, is_heading=True)
            elif line:
                buffer.append(line)

        if buffer:
            yield self.make_element(join_lines(buffer))

    def content2xml(self, content: io.StringIO) -> etree._Element:
        """Transform the ocr text to an intermediate xml document.

        Args:
            content: the content of the ocr_corrected document.

        Returns:
            An etree element.
        """
        document = etree.Element("document")
        etree.SubElement(document, "header")
        body = etree.SubElement(document, "body")

        for para in self.lines2xml(content):
            body.append(para)

        return document

content2xml(content)

Transform the ocr text to an intermediate xml document.

Parameters:

Name Type Description Default
content StringIO

the content of the ocr_corrected document.

 required

Returns:

Type Description
_Element

An etree element.
Source code in corpustools/ocrcontentconverter.py 
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
def content2xml(self, content: io.StringIO) -> etree._Element:
    """Transform the ocr text to an intermediate xml document.

    Args:
        content: the content of the ocr_corrected document.

    Returns:
        An etree element.
    """
    document = etree.Element("document")
    etree.SubElement(document, "header")
    body = etree.SubElement(document, "body")

    for para in self.lines2xml(content):
        body.append(para)

    return document

lines2xml(content)

Turn headings and paragraphs into etree elements.

Parameters:

Name Type Description Default
content StringIO

the content of the ocr_corrected document.

 required

Yields:

Type Description
Iterable[_Element]

An etree element.
Source code in corpustools/ocrcontentconverter.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
def lines2xml(self, content: io.StringIO) -> Iterable[etree._Element]:
    """Turn headings and paragraphs into etree elements.

    Args:
        content: the content of the ocr_corrected document.

    Yields:
        An etree element.
    """
    valid_lines = (
        line.strip()
        for line_no, line in enumerate(content, start=1)
        if line_no not in self.metadata.skip_lines
    )

    buffer: list[str] = []
    for line in valid_lines:
        heading = HEADING_RE.fullmatch(line)

        if (not line or heading) and buffer:
            yield self.make_element(join_lines(buffer))
            buffer.clear()

        if heading:
            title = heading.group("title").strip()
            if title:
                yield self.make_element(title, is_heading=True)
        elif line:
            buffer.append(line)

    if buffer:
        yield self.make_element(join_lines(buffer))

make_element(text, is_heading=False) staticmethod

Make a p element.

Parameters:

Name Type Description Default
text str

the text the element should contain.

 required
is_heading bool

whether the text is a heading.

 False

Returns:

Type Description
_Element

An etree element.
Source code in corpustools/ocrcontentconverter.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
@staticmethod
def make_element(text: str, is_heading: bool = False) -> etree._Element:
    """Make a p element.

    Args:
        text: the text the element should contain.
        is_heading: whether the text is a heading.

    Returns:
        An etree element.
    """
    element = etree.Element("p")
    if is_heading:
        element.set("type", "title")
    element.text = text

    return element

to_unicode()

Read a file into a unicode string.

If the content of the file is not utf-8, pretend the encoding is latin1. The real encoding will be detected later.

Returns:

Type Description
str

The decoded string.
Source code in corpustools/ocrcontentconverter.py 
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
def to_unicode(self) -> str:
    """Read a file into a unicode string.

    If the content of the file is not utf-8, pretend the encoding is
    latin1. The real encoding will be detected later.

    Returns:
        The decoded string.
    """
    try:
        content = codecs.open(self.orig.as_posix(), encoding="utf8").read()
    except ValueError:
        content = codecs.open(self.orig.as_posix(), encoding="latin1").read()

    return content.replace("\r\n", "\n")

convert2intermediate(filename)

Transform an .ocr file to an intermediate xml document.

Parameters:

Name Type Description Default
filename Path

path of the file that should be converted.

 required

Returns:

Type Description
_Element

An etree element.
Source code in corpustools/ocrcontentconverter.py 
158
159
160
161
162
163
164
165
166
167
168
169
def convert2intermediate(filename: Path) -> etree._Element:
    """Transform an .ocr file to an intermediate xml document.

    Args:
        filename: path of the file that should be converted.

    Returns:
        An etree element.
    """
    converter = OcrContentConverter(filename)

    return converter.content2xml(io.StringIO(converter.to_unicode()))

join_lines(lines)

Join the lines of a paragraph into one string.

Words split over two lines are joined again, and the hyphen that split them is removed.

Parameters:

Name Type Description Default
lines list[str]

the lines belonging to one paragraph.

 required

Returns:

Type Description
str

The lines as one string.
Source code in corpustools/ocrcontentconverter.py 
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def join_lines(lines: list[str]) -> str:
    """Join the lines of a paragraph into one string.

    Words split over two lines are joined again, and the hyphen that split
    them is removed.

    Args:
        lines: the lines belonging to one paragraph.

    Returns:
        The lines as one string.
    """
    paragraph = ""

    for line in lines:
        if not paragraph:
            paragraph = line
        elif is_probably_hyphenated(paragraph, line):
            # A word split over two lines, remove the hyphen
            paragraph = f"{paragraph[:-1]}{line}"
        elif paragraph.endswith("-"):
            # A real hyphen, e.g. in a compound, keep it
            paragraph = f"{paragraph}{line}"
        else:
            paragraph = f"{paragraph} {line}"

    return paragraph