Skip to content

convertermanager

This file manages conversion of files to the Giella xml format.

ConverterManager

Manage the conversion of original files to corpus xml.

Class/static variables: _languageguesser (text_cat.Classifier): Language guesser to indicate languages in the converted document. Attributes: write_intermediate (bool): indicate whether intermediate versions of the converted document should be written to disk. goldstandard (bool): indicating whether goldstandard documents should be converted. files (list of str): list of paths to original files that should be converted from original format to xml.

Source code in corpustools/convertermanager.py 
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
class ConverterManager:
    """Manage the conversion of original files to corpus xml.

    Class/static variables:
        _languageguesser (text_cat.Classifier): Language guesser to indicate
            languages in the converted document.
    Attributes:
        write_intermediate (bool): indicate whether intermediate versions
            of the converted document should be written to disk.
        goldstandard (bool): indicating whether goldstandard documents
            should be converted.
        files (list of str): list of paths to original files that should
            be converted from original format to xml.
    """

    _languageguesser = None

    def languageguesser(self):
        """Return our language guesser.
        This is a class variable, but since it takes a while to initialise,
        we don't do it until it's needed."""
        if self._languageguesser is None:
            ConverterManager._languageguesser = text_cat.Classifier(None)
        return self._languageguesser

    def __init__(
        self, lazy_conversion=False, write_intermediate=False, goldstandard=False
    ):
        """Initialise the ConverterManager class.

        Args:
            lazy_conversion (bool): indicate whether conversion depends on the
                fact that metadata have changed since last conversion.
            write_intermediate (bool): indicating whether intermediate versions
                of the converted document should be written to disk.
            goldstandard (bool): indicating whether goldstandard documents
                should be converted.
        """
        self.lazy_conversion = lazy_conversion
        self.write_intermediate = write_intermediate
        self.goldstandard = goldstandard
        self.files = []

    def convert(self, orig_file: CorpusPath):
        """Convert file to corpus xml format.

        Args:
            orig_file: the path to the original file.
        """
        try:
            conv = converter.Converter(orig_file, lazy_conversion=self.lazy_conversion)
            conv.write_complete(self.languageguesser())
        except (
            util.ConversionError,
            ValueError,
            IndexError,
        ) as error:
            LOGGER.warn("Could not convert %s\n%s", orig_file, error)
            raise

    def convert_in_parallel(self, pool_size):
        """Convert files using the multiprocessing module."""
        nfiles = len(self.files)
        futures = {}  # Future -> filename
        print(f"Starting parallel conversion with {pool_size} workers")
        failed = []
        with ProcessPoolExecutor(max_workers=pool_size) as pool:
            for file in self.files:
                fut = pool.submit(partial(self.convert, file))
                futures[fut] = file

            for i, future in enumerate(as_completed(futures), start=1):
                exc = future.exception()
                filename = futures[future]
                if exc is not None:
                    failed.append(filename)
                    print(f"[{i}/{nfiles} FAILED: {filename}")
                    print(exc)
                else:
                    print(f"[{i}/{nfiles}] done: {filename}")

        n_ok = nfiles - len(failed)
        print(f"all done converting. {n_ok} files converted ok, {len(failed)} failed")
        if failed:
            print("the files that failed to convert are:")
            for filename in failed:
                print(filename)

    def convert_serially(self):
        """Convert the files in one process."""
        LOGGER.info("Starting the conversion of %d files", len(self.files))

        for orig_file in self.files:
            LOGGER.debug("converting %s", orig_file)
            self.convert(orig_file)

    def corpus_paths(self, sources: list[str]) -> Iterator[CorpusPath]:
        """Yield all convertible files in sources.

        Args:
            sources (list of str): a list of files or directories where
                convertable files are found.
        Yields:
            Convertible files found in sources.
        """
        LOGGER.info("Collecting files to convert")

        paths = (Path(source) for source in sources)
        for path in paths:
            if path.is_file():
                yield make_corpus_path(path.as_posix())
            elif path.is_dir():
                for xsl_file in path.rglob("*.xsl"):
                    yield make_corpus_path(xsl_file.as_posix())
            else:
                LOGGER.error(
                    "Can not process %s\nThis is neither a file nor a directory.",
                    path,
                )

    def collect_files(self, sources: list[str]):
        """Find all convertible files in sources.

        Args:
            sources (list of str): a list of files or directories where
                convertable files are found.
        """
        LOGGER.info("Collecting files to convert")

        self.files = [
            c_path
            for c_path in self.corpus_paths(sources)
            if c_path.is_convertable(self.goldstandard)
        ]

__init__(lazy_conversion=False, write_intermediate=False, goldstandard=False)

Initialise the ConverterManager class.

Parameters:

Name Type Description Default
lazy_conversion bool

indicate whether conversion depends on the fact that metadata have changed since last conversion.

 False
write_intermediate bool

indicating whether intermediate versions of the converted document should be written to disk.

 False
goldstandard bool

indicating whether goldstandard documents should be converted.

 False
Source code in corpustools/convertermanager.py 
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
def __init__(
    self, lazy_conversion=False, write_intermediate=False, goldstandard=False
):
    """Initialise the ConverterManager class.

    Args:
        lazy_conversion (bool): indicate whether conversion depends on the
            fact that metadata have changed since last conversion.
        write_intermediate (bool): indicating whether intermediate versions
            of the converted document should be written to disk.
        goldstandard (bool): indicating whether goldstandard documents
            should be converted.
    """
    self.lazy_conversion = lazy_conversion
    self.write_intermediate = write_intermediate
    self.goldstandard = goldstandard
    self.files = []

collect_files(sources)

Find all convertible files in sources.

Parameters:

Name Type Description Default
sources list of str

a list of files or directories where convertable files are found.

 required
Source code in corpustools/convertermanager.py 
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def collect_files(self, sources: list[str]):
    """Find all convertible files in sources.

    Args:
        sources (list of str): a list of files or directories where
            convertable files are found.
    """
    LOGGER.info("Collecting files to convert")

    self.files = [
        c_path
        for c_path in self.corpus_paths(sources)
        if c_path.is_convertable(self.goldstandard)
    ]

convert(orig_file)

Convert file to corpus xml format.

Parameters:

Name Type Description Default
orig_file CorpusPath

the path to the original file.

 required
Source code in corpustools/convertermanager.py 
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
def convert(self, orig_file: CorpusPath):
    """Convert file to corpus xml format.

    Args:
        orig_file: the path to the original file.
    """
    try:
        conv = converter.Converter(orig_file, lazy_conversion=self.lazy_conversion)
        conv.write_complete(self.languageguesser())
    except (
        util.ConversionError,
        ValueError,
        IndexError,
    ) as error:
        LOGGER.warn("Could not convert %s\n%s", orig_file, error)
        raise

convert_in_parallel(pool_size)

Convert files using the multiprocessing module.

Source code in corpustools/convertermanager.py 
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def convert_in_parallel(self, pool_size):
    """Convert files using the multiprocessing module."""
    nfiles = len(self.files)
    futures = {}  # Future -> filename
    print(f"Starting parallel conversion with {pool_size} workers")
    failed = []
    with ProcessPoolExecutor(max_workers=pool_size) as pool:
        for file in self.files:
            fut = pool.submit(partial(self.convert, file))
            futures[fut] = file

        for i, future in enumerate(as_completed(futures), start=1):
            exc = future.exception()
            filename = futures[future]
            if exc is not None:
                failed.append(filename)
                print(f"[{i}/{nfiles} FAILED: {filename}")
                print(exc)
            else:
                print(f"[{i}/{nfiles}] done: {filename}")

    n_ok = nfiles - len(failed)
    print(f"all done converting. {n_ok} files converted ok, {len(failed)} failed")
    if failed:
        print("the files that failed to convert are:")
        for filename in failed:
            print(filename)

convert_serially()

Convert the files in one process.

Source code in corpustools/convertermanager.py 
123
124
125
126
127
128
129
def convert_serially(self):
    """Convert the files in one process."""
    LOGGER.info("Starting the conversion of %d files", len(self.files))

    for orig_file in self.files:
        LOGGER.debug("converting %s", orig_file)
        self.convert(orig_file)

corpus_paths(sources)

Yield all convertible files in sources.

Parameters:

Name Type Description Default
sources list of str

a list of files or directories where convertable files are found.

 required

Yields: Convertible files found in sources.

Source code in corpustools/convertermanager.py 
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
def corpus_paths(self, sources: list[str]) -> Iterator[CorpusPath]:
    """Yield all convertible files in sources.

    Args:
        sources (list of str): a list of files or directories where
            convertable files are found.
    Yields:
        Convertible files found in sources.
    """
    LOGGER.info("Collecting files to convert")

    paths = (Path(source) for source in sources)
    for path in paths:
        if path.is_file():
            yield make_corpus_path(path.as_posix())
        elif path.is_dir():
            for xsl_file in path.rglob("*.xsl"):
                yield make_corpus_path(xsl_file.as_posix())
        else:
            LOGGER.error(
                "Can not process %s\nThis is neither a file nor a directory.",
                path,
            )

languageguesser()

Return our language guesser. This is a class variable, but since it takes a while to initialise, we don't do it until it's needed.

Source code in corpustools/convertermanager.py 
52
53
54
55
56
57
58
def languageguesser(self):
    """Return our language guesser.
    This is a class variable, but since it takes a while to initialise,
    we don't do it until it's needed."""
    if self._languageguesser is None:
        ConverterManager._languageguesser = text_cat.Classifier(None)
    return self._languageguesser

main()

Convert documents to giellatekno xml format.

Source code in corpustools/convertermanager.py 
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
def main():
    """Convert documents to giellatekno xml format."""
    LOGGER.setLevel(logging.WARNING)
    try:
        sanity_check()
    except (util.SetupError, util.ExecutableMissingError) as error:
        raise SystemExit(str(error)) from error

    args = parse_options()

    manager = ConverterManager(
        args.lazy_conversion, args.write_intermediate, args.goldstandard
    )
    manager.collect_files(args.sources)

    try:
        if args.serial:
            LOGGER.setLevel(logging.DEBUG)
            manager.convert_serially()
        else:
            manager.convert_in_parallel(args.ncpus)
    except util.ExecutableMissingError as error:
        raise SystemExit(str(error)) from error

parse_options()

Parse the commandline options.

Returns:

Type Description
Namespace

the parsed commandline arguments
Source code in corpustools/convertermanager.py 
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
def parse_options():
    """Parse the commandline options.

    Returns:
        (argparse.Namespace): the parsed commandline arguments
    """
    parser = argparse.ArgumentParser(
        parents=[argparse_version.parser],
        description="Convert original files to giellatekno xml.",
    )

    parser.add_argument("--ncpus", action=NCpus)
    parser.add_argument(
        "--skip-existing",
        action="store_true",
        help="skip converting files that are already converted (that already "
        "exist in the converted/ folder",
    )
    parser.add_argument(
        "--serial",
        action="store_true",
        help="use this for debugging the conversion \
                        process. When this argument is used files will \
                        be converted one by one.",
    )
    parser.add_argument(
        "--lazy-conversion",
        action="store_true",
        help="Reconvert only if metadata have changed.",
    )
    parser.add_argument(
        "--write-intermediate",
        action="store_true",
        help="Write the intermediate XML representation \
                        to ORIGFILE.im.xml, for debugging the XSLT.\
                        (Has no effect if the converted file already exists.)",
    )
    parser.add_argument(
        "--goldstandard",
        action="store_true",
        help="Convert goldstandard and .correct files",
    )
    parser.add_argument(
        "sources",
        nargs="+",
        help="The original file(s) or \
                        directory/ies where the original files exist",
    )

    args = parser.parse_args()

    return args

sanity_check()

Check that needed programs and environment variables are set.

Source code in corpustools/convertermanager.py 
234
235
236
237
238
239
240
241
242
243
244
def sanity_check():
    """Check that needed programs and environment variables are set."""
    util.sanity_check(["pdftotext", "latex2html", "pandoc"])
    if not os.path.isfile(converter.Converter.get_dtd_location()):
        raise util.SetupError(
            "Couldn't find {}\n"
            "Check that GTHOME points at the right directory "
            "(currently: {}).".format(
                converter.Converter.get_dtd_location(), os.environ["GTHOME"]
            )
        )

unwrap_self_convert(arg, **kwarg)

Unpack self from the arguments and call convert again.

This is due to how multiprocess works: http://www.rueckstiess.net/research/snippets/show/ca1d7d90

Source code in corpustools/convertermanager.py 
171
172
173
174
175
176
177
def unwrap_self_convert(arg, **kwarg):
    """Unpack self from the arguments and call convert again.

    This is due to how multiprocess works:
    http://www.rueckstiess.net/research/snippets/show/ca1d7d90
    """
    return ConverterManager.convert(*arg, **kwarg)