namechanger

Classes and functions change names of corpus files.

CorpusFileRemover

Remove an original file and all its derived files.

Source code in /home/anders/projects/CorpusTools/corpustools/namechanger.py

class CorpusFileRemover:
    """Remove an original file and all its derived files."""

    def __init__(self, oldpath):
        """Class to remove corpus files.

        Args:
            oldpath (unicode): the old path
        """
        self.old_corpus_path = corpuspath.make_corpus_path(oldpath)
        p = Path(oldpath)
        if not p.exists():
            raise SystemExit(f"{oldpath} does not exist!")
        self.orig_vcs = versioncontrol.vcs(self.old_corpus_path.orig_corpus_dir)
        self.conv_vcs = versioncontrol.vcs(self.old_corpus_path.converted_corpus_dir)

    def remove_files(self):
        """Remove all the files that are under version control."""
        self.orig_vcs.remove(self.old_corpus_path.orig)
        self.orig_vcs.remove(self.old_corpus_path.xsl)
        self.conv_vcs.remove(self.old_corpus_path.converted)
        for lang in self.old_corpus_path.metadata.get_parallel_texts():
            if os.path.exists(self.old_corpus_path.tmx(lang)):
                self.conv_vcs.remove(self.old_corpus_path.tmx(lang))

__init__(oldpath)

Class to remove corpus files.

Parameters:

Name	Type	Description	Default
oldpath	unicode	the old path	required

Source code in /home/anders/projects/CorpusTools/corpustools/namechanger.py

def __init__(self, oldpath):
    """Class to remove corpus files.

    Args:
        oldpath (unicode): the old path
    """
    self.old_corpus_path = corpuspath.make_corpus_path(oldpath)
    p = Path(oldpath)
    if not p.exists():
        raise SystemExit(f"{oldpath} does not exist!")
    self.orig_vcs = versioncontrol.vcs(self.old_corpus_path.orig_corpus_dir)
    self.conv_vcs = versioncontrol.vcs(self.old_corpus_path.converted_corpus_dir)

remove_files()

Remove all the files that are under version control.

Source code in /home/anders/projects/CorpusTools/corpustools/namechanger.py

def remove_files(self):
    """Remove all the files that are under version control."""
    self.orig_vcs.remove(self.old_corpus_path.orig)
    self.orig_vcs.remove(self.old_corpus_path.xsl)
    self.conv_vcs.remove(self.old_corpus_path.converted)
    for lang in self.old_corpus_path.metadata.get_parallel_texts():
        if os.path.exists(self.old_corpus_path.tmx(lang)):
            self.conv_vcs.remove(self.old_corpus_path.tmx(lang))

NamechangerError

Bases: Exception

This exception is raised when errors occurs in this module.

Source code in /home/anders/projects/CorpusTools/corpustools/namechanger.py

class NamechangerError(Exception):
    """This exception is raised when errors occurs in this module."""

are_duplicates(oldpath, newpath)

Check if oldpath and newpath are duplicates of each other.

Parameters:

Name	Type	Description	Default
oldpath	unicode	old path of the file	required
newpath	unicode	the wanted, new path of the file	required

Returns:

Type	Description
bool	a boolean indicating if the two files are duplicates

Source code in /home/anders/projects/CorpusTools/corpustools/namechanger.py

def are_duplicates(oldpath, newpath):
    """Check if oldpath and newpath are duplicates of each other.

    Args:
        oldpath (unicode): old path of the file
        newpath (unicode): the wanted, new path of the file

    Returns:
        (bool): a boolean indicating if the two files are duplicates
    """
    if os.path.isfile(oldpath) and os.path.isfile(newpath):
        with open(oldpath, "rb") as oldcontent, open(newpath, "rb") as newcontent:
            return compute_hexdigest(oldcontent) == compute_hexdigest(newcontent)
    else:
        return False

compute_hexdigest(afile, blocksize=65536)

Compute the hexdigest of the file in path.

Parameters:

Name	Type	Description	Default
afile	file	a file like object	required

Returns:

Type	Description
str	a hexdigest of the file

Source code in /home/anders/projects/CorpusTools/corpustools/namechanger.py

def compute_hexdigest(afile, blocksize=65536):
    """Compute the hexdigest of the file in path.

    Args:
        afile (file): a file like object

    Returns:
        (str): a hexdigest of the file
    """
    hasher = hashlib.md5()
    buf = afile.read(blocksize)
    while buf:
        hasher.update(buf)
        buf = afile.read(blocksize)

    return hasher.hexdigest()

compute_new_basename(orig_path)

Compute the new path.

Parameters:

Name	Type	Description	Default
orig_path	Path	path to file, basename should possibly be normalised	required

Returns:

Type	Description
pathlib.Path	lower cased, ascii path

Source code in /home/anders/projects/CorpusTools/corpustools/namechanger.py

def compute_new_basename(orig_path):
    """Compute the new path.

    Args:
        orig_path (Path): path to file, basename should possibly be normalised

    Returns:
        (pathlib.Path): lower cased, ascii path
    """
    wanted_basename = normalise_filename(orig_path.name)
    new_path = orig_path.with_name(wanted_basename)
    index = 1
    while os.path.exists(new_path):
        if are_duplicates(orig_path, new_path):
            raise UserWarning(f"{orig_path} and {new_path} are duplicates. ")
        else:
            if "." in wanted_basename:
                dot = wanted_basename.rfind(".")
                extension = wanted_basename[dot:]
                pre_extension = wanted_basename[:dot]
                new_basename = pre_extension + "_" + str(index) + extension
            else:
                new_basename = wanted_basename + str(index)
            new_path = orig_path.with_name(new_basename)
            index += 1

    return new_path

normalise_filename(filename)

Normalise filename to ascii only.

Downcase filename, replace non-ascii characters with ascii ones and remove or replace unwanted characters.

Parameters:

Name	Type	Description	Default
filename	str	name of the file	required

Returns:

Type	Description
str	a downcased string containing only ascii chars

Source code in /home/anders/projects/CorpusTools/corpustools/namechanger.py

def normalise_filename(filename):
    """Normalise filename to ascii only.

    Downcase filename, replace non-ascii characters with ascii ones and
    remove or replace unwanted characters.

    Args:
        filename (str): name of the file

    Returns:
        (str): a downcased string containing only ascii chars
    """
    if os.sep in filename:
        raise NamechangerError(
            "Invalid filename {}.\n"
            "Filename is not allowed to contain {}".format(filename, os.sep)
        )

    # unicode.decode wants a unicode string
    if not isinstance(filename, str):
        filename = filename.decode("utf8")

    # unidecode.unidecode makes ascii only
    # urllib.unquote replaces %xx escapes by their single-character equivalent.
    asciiname = unidecode.unidecode(unquote(filename))

    while asciiname.startswith(("-", "_")):
        asciiname = asciiname[1:]

    unwanted = re.compile("[+ ()'–?,!,<>\"&;&#\\|$]+")

    return unwanted.sub("_", asciiname).lower()