]> git.phdru.name Git - dotfiles.git/commitdiff
bin: Debug and improve filename re-encoding utilities
authorOleg Broytman <phd@phdru.name>
Thu, 24 Oct 2024 13:47:26 +0000 (16:47 +0300)
committerOleg Broytman <phd@phdru.name>
Thu, 24 Oct 2024 13:48:02 +0000 (16:48 +0300)
bin/cleanup-recode.sh
bin/cp_recode_fname
bin/get_encodings [new file with mode: 0755]
bin/recode-filenames-recursive.py
bin/recode_filenames.py
bin/unicode_norm_nfd.py

index 3f8213d2d92cd638d66f9e583f1c2def14dbbe75..4bf86d32cafb752b441bb0427f807ac64bc3dcf2 100755 (executable)
@@ -1,22 +1,15 @@
 #! /bin/sh
+set -e
 
 if ! echo "$LC_CTYPE" | grep -Fiq utf-8; then
-   echo "This script requires UTF-8 locale" 1>&2
-   exit 1
+    echo "This script requires UTF-8 locale" 1>&2
+    exit 1
 fi
 
-from_enc=utf-8
-to_enc=koi8-r
+. get_encodings
 
-while getopts f:t: opt; do
-   case $opt in
-      f ) from_enc="$OPTARG" ;;
-      t ) to_enc="$OPTARG" ;;
-   esac
-done
-shift `expr $OPTIND - 1`
-
-for dir in "${@:-.}"; do
-   cleanup-filenames-recursive.sh "$dir" &&
-   recode-filenames-recursive.py "$from_enc" "$to_enc" "$dir" || exit 1
+for dir in "${filenames:-.}"; do
+    cleanup-filenames-recursive.sh "$dir" &&
+    recode-filenames-recursive.py -f "$from_encoding" -t "$to_encoding" \
+        "$dir" || exit 1
 done
index d6dd74b8f0398574784dc8a25b987f554be81720..13fe8c2726be5015fe09ec0fda94ec6825beaa92 100755 (executable)
@@ -1,34 +1,9 @@
 #! /bin/sh
 set -e
 
-usage() {
-   echo "Usage: $0 [[from_enc] to_enc] filename" >&2
-   exit 1
-}
-
-if [ $# -eq 1 ]; then
-    from_enc="`python3 -c \"from m_lib.defenc import default_encoding; print(default_encoding)"`"
-    if [ "$from_enc" != utf-8 ]; then
-        to_enc=utf-8
-    else
-        usage
-    fi
-    filename="$1"
-elif [ $# -eq 2 ]; then
-    from_enc="`python3 -c \"from m_lib.defenc import default_encoding; print(default_encoding)"`"
-    to_enc="$1"
-    filename="$2"
-elif [ $# -eq 3 ]; then
-    from_enc="$1"
-    to_enc="$2"
-    filename="$3"
-else
-    usage
-fi
-
 cmd="`basename \"$0\"`"
 case "$cmd" in
-    cp_*) cmd="cp -p" ;;
+    cp_*) cmd="cp -ap" ;;
     mv_*) cmd=mv ;;
     *)
         echo "Unknown command $0, aborting" >&2
@@ -36,5 +11,16 @@ case "$cmd" in
     ;;
 esac
 
-filename_recoded=`echo "$filename" | iconv -f "$from_enc" -t "$to_enc"`
-exec $cmd "$filename" "$filename_recoded"
+. get_encodings
+if [ -z "$filenames" ]; then
+    echo "Usage: $0 [-f from_encoding] [-t to_encoding] path [path ...]" >&2
+    exit 1
+fi
+
+for filename in "$filenames"; do
+    filename_recoded=`echo "$filename" |
+        iconv -f "$from_encoding" -t "$to_encoding"`
+    if [ "$filename_recoded" != "$filename" ]; then
+        $cmd "$filename" "$filename_recoded"
+    fi
+done
diff --git a/bin/get_encodings b/bin/get_encodings
new file mode 100755 (executable)
index 0000000..7327926
--- /dev/null
@@ -0,0 +1,64 @@
+#! /bin/sh
+
+case "$0" in
+    */get_encodings)
+        sourced=false
+        ;;
+    *)
+        sourced=true
+        ;;
+esac
+
+from_encoding=
+to_encoding=
+default_encoding="`python3 -c \"from m_lib.defenc import default_encoding; print(default_encoding)"`"
+
+while getopts f:t: opt; do
+   case $opt in
+      f ) from_encoding="$OPTARG" ;;
+      t ) to_encoding="$OPTARG" ;;
+   esac
+done
+shift `expr $OPTIND - 1`
+filenames="$@"
+
+error() {
+    echo "$@" >&2
+    echo "Usage: $0 [-f from_encoding] [-t to_encoding] [path [path ...]]" >&2
+    exit 1
+}
+
+if [ -n "$from_encoding" ]; then
+    if [ -n "$to_encoding" ]; then
+        : # Everything is defined, no need to guess
+    elif [ "$from_encoding" = utf-8 ]; then
+        if [ "$default_encoding" = utf-8 ]; then
+            error Cannot guess to_encoding
+        else
+            to_encoding="$default_encoding"
+        fi
+    else
+        to_encoding=utf-8
+    fi
+elif [ -n "$to_encoding" ]; then
+    if [ "$to_encoding" = "$default_encoding" ]; then
+        if [ "$default_encoding" = utf-8 ]; then
+            error Cannot guess from_encoding
+        else
+            from_encoding=utf-8
+        fi
+    else
+        from_encoding="$default_encoding"
+    fi
+else
+    if [ "$default_encoding" = utf-8 ]; then
+        error Cannot guess encodings
+    else
+        from_encoding="$default_encoding"
+        to_encoding=utf-8
+    fi
+fi
+
+if [ "$sourced" = false ]; then
+    echo "$from_encoding $to_encoding"
+fi
index 54085636506bf1f642663a8fc5697541a762935b..1d73a92cfad9d4c781e843157e79670a55882279 100755 (executable)
@@ -1,57 +1,39 @@
 #! /usr/bin/env python3
 
-import sys, os
+import os
 
-from m_lib.defenc import default_encoding
+from recode_filenames import parse_args, build_recode
 
-if len(sys.argv) == 1:
-    src_encoding = default_encoding
-    if src_encoding == 'utf-8':
-        sys.exit("Usage: %s [[src_enc] dst_enc [start_dir]]" % sys.argv[0])
-    else:
-        dst_encoding = 'utf-8'
-    start_dir = '.'
-elif len(sys.argv) == 2:
-    src_encoding = default_encoding
-    dst_encoding = sys.argv[1]
-    start_dir = '.'
-elif len(sys.argv) == 3:
-    src_encoding = default_encoding
-    dst_encoding = sys.argv[1]
-    start_dir = sys.argv[2]
-elif len(sys.argv) == 4:
-    src_encoding = sys.argv[1]
-    dst_encoding = sys.argv[2]
-    start_dir = sys.argv[3]
-else:
-    sys.exit("Usage: %s [[src_enc] dst_enc [start_dir]]" % sys.argv[0])
-
-# Fake for recode_filenames.py
-sys.argv = ['', src_encoding, dst_encoding]
-from recode_filenames import _recode
+from_encoding, to_encoding, dirnames = parse_args(default='.')
+_recode = build_recode(from_encoding, to_encoding)
 
 
 def _onerror(exc):
     raise exc
 
-plist = list(os.walk(start_dir, topdown=False, onerror=_onerror))
-
 
 save_dir = os.getcwd()
-for dirname, _subdirs, fnames in plist:
-    if dirname == '.':
-        continue
-    os.chdir(dirname)
-    for filename in fnames:
-        # if not exists - it was renamed already
-        if os.path.exists(filename) and \
-                os.path.isfile(filename):
-            newname = _recode(filename)
-            if newname != filename:
-                os.rename(filename, newname)
-    os.chdir('..')
-    dirname = os.path.basename(dirname)
-    newname = _recode(dirname)
-    if newname != dirname:
-        os.rename(dirname, newname)
-    os.chdir(save_dir)
+for start_dir in dirnames:
+    for dirname, _subdirs, fnames in list(
+        os.walk(start_dir, topdown=False, onerror=_onerror)
+    ):
+        if dirname != '.':
+            os.chdir(dirname)
+        for filename in fnames:
+            # if not exists - it was renamed already
+            if os.path.exists(filename) and \
+                    os.path.isfile(filename):
+                newname = _recode(filename)
+                if not isinstance(newname, str):
+                    newname = newname.decode()
+                if newname != filename:
+                    os.rename(filename, newname)
+        if dirname != '.':
+            os.chdir('..')
+            dirname = os.path.basename(dirname)
+            newname = _recode(dirname)
+            if not isinstance(newname, str):
+                newname = newname.decode()
+            if newname != dirname:
+                os.rename(dirname, newname)
+        os.chdir(save_dir)
index b8d3fa04daca8794a10479951126cb9e3a7932ac..bd10bf53a7496e3fd739addc69e42797566c86e8 100755 (executable)
 #! /usr/bin/env python3
 # -*- coding: koi8-r -*-
 
+import argparse
 import sys
 
-src_encoding = sys.argv[1]
-dst_encoding = sys.argv[2]
+from m_lib.defenc import default_encoding
 
-if src_encoding == "translit":
-    if dst_encoding == "koi8-r":
-        from m_lib.rus.lat2rus import lat2koi as _recode
-    elif dst_encoding == "cp1251":
-        from m_lib.rus.lat2rus import lat2win as _recode
+
+def parse_args(default=None):
+    parser = argparse.ArgumentParser(description='Recode filenames')
+    parser.add_argument('-f', '--from-encoding', help='from encoding')
+    parser.add_argument('-t', '--to-encoding', help='to encoding')
+    parser.add_argument('filename', nargs='*' if default else '+',
+                        default=[default], help='filenames to recode')
+    args = parser.parse_args()
+
+    from_encoding = args.from_encoding
+    to_encoding = args.to_encoding
+
+    if from_encoding:
+        if to_encoding:
+            pass  # Everything is defined, no need to guess
+        elif from_encoding == 'utf-8':
+            if default_encoding == 'utf-8':
+                sys.exit('Cannot guess to_encoding')
+            else:
+                to_encoding = default_encoding
+        else:
+            to_encoding = 'utf-8'
+    elif to_encoding:
+        if to_encoding == default_encoding:
+            if default_encoding == 'utf-8':
+                sys.exit('Cannot guess from_encoding')
+            else:
+                from_encoding = 'utf-8'
+        else:
+            from_encoding = default_encoding
     else:
-        raise NotImplementedError("destination encoding must be koi8-r or cp1251, not `%s'" % dst_encoding)
+        if default_encoding == 'utf-8':
+            sys.exit('Cannot guess encodings')
+        else:
+            from_encoding = default_encoding
+            to_encoding = 'utf-8'
+
+    return from_encoding, to_encoding, args.filename
+
+
+def build_recode(from_encoding, to_encoding):
+    if from_encoding == "translit":
+        if to_encoding == "koi8-r":
+            from m_lib.rus.lat2rus import lat2koi as _recode
+        elif to_encoding == "cp1251":
+            from m_lib.rus.lat2rus import lat2win as _recode
+        else:
+            raise NotImplementedError(
+                "destination encoding must be koi8-r or cp1251, "
+                "not `%s'" % to_encoding)
+
+    elif to_encoding == "translit":
+        if from_encoding == "koi8-r":
+            from m_lib.rus.rus2lat import koi2lat as _recode
+        elif from_encoding == "cp1251":
+            from m_lib.rus.rus2lat import win2lat as _recode
+        else:
+            raise NotImplementedError(
+                "source encoding must be koi8-r or cp1251, "
+                "not `%s'" % from_encoding)
+
+        from m_lib.rus.rus2lat import koi2lat_d
+        koi2lat_d["Ъ"] = ''  # remove apostrophs -
+        koi2lat_d["Ь"] = ''  # they are not very good characters in filenames
+        koi2lat_d["ъ"] = ''  # especially on Windoze
+        koi2lat_d["ь"] = ''  # :-)
+
+    elif from_encoding == "url":
+        try:
+            from_encoding, to_encoding = to_encoding.split('/')
+        except ValueError:
+            from_encoding = to_encoding
+        from urllib.parse import unquote
+        from m_lib.opstring import recode
+
+        def _recode(s):
+            s = unquote(s)
+            if from_encoding != to_encoding:
+                s = recode(s, from_encoding, to_encoding, "replace")
+            return s
+
+    elif to_encoding == "url":
+        try:
+            from_encoding, to_encoding = from_encoding.split('/')
+        except ValueError:
+            to_encoding = from_encoding
+        from urllib.parse import quote
+        from m_lib.opstring import recode
+
+        def _recode(s):
+            if from_encoding != to_encoding:
+                s = recode(s, from_encoding, to_encoding, "replace")
+            #                wget treats them as safe
+            #                     vvvvvvvvvvvvv
+            return quote(s, safe=";/?:@&=+$,()'")
 
-elif dst_encoding == "translit":
-    if src_encoding == "koi8-r":
-        from m_lib.rus.rus2lat import koi2lat as _recode
-    elif src_encoding == "cp1251":
-        from m_lib.rus.rus2lat import win2lat as _recode
     else:
-        raise NotImplementedError("source encoding must be koi8-r or cp1251, not `%s'" % src_encoding)
-
-    from m_lib.rus.rus2lat import koi2lat_d
-    koi2lat_d["Ъ"] = '' # remove apostrophs -
-    koi2lat_d["Ь"] = '' # they are not very good characters in filenames
-    koi2lat_d["ъ"] = '' # especially on Windoze
-    koi2lat_d["ь"] = '' # :-)
-
-elif src_encoding == "url":
-    try:
-        src_encoding, dst_encoding = dst_encoding.split('/')
-    except ValueError:
-        src_encoding = dst_encoding
-    from m_lib.opstring import recode
-    import urllib
-    def _recode(s):
-        s = urllib.unquote(s)
-        if src_encoding != dst_encoding:
-            s = recode(s, src_encoding, dst_encoding, "replace")
-        return s
-
-elif dst_encoding == "url":
-    try:
-        src_encoding, dst_encoding = src_encoding.split('/')
-    except ValueError:
-        dst_encoding = src_encoding
-    from m_lib.opstring import recode
-    import urllib
-    def _recode(s):
-        if src_encoding != dst_encoding:
-            s = recode(s, src_encoding, dst_encoding, "replace")
-        return urllib.quote(s, safe=";/?:@&=+$,()'") # wget treats them as safe
-
-else:
-    from m_lib.opstring import recode
-    def _recode(s):
-        return recode(s, src_encoding, dst_encoding, "replace")
+        def _recode(s):
+            return s.encode(to_encoding, "surrogateescape").\
+                decode(from_encoding, "surrogateescape")
+
+    return _recode
 
 
 if __name__ == "__main__":
     import os
-    for filename in sys.argv[3:]:
+    from_encoding, to_encoding, filenames = parse_args()
+    _recode = build_recode(from_encoding, to_encoding)
+    for filename in filenames:
         new_name = _recode(filename)
-        if type(filename) is not type(new_name):
+        if not isinstance(new_name, str):
             new_name = new_name.decode()
         if new_name != filename:
+            sys.stdout.buffer.write(
+                b'%s %s / %s -> %s\n' % (
+                    from_encoding.encode(), to_encoding.encode(),
+                    filename.encode(default_encoding, 'replace'),
+                    new_name.encode())
+                )
             os.rename(filename, new_name)
index 2cca0d9cd7168ae93e6241c710ab190897919d7a..030de472232055aad1d7c63a73b7d65dc5eb4ddd 100755 (executable)
@@ -4,6 +4,8 @@
 import sys
 import unicodedata
 
+from m_lib.defenc import default_encoding
+
 
 def strip_accents(s):
     return ''.join(c for c in unicodedata.normalize('NFD', s)
@@ -20,4 +22,7 @@ if __name__ == '__main__':
     if len(sys.argv) == 1:
         sys.exit('Usage: %s name\n' % sys.argv[0])
     for name in sys.argv[1:]:
-        print(latin1_to_ascii(name))
+        sys.stdout.buffer.write(
+            latin1_to_ascii(name).encode(default_encoding, 'surrogateescape')
+            + b'\n'
+        )