X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=mimedecode.py;h=48a6aba65ff01450825798e128319e0479d4e917;hb=920a25c967df8757fcfca21d1afc521e3f09d1a0;hp=6d4c7ea8cef6856844935337d0798eff27c61da1;hpb=69acf148036c062b05dd115cc45a80e28140f3e9;p=mimedecode.git diff --git a/mimedecode.py b/mimedecode.py index 6d4c7ea..48a6aba 100755 --- a/mimedecode.py +++ b/mimedecode.py @@ -1,10 +1,9 @@ #! /usr/bin/env python """Decode MIME message""" -from mimedecode_version import __version__, __author__, __copyright__, __license__ - import sys, os -import email +from mimedecode_version import __version__, \ + __author__, __copyright__, __license__ me = os.path.basename(sys.argv[0]) @@ -18,7 +17,7 @@ Broytman mimedecode.py version %s, %s def usage(code=0, errormsg=''): version(0) sys.stdout.write("""\ -Usage: %s [-h|--help] [-V|--version] [-cCDP] [-H|--host=hostname] [-f charset] [-d header1[,h2,...]|*[,-h1,...]] [-p header1[,h2,h3,...]:param1[,p2,p3,...]] [-r header] [-R header:param] [--remove-params=header] [-beit mask] [-o output_file] [input_file [output_file]] + Usage: %s [-h|--help] [-V|--version] [-cCDP] [-H|--host=hostname] [-f charset] [-d header1[,h2,...]|*[,-h1,...]] [-p header1[,h2,h3,...]:param1[,p2,p3,...]] [-r header1[,h2,...]|*[,-h1,...]] [-R header1[,h2,h3,...]:param1[,p2,p3,...]] [--set-header header:value] [--set-param header:param=value] [-Bbeit mask] [--save-headers|body|message mask] [-O dest_dir] [-o output_file] [input_file [output_file]] """ % me) if errormsg: sys.stderr.write(errormsg + '\n') @@ -34,12 +33,9 @@ def output_headers(msg): output("\n") # End of headers -def recode(s, charset): - return unicode(s, charset, "replace").encode(gopts.default_encoding, "replace") - def recode_if_needed(s, charset): - if charset and charset.lower() <> gopts.default_encoding: - s = recode(s, charset) + if charset and charset.lower() <> g.default_encoding: + s = unicode(s, charset, "replace").encode(g.default_encoding, "replace") return s @@ -77,9 +73,6 @@ def decode_header(msg, header): set_header(msg, header, new_value) -def _decode_header_param(s): - return recode_if_needed(s[2], s[0]) - def decode_header_param(msg, header, param): "Decode mail header's parameter (if exists) and put it back, if it was encoded" @@ -87,7 +80,7 @@ def decode_header_param(msg, header, param): value = msg.get_param(param, header=header) if value: if isinstance(value, tuple): - new_value = _decode_header_param(value) + new_value = recode_if_needed(value[2], value[0]) else: new_value = _decode_header(value) if new_value <> value: # do not bother to touch msg if not changed @@ -97,26 +90,68 @@ def decode_header_param(msg, header, param): def _get_exceptions(list): return [x[1:].lower() for x in list[1:] if x[0]=='-'] +def _decode_headers_params(msg, header, decode_all_params, param_list): + if decode_all_params: + params = msg.get_params(header=header) + if params: + for param, value in params: + if param not in param_list: + decode_header_param(msg, header, param) + else: + for param in param_list: + decode_header_param(msg, header, param) + +def _remove_headers_params(msg, header, remove_all_params, param_list): + if remove_all_params: + params = msg.get_params(header=header) + if params: + if param_list: + for param, value in params: + if param not in param_list: + msg.del_param(param, header) + else: + value = msg[header] + if value is None: # No such header + return + if ';' not in value: # There are no parameters + return + del msg[header] # Delete all such headers + # Get the value without parameters and set it back + msg[header] = value.split(';')[0].strip() + else: + for param in param_list: + msg.del_param(param, header) + def decode_headers(msg): "Decode message headers according to global options" - for header in gopts.remove_headers: - del msg[header] + for header_list in g.remove_headers: + header_list = header_list.split(',') + if header_list[0] == '*': # Remove all headers except listed + header_list = _get_exceptions(header_list) + for header in msg.keys(): + if header.lower() not in header_list: + del msg[header] + else: # Remove listed headers + for header in header_list: + del msg[header] - for header in gopts.remove_all_params: - value = msg[header] - if value is None: # No such header - continue - if ';' not in value: # There are no parameters - continue - del msg[header] # Delete all such headers - # Get the value without parameters and set it back - msg[header] = value.split(';')[0].strip() - - for header, param in gopts.remove_header_params: - msg.del_param(param, header) - - for header_list in gopts.decode_headers: + for header_list, param_list in g.remove_headers_params: + header_list = header_list.split(',') + param_list = param_list.split(',') + remove_all_params = param_list[0] == '*' # Remove all params except listed + if remove_all_params: + param_list = _get_exceptions(param_list) + if header_list[0] == '*': # Remove for all headers except listed + header_list = _get_exceptions(header_list) + for header in msg.keys(): + if header.lower() not in header_list: + _remove_headers_params(msg, header, remove_all_params, param_list) + else: # Decode for listed headers + for header in header_list: + _remove_headers_params(msg, header, remove_all_params, param_list) + + for header_list in g.decode_headers: header_list = header_list.split(',') if header_list[0] == '*': # Decode all headers except listed header_list = _get_exceptions(header_list) @@ -127,19 +162,20 @@ def decode_headers(msg): for header in header_list: decode_header(msg, header) - for header_list, param_list in gopts.decode_header_params: + for header_list, param_list in g.decode_header_params: header_list = header_list.split(',') param_list = param_list.split(',') + decode_all_params = param_list[0] == '*' # Decode all params except listed + if decode_all_params: + param_list = _get_exceptions(param_list) if header_list[0] == '*': # Decode for all headers except listed header_list = _get_exceptions(header_list) for header in msg.keys(): if header.lower() not in header_list: - for param in param_list: - decode_header_param(msg, header, param) + _decode_headers_params(msg, header, decode_all_params, param_list) else: # Decode for listed headers for header in header_list: - for param in param_list: - decode_header_param(msg, header, param) + _decode_headers_params(msg, header, decode_all_params, param_list) def set_header(msg, header, value): @@ -196,7 +232,7 @@ def decode_body(msg, s): os.remove(filename) set_content_type(msg, "text/plain") - msg["X-MIME-Autoconverted"] = "from %s to text/plain by %s id %s" % (content_type, gopts.host_name, command.split()[0]) + msg["X-MIME-Autoconverted"] = "from %s to text/plain by %s id %s" % (content_type, g.host_name, command.split()[0]) return s @@ -205,11 +241,11 @@ def recode_charset(msg, s): "Recode charset of the message to the default charset" save_charset = charset = msg.get_content_charset() - if charset and charset.lower() <> gopts.default_encoding: + if charset and charset.lower() <> g.default_encoding: s = recode_if_needed(s, charset) content_type = msg.get_content_type() - set_content_type(msg, content_type, gopts.default_encoding) - msg["X-MIME-Autoconverted"] = "from %s to %s by %s id %s" % (save_charset, gopts.default_encoding, gopts.host_name, me) + set_content_type(msg, content_type, g.default_encoding) + msg["X-MIME-Autoconverted"] = "from %s to %s by %s id %s" % (save_charset, g.default_encoding, g.host_name, me) return s @@ -218,54 +254,124 @@ def totext(msg, instring): # Decode body and recode charset s = decode_body(msg, instring) - if gopts.recode_charset: + if g.recode_charset: s = recode_charset(msg, s) output_headers(msg) output(s) + return s + + +mimetypes = None + +def _guess_extension(ctype): + global mimetypes + if mimetypes is None: + import mimetypes + mimetypes.init() + user_mime_type = os.path.expanduser('~/.mime.types') + if os.path.exists(user_mime_type): + mimetypes._db.read(user_mime_type) + return mimetypes.guess_extension(ctype) + +def _save_message(msg, outstring, save_headers=False, save_body=False): + for header, param in ( + ("Content-Disposition", "filename"), + ("Content-Type", "name"), + ): + fname = msg.get_param(param, header=header) + if fname: + if isinstance(fname, tuple): + fname = fname[2] # Do not recode if it isn't recoded yet + try: + for forbidden in chr(0), '/', '\\': + if forbidden in fname: + raise ValueError + except ValueError: + continue + fname = '-' + fname + break + else: + fname = '' + g.save_counter += 1 + fname = str(g.save_counter) + fname + if '.' not in fname: + ext = _guess_extension(msg.get_content_type()) + if ext: fname += ext + + global output + save_output = output + outfile = open_output_file(fname) + output = outfile.write + if save_headers: + output_headers(msg) + if save_body: + output(outstring) + outfile.close() + output = save_output def decode_part(msg): "Decode one part of the message" decode_headers(msg) - encoding = msg["Content-Transfer-Encoding"] - - if encoding in (None, '', '7bit', '8bit', 'binary'): - outstring = str(msg.get_payload()) - else: # Decode from transfer ecoding to text or binary form - outstring = str(msg.get_payload(decode=1)) - set_header(msg, "Content-Transfer-Encoding", "8bit") - msg["X-MIME-Autoconverted"] = "from %s to 8bit by %s id %s" % (encoding, gopts.host_name, me) # Test all mask lists and find what to do with this content type masks = [] ctype = msg.get_content_type() if ctype: masks.append(ctype) - mtype = msg.get_content_maintype() - if mtype: + mtype = ctype.split('/')[0] masks.append(mtype + '/*') masks.append('*/*') + left_binary = False + for content_type in masks: + if content_type in g.totext_mask or \ + content_type in g.decoded_binary_mask: + break + elif content_type in g.binary_mask: + left_binary = True + break + + encoding = msg["Content-Transfer-Encoding"] + if left_binary or encoding in (None, '', '7bit', '8bit', 'binary'): + outstring = msg.get_payload() + else: # Decode from transfer ecoding to text or binary form + outstring = msg.get_payload(decode=1) + set_header(msg, "Content-Transfer-Encoding", "8bit") + msg["X-MIME-Autoconverted"] = "from %s to 8bit by %s id %s" % (encoding, g.host_name, me) + for content_type in masks: - if content_type in gopts.totext_mask: - totext(msg, outstring) - return - elif content_type in gopts.binary_mask: + if content_type in g.totext_mask: + outstring = totext(msg, outstring) + break + elif content_type in g.binary_mask or \ + content_type in g.decoded_binary_mask: output_headers(msg) output(outstring) - return - elif content_type in gopts.ignore_mask: + break + elif content_type in g.ignore_mask: output_headers(msg) - output("\nMessage body of type `%s' skipped.\n" % content_type) - return - elif content_type in gopts.error_mask: - raise ValueError, "content type `%s' prohibited" % content_type + output("\nMessage body of type %s skipped.\n" % ctype) + break + elif content_type in g.error_mask: + break + else: + # Neither content type nor masks were listed - decode by default + outstring = totext(msg, outstring) - # Neither content type nor masks were listed - decode by default - totext(msg, outstring) + for content_type in masks: + if content_type in g.save_headers_mask: + _save_message(msg, outstring, save_headers=True, save_body=False) + elif content_type in g.save_body_mask: + _save_message(msg, outstring, save_headers=False, save_body=True) + elif content_type in g.save_message_mask: + _save_message(msg, outstring, save_headers=True, save_body=True) + for content_type in masks: + if content_type in g.error_mask: + raise ValueError, "content type %s prohibited" % ctype def decode_multipart(msg): "Decode multipart" @@ -275,12 +381,19 @@ def decode_multipart(msg): if msg.preamble: # Preserve the first part, it is probably not a RFC822-message output(msg.preamble) # Usually it is just a few lines of text (MIME warning) + if msg.preamble is not None: + output("\n") + first_subpart = True boundary = msg.get_boundary() for subpart in msg.get_payload(): if boundary: - output("\n--%s\n" % boundary) + if first_subpart: + first_subpart = False + else: + output("\n") + output("--%s\n" % boundary) # Recursively decode all parts of the subpart decode_message(subpart) @@ -303,6 +416,19 @@ def decode_message(msg): output(msg.as_string()) +def open_output_file(filename): + fullpath = os.path.abspath(os.path.join(g.destination_dir, filename)) + full_dir = os.path.dirname(fullpath) + create = not os.path.isdir(full_dir) + if create: + os.makedirs(full_dir) + try: + return open(fullpath, 'w') + except: + if create: + os.removedirs(full_dir) + + class GlobalOptions: from m_lib.defenc import default_encoding recode_charset = 1 # recode charset of message body @@ -322,28 +448,40 @@ class GlobalOptions: # A list of headers to remove remove_headers = [] # A list of headers parameters to remove - remove_header_params = [] - # A list of headers to be stripped of all parameters - remove_all_params = [] + remove_headers_params = [] + + # A list of header/value pairs to set + set_header_value = [] + # A list of header/parameter/value triples to set + set_header_param = [] totext_mask = [] # A list of content-types to decode - binary_mask = [] # A list to pass through + binary_mask = [] # A list of content-types to pass through + decoded_binary_mask = [] # A list of content-types to pass through (content-transfer-decoded) ignore_mask = [] # Ignore (skip, do not decode and do not include into output) error_mask = [] # Raise error if encounter one of these + save_counter = 0 + save_headers_mask = [] + save_body_mask = [] + save_message_mask = [] + input_filename = None output_filename = None + destination_dir = os.curdir -gopts = GlobalOptions +g = GlobalOptions -def get_opt(): +def get_opts(): from getopt import getopt, GetoptError try: options, arguments = getopt(sys.argv[1:], - 'hVcCDPH:f:d:p:r:R:b:e:i:t:o:', - ['help', 'version', 'host=', 'remove-params=']) + 'hVcCDPH:f:d:p:r:R:b:B:e:i:t:O:o:', + ['help', 'version', 'host=', + 'save-headers=', 'save-body=', 'save-message=', + 'set-header=', 'set-param=']) except GetoptError: usage(1) @@ -353,39 +491,56 @@ def get_opt(): elif option in ('-V', '--version'): version() elif option == '-c': - gopts.recode_charset = 1 + g.recode_charset = 1 elif option == '-C': - gopts.recode_charset = 0 + g.recode_charset = 0 elif option in ('-H', '--host'): - gopts.host_name = value + g.host_name = value elif option == '-f': - gopts.default_encoding = value + g.default_encoding = value elif option == '-d': if value.startswith('*'): - gopts.decode_headers = [] - gopts.decode_headers.append(value) + g.decode_headers = [] + g.decode_headers.append(value) elif option == '-D': - gopts.decode_headers = [] + g.decode_headers = [] elif option == '-p': - gopts.decode_header_params.append(value.split(':', 1)) + g.decode_header_params.append(value.split(':', 1)) elif option == '-P': - gopts.decode_header_params = [] + g.decode_header_params = [] elif option == '-r': - gopts.remove_headers.append(value) + g.remove_headers.append(value) elif option == '-R': - gopts.remove_header_params.append(value.split(':', 1)) - elif option == '--remove-params': - gopts.remove_all_params.append(value) + g.remove_headers_params.append(value.split(':', 1)) + elif option == '--set-header': + g.set_header_value.append(value.split(':', 1)) + elif option == '--set-param': + header, value = value.split(':', 1) + if '=' in value: + param, value = value.split('=', 1) + else: + param, value = value.split(':', 1) + g.set_header_param.append((header, param, value)) elif option == '-t': - gopts.totext_mask.append(value) + g.totext_mask.append(value) + elif option == '-B': + g.binary_mask.append(value) elif option == '-b': - gopts.binary_mask.append(value) + g.decoded_binary_mask.append(value) elif option == '-i': - gopts.ignore_mask.append(value) + g.ignore_mask.append(value) elif option == '-e': - gopts.error_mask.append(value) + g.error_mask.append(value) + elif option == '--save-headers': + g.save_headers_mask.append(value) + elif option == '--save-body': + g.save_body_mask.append(value) + elif option == '--save-message': + g.save_message_mask.append(value) + elif option == '-O': + g.destination_dir = value elif option == '-o': - gopts.output_filename = value + g.output_filename = value else: usage(1) @@ -393,39 +548,39 @@ def get_opt(): if __name__ == "__main__": - arguments = get_opt() + arguments = get_opts() la = len(arguments) if la == 0: - gopts.input_filename = '-' + g.input_filename = '-' infile = sys.stdin - if gopts.output_filename: - outfile = open(gopts.output_filename, 'w') + if g.output_filename: + outfile = open_output_file(g.output_filename) else: - gopts.output_filename = '-' + g.output_filename = '-' outfile = sys.stdout elif la in (1, 2): if (arguments[0] == '-'): - gopts.input_filename = '-' + g.input_filename = '-' infile = sys.stdin else: - gopts.input_filename = arguments[0] + g.input_filename = arguments[0] infile = open(arguments[0], 'r') if la == 1: - if gopts.output_filename: - outfile = open(gopts.output_filename, 'w') + if g.output_filename: + outfile = open_output_file(g.output_filename) else: - gopts.output_filename = '-' + g.output_filename = '-' outfile = sys.stdout elif la == 2: - if gopts.output_filename: + if g.output_filename: usage(1, 'Too many output filenames') if (arguments[1] == '-'): - gopts.output_filename = '-' + g.output_filename = '-' outfile = sys.stdout else: - gopts.output_filename = arguments[1] - outfile = open(arguments[1], 'w') + g.output_filename = arguments[1] + outfile = open_output_file(g.output_filename) else: usage(1, 'Too many arguments') @@ -434,15 +589,25 @@ if __name__ == "__main__": usage() usage(1, 'Filtering from console is forbidden') - if not gopts.host_name: + if not g.host_name: import socket - gopts.host_name = socket.gethostname() + g.host_name = socket.gethostname() - gopts.outfile = outfile + g.outfile = outfile output = outfile.write + import email + msg = email.message_from_file(infile) + + for header, value in g.set_header_value: + set_header(msg, header, value) + + for header, param, value in g.set_header_param: + if header in msg: + msg.set_param(param, value, header) + try: - decode_message(email.message_from_file(infile)) + decode_message(msg) finally: infile.close() outfile.close()