public inbox for [email protected]
help / color / mirror / Atom feed[pgAdmin4][PATCH] To add handling of BOM character in query tool
2+ messages / 2 participants
[nested] [flat]
* [pgAdmin4][PATCH] To add handling of BOM character in query tool
@ 2017-05-09 09:33 Murtuza Zabuawala <[email protected]>
2017-05-09 11:07 ` Re: [pgAdmin4][PATCH] To add handling of BOM character in query tool Dave Page <[email protected]>
0 siblings, 1 reply; 2+ messages in thread
From: Murtuza Zabuawala @ 2017-05-09 09:33 UTC (permalink / raw)
To: pgadmin-hackers
Hi,
Please find patch to fix the issue handling BOM characters in user files
when they load their sql files in query tool.
RM#2369
FYI,
I have attached sample BOM file in redmine for testing.
--
Regards,
Murtuza Zabuawala
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
diff --git a/web/pgadmin/misc/file_manager/__init__.py b/web/pgadmin/misc/file_manager/__init__.py
index 6e081b1..85d7fb4 100644
--- a/web/pgadmin/misc/file_manager/__init__.py
+++ b/web/pgadmin/misc/file_manager/__init__.py
@@ -17,6 +17,7 @@ import sys
import time
from sys import platform as _platform
import config
+import codecs
import simplejson as json
from flask import render_template, Response, session, request as req, url_for
@@ -932,6 +933,68 @@ class Filemanager(object):
else:
return newPath, newName
+ @staticmethod
+ def check_file_for_bom_and_binary(filename, enc="utf-8"):
+ """
+ This utility function will check if file is Binary file
+ and/or if it startswith BOM character
+
+ Args:
+ filename: File
+ enc: Encoding for the file
+
+ Returns:
+ Status(Error?), Error message, Binary file flag,
+ BOM character flag and Encoding to open file
+ """
+ status = True
+ err_msg = None
+ is_startswith_bom = False
+
+ # check if file type is text or binary
+ text_chars = bytearray([7, 8, 9, 10, 12, 13, 27]) \
+ + bytearray(range(0x20, 0x7f)) \
+ + bytearray(range(0x80, 0x100))
+
+ def is_binary_string(bytes_data):
+ """Checks if string data is binary"""
+ return bool(
+ bytes_data.translate(None, text_chars)
+ )
+
+ # read the file
+ try:
+
+ with open(filename, 'rb') as f:
+ file_data = f.read(1024)
+
+ # Check for BOM in file data
+ for encoding, boms in \
+ ('utf-8-sig', (codecs.BOM_UTF8,)), \
+ ('utf-16', (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)), \
+ ('utf-32', (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
+ if any(file_data.startswith(bom) for bom in boms):
+ is_startswith_bom = True
+ enc = encoding
+
+ # Check if string is binary
+ is_binary = is_binary_string(file_data)
+
+ except IOError as ex:
+ status = False
+ # we don't want to expose real path of file
+ # so only show error message.
+ if ex.strerror == 'Permission denied':
+ err_msg = u"Error: {0}".format(ex.strerror)
+ else:
+ err_msg = u"Error: {0}".format(str(ex))
+
+ except Exception as ex:
+ status = False
+ err_msg = u"Error: {0}".format(str(ex))
+
+ return status, err_msg, is_binary, is_startswith_bom, enc
+
def addfolder(self, path, name):
"""
Functionality to create new folder
diff --git a/web/pgadmin/tools/sqleditor/__init__.py b/web/pgadmin/tools/sqleditor/__init__.py
index 2d1d4e5..6431b3a 100644
--- a/web/pgadmin/tools/sqleditor/__init__.py
+++ b/web/pgadmin/tools/sqleditor/__init__.py
@@ -12,6 +12,7 @@ import simplejson as json
import os
import pickle
import random
+import codecs
from flask import Response, url_for, render_template, session, request
from flask_babel import gettext
@@ -1220,7 +1221,10 @@ def load_file():
file_path = unquote(file_data['file_name'])
if hasattr(str, 'decode'):
- file_path = unquote(file_data['file_name']).encode('utf-8').decode('utf-8')
+ file_path = unquote(
+ file_data['file_name']
+ ).encode('utf-8').decode('utf-8')
+
# retrieve storage directory path
storage_manager_path = get_storage_directory()
if storage_manager_path:
@@ -1230,45 +1234,27 @@ def load_file():
file_path.lstrip('/').lstrip('\\')
)
- file_data = None
+ status, err_msg, is_binary, \
+ is_startswith_bom, enc = Filemanager.check_file_for_bom_and_binary(
+ file_path
+ )
- # check if file type is text or binary
- textchars = bytearray(
- [7, 8, 9, 10, 12, 13, 27]) + bytearray(
- range(0x20, 0x7f)) + bytearray(range(0x80, 0x100))
+ if not status:
+ return internal_server_error(
+ errormsg=gettext(err_msg)
+ )
- is_binary_string = lambda bytes: bool(
- bytes.translate(None, textchars)
- )
+ if is_binary:
+ return internal_server_error(
+ errormsg=gettext("File type not supported")
+ )
+
+ with codecs.open(file_path, 'r', encoding=enc) as fileObj:
+ data = fileObj.read()
- # read file
- try:
- with open(file_path, 'rb') as fileObj:
- is_binary = is_binary_string(fileObj.read(1024))
- if not is_binary:
- fileObj.seek(0)
- if hasattr(str, 'decode'):
- file_data = fileObj.read().decode('utf-8')
- else:
- file_data = fileObj.read()
- else:
- return internal_server_error(
- errormsg=gettext("File type not supported")
- )
- except IOError as e:
- # we don't want to expose real path of file
- # so only show error message.
- if e.strerror == 'Permission denied':
- err_msg = "Error: {0}".format(e.strerror)
- else:
- err_msg = "Error: {0}".format(e.strerror)
- return internal_server_error(errormsg=err_msg)
- except Exception as e:
- err_msg = "Error: {0}".format(e.strerror)
- return internal_server_error(errormsg=err_msg)
return make_json_response(
data={
- 'status': True, 'result': file_data,
+ 'status': True, 'result': data,
}
)
--
Sent via pgadmin-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgadmin-hackers
Attachments:
[text/plain] fix_bom.diff (5.6K, 3-fix_bom.diff)
download | inline diff:
diff --git a/web/pgadmin/misc/file_manager/__init__.py b/web/pgadmin/misc/file_manager/__init__.py
index 6e081b1..85d7fb4 100644
--- a/web/pgadmin/misc/file_manager/__init__.py
+++ b/web/pgadmin/misc/file_manager/__init__.py
@@ -17,6 +17,7 @@ import sys
import time
from sys import platform as _platform
import config
+import codecs
import simplejson as json
from flask import render_template, Response, session, request as req, url_for
@@ -932,6 +933,68 @@ class Filemanager(object):
else:
return newPath, newName
+ @staticmethod
+ def check_file_for_bom_and_binary(filename, enc="utf-8"):
+ """
+ This utility function will check if file is Binary file
+ and/or if it startswith BOM character
+
+ Args:
+ filename: File
+ enc: Encoding for the file
+
+ Returns:
+ Status(Error?), Error message, Binary file flag,
+ BOM character flag and Encoding to open file
+ """
+ status = True
+ err_msg = None
+ is_startswith_bom = False
+
+ # check if file type is text or binary
+ text_chars = bytearray([7, 8, 9, 10, 12, 13, 27]) \
+ + bytearray(range(0x20, 0x7f)) \
+ + bytearray(range(0x80, 0x100))
+
+ def is_binary_string(bytes_data):
+ """Checks if string data is binary"""
+ return bool(
+ bytes_data.translate(None, text_chars)
+ )
+
+ # read the file
+ try:
+
+ with open(filename, 'rb') as f:
+ file_data = f.read(1024)
+
+ # Check for BOM in file data
+ for encoding, boms in \
+ ('utf-8-sig', (codecs.BOM_UTF8,)), \
+ ('utf-16', (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)), \
+ ('utf-32', (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
+ if any(file_data.startswith(bom) for bom in boms):
+ is_startswith_bom = True
+ enc = encoding
+
+ # Check if string is binary
+ is_binary = is_binary_string(file_data)
+
+ except IOError as ex:
+ status = False
+ # we don't want to expose real path of file
+ # so only show error message.
+ if ex.strerror == 'Permission denied':
+ err_msg = u"Error: {0}".format(ex.strerror)
+ else:
+ err_msg = u"Error: {0}".format(str(ex))
+
+ except Exception as ex:
+ status = False
+ err_msg = u"Error: {0}".format(str(ex))
+
+ return status, err_msg, is_binary, is_startswith_bom, enc
+
def addfolder(self, path, name):
"""
Functionality to create new folder
diff --git a/web/pgadmin/tools/sqleditor/__init__.py b/web/pgadmin/tools/sqleditor/__init__.py
index 2d1d4e5..6431b3a 100644
--- a/web/pgadmin/tools/sqleditor/__init__.py
+++ b/web/pgadmin/tools/sqleditor/__init__.py
@@ -12,6 +12,7 @@ import simplejson as json
import os
import pickle
import random
+import codecs
from flask import Response, url_for, render_template, session, request
from flask_babel import gettext
@@ -1220,7 +1221,10 @@ def load_file():
file_path = unquote(file_data['file_name'])
if hasattr(str, 'decode'):
- file_path = unquote(file_data['file_name']).encode('utf-8').decode('utf-8')
+ file_path = unquote(
+ file_data['file_name']
+ ).encode('utf-8').decode('utf-8')
+
# retrieve storage directory path
storage_manager_path = get_storage_directory()
if storage_manager_path:
@@ -1230,45 +1234,27 @@ def load_file():
file_path.lstrip('/').lstrip('\\')
)
- file_data = None
+ status, err_msg, is_binary, \
+ is_startswith_bom, enc = Filemanager.check_file_for_bom_and_binary(
+ file_path
+ )
- # check if file type is text or binary
- textchars = bytearray(
- [7, 8, 9, 10, 12, 13, 27]) + bytearray(
- range(0x20, 0x7f)) + bytearray(range(0x80, 0x100))
+ if not status:
+ return internal_server_error(
+ errormsg=gettext(err_msg)
+ )
- is_binary_string = lambda bytes: bool(
- bytes.translate(None, textchars)
- )
+ if is_binary:
+ return internal_server_error(
+ errormsg=gettext("File type not supported")
+ )
+
+ with codecs.open(file_path, 'r', encoding=enc) as fileObj:
+ data = fileObj.read()
- # read file
- try:
- with open(file_path, 'rb') as fileObj:
- is_binary = is_binary_string(fileObj.read(1024))
- if not is_binary:
- fileObj.seek(0)
- if hasattr(str, 'decode'):
- file_data = fileObj.read().decode('utf-8')
- else:
- file_data = fileObj.read()
- else:
- return internal_server_error(
- errormsg=gettext("File type not supported")
- )
- except IOError as e:
- # we don't want to expose real path of file
- # so only show error message.
- if e.strerror == 'Permission denied':
- err_msg = "Error: {0}".format(e.strerror)
- else:
- err_msg = "Error: {0}".format(e.strerror)
- return internal_server_error(errormsg=err_msg)
- except Exception as e:
- err_msg = "Error: {0}".format(e.strerror)
- return internal_server_error(errormsg=err_msg)
return make_json_response(
data={
- 'status': True, 'result': file_data,
+ 'status': True, 'result': data,
}
)
^ permalink raw reply [nested|flat] 2+ messages in thread
* Re: [pgAdmin4][PATCH] To add handling of BOM character in query tool
2017-05-09 09:33 [pgAdmin4][PATCH] To add handling of BOM character in query tool Murtuza Zabuawala <[email protected]>
@ 2017-05-09 11:07 ` Dave Page <[email protected]>
0 siblings, 0 replies; 2+ messages in thread
From: Dave Page @ 2017-05-09 11:07 UTC (permalink / raw)
To: Murtuza Zabuawala <[email protected]>; +Cc: pgadmin-hackers
Thanks, patch applied.
On Tue, May 9, 2017 at 10:33 AM, Murtuza Zabuawala <
[email protected]> wrote:
> Hi,
>
> Please find patch to fix the issue handling BOM characters in user files
> when they load their sql files in query tool.
> RM#2369
>
> FYI,
> I have attached sample BOM file in redmine for testing.
>
> --
> Regards,
> Murtuza Zabuawala
> EnterpriseDB: http://www.enterprisedb.com
> The Enterprise PostgreSQL Company
>
>
> --
> Sent via pgadmin-hackers mailing list ([email protected])
> To make changes to your subscription:
> http://www.postgresql.org/mailpref/pgadmin-hackers
>
>
--
Dave Page
Blog: http://pgsnake.blogspot.com
Twitter: @pgsnake
EnterpriseDB UK: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 2+ messages in thread
end of thread, other threads:[~2017-05-09 11:07 UTC | newest]
Thread overview: 2+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2017-05-09 09:33 [pgAdmin4][PATCH] To add handling of BOM character in query tool Murtuza Zabuawala <[email protected]>
2017-05-09 11:07 ` Dave Page <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox