public inbox for [email protected]
help / color / mirror / Atom feed[pgAdmin4][PATCH] To fix the issues with CSV file download
4+ messages / 2 participants
[nested] [flat]
* [pgAdmin4][PATCH] To fix the issues with CSV file download
@ 2017-05-08 12:14 Murtuza Zabuawala <[email protected]>
2017-05-08 12:36 ` Re: [pgAdmin4][PATCH] To fix the issues with CSV file download Dave Page <[email protected]>
0 siblings, 1 reply; 4+ messages in thread
From: Murtuza Zabuawala @ 2017-05-08 12:14 UTC (permalink / raw)
To: pgadmin-hackers
Hi,
Please find the patch to fix the issues in CSV file download feature,
1) To handle non-ascii filenames which we set from table name
RM#2314
2) To handle non-ascii query data
RM#2353
3) To dump JSON type columns properly in csv
RM#2360
Please review.
--
Regards,
Murtuza Zabuawala
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
diff --git a/requirements.txt b/requirements.txt
index 8afeb52..90ed057 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,3 +40,4 @@ SQLAlchemy==1.0.14
sqlparse==0.1.19
Werkzeug==0.9.6
WTForms==2.0.2
+backports.csv==1.0.4; python_version <= '2.7'
\ No newline at end of file
diff --git a/web/pgadmin/tools/sqleditor/__init__.py b/web/pgadmin/tools/sqleditor/__init__.py
index d114988..2d1d4e5 100644
--- a/web/pgadmin/tools/sqleditor/__init__.py
+++ b/web/pgadmin/tools/sqleditor/__init__.py
@@ -1337,7 +1337,8 @@ def save_file():
@login_required
def start_query_download_tool(trans_id):
sync_conn = None
- status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id)
+ status, error_msg, conn, trans_obj, \
+ session_obj = check_transaction_status(trans_id)
if status and conn is not None \
and trans_obj is not None and session_obj is not None:
@@ -1361,11 +1362,15 @@ def start_query_download_tool(trans_id):
del conn.manager.connections[sync_conn.conn_id]
# This returns generator of records.
- status, gen = sync_conn.execute_on_server_as_csv(sql, records=2000)
+ status, gen = sync_conn.execute_on_server_as_csv(
+ sql, records=2000
+ )
if not status:
r = Response('"{0}"'.format(gen), mimetype='text/csv')
- r.headers["Content-Disposition"] = "attachment;filename=error.csv"
+ r.headers[
+ "Content-Disposition"
+ ] = "attachment;filename=error.csv"
r.call_on_close(cleanup)
return r
@@ -1377,7 +1382,18 @@ def start_query_download_tool(trans_id):
import time
filename = str(int(time.time())) + ".csv"
- r.headers["Content-Disposition"] = "attachment;filename={0}".format(filename)
+ # We will try to encode report file name with latin-1
+ # If it fails then we will fallback to default ascii file name
+ # werkzeug only supports latin-1 encoding supported values
+ try:
+ tmp_file_name = filename
+ tmp_file_name.encode('latin-1', 'strict')
+ except UnicodeEncodeError:
+ filename = "download.csv"
+
+ r.headers[
+ "Content-Disposition"
+ ] = "attachment;filename={0}".format(filename)
r.call_on_close(cleanup)
return r
@@ -1388,4 +1404,6 @@ def start_query_download_tool(trans_id):
r.call_on_close(cleanup)
return r
else:
- return internal_server_error(errormsg=gettext("Transaction status check failed."))
+ return internal_server_error(
+ errormsg=gettext("Transaction status check failed.")
+ )
diff --git a/web/pgadmin/utils/driver/psycopg2/__init__.py b/web/pgadmin/utils/driver/psycopg2/__init__.py
index e474817..f2e4b72 100644
--- a/web/pgadmin/utils/driver/psycopg2/__init__.py
+++ b/web/pgadmin/utils/driver/psycopg2/__init__.py
@@ -18,8 +18,8 @@ import os
import random
import select
import sys
-import csv
+import simplejson as json
import psycopg2
import psycopg2.extras
from flask import g, current_app, session
@@ -36,11 +36,15 @@ from ..abstract import BaseDriver, BaseConnection
from .cursor import DictCursor
if sys.version_info < (3,):
+ # Python2 in-built csv module do not handle unicode
+ # backports.csv module ported from PY3 csv module for unicode handling
+ from backports import csv
from StringIO import StringIO
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
else:
from io import StringIO
+ import csv
_ = gettext
@@ -596,7 +600,22 @@ WHERE
if self.async == 1:
self._wait(cur.connection)
- def execute_on_server_as_csv(self, query, params=None, formatted_exception_msg=False, records=2000):
+ def execute_on_server_as_csv(self,
+ query, params=None,
+ formatted_exception_msg=False,
+ records=2000):
+ """
+ To fetch query result and generate CSV output
+
+ Args:
+ query: SQL
+ params: Additional parameters
+ formatted_exception_msg: For exception
+ records: Number of initial records
+
+ Returns:
+ Generator response
+ """
status, cur = self.__cursor(server_cursor=True)
self.row_count = 0
@@ -607,21 +626,26 @@ WHERE
if sys.version_info < (3,) and type(query) == unicode:
query = query.encode('utf-8')
- current_app.logger.log(25,
- u"Execute (with server cursor) for server #{server_id} - {conn_id} (Query-id: {query_id}):\n{query}".format(
- server_id=self.manager.sid,
- conn_id=self.conn_id,
- query=query.decode('utf-8') if sys.version_info < (3,) else query,
- query_id=query_id
- )
- )
+ current_app.logger.log(
+ 25,
+ u"Execute (with server cursor) for server #{server_id} - {conn_id} "
+ u"(Query-id: {query_id}):\n{query}".format(
+ server_id=self.manager.sid,
+ conn_id=self.conn_id,
+ query=query.decode('utf-8') if
+ sys.version_info < (3,) else query,
+ query_id=query_id
+ )
+ )
try:
self.__internal_blocking_execute(cur, query, params)
except psycopg2.Error as pe:
cur.close()
errmsg = self._formatted_exception_msg(pe, formatted_exception_msg)
current_app.logger.error(
- u"failed to execute query ((with server cursor) for the server #{server_id} - {conn_id} (query-id: {query_id}):\nerror message:{errmsg}".format(
+ u"failed to execute query ((with server cursor) "
+ u"for the server #{server_id} - {conn_id} "
+ u"(query-id: {query_id}):\nerror message:{errmsg}".format(
server_id=self.manager.sid,
conn_id=self.conn_id,
query=query,
@@ -631,6 +655,33 @@ WHERE
)
return False, errmsg
+ def handle_json_data(json_columns, results):
+ """
+ [ This is only for Python2.x]
+ This function will be useful to handle json data types.
+ We will dump json data as proper json instead of unicode values
+
+ Args:
+ json_columns: Columns which contains json data
+ results: Query result
+
+ Returns:
+ results
+ """
+ # Only if Python2 and there are columns with JSON type
+ if sys.version_info < (3,) and len(json_columns) > 0:
+ temp_results = []
+ for row in results:
+ res = dict()
+ for k, v in row.items():
+ if k in json_columns:
+ res[k] = json.dumps(v)
+ else:
+ res[k] = v
+ temp_results.append(res)
+ results = temp_results
+ return results
+
def gen():
results = cur.fetchmany(records)
@@ -639,15 +690,26 @@ WHERE
cur.close()
yield gettext('The query executed did not return any data.')
return
-
- header = [c.to_dict()['name'] for c in cur.ordered_description()]
-
+ header = []
+ json_columns = []
+ # json, jsonb, json[], jsonb[]
+ json_types = (114, 199, 3802, 3807)
+ for c in cur.ordered_description():
+ # This is to handle the case in which column name is non-ascii
+ header.append(u"" + c.to_dict()['name'])
+ if c.to_dict()['type_code'] in json_types:
+ json_columns.append(
+ u"" + c.to_dict()['name']
+ )
res_io = StringIO()
csv_writer = csv.DictWriter(
- res_io, fieldnames=header, delimiter=str(','), quoting=csv.QUOTE_NONNUMERIC
+ res_io, fieldnames=header, delimiter=u',',
+ quoting=csv.QUOTE_NONNUMERIC
)
+
csv_writer.writeheader()
+ results = handle_json_data(json_columns, results)
csv_writer.writerows(results)
yield res_io.getvalue()
@@ -662,8 +724,10 @@ WHERE
res_io = StringIO()
csv_writer = csv.DictWriter(
- res_io, fieldnames=header, delimiter=str(','), quoting=csv.QUOTE_NONNUMERIC
+ res_io, fieldnames=header, delimiter=u',',
+ quoting=csv.QUOTE_NONNUMERIC
)
+ results = handle_json_data(json_columns, results)
csv_writer.writerows(results)
yield res_io.getvalue()
@@ -1487,8 +1551,8 @@ class ServerManager(object):
not isinstance(database, unicode):
database = database.decode('utf-8')
if did is not None:
- if did in self.db_info:
- self.db_info[did]['datname']=database
+ if did in self.db_info:
+ self.db_info[did]['datname']=database
else:
if did is None:
database = self.db
--
Sent via pgadmin-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgadmin-hackers
Attachments:
[text/plain] fixes_for_csv_download.diff (9.8K, 3-fixes_for_csv_download.diff)
download | inline diff:
diff --git a/requirements.txt b/requirements.txt
index 8afeb52..90ed057 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,3 +40,4 @@ SQLAlchemy==1.0.14
sqlparse==0.1.19
Werkzeug==0.9.6
WTForms==2.0.2
+backports.csv==1.0.4; python_version <= '2.7'
\ No newline at end of file
diff --git a/web/pgadmin/tools/sqleditor/__init__.py b/web/pgadmin/tools/sqleditor/__init__.py
index d114988..2d1d4e5 100644
--- a/web/pgadmin/tools/sqleditor/__init__.py
+++ b/web/pgadmin/tools/sqleditor/__init__.py
@@ -1337,7 +1337,8 @@ def save_file():
@login_required
def start_query_download_tool(trans_id):
sync_conn = None
- status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id)
+ status, error_msg, conn, trans_obj, \
+ session_obj = check_transaction_status(trans_id)
if status and conn is not None \
and trans_obj is not None and session_obj is not None:
@@ -1361,11 +1362,15 @@ def start_query_download_tool(trans_id):
del conn.manager.connections[sync_conn.conn_id]
# This returns generator of records.
- status, gen = sync_conn.execute_on_server_as_csv(sql, records=2000)
+ status, gen = sync_conn.execute_on_server_as_csv(
+ sql, records=2000
+ )
if not status:
r = Response('"{0}"'.format(gen), mimetype='text/csv')
- r.headers["Content-Disposition"] = "attachment;filename=error.csv"
+ r.headers[
+ "Content-Disposition"
+ ] = "attachment;filename=error.csv"
r.call_on_close(cleanup)
return r
@@ -1377,7 +1382,18 @@ def start_query_download_tool(trans_id):
import time
filename = str(int(time.time())) + ".csv"
- r.headers["Content-Disposition"] = "attachment;filename={0}".format(filename)
+ # We will try to encode report file name with latin-1
+ # If it fails then we will fallback to default ascii file name
+ # werkzeug only supports latin-1 encoding supported values
+ try:
+ tmp_file_name = filename
+ tmp_file_name.encode('latin-1', 'strict')
+ except UnicodeEncodeError:
+ filename = "download.csv"
+
+ r.headers[
+ "Content-Disposition"
+ ] = "attachment;filename={0}".format(filename)
r.call_on_close(cleanup)
return r
@@ -1388,4 +1404,6 @@ def start_query_download_tool(trans_id):
r.call_on_close(cleanup)
return r
else:
- return internal_server_error(errormsg=gettext("Transaction status check failed."))
+ return internal_server_error(
+ errormsg=gettext("Transaction status check failed.")
+ )
diff --git a/web/pgadmin/utils/driver/psycopg2/__init__.py b/web/pgadmin/utils/driver/psycopg2/__init__.py
index e474817..f2e4b72 100644
--- a/web/pgadmin/utils/driver/psycopg2/__init__.py
+++ b/web/pgadmin/utils/driver/psycopg2/__init__.py
@@ -18,8 +18,8 @@ import os
import random
import select
import sys
-import csv
+import simplejson as json
import psycopg2
import psycopg2.extras
from flask import g, current_app, session
@@ -36,11 +36,15 @@ from ..abstract import BaseDriver, BaseConnection
from .cursor import DictCursor
if sys.version_info < (3,):
+ # Python2 in-built csv module do not handle unicode
+ # backports.csv module ported from PY3 csv module for unicode handling
+ from backports import csv
from StringIO import StringIO
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
else:
from io import StringIO
+ import csv
_ = gettext
@@ -596,7 +600,22 @@ WHERE
if self.async == 1:
self._wait(cur.connection)
- def execute_on_server_as_csv(self, query, params=None, formatted_exception_msg=False, records=2000):
+ def execute_on_server_as_csv(self,
+ query, params=None,
+ formatted_exception_msg=False,
+ records=2000):
+ """
+ To fetch query result and generate CSV output
+
+ Args:
+ query: SQL
+ params: Additional parameters
+ formatted_exception_msg: For exception
+ records: Number of initial records
+
+ Returns:
+ Generator response
+ """
status, cur = self.__cursor(server_cursor=True)
self.row_count = 0
@@ -607,21 +626,26 @@ WHERE
if sys.version_info < (3,) and type(query) == unicode:
query = query.encode('utf-8')
- current_app.logger.log(25,
- u"Execute (with server cursor) for server #{server_id} - {conn_id} (Query-id: {query_id}):\n{query}".format(
- server_id=self.manager.sid,
- conn_id=self.conn_id,
- query=query.decode('utf-8') if sys.version_info < (3,) else query,
- query_id=query_id
- )
- )
+ current_app.logger.log(
+ 25,
+ u"Execute (with server cursor) for server #{server_id} - {conn_id} "
+ u"(Query-id: {query_id}):\n{query}".format(
+ server_id=self.manager.sid,
+ conn_id=self.conn_id,
+ query=query.decode('utf-8') if
+ sys.version_info < (3,) else query,
+ query_id=query_id
+ )
+ )
try:
self.__internal_blocking_execute(cur, query, params)
except psycopg2.Error as pe:
cur.close()
errmsg = self._formatted_exception_msg(pe, formatted_exception_msg)
current_app.logger.error(
- u"failed to execute query ((with server cursor) for the server #{server_id} - {conn_id} (query-id: {query_id}):\nerror message:{errmsg}".format(
+ u"failed to execute query ((with server cursor) "
+ u"for the server #{server_id} - {conn_id} "
+ u"(query-id: {query_id}):\nerror message:{errmsg}".format(
server_id=self.manager.sid,
conn_id=self.conn_id,
query=query,
@@ -631,6 +655,33 @@ WHERE
)
return False, errmsg
+ def handle_json_data(json_columns, results):
+ """
+ [ This is only for Python2.x]
+ This function will be useful to handle json data types.
+ We will dump json data as proper json instead of unicode values
+
+ Args:
+ json_columns: Columns which contains json data
+ results: Query result
+
+ Returns:
+ results
+ """
+ # Only if Python2 and there are columns with JSON type
+ if sys.version_info < (3,) and len(json_columns) > 0:
+ temp_results = []
+ for row in results:
+ res = dict()
+ for k, v in row.items():
+ if k in json_columns:
+ res[k] = json.dumps(v)
+ else:
+ res[k] = v
+ temp_results.append(res)
+ results = temp_results
+ return results
+
def gen():
results = cur.fetchmany(records)
@@ -639,15 +690,26 @@ WHERE
cur.close()
yield gettext('The query executed did not return any data.')
return
-
- header = [c.to_dict()['name'] for c in cur.ordered_description()]
-
+ header = []
+ json_columns = []
+ # json, jsonb, json[], jsonb[]
+ json_types = (114, 199, 3802, 3807)
+ for c in cur.ordered_description():
+ # This is to handle the case in which column name is non-ascii
+ header.append(u"" + c.to_dict()['name'])
+ if c.to_dict()['type_code'] in json_types:
+ json_columns.append(
+ u"" + c.to_dict()['name']
+ )
res_io = StringIO()
csv_writer = csv.DictWriter(
- res_io, fieldnames=header, delimiter=str(','), quoting=csv.QUOTE_NONNUMERIC
+ res_io, fieldnames=header, delimiter=u',',
+ quoting=csv.QUOTE_NONNUMERIC
)
+
csv_writer.writeheader()
+ results = handle_json_data(json_columns, results)
csv_writer.writerows(results)
yield res_io.getvalue()
@@ -662,8 +724,10 @@ WHERE
res_io = StringIO()
csv_writer = csv.DictWriter(
- res_io, fieldnames=header, delimiter=str(','), quoting=csv.QUOTE_NONNUMERIC
+ res_io, fieldnames=header, delimiter=u',',
+ quoting=csv.QUOTE_NONNUMERIC
)
+ results = handle_json_data(json_columns, results)
csv_writer.writerows(results)
yield res_io.getvalue()
@@ -1487,8 +1551,8 @@ class ServerManager(object):
not isinstance(database, unicode):
database = database.decode('utf-8')
if did is not None:
- if did in self.db_info:
- self.db_info[did]['datname']=database
+ if did in self.db_info:
+ self.db_info[did]['datname']=database
else:
if did is None:
database = self.db
^ permalink raw reply [nested|flat] 4+ messages in thread
* Re: [pgAdmin4][PATCH] To fix the issues with CSV file download
2017-05-08 12:14 [pgAdmin4][PATCH] To fix the issues with CSV file download Murtuza Zabuawala <[email protected]>
@ 2017-05-08 12:36 ` Dave Page <[email protected]>
2017-05-11 14:21 ` Re: [pgAdmin4][PATCH] To fix the issues with CSV file download Murtuza Zabuawala <[email protected]>
0 siblings, 1 reply; 4+ messages in thread
From: Dave Page @ 2017-05-08 12:36 UTC (permalink / raw)
To: Murtuza Zabuawala <[email protected]>; +Cc: pgadmin-hackers
Thanks, patch applied.
On Mon, May 8, 2017 at 1:14 PM, Murtuza Zabuawala <
[email protected]> wrote:
> Hi,
>
> Please find the patch to fix the issues in CSV file download feature,
>
> 1) To handle non-ascii filenames which we set from table name
> RM#2314
>
> 2) To handle non-ascii query data
> RM#2353
>
> 3) To dump JSON type columns properly in csv
> RM#2360
>
> Please review.
>
> --
> Regards,
> Murtuza Zabuawala
> EnterpriseDB: http://www.enterprisedb.com
> The Enterprise PostgreSQL Company
>
>
> --
> Sent via pgadmin-hackers mailing list ([email protected])
> To make changes to your subscription:
> http://www.postgresql.org/mailpref/pgadmin-hackers
>
>
--
Dave Page
Blog: http://pgsnake.blogspot.com
Twitter: @pgsnake
EnterpriseDB UK: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 4+ messages in thread
* Re: [pgAdmin4][PATCH] To fix the issues with CSV file download
2017-05-08 12:14 [pgAdmin4][PATCH] To fix the issues with CSV file download Murtuza Zabuawala <[email protected]>
2017-05-08 12:36 ` Re: [pgAdmin4][PATCH] To fix the issues with CSV file download Dave Page <[email protected]>
@ 2017-05-11 14:21 ` Murtuza Zabuawala <[email protected]>
2017-05-12 09:38 ` Re: [pgAdmin4][PATCH] To fix the issues with CSV file download Dave Page <[email protected]>
0 siblings, 1 reply; 4+ messages in thread
From: Murtuza Zabuawala @ 2017-05-11 14:21 UTC (permalink / raw)
To: Dave Page <[email protected]>; +Cc: pgadmin-hackers
Hi Dave,
Please find patch to handle non-ascii column names in CSV for Pyhton2.
Thanks to Harshal for pointing out the issue.
--
Regards,
Murtuza Zabuawala
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
On Mon, May 8, 2017 at 6:06 PM, Dave Page <[email protected]> wrote:
> Thanks, patch applied.
>
> On Mon, May 8, 2017 at 1:14 PM, Murtuza Zabuawala <murtuza.zabuawala@
> enterprisedb.com> wrote:
>
>> Hi,
>>
>> Please find the patch to fix the issues in CSV file download feature,
>>
>> 1) To handle non-ascii filenames which we set from table name
>> RM#2314
>>
>> 2) To handle non-ascii query data
>> RM#2353
>>
>> 3) To dump JSON type columns properly in csv
>> RM#2360
>>
>> Please review.
>>
>> --
>> Regards,
>> Murtuza Zabuawala
>> EnterpriseDB: http://www.enterprisedb.com
>> The Enterprise PostgreSQL Company
>>
>>
>> --
>> Sent via pgadmin-hackers mailing list ([email protected])
>> To make changes to your subscription:
>> http://www.postgresql.org/mailpref/pgadmin-hackers
>>
>>
>
>
> --
> Dave Page
> Blog: http://pgsnake.blogspot.com
> Twitter: @pgsnake
>
> EnterpriseDB UK: http://www.enterprisedb.com
> The Enterprise PostgreSQL Company
>
diff --git a/web/pgadmin/utils/driver/psycopg2/__init__.py b/web/pgadmin/utils/driver/psycopg2/__init__.py
index 1f7ccce..502cee4 100644
--- a/web/pgadmin/utils/driver/psycopg2/__init__.py
+++ b/web/pgadmin/utils/driver/psycopg2/__init__.py
@@ -42,9 +42,11 @@ if sys.version_info < (3,):
from StringIO import StringIO
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
+ IS_PY2 = True
else:
from io import StringIO
import csv
+ IS_PY2 = False
_ = gettext
@@ -624,7 +626,7 @@ WHERE
return False, str(cur)
query_id = random.randint(1, 9999999)
- if sys.version_info < (3,) and type(query) == unicode:
+ if IS_PY2 and type(query) == unicode:
query = query.encode('utf-8')
current_app.logger.log(
@@ -670,7 +672,7 @@ WHERE
results
"""
# Only if Python2 and there are columns with JSON type
- if sys.version_info < (3,) and len(json_columns) > 0:
+ if IS_PY2 and len(json_columns) > 0:
temp_results = []
for row in results:
res = dict()
@@ -683,6 +685,26 @@ WHERE
results = temp_results
return results
+ def convert_keys_to_unicode(results, conn_encoding):
+ """
+ [ This is only for Python2.x]
+ We need to convert all keys to unicode as psycopg2
+ sends them as string
+
+ Args:
+ res: Query result set from psycopg2
+ conn_encoding: Connection encoding
+
+ Returns:
+ Result set (With all the keys converted to unicode)
+ """
+ new_results = []
+ for row in results:
+ new_results.append(
+ dict([(k.decode(conn_encoding), v) for k, v in row.items()])
+ )
+ return new_results
+
def gen():
results = cur.fetchmany(records)
@@ -691,17 +713,25 @@ WHERE
cur.close()
yield gettext('The query executed did not return any data.')
return
+
header = []
json_columns = []
+ conn_encoding = cur.connection.encoding
+
# json, jsonb, json[], jsonb[]
json_types = (114, 199, 3802, 3807)
for c in cur.ordered_description():
# This is to handle the case in which column name is non-ascii
- header.append(u"" + c.to_dict()['name'])
+ column_name = c.to_dict()['name']
+ if IS_PY2:
+ column_name = column_name.decode(conn_encoding)
+ header.append(column_name)
if c.to_dict()['type_code'] in json_types:
- json_columns.append(
- u"" + c.to_dict()['name']
- )
+ json_columns.append(column_name)
+
+ if IS_PY2:
+ results = convert_keys_to_unicode(results, conn_encoding)
+
res_io = StringIO()
csv_writer = csv.DictWriter(
@@ -728,6 +758,10 @@ WHERE
res_io, fieldnames=header, delimiter=u',',
quoting=csv.QUOTE_NONNUMERIC
)
+
+ if IS_PY2:
+ results = convert_keys_to_unicode(results, conn_encoding)
+
results = handle_json_data(json_columns, results)
csv_writer.writerows(results)
yield res_io.getvalue()
--
Sent via pgadmin-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgadmin-hackers
Attachments:
[text/plain] RM_2314_v1.diff (3.6K, 3-RM_2314_v1.diff)
download | inline diff:
diff --git a/web/pgadmin/utils/driver/psycopg2/__init__.py b/web/pgadmin/utils/driver/psycopg2/__init__.py
index 1f7ccce..502cee4 100644
--- a/web/pgadmin/utils/driver/psycopg2/__init__.py
+++ b/web/pgadmin/utils/driver/psycopg2/__init__.py
@@ -42,9 +42,11 @@ if sys.version_info < (3,):
from StringIO import StringIO
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
+ IS_PY2 = True
else:
from io import StringIO
import csv
+ IS_PY2 = False
_ = gettext
@@ -624,7 +626,7 @@ WHERE
return False, str(cur)
query_id = random.randint(1, 9999999)
- if sys.version_info < (3,) and type(query) == unicode:
+ if IS_PY2 and type(query) == unicode:
query = query.encode('utf-8')
current_app.logger.log(
@@ -670,7 +672,7 @@ WHERE
results
"""
# Only if Python2 and there are columns with JSON type
- if sys.version_info < (3,) and len(json_columns) > 0:
+ if IS_PY2 and len(json_columns) > 0:
temp_results = []
for row in results:
res = dict()
@@ -683,6 +685,26 @@ WHERE
results = temp_results
return results
+ def convert_keys_to_unicode(results, conn_encoding):
+ """
+ [ This is only for Python2.x]
+ We need to convert all keys to unicode as psycopg2
+ sends them as string
+
+ Args:
+ res: Query result set from psycopg2
+ conn_encoding: Connection encoding
+
+ Returns:
+ Result set (With all the keys converted to unicode)
+ """
+ new_results = []
+ for row in results:
+ new_results.append(
+ dict([(k.decode(conn_encoding), v) for k, v in row.items()])
+ )
+ return new_results
+
def gen():
results = cur.fetchmany(records)
@@ -691,17 +713,25 @@ WHERE
cur.close()
yield gettext('The query executed did not return any data.')
return
+
header = []
json_columns = []
+ conn_encoding = cur.connection.encoding
+
# json, jsonb, json[], jsonb[]
json_types = (114, 199, 3802, 3807)
for c in cur.ordered_description():
# This is to handle the case in which column name is non-ascii
- header.append(u"" + c.to_dict()['name'])
+ column_name = c.to_dict()['name']
+ if IS_PY2:
+ column_name = column_name.decode(conn_encoding)
+ header.append(column_name)
if c.to_dict()['type_code'] in json_types:
- json_columns.append(
- u"" + c.to_dict()['name']
- )
+ json_columns.append(column_name)
+
+ if IS_PY2:
+ results = convert_keys_to_unicode(results, conn_encoding)
+
res_io = StringIO()
csv_writer = csv.DictWriter(
@@ -728,6 +758,10 @@ WHERE
res_io, fieldnames=header, delimiter=u',',
quoting=csv.QUOTE_NONNUMERIC
)
+
+ if IS_PY2:
+ results = convert_keys_to_unicode(results, conn_encoding)
+
results = handle_json_data(json_columns, results)
csv_writer.writerows(results)
yield res_io.getvalue()
^ permalink raw reply [nested|flat] 4+ messages in thread
* Re: [pgAdmin4][PATCH] To fix the issues with CSV file download
2017-05-08 12:14 [pgAdmin4][PATCH] To fix the issues with CSV file download Murtuza Zabuawala <[email protected]>
2017-05-08 12:36 ` Re: [pgAdmin4][PATCH] To fix the issues with CSV file download Dave Page <[email protected]>
2017-05-11 14:21 ` Re: [pgAdmin4][PATCH] To fix the issues with CSV file download Murtuza Zabuawala <[email protected]>
@ 2017-05-12 09:38 ` Dave Page <[email protected]>
0 siblings, 0 replies; 4+ messages in thread
From: Dave Page @ 2017-05-12 09:38 UTC (permalink / raw)
To: Murtuza Zabuawala <[email protected]>; +Cc: pgadmin-hackers
Thanks - patch applied.
On Thu, May 11, 2017 at 3:21 PM, Murtuza Zabuawala <
[email protected]> wrote:
> Hi Dave,
>
> Please find patch to handle non-ascii column names in CSV for Pyhton2.
>
> Thanks to Harshal for pointing out the issue.
>
>
> --
> Regards,
> Murtuza Zabuawala
> EnterpriseDB: http://www.enterprisedb.com
> The Enterprise PostgreSQL Company
>
> On Mon, May 8, 2017 at 6:06 PM, Dave Page <[email protected]> wrote:
>
>> Thanks, patch applied.
>>
>> On Mon, May 8, 2017 at 1:14 PM, Murtuza Zabuawala <
>> [email protected]> wrote:
>>
>>> Hi,
>>>
>>> Please find the patch to fix the issues in CSV file download feature,
>>>
>>> 1) To handle non-ascii filenames which we set from table name
>>> RM#2314
>>>
>>> 2) To handle non-ascii query data
>>> RM#2353
>>>
>>> 3) To dump JSON type columns properly in csv
>>> RM#2360
>>>
>>> Please review.
>>>
>>> --
>>> Regards,
>>> Murtuza Zabuawala
>>> EnterpriseDB: http://www.enterprisedb.com
>>> The Enterprise PostgreSQL Company
>>>
>>>
>>> --
>>> Sent via pgadmin-hackers mailing list ([email protected])
>>> To make changes to your subscription:
>>> http://www.postgresql.org/mailpref/pgadmin-hackers
>>>
>>>
>>
>>
>> --
>> Dave Page
>> Blog: http://pgsnake.blogspot.com
>> Twitter: @pgsnake
>>
>> EnterpriseDB UK: http://www.enterprisedb.com
>> The Enterprise PostgreSQL Company
>>
>
>
--
Dave Page
Blog: http://pgsnake.blogspot.com
Twitter: @pgsnake
EnterpriseDB UK: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
^ permalink raw reply [nested|flat] 4+ messages in thread
end of thread, other threads:[~2017-05-12 09:38 UTC | newest]
Thread overview: 4+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2017-05-08 12:14 [pgAdmin4][PATCH] To fix the issues with CSV file download Murtuza Zabuawala <[email protected]>
2017-05-08 12:36 ` Dave Page <[email protected]>
2017-05-11 14:21 ` Murtuza Zabuawala <[email protected]>
2017-05-12 09:38 ` Dave Page <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox