From 69abfc3ada5d754cb152119c0b4777043657cb6e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Jun 2017 12:29:55 -0400 Subject: [PATCH 01/40] use safe load --- tablib/formats/_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tablib/formats/_yaml.py b/tablib/formats/_yaml.py index 5aecb42..3d17baf 100644 --- a/tablib/formats/_yaml.py +++ b/tablib/formats/_yaml.py @@ -33,7 +33,7 @@ def import_book(dbook, in_stream): dbook.wipe() - for sheet in yaml.load(in_stream): + for sheet in yaml.safe_load(in_stream): data = tablib.Dataset() data.title = sheet['title'] data.dict = sheet['data'] From d89d243a30d453a349d2f630728e72ac452815c4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Jun 2017 12:30:27 -0400 Subject: [PATCH 02/40] v0.11.5 --- tablib/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tablib/core.py b/tablib/core.py index c44c6ac..be648c2 100644 --- a/tablib/core.py +++ b/tablib/core.py @@ -18,11 +18,11 @@ from tablib.compat import OrderedDict, unicode __title__ = 'tablib' -__version__ = '0.11.4' +__version__ = '0.11.5' __build__ = 0x001104 __author__ = 'Kenneth Reitz' __license__ = 'MIT' -__copyright__ = 'Copyright 2016 Kenneth Reitz' +__copyright__ = 'Copyright 2017 Kenneth Reitz' __docformat__ = 'restructuredtext' From a3cd2c9cffaa2940762453f5a3c4f2ae69065aa7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 13 Jun 2017 12:31:42 -0400 Subject: [PATCH 03/40] history --- HISTORY.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index a90a21b..e848c1d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,11 @@ History ------- +0.11.5 (2017-06-13) ++++++++++++++++++++ + +- Use ``yaml.safe_load`` for importing yaml. + 0.11.4 (2017-01-23) +++++++++++++++++++ From 00e2ffa2efd1a7dd47d2a4a2a4303e1a8beb71e3 Mon Sep 17 00:00:00 2001 From: Jason Myers Date: Sat, 26 Aug 2017 20:43:35 -0500 Subject: [PATCH 04/40] Adding initial DataFrames Support Signed-off-by: Jason Myers --- setup.py | 1 + tablib/core.py | 12 ++++++++++++ tablib/formats/__init__.py | 3 ++- tablib/formats/_df.py | 40 ++++++++++++++++++++++++++++++++++++++ test_tablib.py | 1 + 5 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tablib/formats/_df.py diff --git a/setup.py b/setup.py index e46eb88..5691821 100755 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ install = [ 'xlrd', 'xlwt', 'pyyaml', + 'pandas', ] with open('tablib/core.py', 'r') as fd: diff --git a/tablib/core.py b/tablib/core.py index be648c2..13a4514 100644 --- a/tablib/core.py +++ b/tablib/core.py @@ -570,6 +570,18 @@ class Dataset(object): """ pass + @property + def df(): + """A DataFrame representation of the :class:`Dataset` object. + + A dataset object can also be imported by setting the :class:`Dataset.df` attribute: :: + + data = tablib.Dataset() + data.df = DataFrame(np.random.randn(6,4)) + + Import assumes (for now) that headers exist. + """ + pass @property def json(): diff --git a/tablib/formats/__init__.py b/tablib/formats/__init__.py index 5cca19f..94b5bc9 100644 --- a/tablib/formats/__init__.py +++ b/tablib/formats/__init__.py @@ -13,5 +13,6 @@ from . import _xlsx as xlsx from . import _ods as ods from . import _dbf as dbf from . import _latex as latex +from . import _df as df -available = (json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods) +available = (json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods, df) diff --git a/tablib/formats/_df.py b/tablib/formats/_df.py new file mode 100644 index 0000000..5996ce9 --- /dev/null +++ b/tablib/formats/_df.py @@ -0,0 +1,40 @@ +""" Tablib - DataFrame Support. +""" + + +import sys + + +if sys.version_info[0] > 2: + from io import BytesIO +else: + from cStringIO import StringIO as BytesIO + +from pandas import DataFrame + +import tablib + +from tablib.compat import unicode + +title = 'df' +extensions = ('df', ) + +def detect(stream): + """Returns True if given stream is a DataFrame.""" + try: + DataFrame(stream) + return True + except ValueError: + return False + + +def export_set(dset, index=None): + """Returns DataFrame representation of DataBook.""" + dataframe = DataFrame(dset.dict, columns=dset.headers) + return dataframe + + +def import_set(dset, in_stream): + """Returns dataset from DataFrame.""" + dset.wipe() + dset.dict = in_stream.to_dict(orient='records') diff --git a/test_tablib.py b/test_tablib.py index 03a46df..9bd5f12 100755 --- a/test_tablib.py +++ b/test_tablib.py @@ -383,6 +383,7 @@ class TablibTestCase(unittest.TestCase): data.ods data.html data.latex + data.df def test_datetime_append(self): """Passes in a single datetime and a single date and exports.""" From a50ff92ff2aaeae5958728cc4363f0ced23ad772 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 27 Aug 2017 03:26:21 -0400 Subject: [PATCH 05/40] only require pandas if python isn't 2.6 Signed-off-by: Kenneth Reitz --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5691821..ad94b65 100755 --- a/setup.py +++ b/setup.py @@ -47,10 +47,13 @@ install = [ 'unicodecsv', 'xlrd', 'xlwt', - 'pyyaml', - 'pandas', + 'pyyaml' ] +# only require Pandas if Python isn't 2.6. +if not (sys.version_info[0] == 2 and sys.version_info[1] == 6): + install.append('pandas') + with open('tablib/core.py', 'r') as fd: version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) From 5dd74c0104789b2019c97f863e597369ccfed108 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 27 Aug 2017 03:29:44 -0400 Subject: [PATCH 06/40] drop 2.6 --- .travis.yml | 1 - setup.py | 7 ++----- tablib/core.py | 4 ++-- test_tablib.py | 1 - 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e09b14..d78dfd6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - 2.6 - 2.7 - 3.3 - 3.4 diff --git a/setup.py b/setup.py index ad94b65..fd9a626 100755 --- a/setup.py +++ b/setup.py @@ -47,12 +47,10 @@ install = [ 'unicodecsv', 'xlrd', 'xlwt', - 'pyyaml' + 'pyyaml', + 'pandas' ] -# only require Pandas if Python isn't 2.6. -if not (sys.version_info[0] == 2 and sys.version_info[1] == 6): - install.append('pandas') with open('tablib/core.py', 'r') as fd: version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', @@ -75,7 +73,6 @@ setup( 'Natural Language :: English', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', diff --git a/tablib/core.py b/tablib/core.py index 13a4514..dc45a6f 100644 --- a/tablib/core.py +++ b/tablib/core.py @@ -18,8 +18,8 @@ from tablib.compat import OrderedDict, unicode __title__ = 'tablib' -__version__ = '0.11.5' -__build__ = 0x001104 +__version__ = '0.12.0' +__build__ = 0x001200 __author__ = 'Kenneth Reitz' __license__ = 'MIT' __copyright__ = 'Copyright 2017 Kenneth Reitz' diff --git a/test_tablib.py b/test_tablib.py index 9bd5f12..9b47773 100755 --- a/test_tablib.py +++ b/test_tablib.py @@ -5,7 +5,6 @@ import json import unittest import sys -import os import datetime From 7c318adde4c4a2f3254d1fc099b55c04e8ece7e3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 27 Aug 2017 03:41:01 -0400 Subject: [PATCH 07/40] Update README.rst --- README.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 3ba1138..590cc61 100644 --- a/README.rst +++ b/README.rst @@ -86,7 +86,7 @@ JSON! +++++ :: - >>> print data.json + >>> print(data.json) [ { "last_name": "Adams", @@ -105,7 +105,7 @@ YAML! +++++ :: - >>> print data.yaml + >>> print(data.yaml) - {age: 90, first_name: John, last_name: Adams} - {age: 83, first_name: Henry, last_name: Ford} @@ -113,7 +113,7 @@ CSV... ++++++ :: - >>> print data.csv + >>> print(data.csv) first_name,last_name,age John,Adams,90 Henry,Ford,83 @@ -131,6 +131,15 @@ DBF! >>> with open('people.dbf', 'wb') as f: ... f.write(data.dbf) + +Pandas DataFrame! ++++++++++++++++++ + +:: + >>> print(data.df): + first_name last_name age + 0 John Adams 90 + 1 Henry Ford 83 It's that easy. From 34c14aca18c0e1f44fc39bb09df31ee07b7a3fbf Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 27 Aug 2017 03:41:26 -0400 Subject: [PATCH 08/40] Update README.rst --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 590cc61..2d56c54 100644 --- a/README.rst +++ b/README.rst @@ -21,6 +21,7 @@ Output formats supported: - Excel (Sets + Books) - JSON (Sets + Books) - YAML (Sets + Books) +- Pandas DataFrames (Sets) - HTML (Sets) - TSV (Sets) - OSD (Sets) From 44e797d70e18b5ec7add9bf32e8d5b9ab179ece0 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 27 Aug 2017 03:41:53 -0400 Subject: [PATCH 09/40] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 2d56c54..a84cf97 100644 --- a/README.rst +++ b/README.rst @@ -65,13 +65,13 @@ Intelligently add new columns: :: Slice rows: :: - >>> print data[:2] + >>> print(data[:2]) [('John', 'Adams', 90), ('George', 'Washington', 67)] Slice columns by header: :: - >>> print data['first_name'] + >>> print(data['first_name']) ['John', 'George', 'Henry'] Easily delete rows: :: From 412e6902899dda6fa9488d3347f5285a80e89d7b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 27 Aug 2017 03:42:15 -0400 Subject: [PATCH 10/40] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a84cf97..3a54c5c 100644 --- a/README.rst +++ b/README.rst @@ -135,8 +135,8 @@ DBF! Pandas DataFrame! +++++++++++++++++ - :: + >>> print(data.df): first_name last_name age 0 John Adams 90 From e4726cb85cfe9383e7d4012a68638ebb90c58b1e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 27 Aug 2017 03:48:01 -0400 Subject: [PATCH 11/40] update docs Signed-off-by: Kenneth Reitz --- docs/index.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 55e5679..4a0f3f0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,18 +29,23 @@ Tablib is an :ref:`MIT Licensed ` format-agnostic tabular dataset library, >>> data = tablib.Dataset(headers=['First Name', 'Last Name', 'Age']) >>> for i in [('Kenneth', 'Reitz', 22), ('Bessie', 'Monke', 21)]: ... data.append(i) - - >>> print data.json + + >>> print(data.json) [{"Last Name": "Reitz", "First Name": "Kenneth", "Age": 22}, {"Last Name": "Monke", "First Name": "Bessie", "Age": 21}] - >>> print data.yaml + >>> print(data.yaml) - {Age: 22, First Name: Kenneth, Last Name: Reitz} - {Age: 21, First Name: Bessie, Last Name: Monke} >>> data.xlsx + >>> data.df + First Name Last Name Age + 0 Kenneth Reitz 22 + 1 Bessie Monke 21 + Testimonials ------------ From bb0abc863eb5290527113c11af71c6379f960ef6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 27 Aug 2017 03:49:29 -0400 Subject: [PATCH 12/40] bunk requirements file Signed-off-by: Kenneth Reitz --- requirements.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2fab040 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +certifi==2017.7.27.1 +chardet==3.0.4 +et-xmlfile==1.0.1 +idna==2.6 +jdcal==1.3 +numpy==1.13.1 +odfpy==1.3.5 +openpyxl==2.4.8 +pandas==0.20.3 +pkginfo==1.4.1 +python-dateutil==2.6.1 +pytz==2017.2 +PyYAML==3.12 +requests==2.18.4 +requests-toolbelt==0.8.0 +six==1.10.0 +tqdm==4.15.0 +unicodecsv==0.14.1 +urllib3==1.22 +xlrd==1.1.0 +xlwt==1.3.0 From 36fa7ef097895fb0338515246beffed1557b1510 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 27 Aug 2017 03:56:14 -0400 Subject: [PATCH 13/40] update docs Signed-off-by: Kenneth Reitz --- docs/intro.rst | 3 +-- docs/tutorial.rst | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/intro.rst b/docs/intro.rst index e3da4dc..6af436d 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -49,7 +49,7 @@ Tablib is released under terms of `The MIT License`_. Tablib License -------------- -Copyright 2016 Kenneth Reitz +Copyright 2017 Kenneth Reitz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -77,7 +77,6 @@ Pythons Supported At this time, the following Python platforms are officially supported: -* cPython 2.6 * cPython 2.7 * cPython 3.3 * cPython 3.4 diff --git a/docs/tutorial.rst b/docs/tutorial.rst index d552e21..1a092e9 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -139,6 +139,14 @@ Tablib's killer feature is the ability to export your :class:`Dataset` objects i +**Pandas DataFrame** :: + + >>> data.df + First Name Last Name Age + 0 Kenneth Reitz 22 + 1 Bessie Monke 21 + + ------------------------ Selecting Rows & Columns ------------------------ From 56005d80222a487178742b70f8858e4fd96fd0e3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 28 Aug 2017 01:02:49 -0400 Subject: [PATCH 14/40] Update README.rst --- README.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 3a54c5c..daa1b91 100644 --- a/README.rst +++ b/README.rst @@ -87,7 +87,7 @@ JSON! +++++ :: - >>> print(data.json) + >>> print(data.export('json')) [ { "last_name": "Adams", @@ -106,7 +106,7 @@ YAML! +++++ :: - >>> print(data.yaml) + >>> print(data.export('yaml')) - {age: 90, first_name: John, last_name: Adams} - {age: 83, first_name: Henry, last_name: Ford} @@ -114,7 +114,7 @@ CSV... ++++++ :: - >>> print(data.csv) + >>> print(data.export('csv')) first_name,last_name,age John,Adams,90 Henry,Ford,83 @@ -124,20 +124,20 @@ EXCEL! :: >>> with open('people.xls', 'wb') as f: - ... f.write(data.xls) + ... f.write(data.export('xls')) DBF! ++++ :: >>> with open('people.dbf', 'wb') as f: - ... f.write(data.dbf) + ... f.write(data.export('dbf')) Pandas DataFrame! +++++++++++++++++ :: - >>> print(data.df): + >>> print(data.export('df')): first_name last_name age 0 John Adams 90 1 Henry Ford 83 From ab6633549f2f2f331736f3f974483c0e39a451f2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 28 Aug 2017 01:04:16 -0400 Subject: [PATCH 15/40] Update index.rst --- docs/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4a0f3f0..fb6db95 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,17 +31,17 @@ Tablib is an :ref:`MIT Licensed ` format-agnostic tabular dataset library, ... data.append(i) - >>> print(data.json) + >>> print(data.export('json')) [{"Last Name": "Reitz", "First Name": "Kenneth", "Age": 22}, {"Last Name": "Monke", "First Name": "Bessie", "Age": 21}] >>> print(data.yaml) - {Age: 22, First Name: Kenneth, Last Name: Reitz} - {Age: 21, First Name: Bessie, Last Name: Monke} - >>> data.xlsx + >>> data.export('xlsx') - >>> data.df + >>> data.export('df') First Name Last Name Age 0 Kenneth Reitz 22 1 Bessie Monke 21 From ec54918f4a7303eba65e7575be52befed9c205c7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 28 Aug 2017 01:06:43 -0400 Subject: [PATCH 16/40] Update tutorial.rst --- docs/tutorial.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 1a092e9..23d2773 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -115,33 +115,33 @@ Tablib's killer feature is the ability to export your :class:`Dataset` objects i **Comma-Separated Values** :: - >>> data.csv + >>> data.export('csv') Last Name,First Name,Age Reitz,Kenneth,22 Monke,Bessie,20 **JavaScript Object Notation** :: - >>> data.json + >>> data.export('json') [{"Last Name": "Reitz", "First Name": "Kenneth", "Age": 22}, {"Last Name": "Monke", "First Name": "Bessie", "Age": 20}] **YAML Ain't Markup Language** :: - >>> data.yaml + >>> data.export('yaml') - {Age: 22, First Name: Kenneth, Last Name: Reitz} - {Age: 20, First Name: Bessie, Last Name: Monke} **Microsoft Excel** :: - >>> data.xls + >>> data.export('xls') **Pandas DataFrame** :: - >>> data.df + >>> data.export('df') First Name Last Name Age 0 Kenneth Reitz 22 1 Bessie Monke 21 @@ -224,7 +224,7 @@ Let's add a dynamic column to our :class:`Dataset` object. In this example, we h Let's have a look at our data. :: - >>> data.yaml + >>> data.export('yaml') - {Age: 22, First Name: Kenneth, Grade: 0.6, Last Name: Reitz} - {Age: 20, First Name: Bessie, Grade: 0.75, Last Name: Monke} @@ -254,7 +254,7 @@ For example, we can use the data available in the row to guess the gender of a s Adding this function to our dataset as a dynamic column would result in: :: - >>> data.yaml + >>> data.export('yaml') - {Age: 22, First Name: Kenneth, Gender: Male, Last Name: Reitz} - {Age: 20, First Name: Bessie, Gender: Female, Last Name: Monke} @@ -354,7 +354,7 @@ When, it's often useful to create a blank row containing information on the upco # Write spreadsheet to disk with open('grades.xls', 'wb') as f: - f.write(tests.xls) + f.write(tests.export('xls')) The resulting **tests.xls** will have the following layout: From 69edb9def37be7f38a4fed6d6bf10a618eaec8e3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 28 Aug 2017 01:14:36 -0400 Subject: [PATCH 17/40] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index fb6db95..90289e2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,7 +34,7 @@ Tablib is an :ref:`MIT Licensed ` format-agnostic tabular dataset library, >>> print(data.export('json')) [{"Last Name": "Reitz", "First Name": "Kenneth", "Age": 22}, {"Last Name": "Monke", "First Name": "Bessie", "Age": 21}] - >>> print(data.yaml) + >>> print(data.export('yaml')) - {Age: 22, First Name: Kenneth, Last Name: Reitz} - {Age: 21, First Name: Bessie, Last Name: Monke} From b09fface1be24bd4025b705f7f8495efb398991d Mon Sep 17 00:00:00 2001 From: Ryan Castner Date: Fri, 1 Sep 2017 13:20:54 -0400 Subject: [PATCH 18/40] Make pandas an optional install --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fd9a626..d251352 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,6 @@ install = [ 'xlrd', 'xlwt', 'pyyaml', - 'pandas' ] @@ -81,4 +80,7 @@ setup( ], tests_require=['pytest'], install_requires=install, + extras_require={ + 'pandas': ['pandas'], + }, ) From 7f1db4023f2310529822d721379b1019aaf320fc Mon Sep 17 00:00:00 2001 From: Ryan Castner Date: Fri, 1 Sep 2017 13:21:21 -0400 Subject: [PATCH 19/40] Raise NotImplementedError if pandas is not installed --- tablib/formats/_df.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tablib/formats/_df.py b/tablib/formats/_df.py index 5996ce9..44b967f 100644 --- a/tablib/formats/_df.py +++ b/tablib/formats/_df.py @@ -10,7 +10,10 @@ if sys.version_info[0] > 2: else: from cStringIO import StringIO as BytesIO -from pandas import DataFrame +try: + from pandas import DataFrame +except ImportError: + DataFrame = None import tablib @@ -21,6 +24,8 @@ extensions = ('df', ) def detect(stream): """Returns True if given stream is a DataFrame.""" + if DataFrame is None: + return False try: DataFrame(stream) return True @@ -30,6 +35,10 @@ def detect(stream): def export_set(dset, index=None): """Returns DataFrame representation of DataBook.""" + if DataFrame is None: + raise NotImplementedError( + 'DataFrame Format requires `pandas` to be installed.' + ' Try `pip install tablib[pandas]`.') dataframe = DataFrame(dset.dict, columns=dset.headers) return dataframe From 38183938dc7332bcbdfeef4b8631ead8e32139e0 Mon Sep 17 00:00:00 2001 From: Ryan Castner Date: Fri, 1 Sep 2017 13:33:28 -0400 Subject: [PATCH 20/40] Change how travis installs to get all test dependencies --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d78dfd6..1948f07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,5 @@ python: - 3.5 - 3.6 install: - - python setup.py install + - pip install -r requirements.txt script: python test_tablib.py From edbb16ec97bbecc679b0ebc9de6f09f7b942bf8b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 1 Sep 2017 15:37:00 -0400 Subject: [PATCH 21/40] next version --- tablib/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tablib/core.py b/tablib/core.py index dc45a6f..8c49f2e 100644 --- a/tablib/core.py +++ b/tablib/core.py @@ -18,8 +18,8 @@ from tablib.compat import OrderedDict, unicode __title__ = 'tablib' -__version__ = '0.12.0' -__build__ = 0x001200 +__version__ = '0.12.1' +__build__ = 0x001201 __author__ = 'Kenneth Reitz' __license__ = 'MIT' __copyright__ = 'Copyright 2017 Kenneth Reitz' From 4c300e65a50eef72b91fa1909c9f68679723955e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 1 Sep 2017 15:42:51 -0400 Subject: [PATCH 22/40] update install instructions Signed-off-by: Kenneth Reitz --- README.rst | 6 +++--- docs/install.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index daa1b91..28b305f 100644 --- a/README.rst +++ b/README.rst @@ -132,10 +132,10 @@ DBF! >>> with open('people.dbf', 'wb') as f: ... f.write(data.export('dbf')) - + Pandas DataFrame! +++++++++++++++++ -:: +:: >>> print(data.export('df')): first_name last_name age @@ -150,7 +150,7 @@ Installation To install tablib, simply: :: - $ pip install tablib + $ pip install tablib[pandas] Make sure to check out `Tablib on PyPi `_! diff --git a/docs/install.rst b/docs/install.rst index 365cca8..4dab923 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -16,7 +16,7 @@ Distribute & Pip Of course, the recommended way to install Tablib is with `pip `_:: - $ pip install tablib + $ pip install tablib[pandas] ------------------- From 61063e2b09a5cd26926311a34826fd33be0b1757 Mon Sep 17 00:00:00 2001 From: DougHudgeon Date: Mon, 25 Jun 2018 14:17:34 +1000 Subject: [PATCH 23/40] Updated xlsx format to use openpyxl's .active property --- tablib/formats/_xlsx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tablib/formats/_xlsx.py b/tablib/formats/_xlsx.py index 20f55df..67aa5f4 100644 --- a/tablib/formats/_xlsx.py +++ b/tablib/formats/_xlsx.py @@ -71,7 +71,7 @@ def import_set(dset, in_stream, headers=True): dset.wipe() xls_book = openpyxl.reader.excel.load_workbook(BytesIO(in_stream)) - sheet = xls_book.get_active_sheet() + sheet = xls_book.active dset.title = sheet.title From 4c5d0b1a457b55c943fbff888459ad224c8a5bcd Mon Sep 17 00:00:00 2001 From: DougHudgeon Date: Mon, 25 Jun 2018 14:25:50 +1000 Subject: [PATCH 24/40] Instructions for opening Excel workbook and reading the first sheet --- docs/tutorial.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 23d2773..1fe11ee 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -289,6 +289,14 @@ Now that we have extra meta-data on our rows, we can easily filter our :class:`D It's that simple. The original :class:`Dataset` is untouched. +Open an Excel Workbook and read first sheet +-------------------------------- + +To open an Excel 2007 and later workbook with a single sheet (or a workbook with multiple sheets but you just want the first sheet), use the following: + +data = tablib.Dataset() +data.xlsx = open('my_excel_file.xlsx', 'rb').read() +print(data) Excel Workbook With Multiple Sheets ------------------------------------ From f812c29275c7f47b08f8f1b962625545ec4e22ad Mon Sep 17 00:00:00 2001 From: DougHudgeon Date: Tue, 26 Jun 2018 10:33:21 +1000 Subject: [PATCH 25/40] Add instructions for handling csv line endings in Windows in Python 3 --- tablib/core.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tablib/core.py b/tablib/core.py index 8c49f2e..e0d61f0 100644 --- a/tablib/core.py +++ b/tablib/core.py @@ -526,9 +526,9 @@ class Dataset(object): Import assumes (for now) that headers exist. - .. admonition:: Binary Warning + .. admonition:: Binary Warning for Python 2 - :class:`Dataset.csv` uses \\r\\n line endings by default, so make + :class:`Dataset.csv` uses \\r\\n line endings by default so, in Python 2, make sure to write in binary mode:: with open('output.csv', 'wb') as f: @@ -536,6 +536,18 @@ class Dataset(object): If you do not do this, and you export the file on Windows, your CSV file will open in Excel with a blank line between each row. + + .. admonition:: Line endings for Python 3 + + :class:`Dataset.csv` uses \\r\\n line endings by default so, in Python 3, make + sure to include newline='' otherwise you will get a blank line between each row + when you open the file in Excel:: + + with open('output.csv', 'w', newline='') as f: + f.write(data.csv) + + If you do not do this, and you export the file on Windows, your + CSV file will open in Excel with a blank line between each row. """ pass From ac3cf67620c89f71b73bed891355e8dc0491a541 Mon Sep 17 00:00:00 2001 From: Gregory Bataille Date: Wed, 12 Sep 2018 13:34:55 +0200 Subject: [PATCH 26/40] fix(): remove openpyxl warning by properly accessing cells (#296) --- tablib/formats/_xlsx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tablib/formats/_xlsx.py b/tablib/formats/_xlsx.py index 20f55df..eb921f9 100644 --- a/tablib/formats/_xlsx.py +++ b/tablib/formats/_xlsx.py @@ -119,7 +119,7 @@ def dset_sheet(dataset, ws, freeze_panes=True): row_number = i + 1 for j, col in enumerate(row): col_idx = get_column_letter(j + 1) - cell = ws.cell('%s%s' % (col_idx, row_number)) + cell = ws['%s%s' % (col_idx, row_number)] # bold headers if (row_number == 1) and dataset.headers: From 4749760e6f111dda398ddd8b3319eeaac1e3bace Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Wed, 12 Sep 2018 15:22:06 -0300 Subject: [PATCH 27/40] Typo: OSD -> ODS Fix #330 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 28b305f..2e51e04 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ Output formats supported: - Pandas DataFrames (Sets) - HTML (Sets) - TSV (Sets) -- OSD (Sets) +- ODS (Sets) - CSV (Sets) - DBF (Sets) From 75f1bafd69add49d969b0f8e579baf3659f16100 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 12 Sep 2018 20:24:37 +0200 Subject: [PATCH 28/40] Removed Python 3.3 support (#310) --- .travis.yml | 1 - setup.py | 1 - tox.ini | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1948f07..3f8730d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - 2.7 - - 3.3 - 3.4 - 3.5 - 3.6 diff --git a/setup.py b/setup.py index d251352..ed77fed 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,6 @@ setup( 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/tox.ini b/tox.ini index 3e1d6a2..32dce59 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py27, py33, py34, py35, py36, pypy +envlist = py27, py34, py35, py36, pypy [testenv] commands = python setup.py test From 38486231cc4dc46d1ff055a38fe7c236cbe5c8aa Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Wed, 12 Sep 2018 20:27:10 +0200 Subject: [PATCH 29/40] reStructuredText (#336) * median for Python 2 * More compat * Support reStructuredText * Tests --- tablib/compat.py | 5 +- tablib/formats/__init__.py | 3 +- tablib/formats/_rst.py | 273 ++++++++++++++++++++++++++++++++++ tablib/packages/statistics.py | 24 +++ test_tablib.py | 22 +++ 5 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 tablib/formats/_rst.py create mode 100644 tablib/packages/statistics.py diff --git a/tablib/compat.py b/tablib/compat.py index 43e0bbc..d60ce72 100644 --- a/tablib/compat.py +++ b/tablib/compat.py @@ -22,6 +22,8 @@ except ImportError: if is_py3: from io import BytesIO + from itertools import zip_longest as izip_longest + from statistics import median from tablib.packages import markup3 as markup import tablib.packages.dbfpy3 as dbfpy @@ -39,7 +41,8 @@ else: from cStringIO import StringIO as BytesIO from cStringIO import StringIO from tablib.packages import markup - from itertools import ifilter + from tablib.packages.statistics import median + from itertools import ifilter, izip_longest import unicodecsv as csv import tablib.packages.dbfpy as dbfpy diff --git a/tablib/formats/__init__.py b/tablib/formats/__init__.py index 94b5bc9..3eb747e 100644 --- a/tablib/formats/__init__.py +++ b/tablib/formats/__init__.py @@ -14,5 +14,6 @@ from . import _ods as ods from . import _dbf as dbf from . import _latex as latex from . import _df as df +from . import _rst as rst -available = (json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods, df) +available = (json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods, df, rst) diff --git a/tablib/formats/_rst.py b/tablib/formats/_rst.py new file mode 100644 index 0000000..4b53ad7 --- /dev/null +++ b/tablib/formats/_rst.py @@ -0,0 +1,273 @@ +# -*- coding: utf-8 -*- + +""" Tablib - reStructuredText Support +""" +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from textwrap import TextWrapper + +from tablib.compat import ( + median, + unicode, + izip_longest, +) + + +title = 'rst' +extensions = ('rst',) + + +MAX_TABLE_WIDTH = 80 # Roughly. It may be wider to avoid breaking words. + + +JUSTIFY_LEFT = 'left' +JUSTIFY_CENTER = 'center' +JUSTIFY_RIGHT = 'right' +JUSTIFY_VALUES = (JUSTIFY_LEFT, JUSTIFY_CENTER, JUSTIFY_RIGHT) + + +def to_unicode(value): + if isinstance(value, bytes): + return value.decode('utf-8') + return unicode(value) + + +def _max_word_len(text): + """ + Return the length of the longest word in `text`. + + + >>> _max_word_len('Python Module for Tabular Datasets') + 8 + + """ + return max((len(word) for word in text.split())) + + +def _get_column_string_lengths(dataset): + """ + Returns a list of string lengths of each column, and a list of + maximum word lengths. + """ + if dataset.headers: + column_lengths = [[len(h)] for h in dataset.headers] + word_lens = [_max_word_len(h) for h in dataset.headers] + else: + column_lengths = [[] for _ in range(dataset.width)] + word_lens = [0 for _ in range(dataset.width)] + for row in dataset.dict: + values = iter(row.values() if hasattr(row, 'values') else row) + for i, val in enumerate(values): + text = to_unicode(val) + column_lengths[i].append(len(text)) + word_lens[i] = max(word_lens[i], _max_word_len(text)) + return column_lengths, word_lens + + +def _row_to_lines(values, widths, wrapper, sep='|', justify=JUSTIFY_LEFT): + """ + Returns a table row of wrapped values as a list of lines + """ + if justify not in JUSTIFY_VALUES: + raise ValueError('Value of "justify" must be one of "{}"'.format( + '", "'.join(JUSTIFY_VALUES) + )) + if justify == JUSTIFY_LEFT: + just = lambda text, width: text.ljust(width) + elif justify == JUSTIFY_CENTER: + just = lambda text, width: text.center(width) + else: + just = lambda text, width: text.rjust(width) + lpad = sep + ' ' if sep else '' + rpad = ' ' + sep if sep else '' + pad = ' ' + sep + ' ' + cells = [] + for value, width in zip(values, widths): + wrapper.width = width + text = to_unicode(value) + cell = wrapper.wrap(text) + cells.append(cell) + lines = izip_longest(*cells, fillvalue='') + lines = ( + (just(cell_line, widths[i]) for i, cell_line in enumerate(line)) + for line in lines + ) + lines = [''.join((lpad, pad.join(line), rpad)) for line in lines] + return lines + + +def _get_column_widths(dataset, max_table_width=MAX_TABLE_WIDTH, pad_len=3): + """ + Returns a list of column widths proportional to the median length + of the text in their cells. + """ + str_lens, word_lens = _get_column_string_lengths(dataset) + median_lens = [int(median(lens)) for lens in str_lens] + total = sum(median_lens) + if total > max_table_width - (pad_len * len(median_lens)): + column_widths = (max_table_width * l // total for l in median_lens) + else: + column_widths = (l for l in median_lens) + # Allow for separator and padding: + column_widths = (w - pad_len if w > pad_len else w for w in column_widths) + # Rather widen table than break words: + column_widths = [max(w, l) for w, l in zip(column_widths, word_lens)] + return column_widths + + +def export_set_as_simple_table(dataset, column_widths=None): + """ + Returns reStructuredText grid table representation of dataset. + """ + lines = [] + wrapper = TextWrapper() + if column_widths is None: + column_widths = _get_column_widths(dataset, pad_len=2) + border = ' '.join(['=' * w for w in column_widths]) + + lines.append(border) + if dataset.headers: + lines.extend(_row_to_lines( + dataset.headers, + column_widths, + wrapper, + sep='', + justify=JUSTIFY_CENTER, + )) + lines.append(border) + for row in dataset.dict: + values = iter(row.values() if hasattr(row, 'values') else row) + lines.extend(_row_to_lines(values, column_widths, wrapper, '')) + lines.append(border) + return '\n'.join(lines) + + +def export_set_as_grid_table(dataset, column_widths=None): + """ + Returns reStructuredText grid table representation of dataset. + + + >>> from tablib import Dataset + >>> from tablib.formats import rst + >>> bits = ((0, 0), (1, 0), (0, 1), (1, 1)) + >>> data = Dataset() + >>> data.headers = ['A', 'B', 'A and B'] + >>> for a, b in bits: + ... data.append([bool(a), bool(b), bool(a * b)]) + >>> print(rst.export_set(data, force_grid=True)) + +-------+-------+-------+ + | A | B | A and | + | | | B | + +=======+=======+=======+ + | False | False | False | + +-------+-------+-------+ + | True | False | False | + +-------+-------+-------+ + | False | True | False | + +-------+-------+-------+ + | True | True | True | + +-------+-------+-------+ + + """ + lines = [] + wrapper = TextWrapper() + if column_widths is None: + column_widths = _get_column_widths(dataset) + header_sep = '+=' + '=+='.join(['=' * w for w in column_widths]) + '=+' + row_sep = '+-' + '-+-'.join(['-' * w for w in column_widths]) + '-+' + + lines.append(row_sep) + if dataset.headers: + lines.extend(_row_to_lines( + dataset.headers, + column_widths, + wrapper, + justify=JUSTIFY_CENTER, + )) + lines.append(header_sep) + for row in dataset.dict: + values = iter(row.values() if hasattr(row, 'values') else row) + lines.extend(_row_to_lines(values, column_widths, wrapper)) + lines.append(row_sep) + return '\n'.join(lines) + + +def _use_simple_table(head0, col0, width0): + """ + Use a simple table if the text in the first column is never wrapped + + + >>> _use_simple_table('menu', ['egg', 'bacon'], 10) + True + >>> _use_simple_table(None, ['lobster thermidor', 'spam'], 10) + False + + """ + if head0 is not None: + head0 = to_unicode(head0) + if len(head0) > width0: + return False + for cell in col0: + cell = to_unicode(cell) + if len(cell) > width0: + return False + return True + + +def export_set(dataset, **kwargs): + """ + Returns reStructuredText table representation of dataset. + + Returns a simple table if the text in the first column is never + wrapped, otherwise returns a grid table. + + + >>> from tablib import Dataset + >>> bits = ((0, 0), (1, 0), (0, 1), (1, 1)) + >>> data = Dataset() + >>> data.headers = ['A', 'B', 'A and B'] + >>> for a, b in bits: + ... data.append([bool(a), bool(b), bool(a * b)]) + >>> table = data.rst + >>> table.split('\\n') == [ + ... '===== ===== =====', + ... ' A B A and', + ... ' B ', + ... '===== ===== =====', + ... 'False False False', + ... 'True False False', + ... 'False True False', + ... 'True True True ', + ... '===== ===== =====', + ... ] + True + + """ + if not dataset.dict: + return '' + force_grid = kwargs.get('force_grid', False) + max_table_width = kwargs.get('max_table_width', MAX_TABLE_WIDTH) + column_widths = _get_column_widths(dataset, max_table_width) + + use_simple_table = _use_simple_table( + dataset.headers[0] if dataset.headers else None, + dataset.get_col(0), + column_widths[0], + ) + if use_simple_table and not force_grid: + return export_set_as_simple_table(dataset, column_widths) + else: + return export_set_as_grid_table(dataset, column_widths) + + +def export_book(databook): + """ + reStructuredText representation of a Databook. + + Tables are separated by a blank line. All tables use the grid + format. + """ + return '\n\n'.join(export_set(dataset, force_grid=True) + for dataset in databook._datasets) diff --git a/tablib/packages/statistics.py b/tablib/packages/statistics.py new file mode 100644 index 0000000..e97a6c9 --- /dev/null +++ b/tablib/packages/statistics.py @@ -0,0 +1,24 @@ +from __future__ import division + + +def median(data): + """ + Return the median (middle value) of numeric data, using the common + "mean of middle two" method. If data is empty, ValueError is raised. + + Mimics the behaviour of Python3's statistics.median + + >>> median([1, 3, 5]) + 3 + >>> median([1, 3, 5, 7]) + 4.0 + + """ + data = sorted(data) + n = len(data) + if not n: + raise ValueError("No median for empty data") + i = n // 2 + if n % 2: + return data[i] + return (data[i - 1] + data[i]) / 2 diff --git a/test_tablib.py b/test_tablib.py index 9b47773..84cc46a 100755 --- a/test_tablib.py +++ b/test_tablib.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- """Tests for Tablib.""" +import doctest import json import unittest import sys @@ -383,6 +384,7 @@ class TablibTestCase(unittest.TestCase): data.html data.latex data.df + data.rst def test_datetime_append(self): """Passes in a single datetime and a single date and exports.""" @@ -403,6 +405,7 @@ class TablibTestCase(unittest.TestCase): data.ods data.html data.latex + data.rst def test_book_export_no_exceptions(self): """Test that various exports don't error out.""" @@ -416,6 +419,7 @@ class TablibTestCase(unittest.TestCase): book.xlsx book.ods book.html + data.rst def test_json_import_set(self): """Generate and import JSON set serialization.""" @@ -961,6 +965,24 @@ class TablibTestCase(unittest.TestCase): self.founders.append(('First\nSecond', 'Name', 42)) self.founders.export('xlsx') + def test_rst_force_grid(self): + data.append(self.john) + data.append(self.george) + data.headers = self.headers + + simple = tablib.formats._rst.export_set(data) + grid = tablib.formats._rst.export_set(data, force_grid=True) + self.assertNotEqual(simple, grid) + self.assertNotIn('+', simple) + self.assertIn('+', grid) + + +class DocTests(unittest.TestCase): + + def test_rst_formatter_doctests(self): + results = doctest.testmod(tablib.formats._rst) + self.assertEqual(results.failed, 0) + if __name__ == '__main__': unittest.main() From 3d5943a8a43a721bea1f3e33dc669a581a095c53 Mon Sep 17 00:00:00 2001 From: Bruno Soares Date: Wed, 12 Sep 2018 15:49:46 -0300 Subject: [PATCH 30/40] Fix: Circular reference detected error (#332) * Rename function name * Add uuid handler on json dumps * Add myself to authors --- AUTHORS | 1 + tablib/formats/_json.py | 10 +++++----- test_tablib.py | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index 8afb539..65463bd 100644 --- a/AUTHORS +++ b/AUTHORS @@ -34,3 +34,4 @@ Patches and Suggestions - Mathias Loesch - Tushar Makkar - Andrii Soldatenko +- Bruno Soares diff --git a/tablib/formats/_json.py b/tablib/formats/_json.py index a3d6cc3..a3b88f8 100644 --- a/tablib/formats/_json.py +++ b/tablib/formats/_json.py @@ -3,6 +3,7 @@ """ Tablib - JSON Support """ import decimal +from uuid import UUID import tablib @@ -15,24 +16,23 @@ title = 'json' extensions = ('json', 'jsn') -def date_handler(obj): - if isinstance(obj, decimal.Decimal): +def serialize_objects_handler(obj): + if isinstance(obj, decimal.Decimal) or isinstance(obj, UUID): return str(obj) elif hasattr(obj, 'isoformat'): return obj.isoformat() else: return obj - # return obj.isoformat() if hasattr(obj, 'isoformat') else obj def export_set(dataset): """Returns JSON representation of Dataset.""" - return json.dumps(dataset.dict, default=date_handler) + return json.dumps(dataset.dict, default=serialize_objects_handler) def export_book(databook): """Returns JSON representation of Databook.""" - return json.dumps(databook._package(), default=date_handler) + return json.dumps(databook._package(), default=serialize_objects_handler) def import_set(dset, in_stream): diff --git a/test_tablib.py b/test_tablib.py index 84cc46a..b945a2d 100755 --- a/test_tablib.py +++ b/test_tablib.py @@ -6,6 +6,7 @@ import doctest import json import unittest import sys +from uuid import uuid4 import datetime @@ -226,6 +227,22 @@ class TablibTestCase(unittest.TestCase): # Delete from invalid index self.assertRaises(IndexError, self.founders.__delitem__, 3) + + def test_json_export(self): + """Verify exporting dataset object as JSON""" + + address_id = uuid4() + headers = self.headers + ('address_id',) + founders = tablib.Dataset(headers=headers, title='Founders') + founders.append(('John', 'Adams', 90, address_id)) + founders_json = founders.export('json') + + expected_json = ( + '[{"first_name": "John", "last_name": "Adams", "gpa": 90, ' + '"address_id": "%s"}]' % str(address_id) + ) + + self.assertEqual(founders_json, expected_json) def test_csv_export(self): """Verify exporting dataset object as CSV.""" From 4f8949417e5126f8416b61d7feb6cd666bae4fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20=22RooTer=22=20Urba=C5=84ski?= Date: Wed, 12 Sep 2018 21:15:20 +0200 Subject: [PATCH 31/40] ujson presence no longer breaks tablib (resolves #297) (#311) --- docs/install.rst | 10 ---------- setup.py | 9 --------- tablib/formats/_json.py | 5 +---- 3 files changed, 1 insertion(+), 23 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 4dab923..a236b87 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -40,16 +40,6 @@ To download the full source history from Git, see :ref:`Source Control `. .. _zipball: http://github.com/kennethreitz/tablib/zipball/master -.. _speed-extensions: -Speed Extensions ----------------- - -You can gain some speed improvement by optionally installing the ujson_ library. -Tablib will fallback to the standard `json` module if it doesn't find ``ujson``. - -.. _ujson: https://pypi.python.org/pypi/ujson - - .. _updates: Staying Updated --------------- diff --git a/setup.py b/setup.py index ed77fed..0f9da92 100755 --- a/setup.py +++ b/setup.py @@ -14,15 +14,6 @@ if sys.argv[-1] == 'publish': os.system("python setup.py sdist upload") sys.exit() -if sys.argv[-1] == 'speedups': - try: - __import__('pip') - except ImportError: - print('Pip required.') - sys.exit(1) - - os.system('pip install ujson') - sys.exit() if sys.argv[-1] == 'test': try: diff --git a/tablib/formats/_json.py b/tablib/formats/_json.py index a3b88f8..bbd2c96 100644 --- a/tablib/formats/_json.py +++ b/tablib/formats/_json.py @@ -3,14 +3,11 @@ """ Tablib - JSON Support """ import decimal +import json from uuid import UUID import tablib -try: - import ujson as json -except ImportError: - import json title = 'json' extensions = ('json', 'jsn') From 359007444c2751d1cc325854ec86cdfe0a38b7bb Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 17 Sep 2018 08:13:48 -0400 Subject: [PATCH 32/40] Update README.rst --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 2e51e04..0c48c82 100644 --- a/README.rst +++ b/README.rst @@ -30,6 +30,8 @@ Output formats supported: Note that tablib *purposefully* excludes XML support. It always will. (Note: This is a joke. Pull requests are welcome.) +**If you're interested in financially supporting Kenneth Reitz open source, consider [visiting this link](https://cash.me/$KennethReitz). Your support helps tremendously with sustainability of motivation, as Open Source is no longer part of my day job.** + Overview -------- From 5a359ba4def1611b34df003bec58439a125d4e56 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 17 Sep 2018 08:14:12 -0400 Subject: [PATCH 33/40] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0c48c82..6258001 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ Output formats supported: Note that tablib *purposefully* excludes XML support. It always will. (Note: This is a joke. Pull requests are welcome.) -**If you're interested in financially supporting Kenneth Reitz open source, consider [visiting this link](https://cash.me/$KennethReitz). Your support helps tremendously with sustainability of motivation, as Open Source is no longer part of my day job.** +If you're interested in financially supporting Kenneth Reitz open source, consider `visiting this link `_. Your support helps tremendously with sustainability of motivation, as Open Source is no longer part of my day job. Overview -------- From d38549ef1e43c6c296b7f3d44210d61e34e6c4c9 Mon Sep 17 00:00:00 2001 From: lepuchi Date: Tue, 2 Oct 2018 23:26:19 +0530 Subject: [PATCH 34/40] only add row if it exists --- tablib/formats/_csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tablib/formats/_csv.py b/tablib/formats/_csv.py index 994b23b..06e7830 100644 --- a/tablib/formats/_csv.py +++ b/tablib/formats/_csv.py @@ -44,7 +44,7 @@ def import_set(dset, in_stream, headers=True, **kwargs): if (i == 0) and (headers): dset.headers = row - else: + elif row: dset.append(row) From a28a057559bcfb39612e91af59f0e80a1796caf6 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sat, 6 Oct 2018 19:19:09 +0900 Subject: [PATCH 35/40] Replace a deprecated method call Workbook.remove_sheet method deprecated since openpyxl 2.4.0 --- AUTHORS | 1 + setup.py | 2 +- tablib/formats/_xlsx.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 65463bd..e574b85 100644 --- a/AUTHORS +++ b/AUTHORS @@ -35,3 +35,4 @@ Patches and Suggestions - Tushar Makkar - Andrii Soldatenko - Bruno Soares +- Tsuyoshi Hombashi diff --git a/setup.py b/setup.py index 0f9da92..2d1b0fa 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ packages = [ install = [ 'odfpy', - 'openpyxl', + 'openpyxl>=2.4.0', 'unicodecsv', 'xlrd', 'xlwt', diff --git a/tablib/formats/_xlsx.py b/tablib/formats/_xlsx.py index eb921f9..1d29ba8 100644 --- a/tablib/formats/_xlsx.py +++ b/tablib/formats/_xlsx.py @@ -52,7 +52,7 @@ def export_book(databook, freeze_panes=True): wb = Workbook() for sheet in wb.worksheets: - wb.remove_sheet(sheet) + wb.remove(sheet) for i, dset in enumerate(databook._datasets): ws = wb.create_sheet() ws.title = dset.title if dset.title else 'Sheet%s' % (i) From dd2ba714d3eec5e6a14bdac27b4e6467b6dfb439 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Tue, 1 Jan 2019 09:58:53 -0800 Subject: [PATCH 36/40] Remove vendored ordereddict package Now that Python 2.6 support has been dropped, can remove the vendored ordereddict package. Use the stdlib collections.OrderedDict instead. --- NOTICE | 28 +------- tablib/compat.py | 7 -- tablib/core.py | 3 +- tablib/packages/ordereddict.py | 127 --------------------------------- 4 files changed, 3 insertions(+), 162 deletions(-) delete mode 100644 tablib/packages/ordereddict.py diff --git a/NOTICE b/NOTICE index 4bdbb05..6966f47 100644 --- a/NOTICE +++ b/NOTICE @@ -1,32 +1,6 @@ -Tablib includes some vendorized python libraries: ordereddict, markup. +Tablib includes some vendorized Python libraries: markup. Markup License ============== Markup is in the public domain. - - -OrderedDict License -=================== - -Copyright (c) 2009 Raymond Hettinger - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation files -(the "Software"), to deal in the Software without restriction, -including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, -and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. diff --git a/tablib/compat.py b/tablib/compat.py index d60ce72..916d6ab 100644 --- a/tablib/compat.py +++ b/tablib/compat.py @@ -13,13 +13,6 @@ import sys is_py3 = (sys.version_info[0] > 2) - -try: - from collections import OrderedDict -except ImportError: - from tablib.packages.ordereddict import OrderedDict - - if is_py3: from io import BytesIO from itertools import zip_longest as izip_longest diff --git a/tablib/core.py b/tablib/core.py index 8c49f2e..f9951a9 100644 --- a/tablib/core.py +++ b/tablib/core.py @@ -9,12 +9,13 @@ :license: MIT, see LICENSE for more details. """ +from collections import OrderedDict from copy import copy from operator import itemgetter from tablib import formats -from tablib.compat import OrderedDict, unicode +from tablib.compat import unicode __title__ = 'tablib' diff --git a/tablib/packages/ordereddict.py b/tablib/packages/ordereddict.py deleted file mode 100644 index a5b896d..0000000 --- a/tablib/packages/ordereddict.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (c) 2009 Raymond Hettinger -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -from UserDict import DictMixin - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - if last: - key = next(reversed(self)) - else: - key = next(iter(self)) - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, list(self.items())) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - if len(self) != len(other): - return False - for p, q in zip(list(self.items()), list(other.items())): - if p != q: - return False - return True - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other From 3e4d6fb5aa6d53ebf09b5b017c0deb3328e66064 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Tue, 1 Jan 2019 10:28:29 -0800 Subject: [PATCH 37/40] Include pandas dependency when testing with tox Allows all tests to pass. As pandas is defined as an 'extra', use tox's 'extras' feature. This requires tox 2.4+, so document that as well. https://tox.readthedocs.io/en/latest/config.html#conf-extras --- tox.ini | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 32dce59..4dabf0c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,8 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - [tox] +minversion = 2.4 envlist = py27, py34, py35, py36, pypy [testenv] -commands = python setup.py test deps = pytest +extras = pandas +commands = python setup.py test From c650b67e064573e49c4a2529c32309e8fc974fa3 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Tue, 1 Jan 2019 10:32:08 -0800 Subject: [PATCH 38/40] Enable pip cache in Travis CI Reduce load on PyPI servers and slightly speed up builds. For more information, see: https://docs.travis-ci.com/user/caching/#pip-cache --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3f8730d..53af531 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +cache: pip python: - 2.7 - 3.4 From 499ce52304b8994500ba0e81bd23d9ac6a2038cf Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Tue, 1 Jan 2019 10:40:29 -0800 Subject: [PATCH 39/40] Remove unused compat entries Organize both the Python2 & Python3 sections in the same order so they are easier to compare. Removed: - basestring - ifilter - bytes --- tablib/compat.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tablib/compat.py b/tablib/compat.py index d60ce72..2f97650 100644 --- a/tablib/compat.py +++ b/tablib/compat.py @@ -22,19 +22,14 @@ except ImportError: if is_py3: from io import BytesIO - from itertools import zip_longest as izip_longest - from statistics import median + from io import StringIO from tablib.packages import markup3 as markup + from statistics import median + from itertools import zip_longest as izip_longest + import csv import tablib.packages.dbfpy3 as dbfpy - import csv - from io import StringIO - # py3 mappings - - ifilter = filter unicode = str - bytes = bytes - basestring = str xrange = range else: @@ -42,8 +37,7 @@ else: from cStringIO import StringIO from tablib.packages import markup from tablib.packages.statistics import median - from itertools import ifilter, izip_longest - + from itertools import izip_longest import unicodecsv as csv import tablib.packages.dbfpy as dbfpy From 102073c426d7a76384e75379abbf1a3b0ed78662 Mon Sep 17 00:00:00 2001 From: Mathias Loesch Date: Wed, 23 Jan 2019 14:18:01 +0100 Subject: [PATCH 40/40] Add Jira table export --- README.rst | 1 + tablib/core.py | 8 +++++++- tablib/formats/__init__.py | 3 ++- tablib/formats/_jira.py | 39 ++++++++++++++++++++++++++++++++++++++ test_tablib.py | 19 +++++++++++++++++++ 5 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 tablib/formats/_jira.py diff --git a/README.rst b/README.rst index 6258001..d9190bb 100644 --- a/README.rst +++ b/README.rst @@ -23,6 +23,7 @@ Output formats supported: - YAML (Sets + Books) - Pandas DataFrames (Sets) - HTML (Sets) +- Jira (Sets) - TSV (Sets) - ODS (Sets) - CSV (Sets) diff --git a/tablib/core.py b/tablib/core.py index 8c49f2e..3dc23ea 100644 --- a/tablib/core.py +++ b/tablib/core.py @@ -631,7 +631,6 @@ class Dataset(object): """ pass - @property def latex(): """A LaTeX booktabs representation of the :class:`Dataset` object. If a @@ -641,6 +640,13 @@ class Dataset(object): """ pass + @property + def jira(): + """A Jira table representation of the :class:`Dataset` object. + + .. note:: This method can be used for export only. + """ + pass # ---- # Rows diff --git a/tablib/formats/__init__.py b/tablib/formats/__init__.py index 3eb747e..418e607 100644 --- a/tablib/formats/__init__.py +++ b/tablib/formats/__init__.py @@ -15,5 +15,6 @@ from . import _dbf as dbf from . import _latex as latex from . import _df as df from . import _rst as rst +from . import _jira as jira -available = (json, xls, yaml, csv, dbf, tsv, html, latex, xlsx, ods, df, rst) +available = (json, xls, yaml, csv, dbf, tsv, html, jira, latex, xlsx, ods, df, rst) diff --git a/tablib/formats/_jira.py b/tablib/formats/_jira.py new file mode 100644 index 0000000..55fce52 --- /dev/null +++ b/tablib/formats/_jira.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +"""Tablib - Jira table export support. + + Generates a Jira table from the dataset. +""" +from tablib.compat import unicode + +title = 'jira' + + +def export_set(dataset): + """Formats the dataset according to the Jira table syntax: + + ||heading 1||heading 2||heading 3|| + |col A1|col A2|col A3| + |col B1|col B2|col B3| + + :param dataset: dataset to serialize + :type dataset: tablib.core.Dataset + """ + + header = _get_header(dataset.headers) if dataset.headers else '' + body = _get_body(dataset) + return '%s\n%s' % (header, body) if header else body + + +def _get_body(dataset): + return '\n'.join([_serialize_row(row) for row in dataset]) + + +def _get_header(headers): + return _serialize_row(headers, delimiter='||') + + +def _serialize_row(row, delimiter='|'): + return '%s%s%s' % (delimiter, + delimiter.join([unicode(item) if item else ' ' for item in row]), + delimiter) diff --git a/test_tablib.py b/test_tablib.py index b945a2d..57b1b39 100755 --- a/test_tablib.py +++ b/test_tablib.py @@ -316,6 +316,23 @@ class TablibTestCase(unittest.TestCase): self.assertEqual(html, d.html) + def test_jira_export(self): + + expected = """||first_name||last_name||gpa|| +|John|Adams|90| +|George|Washington|67| +|Thomas|Jefferson|50|""" + self.assertEqual(expected, self.founders.jira) + + def test_jira_export_no_headers(self): + self.assertEqual('|a|b|c|', tablib.Dataset(['a', 'b', 'c']).jira) + + def test_jira_export_none_and_empty_values(self): + self.assertEqual('| | |c|', tablib.Dataset(['', None, 'c']).jira) + + def test_jira_export_empty_dataset(self): + self.assertTrue(tablib.Dataset().jira is not None) + def test_latex_export(self): """LaTeX export""" @@ -399,6 +416,7 @@ class TablibTestCase(unittest.TestCase): data.xlsx data.ods data.html + data.jira data.latex data.df data.rst @@ -421,6 +439,7 @@ class TablibTestCase(unittest.TestCase): data.xlsx data.ods data.html + data.jira data.latex data.rst