修訂 | 26c13ecbcedd69a4e8b686d7a015af353eabceef (tree) |
---|---|
時間 | 2022-06-21 13:48:50 |
作者 | rdnielsen |
Commiter | rdnielsen |
Modified import of Feather and Parquet data to replace 'nan' values in a pandas DataFrame with 'None' values in a Python list.
@@ -1688,14 +1688,17 @@ | ||
1688 | 1688 | single: Exporting; feather |
1689 | 1689 | |
1690 | 1690 | FEATHER |
1691 | - The feather binary file format established by the `Apache Arrow | |
1691 | + The Feather binary file format established by the `Apache Arrow | |
1692 | 1692 | <https://arrow.apache.org/>`_ project. The "APPEND" and "DESCRIPTION" |
1693 | 1693 | keywords are ignored when this format is used. Exporting data in |
1694 | 1694 | this format requires that the entire data set be first converted |
1695 | 1695 | to a `pandas <https://pandas.pydata.org/>`_ data frame in memory, |
1696 | 1696 | so there is a system-specific limit to the size of the data set |
1697 | 1697 | that can be exported in this format. The *feather* and *pandas* |
1698 | - libraries must be installed to export data in the feather format. | |
1698 | + libraries must be installed to export data in the Feather format. | |
1699 | + Not all data types that may be present in a database can necessarily | |
1700 | + be exported to a Feather data file, so some data types, like | |
1701 | + timestamps, may have to be converted to character data before export. | |
1699 | 1702 | Data exported in feather format cannot be written into a zipfile. |
1700 | 1703 | |
1701 | 1704 | .. index:: |
@@ -27,12 +27,12 @@ | ||
27 | 27 | # |
28 | 28 | # =============================================================================== |
29 | 29 | |
30 | -__version__ = "1.101.1" | |
31 | -__vdate = "2022-06-18" | |
30 | +__version__ = "1.101.2" | |
31 | +__vdate = "2022-06-20" | |
32 | 32 | |
33 | 33 | primary_vno = 1 |
34 | 34 | secondary_vno = 101 |
35 | -tertiary_vno = 1 | |
35 | +tertiary_vno = 2 | |
36 | 36 | |
37 | 37 | import os |
38 | 38 | import os.path |
@@ -13854,12 +13854,14 @@ | ||
13854 | 13854 | |
13855 | 13855 | def import_feather(db, schemaname, tablename, filename, is_new): |
13856 | 13856 | try: |
13857 | + import numpy as np | |
13857 | 13858 | import pandas as pd |
13858 | 13859 | import pyarrow.feather |
13859 | 13860 | except: |
13860 | 13861 | raise ErrInfo("exception", exception_msg=exception_desc(), other_msg=u"The pandas and pyarrow Python libraries must be installed to import data from the Feather format.") |
13861 | 13862 | #df = pyarrow.feather.read_feather(filename) |
13862 | 13863 | df = pd.read_feather(filename) |
13864 | + df = df.replace({np.nan: None}) | |
13863 | 13865 | hdrs = df.columns.values.tolist() |
13864 | 13866 | data = df.values.tolist() |
13865 | 13867 | import_data_table(db, schemaname, tablename, is_new, hdrs, data) |
@@ -13867,10 +13869,12 @@ | ||
13867 | 13869 | |
13868 | 13870 | def import_parquet(db, schemaname, tablename, filename, is_new): |
13869 | 13871 | try: |
13872 | + import numpy as np | |
13870 | 13873 | import pandas as pd |
13871 | 13874 | except: |
13872 | 13875 | raise ErrInfo("exception", exception_msg=exception_desc(), other_msg=u"The pandas and fastparquet or pyarrow Python libraries must be installed to import data from the Parquet format.") |
13873 | 13876 | df = pd.read_parquet(filename) |
13877 | + df = df.replace({np.nan: None}) | |
13874 | 13878 | hdrs = df.columns.values.tolist() |
13875 | 13879 | data = df.values.tolist() |
13876 | 13880 | import_data_table(db, schemaname, tablename, is_new, hdrs, data) |
@@ -5,7 +5,7 @@ | ||
5 | 5 | long_description = f.read() |
6 | 6 | |
7 | 7 | setuptools.setup(name='execsql', |
8 | - version='1.101.1', | |
8 | + version='1.101.2', | |
9 | 9 | description="Runs a SQL script against a PostgreSQL, MS-Access, SQLite, MS-SQL-Server, MySQL, MariaDB, Firebird, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables. Data can be exported in 18 different formats, including CSV, TSV, ODS, HTML, JSON, LaTeX, and Markdown tables, and using custom templates.", |
10 | 10 | author='Dreas Nielsen', |
11 | 11 | author_email='dreas.nielsen@gmail.com', |